#!/usr/bin/env python ################################################################################ # # Copyright (c) 2017, Perforce Software, Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # DATE # # $Date: 2017/11/30 $ # # SYNOPSIS # # IntegHistory.py [-p p4port] [-u p4user] [-c p4client] depotFile[revRange] # * depotFile must be in depot syntax # * depotFile with spaces requires quote # * wildcards and revision specifiers are supported # # DESCRIPTION # # This script collects the full integration history of files. # The result is stored in a gzipped file (IntegHistory.log.gz) that can # be used by IntegHistoryRebuild.py to rebuild a partial Helix server. # # REQUIREMENT # # Command line p4 2013.2 or greater (check "p4 -V") # ################################################################################ from __future__ import print_function from collections import OrderedDict, deque import subprocess import argparse import ast import sys import re import gzip import shutil import os python3 = sys.version_info[0] >= 3 minP4Version = 2013.2 class IntegHistory: def __init__(self, depotFiles, logName, verbose = False): self.depotFiles = depotFiles self.logName = logName self.caseHandling = b"insensitive" self.isSuper = False self.verbose = verbose self.files = [] self.integed = [] self.depots = {} self.streams = [] self.changes = {} self.depotRE = re.compile(b'\/\/(.*?)/.*') self.filelog1RE = re.compile(b'\.\.\. #(\d+) change (\d+)') self.filelog2RE = re.compile(b'\.\.\. \.\.\. .* (\/\/.*?)#') try: self.log = open(logName, "w") # for Python 2 compatibility purpose self.output = getattr(self.log, 'buffer', self.log) except IOError: sys.exit("Error: can\'t create " + logName) try: self.output.write(b"['files', '") for i in range(len(self.depotFiles)): self.output.write(self.depotFiles[i]) if i+1 < len(self.depotFiles): self.output.write(b"', '") self.output.write(b"']\n") except IOError: sys.exit("Error: can\'t write in " + logName) match = re.match(b'.*\/(\d+.\d)\/.*',self.runCmd(b'p4 -V', False)[-1]) if match and float(match.group(1)) < minP4Version: sys.exit("Error: minimum supported P4 version is " + str(minP4Version)) def runCmd(self, cmd, toLog = True): tagged = False if re.search(b'-ztag', cmd): tagged = True filtered = False if re.search(b'-ztag -F', cmd): filtered = True result = [] if toLog: result.append(cmd) if sys.platform == 'win32': cmd = cmd.decode() process = subprocess.Popen(cmd, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE) (output, err) = process.communicate() exitCode = process.wait() if self.verbose: sys.stderr.write(cmd + "\n") if output: sys.stderr.write(output) if err: sys.stderr.write(err) if exitCode == 0 and output: if filtered or not tagged: result = result + list(output.splitlines()) else: out = deque(output.splitlines()) line = out.popleft() block = line while out: nextLine = out.popleft() match = re.search(b'^\.\.\. ', nextLine) while out and not match: block = block + b"\\n" + nextLine nextLine = out.popleft() match = re.search(b'^\.\.\. ', nextLine) result.append(block) line = nextLine block = line if toLog == True: if result: self.output.write(b"[") for i,item in enumerate(result): self.output.write(b"'") self.output.write(item.replace(b"\\",b"\\\\").replace(b"\\\\n",b"\\n").replace(b"'",b"\\'")) self.output.write(b"'") if i < len(result)-1: self.output.write(b", ") self.output.write(b"]\n") return(result) def getConfigData(self): for line in self.runCmd(b'p4 -ztag info'): match = re.search(b'\.\.\. caseHandling (.*)', line.strip()) if match: self.caseHandling = match.group(1) break self.runCmd(b'p4 set') self.runCmd(b'p4 -ztag user -o') self.runCmd(b'p4 -ztag client -o') self.runCmd(b'p4 -ztag protects') if re.search(b'super', self.runCmd(b"p4 -ztag -F%permMax% protects -m")[-1]): self.isSuper = True self.runCmd(b'p4 -ztag configure show') self.runCmd(b'p4 counters -e change') self.runCmd(b'p4 counters -e traits') def getFilesHistory(self, depotFile): if (self.caseHandling == b"insensitive" and depotFile.lower() not in map(lambda name: name.lower(), self.files)) or \ (self.caseHandling == b"sensitive" and depotFile not in self.files): filelog = self.runCmd(b'p4 filelog -1 -l -t "' + depotFile + b'"', False) if len(filelog) > 2: self.files.append(depotFile) match = self.depotRE.match(depotFile) if match: depot = match.group(1) if depot not in self.depots: streamDepth = None result = self.runCmd(b'p4 -ztag -F "%Type% %StreamDepth%" depot -o "' + depot + b'"', False) if result: if result[0][0:6] == "stream": if len(result[0]) > 7: streamDepth = result[0][7:] else: # for release with stream depot and no StreamDepth field streamDepth = "//" + depot + "/1" self.depots[depot] = streamDepth else: streamDepth = self.depots[depot] if streamDepth: stream = b"" nbSlash = 0 for i, char in enumerate(depotFile): if (not python3 and char == '/') or (python3 and char == 47): nbSlash += 1 if nbSlash > streamDepth.count(b'/'): stream = depotFile[:i] break if stream not in self.streams: self.streams.append(stream) for line in filelog: match = self.filelog1RE.match(line.strip()) if match: if not match.group(2) in self.changes: self.changes[match.group(2)] = [] self.changes[match.group(2)].append(depotFile + b"#" + match.group(1)) else: match = self.filelog2RE.match(line.strip()) if match: self.getFilesHistory(match.group(1)) def getDepotsData(self): for depot in self.depots: self.runCmd(b'p4 -ztag depot -o "' + depot + b'"') def getStreamsData(self): for stream in self.streams: if self.runCmd(b'p4 streams -F Stream="' + stream + b'"', False): result = self.runCmd(b'p4 -ztag -F %Parent% stream -o "' + stream + b'"', False) if result: self.addParentStream(result[0]) else: self.streams.remove(stream) for stream in self.streams: self.runCmd(b'p4 -ztag stream -o "' + stream + b'"') self.runCmd(b'p4 -ztag istat -s "' + stream + b'"') def addParentStream(self, stream): if stream not in self.streams and not stream == "none": self.streams.append(stream) result = self.runCmd(b'p4 -ztag -F %Parent% stream -o "' + stream + b'"', False) if result: self.addParentStream(result[0]) def getFilesData(self): flags = b"-Ol -Oi" if self.isSuper: flags = flags + b" -Oc" for change in self.changes: for depotFile in self.changes[change]: if python3: filename = bytearray(depotFile)[0:depotFile.find(b"#")] else: filename = depotFile[0:depotFile.find(b"#")] self.runCmd(b'p4 -ztag fstat ' + flags + b' "' + filename + b'@=' + change + b'"') def getIntegedData(self): for change in self.changes: for depotFile in self.changes[change]: if python3: filename = bytearray(depotFile)[0:depotFile.find(b"#")] else: filename = depotFile[0:depotFile.find(b"#")] for depotFile in self.runCmd(b'p4 -ztag -F%depotFile% files "' + filename + b'@=' + change + b'"', False): if depotFile not in self.integed: self.runCmd(b'p4 -ztag integed "' + depotFile + b'"') self.integed.append(depotFile) def getChangesData(self): for change in self.changes: self.runCmd(b'p4 -ztag changes -l @=' + change) def getData(self): self.getConfigData() for depotFile in self.depotFiles: for df in self.runCmd(b'p4 -ztag -F%depotFile% files "' + depotFile + b'"', False): self.getFilesHistory(df) if len(self.files) == 0: sys.exit("Error: No file found, check " + self.logName) self.changes = OrderedDict(sorted(self.changes.items(), key=lambda t: t[0])) self.getDepotsData() self.getStreamsData() self.getFilesData() self.getIntegedData() self.getChangesData() self.log.close() try: with open(self.logName, 'rb') as uncompressed: with gzip.open(self.logName + ".gz", 'wb') as compressed: shutil.copyfileobj(uncompressed, compressed) if os.path.isfile(self.logName): os.remove(self.logName) return self.logName + ".gz" except: return self.logName def setP4Env(p4port, p4user, p4client): output = subprocess.check_output(["p4", "set"]) for line in output.splitlines(): match = re.search(b"P4PORT=(.*?) .*", line) if match: if not p4port: p4port = match.group(1) else: match = re.search(b"P4USER=(.*?) .*", line) if match: if not p4user: p4user = match.group(1) else: match = re.search(b"P4CLIENT=(.*?) .*", line) if match: if not p4client: p4client = match.group(1) os.environ["P4CONFIG"] = "" if p4port: os.environ["P4PORT"] = p4port if p4user: os.environ["P4USER"] = p4user if p4client: os.environ["P4CLIENT"] = p4client def main(): parser = argparse.ArgumentParser(prog='IntegHistory', usage='%(prog)s [-v] [-P p4_path] [-p p4port] [-u p4user] [-c p4client] depotFile[revRange] ...') if python3: parser.add_argument("depotFiles", nargs="*", help="log or compressed log", type=os.fsencode) else: parser.add_argument("depotFiles", nargs="*", help="log or compressed log") parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true") parser.add_argument("-P", "--path", dest="path", help="P4 path value") parser.add_argument("-p", "--p4port", dest="p4port", help="server value") parser.add_argument("-u", "--p4user", dest="p4user", help="user value") parser.add_argument("-c", "--p4client", dest="p4client", help="client value") args = parser.parse_args() if not args.depotFiles: sys.exit(parser.print_help()) for depotFile in args.depotFiles: if not re.search(b'\/\/.*', depotFile): sys.exit(parser.print_help()) if args.p4port or args.p4user or args.p4client: setP4Env(args.p4port, args.p4user, args.p4client) if args.path: os.environ["PATH"] = args.path + ":" + os.environ["PATH"] logName = os.path.abspath(os.getcwd() + "/IntegHistory.log") history = IntegHistory(args.depotFiles, logName, args.verbose) print("Info: " + history.getData() + " created successfully") if __name__ == '__main__': main()