#!/usr/bin/env python ################################################################################ # # Copyright (c) 2020, Perforce Software, Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # DATE # # $Date: 2021/06/17 $ # # SYNOPSIS # # IntegHistory.py [-v] [-o] [-p p4port] [-u p4user] [-c p4client] depotFile(s)[revRange] # * depotFile must be in depot syntax # * depotFile with spaces requires quote # * multiple depotFiles allowed # * wildcards and revision specifiers are supported # # DESCRIPTION # # This script collects the full integration history of files. # The result is stored in a gzipped file (IntegHistory.log.gz) that can # be used by IntegHistoryRebuild.py to rebuild a partial Helix server. # # REQUIREMENT # # Command line p4 2013.2 or greater (check "p4 -V") # ################################################################################ from __future__ import print_function from collections import OrderedDict, deque import subprocess import argparse import ast import sys import re import gzip import shutil import os import random import string python3 = sys.version_info[0] >= 3 minP4Version = 2013.2 class IntegHistory: def __init__(self, depotFiles, logName, verbose = False, obfuscate = False): self.depotFiles = depotFiles self.logName = logName self.caseHandling = b"insensitive" self.isSuper = False self.verbose = verbose self.obfuscate = obfuscate self.obfuscateDict = {} self.files = [] self.integed = [] self.depots = {} self.streams = [] self.changes = {} self.depotRE = re.compile(b'\/\/(.*?)/.*') self.filelog1RE = re.compile(b'\.\.\. #(\d+) change (\d+)') self.filelog2RE = re.compile(b'\.\.\. \.\.\. .* (\/\/.*?)#') self.importRE = re.compile(b'... Paths\d+ import.* (.*)') try: self.log = open(logName, "w") # for Python 2 compatibility purpose self.output = getattr(self.log, 'buffer', self.log) except IOError: sys.exit("Error: can\'t create " + logName) try: self.output.write(b"['files', '") for i in range(len(self.depotFiles)): depotFile = self.depotFiles[i] if self.obfuscate: depotFile = self.obfuscateFilepath(depotFile) self.output.write(depotFile) if i+1 < len(self.depotFiles): self.output.write(b"', '") self.output.write(b"']\n") except IOError: sys.exit("Error: can\'t write in " + logName) match = re.match(b'.*\/(\d+.\d)\/.*',self.runCmd(b'p4 -V', False)[-1]) if match and float(match.group(1)) < minP4Version: sys.exit("Error: minimum supported P4 version is " + str(minP4Version)) def obfuscateSave(self, filename): with gzip.open(filename + ".gz", 'wb') as gzfile: for depotFile in self.files: listStr = [] for str in depotFile.split('/'): if not str == "": listStr.append(self.obfuscateDict[str]) else: listStr.append("") gzfile.write(depotFile + " <=> " + '/'.join(listStr) + "\n") def obfuscateFilepath(self, filepath): strList = [] for str in filepath.split('/'): obfuscatedStr = "" if not str == "": if str in self.obfuscateDict: obfuscatedStr = self.obfuscateDict[str] else: obfuscatedStr = ''.join(((random.choice(string.letters) if char.isalpha() else char) for char in str)) self.obfuscateDict[str] = obfuscatedStr strList.append(obfuscatedStr) return('/'.join(strList)) def obfuscateOutput(self, line): match = re.match(b'(... (?:desc|Description) )(.*)', line) if match: line = match.group(1) + "obfuscated description\\n" else: match = re.match(b'(... (?:clientFile|clientRoot|clientCwd|serverRoot|Root|Value) )(.*)', line) if match: line = match.group(1) + "/obfuscated/path" else: match = re.match(b'(... (?:Depot|depotFile|lbrFile|StreamDepth|stream|Stream|Name|path\d+ .*?|toFile|fromFile|path|user.*|client.*|User|Client|Owner|Email|FullName|Host|View\d+|Host|Map) )(.*)', line) if match: line = match.group(1) + self.obfuscateFilepath(match.group(2)) else: match = re.match(b'(... (?:parent|Parent|baseParent|parentBase) )(.*)', line) if match and not match.group(2) == "none": line = match.group(1) + self.obfuscateFilepath(match.group(2)) else: match = re.match(b'(p4 .* (?:depot .*|stream .*|istat .*|integed) \")(.*)(\")', line) if match: line = match.group(1) + self.obfuscateFilepath(match.group(2)) + match.group(3) else: match = re.match(b'(p4 .* fstat .* \")(.*)(@.*\")', line) if match: line = match.group(1) + self.obfuscateFilepath(match.group(2)) + match.group(3) return(line) def runCmd(self, cmd, toLog = True): # for Python 2 compatibility purpose stderr = getattr(sys.stderr, 'buffer', sys.stderr) tagged = False if re.search(b'-ztag', cmd): tagged = True filtered = False if re.search(b'-ztag -F', cmd): filtered = True result = [] if toLog: result.append(cmd) if sys.platform == 'win32': process = subprocess.Popen(cmd.decode(), shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE) else: process = subprocess.Popen(cmd, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE) (output, err) = process.communicate() exitCode = process.wait() if self.verbose or (err and not b"no file(s) integrated" in err): stderr.write(cmd + b'\n') if self.verbose: stderr.write(output) if err and (not b"no file(s) integrated" in err or self.verbose): stderr.write(b'Error: ' + err) if b'Perforce password (P4PASSWD) invalid or unset.' in err: sys.exit() if exitCode == 0 and output: if filtered or not tagged: result = result + list(output.splitlines()) else: out = deque(output.splitlines()) line = out.popleft() block = line while out: nextLine = out.popleft() match = re.search(b'^\.\.\. ', nextLine) while out and not match: block = block + b"\\n" + nextLine nextLine = out.popleft() match = re.search(b'^\.\.\. ', nextLine) result.append(block) line = nextLine block = line if toLog == True: if result: self.output.write(b'[') for i,item in enumerate(result): if self.obfuscate: item = self.obfuscateOutput(item) self.output.write(b"'") self.output.write(item.replace(b"\\",b"\\\\").replace(b"\\\\n",b"\\n").replace(b"'",b"\\'")) self.output.write(b"'") if i < len(result)-1: self.output.write(b", ") self.output.write(b"]\n") return(result) def getConfigData(self): for line in self.runCmd(b'p4 -ztag info'): match = re.search(b'\.\.\. caseHandling (.*)', line.strip()) if match: self.caseHandling = match.group(1) break self.runCmd(b'p4 set') self.runCmd(b'p4 -ztag user -o') for line in self.runCmd(b'p4 -ztag client -o'): match = re.search(b'\.\.\. Stream (.*)', line.strip()) if match: self.streams.append(match.group(1)) self.runCmd(b'p4 -ztag protects') if re.search(b'... permMax super', self.runCmd(b"p4 -ztag protects -m")[-1]): self.isSuper = True self.runCmd(b'p4 -ztag configure show') self.runCmd(b'p4 counters -e change') self.runCmd(b'p4 counters -e traits') def addParentStream(self, stream): stream = stream.replace(b"(",b"\(").replace(b")",b"\)").replace(b"&",b"\&") for parent in self.runCmd(b'p4 -F %Parent% streams -m1 -F Stream="' + stream + b'"', False): if not parent == b"none" and parent not in self.streams: self.streams.append(parent) self.addParentStream(parent) def addVirtualStream(self, stream): stream = stream.replace(b"(",b"\(").replace(b")",b"\)").replace(b"&",b"\&") for virtual in self.runCmd(b'p4 -F %Stream% streams -F "Parent=' + stream + b' & Type=virtual"', False): if virtual not in self.streams: self.streams.append(virtual) self.addVirtualStream(virtual) def getFilesHistory(self, depotFile): if (self.caseHandling == b"insensitive" and depotFile.lower() not in map(lambda name: name.lower(), self.files)) or \ (self.caseHandling == b"sensitive" and depotFile not in self.files): filelog = self.runCmd(b'p4 filelog -1 -l -t "' + depotFile + b'"', False) if len(filelog) > 2: self.files.append(depotFile) self.parseDepotFile(depotFile) for line in filelog: match = self.filelog1RE.match(line.strip()) if match: if match.group(2) not in self.changes: self.changes[match.group(2)] = [] self.changes[match.group(2)].append(depotFile + b'#' + match.group(1)) else: match = self.filelog2RE.match(line.strip()) if match: self.getFilesHistory(match.group(1)) def parseDepotFile(self, depotFile): match = self.depotRE.match(depotFile) if match: depot = match.group(1) if depot not in self.depots: streamDepth = None result = self.runCmd(b'p4 -ztag -F "%Type% %StreamDepth%" depot -o "' + depot + b'"', False) if result: if result[0][0:6] == b'stream': if len(result[0]) > 7: streamDepth = result[0][7:] else: # for release with stream depot and no StreamDepth field streamDepth = b'//' + depot + b'/1' self.depots[depot] = streamDepth else: streamDepth = self.depots[depot] if streamDepth: stream = b"" nbSlash = 0 for i, char in enumerate(depotFile): if (not python3 and char == b'/') or (python3 and char == 47): nbSlash += 1 if nbSlash > streamDepth.count(b'/'): stream = depotFile[:i] break stream = stream.replace(b"(",b"\(").replace(b")",b"\)").replace(b"&",b"\&") if stream not in self.streams and self.runCmd(b'p4 streams -F "Stream=' + stream + b'"', False): self.streams.append(stream) self.addVirtualStream(stream) self.addParentStream(stream) def getDepotsData(self): for depot in self.depots: self.runCmd(b'p4 -ztag depot -o "' + depot + b'"') def getStreamsData(self): for stream in self.streams: spec = self.runCmd(b'p4 -ztag stream -o "' + stream + b'"') if b'... Type virtual' not in spec: self.runCmd(b'p4 -ztag istat -s "' + stream + b'"') for line in spec: match = self.importRE.match(line) if match: self.parseDepotFile(match.group(1)) def getFilesData(self): flags = b'-Ol -Oi' if self.isSuper: flags = flags + b' -Oc' for change in self.changes: for depotFile in self.changes[change]: if python3: filename = bytearray(depotFile)[0:depotFile.find(b'#')] else: filename = depotFile[0:depotFile.find(b'#')] self.runCmd(b'p4 -ztag fstat ' + flags + b' "' + filename + b'@=' + change + b'"') def getIntegedData(self): for change in self.changes: for depotFile in self.changes[change]: if python3: filename = bytearray(depotFile)[0:depotFile.find(b'#')] else: filename = depotFile[0:depotFile.find(b'#')] for depotFile in self.runCmd(b'p4 -ztag -F%depotFile% files "' + filename + b'@=' + change + b'"', False): if depotFile not in self.integed: self.runCmd(b'p4 -ztag integed "' + depotFile + b'"') self.integed.append(depotFile) def getChangesData(self): for change in self.changes: self.runCmd(b'p4 -ztag changes -l @=' + change) def getData(self): self.getConfigData() for depotFile in self.depotFiles: for df in self.runCmd(b'p4 -ztag -F%depotFile% files "' + depotFile + b'"', False): self.getFilesHistory(df) if len(self.files) == 0: sys.exit("Error: No file found, check " + self.logName) self.changes = OrderedDict(sorted(self.changes.items(), key=lambda t: t[0])) self.getStreamsData() self.getDepotsData() self.getFilesData() self.getIntegedData() self.getChangesData() self.log.close() if (self.obfuscate): self.obfuscateSave("IntegHistoryObfuscated.txt") try: with open(self.logName, 'rb') as uncompressed: with gzip.open(self.logName + ".gz", 'wb') as compressed: shutil.copyfileobj(uncompressed, compressed) if os.path.isfile(self.logName): os.remove(self.logName) return self.logName + ".gz" except: return self.logName def setP4Env(p4port, p4user, p4client): output = subprocess.check_output(["p4", "set"], universal_newlines=True) for line in output.splitlines(): match = re.search("P4PORT=(.*?) .*", line) if match: if not p4port: p4port = match.group(1) else: match = re.search("P4USER=(.*?) .*", line) if match: if not p4user: p4user = match.group(1) else: match = re.search("P4CLIENT=(.*?) .*", line) if match: if not p4client: p4client = match.group(1) os.environ["P4CONFIG"] = "" if p4port: os.environ["P4PORT"] = p4port if p4user: os.environ["P4USER"] = p4user if p4client: os.environ["P4CLIENT"] = p4client def main(): parser = argparse.ArgumentParser(prog='IntegHistory', usage='%(prog)s [-v] [-o] [-P p4_path] [-p p4port] [-u p4user] [-c p4client] depotFiles(s)[revRange]') if python3: parser.add_argument("depotFiles", nargs="*", help="list of files in depot syntax separated with spaces", type=os.fsencode) else: parser.add_argument("depotFiles", nargs="*", help="list of files in depot syntax separated with spaces") parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true") parser.add_argument("-o", "--obfuscate", help="obfuscate command output", action="store_true") parser.add_argument("-P", "--path", dest="path", help="P4 path value") parser.add_argument("-p", "--p4port", dest="p4port", help="server value") parser.add_argument("-u", "--p4user", dest="p4user", help="user value") parser.add_argument("-c", "--p4client", dest="p4client", help="client value") args = parser.parse_args() if not args.depotFiles: sys.exit(parser.print_help()) for depotFile in args.depotFiles: if not re.search(b'\/\/.*', depotFile): sys.exit(parser.print_help()) if args.p4port or args.p4user or args.p4client: setP4Env(args.p4port, args.p4user, args.p4client) if args.path: os.environ["PATH"] = args.path + ":" + os.environ["PATH"] logName = os.path.abspath(os.getcwd() + "/IntegHistory.log") history = IntegHistory(args.depotFiles, logName, args.verbose, args.obfuscate) print("Info: " + history.getData() + " created successfully") if __name__ == '__main__': main()