#!/usr/bin/env python ################################################################################ # # Copyright (c) 2017, Perforce Software, Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # DATE # # $Date: 2017/07/31 $ # # SYNOPSIS # # IntegHistory.py [-p p4port] [-u p4user] [-c p4client] depotFile[revRange] # * depotFile must be in depot syntax # * depotFile with spaces requires quote # * wildcards and revision specifiers are supported # # DESCRIPTION # # This script collects the full integration history of files. # The result is stored in a gzipped file (IntegHistory.log.gz) that can # be used by IntegHistoryRebuild.py to rebuild a partial Helix server. # ################################################################################ from __future__ import print_function from collections import OrderedDict, deque import subprocess import argparse import ast import sys import re import gzip import shutil import os python3 = sys.version_info[0] >= 3 class IntegHistory: def __init__(self, depotFiles, logName): self.depotFiles = depotFiles self.logName = logName self.caseHandling = b"insensitive" self.isSuper = False self.files = [] self.integed = [] self.depots = {} self.streams = [] self.changes = {} self.depotRE = re.compile(b'\/\/(.*?)/.*') self.filelog1RE = re.compile(b'\.\.\. #(\d+) change (\d+)') self.filelog2RE = re.compile(b'\.\.\. \.\.\. .* (\/\/.*?)#') try: self.log = open(logName, "w") # In order to output bytes to stdout, sys.stdout.buffer must be used in Python 3 # but this is not compatible with Python 2, so to make it compatible: self.output = getattr(self.log, 'buffer', self.log) except IOError: sys.exit("Error: can\'t create " + logName) try: self.output.write(b"['files', '") for i in range(len(self.depotFiles)): self.output.write(self.depotFiles[i]) if i+1 < len(self.depotFiles): self.output.write("', '") self.output.write(b"']\n") except IOError: sys.exit("Error: can\'t write in " + logName) def runCmd(self, cmd, toLog = True): tagged = False if re.search(b'-ztag', cmd): tagged = True filtered = False if re.search(b'-ztag -F', cmd): filtered = True result = [] if toLog: result.append(cmd) process = subprocess.Popen(cmd, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE) (output, err) = process.communicate() exitCode = process.wait() if exitCode == 0 and output: if filtered or not tagged: result = result + list(output.splitlines()) else: out = deque(output.splitlines()) line = out.popleft() block = line while out: nextLine = out.popleft() match = re.search(b'^\.\.\. ', nextLine) while out and not match: block = block + b"\\n" + nextLine nextLine = out.popleft() match = re.search(b'^\.\.\. ', nextLine) result.append(block) line = nextLine block = line if toLog == True: if result: self.output.write(b"[") for i,item in enumerate(result): self.output.write(b"'") self.output.write(item.replace(b"'",b"\\'")) self.output.write(b"'") if i < len(result)-1: self.output.write(b", ") self.output.write(b"]\n") return(result) def getConfigData(self): for line in self.runCmd(b'p4 -ztag info'): match = re.search(b'\.\.\. caseHandling (.*)', line.strip()) if match: self.caseHandling = match.group(1) break self.runCmd(b'p4 set') self.runCmd(b'p4 -ztag user -o') self.runCmd(b'p4 -ztag client -o') self.runCmd(b'p4 -ztag protects') if self.runCmd(b"p4 -ztag -F%permMax% protects -m")[0] == "super": self.isSuper = True self.runCmd(b'p4 -ztag configure show') self.runCmd(b'p4 counters -e change') self.runCmd(b'p4 counters -e traits') def getFilesHistory(self, depotFile): if (self.caseHandling == b"insensitive" and depotFile.lower() not in map(lambda name: name.lower(), self.files)) or \ (self.caseHandling == b"sensitive" and depotFile not in self.files): filelog = self.runCmd(b'p4 filelog -1 -l -t "' + depotFile + b'"', False) if len(filelog) > 2: self.files.append(depotFile) match = self.depotRE.match(depotFile) if match: depot = match.group(1) if depot not in self.depots: streamDepth = None result = self.runCmd(b'p4 -ztag -F%StreamDepth% depot -o "' + depot + b'"', False) if result: streamDepth = result[0] self.depots[depot] = streamDepth else: streamDepth = self.depots[depot] if streamDepth: i = n = 0 stream = b"" while n < streamDepth.count(b'/'): i = i + 1 if depotFile[i] == b"/": n = n + 1 stream = depotFile[:i] if stream not in self.streams: self.streams.append(stream) for line in filelog: match = self.filelog1RE.match(line.strip()) if match: if not match.group(2) in self.changes: self.changes[match.group(2)] = [] self.changes[match.group(2)].append(depotFile + b"#" + match.group(1)) else: match = self.filelog2RE.match(line.strip()) if match: self.getFilesHistory(match.group(1)) def getDepotsData(self): for depot in self.depots: self.runCmd(b'p4 -ztag depot -o "' + depot + b'"') def getStreamsData(self): for stream in self.streams: if self.runCmd(b'p4 streams -F Stream="' + stream + b'"', False)[0]: result = self.runCmd(b'p4 -ztag -F %Parent% stream -o "' + stream + b'"', False) if result: self.addParentStream(result[0]) else: self.streams.remove(stream) for stream in self.streams: self.runCmd(b'p4 -ztag stream -o "' + stream + b'"') self.runCmd(b'p4 -ztag istat -s "' + stream + b'"') def addParentStream(self, stream): if stream not in self.streams and not stream == "none": self.streams.append(stream) result = self.runCmd(b'p4 -ztag -F %Parent% stream -o "' + stream + b'"', False) if result: self.addParentStream(result[0]) def getFilesData(self): flags = b"-Ol -Oi" if self.isSuper: flags = flags + b" -Oc" for change in self.changes: for depotFile in self.changes[change]: if python3: filename = bytearray(depotFile)[0:depotFile.find(b"#")] else: filename = depotFile[0:depotFile.find(b"#")] self.runCmd(b'p4 -ztag fstat ' + flags + b' "' + filename + b'@=' + change + b'"') def getIntegedData(self): for change in self.changes: for depotFile in self.changes[change]: if python3: filename = bytearray(depotFile)[0:depotFile.find(b"#")] else: filename = depotFile[0:depotFile.find(b"#")] for depotFile in self.runCmd(b'p4 -ztag -F%depotFile% files "' + filename + b'@=' + change + b'"', False): if depotFile not in self.integed: self.runCmd(b'p4 -ztag integed "' + depotFile + b'"') self.integed.append(depotFile) def getChangesData(self): for change in self.changes: self.runCmd(b'p4 -ztag changes -l @=' + change) def getData(self): self.getConfigData() for depotFile in self.depotFiles: for df in self.runCmd(b'p4 -ztag -F%depotFile% files "' + depotFile + b'"', False): self.getFilesHistory(df) if len(self.files) == 0: sys.exit("Error: No file found, check " + self.logName) self.changes = OrderedDict(sorted(self.changes.items(), key=lambda t: t[0])) self.getDepotsData() self.getStreamsData() self.getFilesData() self.getIntegedData() self.getChangesData() self.log.close() try: with open(self.logName, 'rb') as uncompressed: with gzip.open(self.logName + ".gz", 'wb') as compressed: shutil.copyfileobj(uncompressed, compressed) os.remove(self.logName) return self.logName + ".gz" except: return self.logName def setP4Env(p4port, p4user, p4client): output = subprocess.check_output(["p4", "set"]) for line in output.splitlines(): match = re.search(b"P4PORT=(.*?) .*", line) if match: if not p4port: p4port = match.group(1) else: match = re.search(b"P4USER=(.*?) .*", line) if match: if not p4user: p4user = match.group(1) else: match = re.search(b"P4CLIENT=(.*?) .*", line) if match: if not p4client: p4client = match.group(1) os.environ["P4CONFIG"] = "" os.environ["P4PORT"] = p4port os.environ["P4USER"] = p4user os.environ["P4CLIENT"] = p4client def cmdUsage(): sys.exit('Usage: -p p4port -u p4user -c p4client depotfile[revRange] ...') def main(): parser = argparse.ArgumentParser(prog='IntegHistory', usage='%(prog)s -[-p p4port] [-u p4user] [-c p4client] depotFile[revRange] ...') if python3: parser.add_argument("depotFiles", nargs="*", help="log or compressed log", type=os.fsencode) else: parser.add_argument("depotFiles", nargs="*", help="log or compressed log") parser.add_argument("-p", "--p4port", dest="p4port", help="server value") parser.add_argument("-u", "--p4user", dest="p4user", help="user value") parser.add_argument("-c", "--p4client", dest="p4client", help="client value") args = parser.parse_args() for depotFile in args.depotFiles: if not re.search(b'\/\/.*', depotFile): cmdUsage() if args.p4port or args.p4user or args.p4client: setP4Env(args.p4port, args.p4user, args.p4client) logName = os.path.abspath(os.getcwd() + "/IntegHistory.log") history = IntegHistory(args.depotFiles, logName) print("Info: " + history.getData() + " created successfully") if __name__ == '__main__': main()
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#43 | 30126 | Pascal Soccard |
All the backslashes need to be escaped to prevent eval issues when running the IntegHistoryRebuild.py script |
||
#42 | 30088 | Pascal Soccard | Reworked how to escape \n as a user can be set to "ABC\nick" | ||
#41 | 30087 | Pascal Soccard |
Reverted partially change 25690 as from my latest tests, the & character does not need to be escaped. |
||
#40 | 29853 | Pascal Soccard |
For server version > 2016.2, use "p4 fstat -Ob" instead of "p4 fstat -Oc" |
||
#39 | 29793 | Pascal Soccard |
Removed obfuscated mode as it is too difficult to analyse integration issue with obfuscated data. |
||
#38 | 29474 | Pascal Soccard |
Fixed branch argument for Python 3 (TypeError: can't concat str to bytes) |
||
#37 | 28877 | Pascal Soccard | Added feature for collecting a branch spec | ||
#36 | 28689 | Pascal Soccard |
Added support for "p4 -ztag -zconfigurables info". The -zconfigurables option was introduced with 2021.2 and exposes some configurables without having super privileges. This is useful to get the dm.integ configurables which are not exposed by "p4 -ztag info" like the dm.integ.undo configurable. |
||
#35 | 28610 | Pascal Soccard |
Added experimental quick mode to exclude "into" integration data. In order to collect the full integration history when using this mode, it is Requires to run "IntegHistory.py -q //source //target |
||
#34 | 28488 | Pascal Soccard | Fixed obfuscate issues | ||
#33 | 27889 | Pascal Soccard |
Fixed db.storage records population (maxCommitChange needs to be set) and use wrong "p4 storage" flag. |
||
#32 | 27816 | Pascal Soccard |
Replay could have failed if the user's client was refeirring to a stream unrelated to the involved files |
||
#31 | 26535 | Pascal Soccard | Fixed missing b"" in previous change | ||
#30 | 26504 | Pascal Soccard | Only show the "no file(s) integrated" error in verbose mode | ||
#29 | 26424 | Pascal Soccard | Fixed istat error for ("is a virtual stream") | ||
#28 | 26280 | Pascal Soccard | Fixed missing byte string conversion | ||
#27 | 26261 | Pascal Soccard | Fixed highest privilege level detection | ||
#26 | 26260 | Pascal Soccard |
Reworked errors sent to stderr due to differences between Python 2 and Python 3 on Windows |
||
#25 | 26195 | Pascal Soccard | Retrieve depot/stream data for stream import/import+ mappings | ||
#24 | 26113 | Pascal Soccard |
Reworked how virtual stream spec is collected Errors are now always reported (not just in verbose mode) |
||
#23 | 25729 | Pascal Soccard | Fixed encoding issue specific to Windows | ||
#22 | 25690 | Pascal Soccard |
Fixed "p4 streams" filtering issue when a stream name includes the following characters: ( ) & These characters needs to be escaped in the filter string. |
||
#21 | 25055 | Pascal Soccard |
Added a new user data Obfuscate feature (-o flag) An IntegHistoryObfuscated.txt.gz file is created which list how the depot file names have been converted (to be kept by the user running the IntegHistory.py scritp) |
||
#20 | 24437 | Pascal Soccard | Fixed -v output issue for Python 3 | ||
#19 | 23437 | Pascal Soccard | Fixed stream identification problem affecting Windows | ||
#18 | 23300 | Pascal Soccard | Fixed incorrect stream depth command | ||
#17 | 22964 | Pascal Soccard |
Added new options (verbosity and P4 path) Fixed StreamDepth check for older version Escaped \ for Windows path |
||
#16 | 22883 | Pascal Soccard | More Windows and Python 2 vs Python 3 fixes | ||
#15 | 22882 | Pascal Soccard |
Reintroduced p4 version check and added fix for Popen; on Windows it requires a string rather than a bytes which works fine on Linux |
||
#14 | 22880 | Pascal Soccard | Reverted p4 version check as it was causing an issue with Python 3.5.3 | ||
#13 | 22878 | Pascal Soccard | Fixed super user detection again | ||
#12 | 22877 | Pascal Soccard | Fixed Python 2.7 version check problem introduced by previous change | ||
#11 | 22876 | Pascal Soccard | Added minimum p4 version check (>=2013.2) | ||
#10 | 22867 | Pascal Soccard | Fixed check for super privileges detection | ||
#9 | 22630 | Pascal Soccard |
Fixed variable spelling mistake in a use case I have not tested after previous change (multiple files arguments). |
||
#8 | 22577 | Pascal Soccard |
Using bytes rather than string, so the script can now deal properly with non utf8 characters. IntegHistoryRebuild.py run with Python 3 will still fail when non utf8 characters are encountered. Use Python 2.7 until I have figured out how to fix this issue. |
||
#7 | 22469 | Pascal Soccard |
Added support for task stream Added support for undo actions |
||
#6 | 22353 | Pascal Soccard | Fixed previous change; parent stream were not processed properly | ||
#5 | 22323 | Pascal Soccard |
Reworked script to handle lbr values for branched move/delete db.rev records as "p4 fstat -Oc" does't provide these data. Fixed use of stream depth which was not taking into account. |
||
#4 | 22291 | Pascal Soccard | Added new P4 environment variables for command options | ||
#3 | 19319 | Pascal Soccard |
Moved recently added compressed code out of main(), so it can be called by IntegHistoryRebuild.py |
||
#2 | 19275 | Pascal Soccard | Changed compression method as it was causing some issues on Windows | ||
#1 | 18490 | Pascal Soccard |
This tool recreates a partial Helix server (metadata and dummy depot files) using data collected from a Helix server using p4 commands. |