#!/usr/bin/env python3 ################################################################################ # # Copyright (c) 2017, Perforce Software, Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # DATE # # $Date: 2018/09/04 $ # # SYNOPSIS # # IntegHistoryRebuild.py # # DESCRIPTION # # This script creates a Helix server (metadata and dummy depot files) from # data collected by the IntegHistory.py script (stored in an # IntegHistory.log.gz file). # # The script was originally designed as a support tool to recreate quickly # integration issues without the need of a checkpoint. # # The Helix server is created in the IntegHistoryRebuildRoot directory # under the current directory (automatically removed if it exists). # The created Helix server can be accessed directly via rsh using the # P4CONFIG file created in the P4ROOT directory (P4CONFIG must be set): # P4PORT = rsh:p4d -r IntegHistoryRebuildRoot -L log -vserver=3 -i # P4USER = perforce # P4CLIENT = IntegHistoryRebuild # # REQUIREMENTS # # * Python (tested with 2.7 and 3.4) # * P4Python # * Helix p4d executable in your path # * IntegHistory.py in your current directory or in your PYTHONPATH # or in your path (Linux only) # ################################################################################ from __future__ import print_function from P4 import P4, P4Exception import re import sys import os import shutil import time import mimetypes import gzip import difflib import binascii from subprocess import Popen, PIPE, STDOUT import shlex sys.path.append('.') import argparse try: import IntegHistory except ImportError: pass python3 = sys.version_info[0] >= 3 class DbSchema: def __init__(self, schema): self.table = schema["table"] self.version = schema["version"] self.name = schema["name"] self.type = schema["type"] class DbRecord: def __init__(self, schema): self.schema = schema self.row = {} for i in range(len(self.schema.name)): if self.schema.type[i] == "key" or self.schema.type[i] == "text": self.row[self.schema.name[i]] = "" else: self.row[self.schema.name[i]] = 0 def patch(self, jnl, how): if how == "put": jnl.write("@pv@ ") elif how == "replace": jnl.write("@rv@ ") elif how == "remove": jnl.write("@dv@ ") jnl.write(self.schema.version + " @" + self.schema.table + "@ ") for i in range(len(self.schema.name)): if self.schema.type[i] == "key" or self.schema.type[i] == "text": jnl.write("@") jnl.write(str(self.row[self.schema.name[i]]).replace("@","@@")) if self.schema.type[i] == "key" or self.schema.type[i] == "text": jnl.write("@ ") else: jnl.write(" ") jnl.write("\n") class DbConfig: def __init__(self, schema): self.dbConfigDbSchema = DbSchema(schema) self.dbConfigList = [] def add(self, name, value): if name not in self.dbConfigList: record = DbRecord(self.dbConfigDbSchema) record.row["CFsname"] = "any" record.row["CFname"] = name record.row["CFvalue"] = value self.dbConfigList.append(record) def remove(self, name): for i in range(len(self.dbConfigList)): if self.dbConfigList[i].row["CFname"] == name: del(self.dbConfigList[i]) break def patch(self, jnl): for dbConfig in self.dbConfigList: dbConfig.patch(jnl, "put") class DbCounters: def __init__(self, schema): self.dbCountersDbSchema = DbSchema(schema) self.dbCountersList = [] self.maxChangelist = 0 def add(self, name, value): if not name in self.dbCountersList: record = DbRecord(self.dbCountersDbSchema) record.row["COname"] = name record.row["COvalue"] = value self.dbCountersList.append(record) if name == "change": self.maxChangelist = value def patch(self, jnl): for counter in self.dbCountersList: counter.patch(jnl, "put") class DbUser: def __init__(self, schema): self.dbUserDbSchema = DbSchema(schema) self.dbUserList = [] def add(self, user, email, update, access, fullname): if not user in self.dbUserList: record = DbRecord(self.dbUserDbSchema) record.row["USuser"] = user record.row["USemail"] = email record.row["USupdate"] = update record.row["USaccess"] = access record.row["USfullname"] = fullname self.dbUserList.append(record) def patch(self, jnl): for user in self.dbUserList: user.patch(jnl, "put") class DbDepot: def __init__(self, schema): self.dbDepotDbSchema = DbSchema(schema) self.dbDepotList = [] def add(self, name, type, depth, map): if name not in self.dbDepotList: record = DbRecord(self.dbDepotDbSchema) record.row["DPname"] = name record.row["DPtype"] = type record.row["DPextra"] = depth record.row["DPmap"] = map self.dbDepotList.append(record) def getStreamPath(self, depotFile): streamPath = "" streamPath = 0 streamDepth = 0 depot = depotFile.split('/')[2] for d in self.dbDepotList: if d.row["DPname"] == depot: if d.row["DPextra"]: streamDepth = int(d.row["DPextra"]) break if streamDepth > 0: nbSlash = 0 for i, char in enumerate(depotFile): if (not python3 and char == '/') or (python3 and char == 47): nbSlash += 1 if nbSlash > streamDepth + 2: streamPath = depotFile[:i] break return(streamPath) def patch(self, jnl): for depot in self.dbDepotList: depot.patch(jnl, "put") class DbStream: def __init__(self, schema): self.dbStreamDbSchema = DbSchema(schema) self.dbStreamDict = {} def add(self, STstream, STparent, STtitle, STtype, STpreview): if STstream not in self.dbStreamDict: record = DbRecord(self.dbStreamDbSchema) record.row["STstream"] = STstream record.row["STparent"] = STparent record.row["STtitle"] = STtitle record.row["STtype"] = STtype record.row["STpreview"] = STpreview self.dbStreamDict[STstream] = record def update(self, STstream, STchange, STcopychg, STmergechg, SThighchg, SThash, STstatus): self.dbStreamDict[STstream].row["STchange"] = STchange.replace("default","0") self.dbStreamDict[STstream].row["STcopychg"] = STcopychg.replace("default","0") self.dbStreamDict[STstream].row["STmergechg"] = STmergechg.replace("default","0") self.dbStreamDict[STstream].row["SThighchg"] = SThighchg.replace("default","0") self.dbStreamDict[STstream].row["SThash"] = SThash self.dbStreamDict[STstream].row["STstatus"] = STstatus def isTaskStream(self, stream): isTask = False if stream in self.dbStreamDict: type = self.dbStreamDict[stream].row["STtype"] if type == 4 or type == 5: isTask = True return(isTask) def patch(self, jnl): for stream in self.dbStreamDict: self.dbStreamDict[stream].patch(jnl, "put") class DbDomain: def __init__(self, schema): self.dbDomainDbSchema = DbSchema(schema) self.dbDomainList = [] def add(self, name, type, mount, owner, update, access, options, desc): if name not in self.dbDomainList: record = DbRecord(self.dbDomainDbSchema) record.row["DOname"] = name record.row["DOtype"] = type record.row["DOmount"] = mount record.row["DOowner"] = owner record.row["DOupdate"] = update record.row["DOaccess"] = access record.row["DOoptions"] = options record.row["DOdesc"] = desc self.dbDomainList.append(record) def patch(self, jnl): for domain in self.dbDomainList: domain.patch(jnl, "put") class DbTemplate: def __init__(self, schema): self.dbTemplateDbSchema = DbSchema(schema) self.dbTemplateList = [] self.seq = 0 def add(self, name, change, parent, type, path, vfile, dfile, cmap): record = DbRecord(self.dbTemplateDbSchema) record.row["TVname"] = name record.row["TVchange"] = change record.row["TVseq"] = self.seq record.row["TVparent"] = parent record.row["TVtype"] = type record.row["TVpath"] = path record.row["TVvfile"] = vfile record.row["TVdfile"] = dfile if "TVcmap" in self.dbTemplateDbSchema.name: if self.dbTemplateDbSchema.type[self.dbTemplateDbSchema.name.index("TVcmap")] == "int" and cmap == "": record.row["TVcmap"] = -1 else: record.row["TVcmap"] = cmap self.dbTemplateList.append(record) self.seq = self.seq + 1 def patch(self, jnl): for template in self.dbTemplateList: template.patch(jnl, "put") class DbView: def __init__(self, schema): self.dbViewDbSchema = DbSchema(schema) self.dbViewList = [] def add(self, name, seq, mapflag, vfile, dfile): record = DbRecord(self.dbViewDbSchema) record.row["VIname"] = name record.row["VIseq"] = seq record.row["VImapflag"] = mapflag record.row["VIvfile"] = vfile record.row["VIdfile"] = dfile self.dbViewList.append(record) def patch(self, jnl): for view in self.dbViewList: view.patch(jnl, "put") class DbInteged: def __init__(self, schema): self.dbIntegedDbSchema = DbSchema(schema) self.dbIntegedList = [] def add(self, tfile, ffile, sfrev, efrev, strev, etrev, how, change): record = DbRecord(self.dbIntegedDbSchema) record.row["INtfile"] = tfile record.row["INffile"] = ffile record.row["INsfrev"] = sfrev record.row["INefrev"] = efrev record.row["INstrev"] = strev record.row["INetrev"] = etrev record.row["INhow"] = how record.row["INchange"] = change self.dbIntegedList.append(record) def patch(self, jnl): for integed in self.dbIntegedList: integed.patch(jnl, "put") class DbIntegtx: def __init__(self, schema): self.dbIntegtxDbSchema = DbSchema(schema) self.dbIntegtxList = [] def add(self, tfile, ffile, sfrev, efrev, strev, etrev, how, change): record = DbRecord(self.dbIntegtxDbSchema) record.row["INtfile"] = tfile record.row["INffile"] = ffile record.row["INsfrev"] = sfrev record.row["INefrev"] = efrev record.row["INstrev"] = strev record.row["INetrev"] = etrev record.row["INhow"] = how record.row["INchange"] = change self.dbIntegtxList.append(record) def patch(self, jnl): for integed in self.dbIntegtxList: integed.patch(jnl, "put") class DbRev: def __init__(self, schema): self.dbRevDbSchema = DbSchema(schema) self.dbRevList = [] def isIncluded(self, other): included = False for rev in self.dbRevList: if (rev.row["REdfile"] == other.row["REdfile"] and rev.row["RErev"] == other.row["RErev"]): included = True break return(included) def add(self, dfile, rev, type, action, change, date, modtime, digest, size, traitlot, alazy, afile, arev, atype): record = DbRecord(self.dbRevDbSchema) record.row["REdfile"] = dfile record.row["RErev"] = rev record.row["REtype"] = type record.row["REaction"] = action record.row["REchange"] = change record.row["REdate"] = date record.row["REmodtime"] = modtime record.row["REdigest"] = digest record.row["REsize"] = size record.row["REtraitlot"] = traitlot record.row["REalazy"] = alazy record.row["REafile"] = afile record.row["REarev"] = arev record.row["REatype"] = atype if not self.isIncluded(record): self.dbRevList.append(record) def patch(self, jnl): for rev in self.dbRevList: rev.patch(jnl, "put") def getAction(self, dfile, revnum): action = -1 for rev in self.dbRevList: if rev.row["REdfile"] == dfile and rev.row["RErev"] == revnum: action = rev.row["REaction"] break return(action) def getREafile(self, dfile, revnum): afile = "" for rev in self.dbRevList: if rev.row["REdfile"] == dfile and rev.row["RErev"] == revnum: afile = rev.row["REafile"] break return(afile) def getREarev(self, dfile, revnum): arev = -1 for rev in self.dbRevList: if rev.row["REdfile"] == dfile and rev.row["RErev"] == revnum: arev = rev.row["REarev"] break return(arev) def updateDeleteLbr(self, srcFile, srcRev, tgtFile, tgtRev): afile = arev = size = "" for rev in self.dbRevList: if rev.row["REdfile"] == srcFile and rev.row["RErev"] == srcRev: afile = rev.row["REafile"] arev = rev.row["REarev"] size = rev.row["REsize"] break; if afile: index = 0 for rev in self.dbRevList: if rev.row["REdfile"] == tgtFile and rev.row["RErev"] == tgtRev: self.dbRevList[index].row["REafile"] = afile self.dbRevList[index].row["REarev"] = arev self.dbRevList[index].row["REsize"] = size break; index += 1 def getList(self): return(sorted(self.dbRevList, key = lambda rev: (rev.row["REdfile"], int(re.sub('^1\.', '', rev.row["REarev"]))), reverse=True)) class DbRevtx: def __init__(self, schema): self.dbRevtxDbSchema = DbSchema(schema) self.dbRevtxList = [] def isIncluded(self, other): included = False for rev in self.dbRevtxList: if (rev.row["REdfile"] == other.row["REdfile"] and rev.row["RErev"] == other.row["RErev"]): included = True break return(included) def add(self, dfile, rev, type, action, change, date, modtime, digest, size, traitlot, alazy, afile, arev, atype): record = DbRecord(self.dbRevtxDbSchema) record.row["REdfile"] = dfile record.row["RErev"] = rev record.row["REtype"] = type record.row["REaction"] = action record.row["REchange"] = change record.row["REdate"] = date record.row["REmodtime"] = modtime record.row["REdigest"] = digest record.row["REsize"] = size record.row["REtraitlot"] = traitlot record.row["REalazy"] = alazy record.row["REafile"] = afile record.row["REarev"] = arev record.row["REatype"] = atype if not self.isIncluded(record): self.dbRevtxList.append(record) def patch(self, jnl): for rev in self.dbRevtxList: rev.patch(jnl, "put") def getREalazy(self, dfile, revnum): lazy = -1 for rev in self.dbRevtxList: if rev.row["REdfile"] == dfile and rev.row["RErev"] == revnum: lazy = rev.row["REalazy"] break return(lazy) class DbTraits: def __init__(self, schema): self.dbTraitsDbSchema = DbSchema(schema) self.traitList = [] def add(self, traitlot, name, type, value): record = DbRecord(self.dbTraitsDbSchema) record.row["TTtraitlot"] = traitlot record.row["TTname"] = name record.row["TTtype"] = type record.row["TTvalue"] = str(int(len(value) / 2)) + " " + value self.traitList.append(record) def patch(self, jnl): for trait in self.traitList: trait.patch(jnl, "put") class DbChange: def __init__(self, schema): self.dbChangeDbSchema = DbSchema(schema) self.dbChangeList = [] def add(self, change, key, client, user, date, status, desc, root): if not change in self.dbChangeList: record = DbRecord(self.dbChangeDbSchema) record.row["CHchange"] = change record.row["CHkey"] = key record.row["CHclient"] = client record.row["CHuser"] = user record.row["CHdate"] = date record.row["CHstatus"] = status record.row["CHdesc"] = desc record.row["CHroot"] = root self.dbChangeList.append(record) def patch(self, jnl): for change in self.dbChangeList: change.patch(jnl, "put") class DbDesc: def __init__(self, schema): self.dbDescDbSchema = DbSchema(schema) self.dbDescList = [] def add(self, key, desc): if not key in self.dbDescList: record = DbRecord(self.dbDescDbSchema) record.row["DEkey"] = key record.row["DEdesc"] = desc self.dbDescList.append(record) def patch(self, jnl): for desc in self.dbDescList: desc.patch(jnl, "put") class DbProtect: def __init__(self, schema): self.dbProtectDbSchema = DbSchema(schema) self.dbProtectList = [] def add(self, seq, group, user, host, perm, mapflag, dfile): if not seq in self.dbProtectList: record = DbRecord(self.dbProtectDbSchema) record.row["PRseq"] = seq record.row["PRgroup"] = group record.row["PRuser"] = user record.row["PRhost"] = host record.row["PRperm"] = perm record.row["PRmapflag"] = mapflag record.row["PRdfile"] = dfile self.dbProtectList.append(record) def patch(self, jnl): for protect in self.dbProtectList: protect.patch(jnl, "put") class IntegHistoryRebuild(): def __init__(self, logName): self.logPath = os.getcwd() if logName: if not os.path.dirname(logName): self.logName = os.path.abspath(self.logPath + "/" + logName) else: self.logName = logName else: self.logName = os.path.abspath(self.logPath + "/IntegHistory.log.gz") try: if mimetypes.guess_type(self.logName)[1] == 'gzip': self.log = gzip.open(self.logName, "rt") else: self.log = open(self.logName) except IOError: sys.exit("Error: can\'t open " + self.logName) self.jnlPatch = os.path.abspath(self.logPath + "/" + "jnl.patch") self.serverRoot = os.path.abspath(os.getcwd() + "/" + "IntegHistoryRebuildRoot") self.user = "perforce" self.client = "IntegHistoryRebuild" self.files = "" self.createServerRoot() self.port = "rsh:p4d -r \""+ self.serverRoot + "\" " + " -L log -vserver=3 -i" p4 = P4() p4.port = self.port try: p4.connect() self.dbConfig = DbConfig(p4.run_dbschema("db.config")[0]) self.dbCounters = DbCounters(p4.run_dbschema("db.counters")[0]) self.dbUser = DbUser(p4.run_dbschema("db.user")[0]) self.dbDepot = DbDepot(p4.run_dbschema("db.depot")[0]) self.dbStream = DbStream(p4.run_dbschema("db.stream")[0]) self.dbDomain = DbDomain(p4.run_dbschema("db.domain")[0]) self.dbTemplate = DbTemplate(p4.run_dbschema("db.template")[0]) self.dbView = DbView(p4.run_dbschema("db.view")[0]) self.dbInteged = DbInteged(p4.run_dbschema("db.integed")[0]) self.dbIntegtx = DbIntegtx(p4.run_dbschema("db.integtx")[0]) self.dbRev = DbRev(p4.run_dbschema("db.rev")[0]) self.dbRevtx = DbRevtx(p4.run_dbschema("db.revtx")[0]) self.dbTraits = DbTraits(p4.run_dbschema("db.traits")[0]) self.dbChange = DbChange(p4.run_dbschema("db.change")[0]) self.dbDesc = DbDesc(p4.run_dbschema("db.desc")[0]) self.dbProtect = DbProtect(p4.run_dbschema("db.protect")[0]) except P4Exception: sys.exit("Error: " + "".join(p4.errors)) p4.disconnect() self.info = {} def toEpoch(self, date): return(str(int(time.mktime(time.strptime(date,'%Y/%m/%d %H:%M:%S')) - time.timezone))) def parseInfo(self, output): for result in output: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: self.info[match.group(1)] = match.group(2) def parseSet(self, output): for result in output: match = re.match(r'(.*?)=(.*?) .*', result, re.DOTALL) if match: self.info[match.group(1)] = match.group(2) def parseProtects(self, output): self.info["permMax"] = output[0] def parseConfigure(self, output): type = name = "" for result in output: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: if match.group(1) == "Type": name = match.group(2) elif match.group(1) == "Name": name = match.group(2) elif (match.group(1) == "Value" and not type == "default") and (name == "dm.integ.engine"): self.dbConfig.add(name, match.group(2)) def parseCounters(self, output): for result in output: match = re.match(r'(.*) = (.*)', result, re.DOTALL) if match: self.dbCounters.add(match.group(1), match.group(2)) def parseClient(self, output): for result in output: match = re.match(r'\.\.\. Stream (.*)', result, re.DOTALL) if match: self.info["Stream"] = match.group(1) def integHow(self, how, action): value = 0 if how == "merge from": value = 0 elif how == "merge into": value = 1 elif how == "branch from" and action != 2: value = 2 elif how == "branch into": value = 3 elif how == "copy from": value = 4 elif how == "copy into": value = 5 elif how == "ignored" and action != 2: # target is not a deleted dbRevision value = 6 elif how == "ignored by" and action != 2: # source is not a deleted dbRevision value = 7 elif how == "delete from": value = 8 elif how == "delete into": value = 9 elif how == "edit into": value = 10 elif how == "add into": value = 11 elif how == "edit from": value = 12 elif how == "add from": value = 13 elif how == "moved from": value = 14 elif how == "moved into": value = 15 elif how == "ignored": # target is a deleted dbRevision value = 16 elif how == "ignored by": # ? value = 17 elif how == "ignored": # ? value = 18 elif how == "ignored by": # source is a deleted dbRevision value = 19 elif how == "branch from": # deleted branch value = 20 elif how == "delete into": # deleted branch value = 21 elif how == "undid": # undo: opposite of merge value = 22 elif how == "undone by": # undo reverse value = 23 elif how == "moved from/undid": # move undo: move + undo as one record value = 24 elif how == "moved into/undone": # move undo reverse value = 25 return(value) def addInteged(self, dict): startToRev = dict["startToRev"].replace("#","").replace("none", "0") endToRev = dict["endToRev"].replace("#","").replace("none", "0") startFromRev = dict["startFromRev"].replace("#","").replace("none", "0") endFromRev = dict["endFromRev"].replace("#","").replace("none", "0") action = self.dbRev.getAction(dict["toFile"], endFromRev) # Need to patch lbr values for branched move/delete db.rev records as "p4 fstat -Oc" does't provide these data if (dict["how"] == "delete into"): self.dbRev.updateDeleteLbr(dict["toFile"], str(int(endToRev) - 1), dict["fromFile"], endFromRev) lazy = self.dbRevtx.getREalazy(dict["toFile"], endToRev) isTask = isSparse = False if lazy == -1: lazy = self.dbRevtx.getREalazy(dict["fromFile"], endFromRev) if not lazy == -1: if (lazy & 0x0002) == 0x0002: isSparse = True if (lazy & 0x0004) == 0x0004: isTask = True if isTask: self.dbIntegtx.add(dict["toFile"], dict["fromFile"], startFromRev, endFromRev, startToRev, endToRev, self.integHow(dict["how"], action), dict["change"]) # need to create an extra dbIntegtx because "p4 integed" does not report it if not isSparse: self.dbIntegtx.add(dict["fromFile"], dict["toFile"], startToRev, endToRev, startFromRev, endFromRev, self.integHow(dict["how"], action) + 1, dict["change"]) if not isTask or isSparse: self.dbInteged.add(dict["toFile"], dict["fromFile"], startFromRev, endFromRev, startToRev, endToRev, self.integHow(dict["how"], action), dict["change"]) def parseInteged(self, output): dict = {} for result in output: match = re.match(r'\.\.\. (toFile) (.*)', result, re.DOTALL) if match: if dict: self.addInteged(dict) dict = {} dict[match.group(1)] = match.group(2) else: match = re.search('\.\.\. (.*?) (.*)', result) if match: dict[match.group(1)] = match.group(2) if dict: self.addInteged(dict) def fileType(self, type): value = 0 if re.search('.*text.*', type): value = 0x0000000 type = type.replace("text","") elif re.search('.*binary.*', type): if re.search('.*F.*', type): value = 0x0010000 else: value = 0x0010003 type = type.replace("binary","") elif re.search('.*unicode.*', type): value = 0x0080000 type = type.replace("unicode","") elif re.search('.*symlink.*', type): value = 0x0040000 type = type.replace("symlink","") elif re.search('.*apple.*', type): value = 0x000C0000 type = type.replace("apple","") elif re.search('.*resource.*', type): value = 0x00050000 type = type.replace("resource","") elif re.search('.*utf16.*', type): value = 0x1080000 type = type.replace("utf16","") elif re.search('.*utf8.*', type): value = 0x1040000 type = type.replace("utf8","") if re.search('.*x.*', type): value = value | 0x0020000 if re.search('.*ko.*', type): value = value | 0x0010 elif re.search('.*k.*', type): value = value | 0x0020 if re.search('.*\+.*l.*', type): value = value | 0x0040 if re.search('.*w.*', type): value = value | 0x0100000 if re.search('.*m.*', type): value = value | 0x0200000 if re.search('.*C.*|.*c.*', type): value = value | 0x0000003 if re.search('.*u.*', type): value = value | 0x0000001 if re.search('.*F.*', type): value = value | 0x0000001 if re.search('^l.*', type): value = value | 0x0000001 if re.search('.*D.*', type): value = value & 0x10D0000 if re.search('.*X.*', type): value = value | 0x0000008 match = re.search('.*S(\d+).*|.*S.*', type) if match: value = value | 0x0080 if match.group(1) == "2": value = value | 0x0100 if match.group(1) == "3": value = value | 0x0200 if match.group(1) == "4": value = value | 0x0300 if match.group(1) == "5": value = value | 0x0400 if match.group(1) == "6": value = value | 0x0500 if match.group(1) == "7": value = value | 0x0600 if match.group(1) == "8": value = value | 0x0700 if match.group(1) == "9": value = value | 0x0800 if match.group(1) == "10": value = value | 0x0900 if match.group(1) == "16": value = value | 0x0A00 if match.group(1) == "32": value = value | 0x0B00 if match.group(1) == "64": value = value | 0x0C00 if match.group(1) == "128": value = value | 0x0D00 if match.group(1) == "256": value = value | 0x0E00 if match.group(1) == "512": value = value | 0x0F00 return(value) def actionType(self, action): value = 0 if action == "add": value = 0 elif action == "edit": value = 1 elif action == "delete": value = 2 elif action == "branch": value = 3 elif action == "integrate": value = 4 elif action == "import": value = 5 elif action == "purge": value = 6 elif action == "move/delete": value = 7 elif action == "move/add": value = 8 elif action == "archive": value = 9 return(value) def getRevStatus(self, lazy, sparse, task, charset): value = 0 if charset == "none": value = 0 elif charset == "utf8": value = 1 elif charset == "iso8859-1": value = 2 elif charset == "utf16-nobom": value = 3 elif charset == "shiftjis": value = 4 elif charset == "eucjp": value = 5 elif charset == "winansi": value = 6 elif charset == "cp850": value = 7 elif charset == "macosroman": value = 8 elif charset == "iso8859-15": value = 9 elif charset == "iso8859-5": value = 10 elif charset == "koi8-r": value = 11 elif charset == "cp1251": value = 12 elif charset == "utf16le": value = 13 elif charset == "utf16be": value = 14 elif charset == "utf16le-bom": value = 15 elif charset == "utf16be-bom": value = 16 elif charset == "utf16-bom": value = 17 elif charset == "utf8-bom": value = 18 elif charset == "utf32-nobom": value = 19 elif charset == "utf32le": value = 20 elif charset == "utf32be": value = 21 elif charset == "utf32le-bom": value = 22 elif charset == "utf32be-bom": value = 23 elif charset == "utf32": value = 24 elif charset == "UTF_8_UNCHECKED": value = 25 elif charset == "UTF_8_UNCHECKED_BOM": value = 26 elif charset == "cp949": value = 27 elif charset == "cp936": value = 28 elif charset == "cp950": value = 29 elif charset == "cp850": value = 30 elif charset == "cp858": value = 31 elif charset == "cp1253": value = 32 elif charset == "iso8859-7": value = 33 value = value << 24 if lazy == "1": value = value | 0x0001 if sparse == True: value = value | 0x0002 if task == True: value = value | 0x0004 return(value) def addRev(self, dict, traitsdict): if "depotFile" in dict: if traitsdict: for traitLot in traitsdict: traitType = traitsdict[traitLot]["type"] traitName = traitsdict[traitLot]["name"] if python3: traitValue = binascii.b2a_hex(bytes(traitsdict[traitLot]["value"],'ascii')).decode('ascii').upper() refsValue = binascii.b2a_hex(bytes(traitsdict[traitLot]["refs"],'ascii')).decode('ascii').upper() else: traitValue = binascii.b2a_hex(bytes(traitsdict[traitLot]["value"])).upper() refsValue = binascii.b2a_hex(bytes(traitsdict[traitLot]["refs"])).upper() self.dbTraits.add(traitLot, traitName, traitType, traitValue) self.dbTraits.add(traitLot, "refs", 0, refsValue) if "lbrFile" not in dict: if dict["headAction"] == "move/delete" and re.search('.*\+.*S(\d+).*|.*\+.*S.*', dict["headType"]): dict["lbrFile"] = self.dbRev.getREafile(dict["depotFile"], str(int(dict["headRev"])-1)) dict["lbrRev"] = self.dbRev.getREarev(dict["depotFile"], str(int(dict["headRev"])-1)) else: dict["lbrFile"] = dict["depotFile"] dict["lbrRev"] = "1." + dict["headChange"] dict["lbrType"] = dict["headType"] lazy = self.getRevStatus(dict["lbrIsLazy"], dict["isSparse"], dict["isTask"], dict["headCharset"]) streamPath = self.dbDepot.getStreamPath(dict["depotFile"]) isTask = self.dbStream.isTaskStream(streamPath) if isTask: self.dbRevtx.add(dict["depotFile"], dict["headRev"], self.fileType(dict["headType"]), self.actionType(dict["headAction"]), dict["headChange"], dict["headTime"], dict["headModTime"], dict["digest"], dict["fileSize"], dict["traitLot"], lazy, dict["lbrFile"], dict["lbrRev"], self.fileType(dict["lbrType"])) if not isTask or dict["isSparse"]: self.dbRev.add(dict["depotFile"], dict["headRev"], self.fileType(dict["headType"]), self.actionType(dict["headAction"]), dict["headChange"], dict["headTime"], dict["headModTime"], dict["digest"], dict["fileSize"], dict["traitLot"], lazy, dict["lbrFile"], dict["lbrRev"], self.fileType(dict["lbrType"])) def parseFstat(self, output): dict = {} traitsdict = {} for result in output: match = re.match(r'\.\.\. depotFile (.*)', result, re.DOTALL) if match: self.addRev(dict, traitsdict) dict = {} traitsdict = {} dict["depotFile"] = match.group(1) dict["traitLot"] = 0 dict["digest"] = "00000000000000000000000000000000" dict["fileSize"] = "-1" dict["lbrIsLazy"] = 0 dict["isTask"] = False dict["isSparse"] = False dict["headCharset"] = "none" else: match = re.match(r'\.\.\. attr(\d+?)-refs (.*)', result, re.DOTALL) if match: if not match.group(1) in traitsdict: dict["traitLot"] = match.group(1) traitsdict[match.group(1)] = {} traitsdict[match.group(1)]["refs"] = match.group(2) else: match = re.match(r'\.\.\. attr(\d+?)-(\w+?) (.*)', result, re.DOTALL) if match: if not match.group(1) in traitsdict: dict["traitLot"] = match.group(1) traitsdict[match.group(1)] = {} traitsdict[match.group(1)]["type"] = 1 traitsdict[match.group(1)]["name"] = match.group(2) traitsdict[match.group(1)]["value"] = match.group(3) else: match = re.match(r'\.\.\. attrProp(\d+?)-(\w+?) (.*)', result, re.DOTALL) if match: if not match.group(1) in traitsdict: dict["traitLot"] = match.group(1) traitsdict[match.group(1)] = {} traitsdict[match.group(1)]["type"] = 2 traitsdict[match.group(1)]["name"] = match.group(2) traitsdict[match.group(1)]["value"] = match.group(3) else: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: if not match.group(2): dict[match.group(1)] = True else: dict[match.group(1)] = match.group(2) if dict: self.addRev(dict, traitsdict) def getStatus(self, status, type): value = 0 if status == "pending" and type == "public": value = 0 elif status == "submitted" and type == "public": value = 1 elif status == "shelved" and type == "public": value = 2 elif status == "pending" and type == "restricted": value = 4 elif status == "submitted" and type == "restricted": value = 5 elif status == "shelved" and type == "restricted": value = 8 elif status == "shelved" and type == "promoted": value = 16 return(value) def parseChanges(self, output): dict = {} for result in output: match = re.match(r'\.\.\. (desc)(\n.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) else: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) if dict: if not "oldChange" in dict: dict["oldChange"] = dict["change"] if not "desc" in dict: dict["desc"] = "" self.dbChange.add(dict["change"], dict["oldChange"], dict["client"], dict["user"], dict["time"], self.getStatus(dict["status"], dict["changeType"]), dict["desc"][:31], dict["path"]) self.dbDesc.add(dict["oldChange"], dict["desc"]) def depotType(self, type): value = 0 if type == "local": value = 0 elif type == "remote": value = 1 elif type == "spec": value = 2 elif type == "stream": value = 3 elif type == "archive": value = 4 elif type == "unload": value = 5 elif type == "tangent": value = 6 elif type == "graph": value = 6 return(value) def parseDepot(self, output): dict = {} for result in output: match = re.match(r'\.\.\. (Description)(\n.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) else: match = re.match(r'\.\.\. (StreamDepth) .*/(.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) else: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) if dict: if not "Owner" in dict: dict["Owner"] = "" if not "StreamDepth" in dict: dict["StreamDepth"] = "" if not "Description" in dict: dict["Description"] = "" self.dbDepot.add(dict["Depot"], self.depotType(dict["Type"]), dict["StreamDepth"], dict["Depot"] + "/...") self.dbDomain.add(dict["Depot"], "100", "", dict["Owner"], self.toEpoch(dict["Date"]), self.toEpoch(dict["Date"]), 0, dict["Description"]) def streamType(self, type): value = 0 if type == "mainline": value = 0 elif type == "release": value = 1 elif type == "development": value = 2 elif type == "virtual": value = 3 elif type == "task": value = 4 elif type == "task - unloaded": value = 5 return(value) def streamPathType(self, type): value = 0 if type == "share": value = 0 elif type == "isolate": value = 1 elif type == "import": value = 2 elif type == "exclude": value = 3 elif type == "Remapped": value = 4 elif type == "Ignored": value = 5 elif type == "exclude": value = 6 elif type == "import+": value = 7 return(value) def streamOptions(self, options): value = 0 if re.search('.*ownersubmit.*', options): value = value | 0x0001 if re.search('.* locked.*', options): value = value | 0x0002 if re.search('.* toparent.*', options): value = value | 0x0004 if re.search('.* fromparent.*', options): value = value | 0x0008 if re.search('.*mergeany.*', options): value = value | 0x0010 return(value) def parseStream(self, output): dict = {} mappings = {} for result in output: match = re.match(r'\.\.\. (\w+?)(\d+) (.*)', result, re.DOTALL) if match: key = match.group(1) index = int(match.group(2)) args = match.group(3) if not key in mappings: mappings[key] = {} match = re.match(r'(.*?) "(.*?)" "(.*)"', args, re.DOTALL) if match: mappings[key][index] = [match.group(1), match.group(2), match.group(3)] else: match = re.match(r'(.*?) "(.*?)" (.*)', args, re.DOTALL) if match: mappings[key][index] = [match.group(1), match.group(2), match.group(3)] else: match = re.match(r'(.*?) (.*?) "(.*)"', args, re.DOTALL) if match: mappings[key][index] = [match.group(1), match.group(2), match.group(3)] else: match = re.match(r'(.*?) (.*?) (.*)', args, re.DOTALL) if match: mappings[key][index] = [match.group(1), match.group(2), match.group(3)] else: match = re.match(r'(.*? .*)', args, re.DOTALL) if match: mappings[key][index] = match.group(1).split() else: mappings[key][index] = [args, ""] else: match = re.match(r'\.\.\. (Description)(\n.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) else: match = re.match(r'\.\.\. (\w+?) (.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) if dict: update = access = 0 if "Update" in dict: update = self.toEpoch(dict["Update"]) if "Access" in dict: access = self.toEpoch(dict["Access"]) if not "Description" in dict: dict["Description"] = "" description = dict["Description"] self.dbStream.add(dict["Stream"], dict["Parent"], dict["Name"], self.streamType(dict["Type"]), update) self.dbDomain.add(dict["Stream"], "115", "", dict["Owner"], update, access, self.streamOptions(dict["Options"]), description) for key in mappings: for seq in sorted(mappings[key]): cmap = "" if key == "Paths": vfile = dfile = "" list = mappings[key][seq] if len(list) > 1: vfile = list[1] if len(list) > 2: if list[2].find("@") == -1: dfile = list[2] else: dfile = list[2].split("@")[0] cmap = list[2].split("@")[1] self.dbTemplate.add(dict["Stream"], self.dbCounters.maxChangelist, dict["Parent"], self.streamType(dict["Type"]), self.streamPathType(list[0]), vfile, dfile, cmap) elif key == "Remapped": list = mappings[key][seq] vfile = list[1] dfile = list[0] self.dbTemplate.add(dict["Stream"], self.dbCounters.maxChangelist, dict["Parent"], self.streamType(dict["Type"]), self.streamPathType(key), vfile, dfile, cmap) elif key == "Ignored": list = mappings[key][seq] dfile = "..." + list[0] self.dbTemplate.add(dict["Stream"], self.dbCounters.maxChangelist, dict["Parent"], self.streamType(dict["Type"]), self.streamPathType(key), "", dfile, cmap) def parseIstat(self, output): dict = {} for result in output: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) if dict: self.dbStream.update(dict["stream"], dict["change"], dict["copyParent"], dict["mergeParent"], dict["mergeHighVal"], dict["branchHash"], dict["status"]) def parse(self): for line in self.log: list = eval(str(line)) cmd = list[0] output = list[1:] if cmd == "files": self.files = ' '.join(file for file in output) else: match = re.search('^p4.*? (.*)', cmd) if match: if re.search(" info", cmd): self.parseInfo(output) elif re.search(" set", cmd): self.parseSet(output) elif re.search(" configure ", cmd): self.parseConfigure(output) elif re.search(" integed ", cmd): self.parseInteged(output) elif re.search(" fstat ", cmd): self.parseFstat(output) elif re.search(" depot ", cmd): self.parseDepot(output) elif re.search(" stream ", cmd): self.parseStream(output) elif re.search(" istat ", cmd): self.parseIstat(output) elif re.search(" changes ", cmd): self.parseChanges(output) elif re.search(" counters", cmd): self.parseCounters(output) elif re.search(" protects -m", cmd): self.parseProtects(output) elif re.search(" client ", cmd): self.parseClient(output) self.log.close() now = self.toEpoch(time.strftime('%Y/%m/%d %H:%M:%S',time.localtime())) self.dbUser.add("super", "super@super" , now, now, "super") self.dbProtect.add(1, 0, "super", "*", 255, 0, "//...") self.dbConfig.add("lbr.verify.out", 0) if "permMax" in self.info: privilege = self.info["permMax"] if privilege == "super": level = 255 elif privilege == "admin": level = 191 elif privilege == "review": level = 39 elif privilege == "write": level = 31 elif privilege == "open": level = 15 elif privilege == "read": level = 7 elif privilege == "list": level = 1 self.dbProtect.add(0, 0, self.user, "*", level, 0, "//...") if "unicode" in self.info: self.dbConfig.add("unicode", "1") self.dbConfig.add("server.filecharset", 1) if "integEngine" in self.info: self.dbConfig.add("dm.integ.engine", self.info["integEngine"]) self.dbUser.add(self.user, self.user + "@" + self.user, now, now, self.user) def createClient(self): p4 = P4() p4.port = self.port p4.user = self.user p4.client = self.client p4.connect() client = p4.fetch_client() if "Stream" in self.info: client["Stream"] = self.info["Stream"] p4.save_client(client) p4.disconnect() def createJnlPatch(self): jnlFile = open(self.jnlPatch, "wt") self.dbConfig.patch(jnlFile) self.dbCounters.patch(jnlFile) self.dbDepot.patch(jnlFile) self.dbUser.patch(jnlFile) self.dbStream.patch(jnlFile) self.dbDomain.patch(jnlFile) self.dbTemplate.patch(jnlFile) self.dbView.patch(jnlFile) self.dbInteged.patch(jnlFile) self.dbIntegtx.patch(jnlFile) self.dbRev.patch(jnlFile) self.dbRevtx.patch(jnlFile) self.dbTraits.patch(jnlFile) self.dbChange.patch(jnlFile) self.dbDesc.patch(jnlFile) self.dbProtect.patch(jnlFile) jnlFile.close() def getServerRoot(self): return(self.serverRoot) def createServerRoot(self): shutil.rmtree(self.serverRoot, ignore_errors = True) if not os.path.exists(self.serverRoot): os.mkdir(self.serverRoot) os.chdir(self.serverRoot) def createServer(self): caseFlag = "-C0" if self.info["caseHandling"] == "insensitive": caseFlag = "-C1" self.createServerRoot() shutil.move(self.jnlPatch, self.serverRoot) os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -jr jnl.patch > log 2>&1") os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -xu >> log 2>&1") os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -xx >> log 2>&1") os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -jr jnl.fix >> log 2>&1") os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -jc >> log 2>&1") self.port = "rsh:p4d -r \""+ self.serverRoot + "\" " + caseFlag + " -L log -vserver=3 -i" p4 = P4() if p4.env('P4CONFIG'): p4config = os.path.abspath(self.serverRoot + "/" + p4.env('P4CONFIG')) configFile = open(p4config, "wt") print("P4PORT="+ self.port, file=configFile) print("P4USER=" + self.user, file=configFile) print("P4CLIENT=" + self.client, file=configFile) if "unicode" in self.info: p4charset = "utf8" if "P4CHARSET" in self.info: p4charset = self.info["P4CHARSET"] print("P4CHARSET=" + p4charset, file=configFile) configFile.close() self.createClient() def createFullFile(self, fullFile, digest, fileSize): fullDir = os.path.dirname(fullFile) if not os.path.exists(fullDir): os.makedirs(fullDir) full = open(fullFile, "wt") print(digest + " " + fileSize, file=full) full.close() def createGzipFile(self, gzipFile, digest, fileSize): gzipDir = os.path.dirname(gzipFile) if not os.path.exists(gzipDir): os.makedirs(gzipDir) gz = gzip.open(gzipFile, "wt") print(digest + " " + fileSize, file=gz) gz.close() def createRCSFile(self, rcsFile, lbrRev, date, digest, fileSize): rcsDir = os.path.dirname(rcsFile) if not os.path.exists(rcsDir): os.makedirs(rcsDir) rcs = open(rcsFile, "wt") print("head " + lbrRev + ";", file=rcs) print("access ;", file=rcs) print("symbols ;", file=rcs) print("locks ;comment @@;", file=rcs) print("", file=rcs) print("", file=rcs) print(lbrRev, file=rcs) print("date " + date + "; author p4; state Exp;", file=rcs) print("branches ;", file=rcs) print("next ;", file=rcs) print("", file=rcs) print("", file=rcs) print("desc", file=rcs) print("@@", file=rcs) print("", file=rcs) print("", file=rcs) print(lbrRev, file=rcs) print("log", file=rcs) print("@@", file=rcs) print("text", file=rcs) print("@" + digest + " " + fileSize, file=rcs) print("@", file=rcs) rcs.close() def updateRCSFile(self, rcsFile, lbrRev, date, digest, fileSize): if not os.path.exists(rcsFile): self.createRCSFile(rcsFile, lbrRev, date, digest, fileSize) else: rcs = open(rcsFile, "rt") rcstmp = open(rcsFile+".tmp", "wt") for line in rcs.read().splitlines(): match = re.search('^next ;', line) if match: print("next "+ lbrRev + ";", file=rcstmp) print("", file=rcstmp) print(lbrRev, file=rcstmp) print("date " + date + "; author p4; state Exp;", file=rcstmp) print("branches ;", file=rcstmp) print("next ;", file=rcstmp) else: print(line, file=rcstmp); print("", file=rcstmp) print("", file=rcstmp) print(lbrRev, file=rcstmp) print("log", file=rcstmp) print("@@", file=rcstmp) print("text", file=rcstmp) print("@d1 1", file=rcstmp) print("a1 1", file=rcstmp) print(digest + " " + fileSize, file=rcstmp) print("@", file=rcstmp) rcs.close() rcstmp.close() os.remove(rcsFile) os.rename(rcsFile+".tmp", rcsFile) def createServerDepotFiles(self): for rev in self.dbRev.getList(): if not int(rev.row["REalazy"]) & 0x0001 == 1 and not (rev.row["REaction"] == 2 or rev.row["REaction"] == 7): serverFileType = rev.row["REatype"] & 0x000F reafile = rev.row["REafile"] if self.info["caseHandling"] == "insensitive": reafile = reafile.lower() if serverFileType == 0: rcsFile = os.path.abspath(reafile.replace("//", self.serverRoot + "/") + ",v") self.updateRCSFile(rcsFile, rev.row["REarev"], time.strftime("%Y.%m.%d.%H.%M.%S", time.localtime(int(rev.row["REdate"]))), rev.row["REdigest"], rev.row["REsize"]) elif serverFileType == 1: fullFile = os.path.abspath(reafile.replace("//", self.serverRoot + "/") + ",d/" + rev.row["REarev"]) self.createFullFile(fullFile, rev.row["REdigest"], rev.row["REsize"]) elif serverFileType == 3: gzipFile = os.path.abspath(reafile.replace("//", self.serverRoot + "/") + ",d/" + rev.row["REarev"] + ".gz") self.createGzipFile(gzipFile, rev.row["REdigest"], rev.row["REsize"]) def filetolist(self, file): alist = [] if mimetypes.guess_type(file)[1] == 'gzip': f = gzip.open(file, "rt") else: f = open(file) lines = f.readlines() lines.sort() for line in lines: line = line.strip() match = re.search(r'\'... (serverVersion .*?\')', line) if match: line = match.group(1) line = re.sub(r', \'\.\.\. Map .*?\'', '', line) line = re.sub(r', \'\.\.\. movedFile .*?\'', '', line) line = re.sub(r', \'\.\.\. clientFile .*?\'', '', line) line = re.sub(r', \'\.\.\. isMapped \'', '', line) line = re.sub(r', \'\.\.\. haveRev .*?\'', '', line) line = re.sub(r', \'\.\.\. action .*?\'', '', line) line = re.sub(r', \'\.\.\. actionOwner .*?\'', '', line) line = re.sub(r', \'\.\.\. resolved \'', '', line) line = re.sub(r', \'\.\.\. unresolved \'', '', line) line = re.sub(r', \'\.\.\. reresolvable \'', '', line) line = re.sub(r', \'\.\.\. change .*?\'', '', line) line = re.sub(r', \'\.\.\. type .*?\'', '', line) line = re.sub(r', \'\.\.\. charset .*?\'', '', line) line = re.sub(r', \'\.\.\. Type .*?\'', '', line) line = re.sub(r', \'\.\.\. Date .*?\'', '', line) line = re.sub(r', \'\.\.\. time .*?\'', '', line) line = re.sub(r', \'\.\.\. headTime .*?\'', '', line) line = re.sub(r', \'\.\.\. headModTime .*?\'', '', line) line = re.sub(r', \'\.\.\. Update .*?\'', '', line) line = re.sub(r', \'\.\.\. Access .*?\'', '', line) line = re.sub(r', \'\.\.\. workRev .*?\'', '', line) line = re.sub(r', \'\.\.\. \.\.\. other.*?\'', '', line) if (not re.match(r'\[\'language\'', line) and not re.match(r'\[\'p4 set\'', line) and not re.match(r'\[\'p4 \-ztag user', line) and not re.match(r'\[\'p4 \-ztag client', line) and not re.match(r'\[\'p4 \-ztag protects', line) and not re.match(r'\[\'p4 \-ztag configure show\'', line)): alist.append(line) f.close() return(alist) def validation(self): logNameRebuild = os.path.abspath(self.serverRoot + "/IntegHistory.log") try: # first try to run IntegHistory as a module # therefore IntegHistory.py must be in the current directory history = IntegHistory.IntegHistory([self.files], logNameRebuild) logNameRebuild = history.getData() except: # On Linux, the "IntegHistory.py" script can be in the user PATH process = Popen("IntegHistory.py " + self.files, shell = True, stdout = PIPE) (output, err) = process.communicate() exitCode = process.wait() if not os.path.exists(logNameRebuild): logNameRebuild = os.path.abspath(self.serverRoot + "/IntegHistory.log.gz") if not os.path.exists(logNameRebuild): print(" ==> Validation failed: Cannot find the IntegHistory.py script") else: fromlines = self.filetolist(self.logName) tolines = self.filetolist(logNameRebuild) validationFilename = os.path.abspath(self.serverRoot + "/validation.log") validationFile = open(validationFilename, "w") errors = False for diff in difflib.unified_diff(fromlines, tolines, fromfile=self.logName, tofile=logNameRebuild, n=0): validationFile.write(diff.strip() + "\n") if not re.match(r'\-\-\-', diff) and not re.match(r'\+\+\+', diff): errors = True validationFile.close() if errors: print(" ==> problems detected during validation, review differences against original IntegHistory log in " + validationFilename) def main(): parser = argparse.ArgumentParser(prog='IntegHistoryRebuild', usage='%(prog)s [IntegHistory-output-filename]') parser.add_argument("logName", nargs="?", help="IntegHistory output filename", default="IntegHistory.log.gz") args = parser.parse_args() logName = args.logName p4 = P4() if not p4.env('P4CONFIG'): sys.exit("Error: P4CONFIG variable is needed and must be set!") history = IntegHistoryRebuild(logName) print("Reading server data from " + logName + "...") history.parse() print("Creating rebuilt server...") history.createJnlPatch() history.createServer() print("Creating dummy server depot files...") history.createServerDepotFiles() print("Validating rebuilt server...") history.validation() print("Completed!") print("Server root=" + history.getServerRoot()) print("P4CONFIG file=" + os.path.abspath(history.getServerRoot() + "/" + p4.env('P4CONFIG'))) if __name__ == "__main__": main()