#!/usr/bin/env python ################################################################################ # # Copyright (c) 2016, Perforce Software, Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # DATE # # $Date: 2016/04/15 $ # # SYNOPSIS # # IntegHistoryRebuild.py # # DESCRIPTION # # This script creates a Helix server (metadata and dummy depot files) from # data collected by the IntegHistory.py script (stored in an # IntegHistory.log.gz file). # # The script was originally designed as a support tool to recreate quickly # integration issues without the need of a checkpoint. # # The Helix server is created in the IntegHistoryRebuildRoot directory # under the current directory (automatically removed if it exists). # The created Helix server can be accessed directly via rsh using the # P4CONFIG file created in the P4ROOT directory (P4CONFIG must be set): # P4PORT = rsh:p4d -r IntegHistoryRebuildRoot -L log -vserver=3 -i # P4USER = perforce # P4CLIENT = IntegHistoryRebuild # # REQUIREMENTS # # * Python (tested with 2.7 and 3.4) # * P4Python # * Helix p4d executable in your path # * IntegHistory.py in your current directory or in your PYTHONPATH # or in your path (Linux only) # ################################################################################ from __future__ import print_function from P4 import P4, P4Exception import re import sys import os import shutil import time import mimetypes import gzip import difflib import binascii from subprocess import Popen, PIPE, STDOUT import shlex sys.path.append('.') import argparse try: import IntegHistory except ImportError: pass class DbSchema: def __init__(self, schema): self.table = schema["table"] self.version = schema["version"] self.name = schema["name"] self.type = schema["type"] class DbRecord: def __init__(self, schema): self.schema = schema self.row = {} for i in range(len(self.schema.name)): if self.schema.type[i] == "key" or self.schema.type[i] == "text": self.row[self.schema.name[i]] = "" else: self.row[self.schema.name[i]] = 0 def patch(self, jnl, how): if how == "put": jnl.write("@pv@ ") elif how == "replace": jnl.write("@rv@ ") elif how == "remove": jnl.write("@dv@ ") jnl.write(self.schema.version + " @" + self.schema.table + "@ ") for i in range(len(self.schema.name)): if self.schema.type[i] == "key" or self.schema.type[i] == "text": jnl.write("@") jnl.write(str(self.row[self.schema.name[i]]).replace("@","@@")) if self.schema.type[i] == "key" or self.schema.type[i] == "text": jnl.write("@ ") else: jnl.write(" ") jnl.write("\n") class DbConfig: def __init__(self, schema): self.dbConfigDbSchema = DbSchema(schema) self.dbConfigList = [] def add(self, name, value): if name not in self.dbConfigList: record = DbRecord(self.dbConfigDbSchema) record.row["CFsname"] = "any" record.row["CFname"] = name record.row["CFvalue"] = value self.dbConfigList.append(record) def remove(self, name): for i in range(len(self.dbConfigList)): if self.dbConfigList[i].row["CFname"] == name: del(self.dbConfigList[i]) break def patch(self, jnl): for dbConfig in self.dbConfigList: dbConfig.patch(jnl, "put") class DbCounters: def __init__(self, schema): self.dbCountersDbSchema = DbSchema(schema) self.dbCountersList = [] self.maxChangelist = 0 def add(self, name, value): if not name in self.dbCountersList: record = DbRecord(self.dbCountersDbSchema) record.row["COname"] = name record.row["COvalue"] = value self.dbCountersList.append(record) if name == "change": self.maxChangelist = value def patch(self, jnl): for counter in self.dbCountersList: counter.patch(jnl, "put") class DbUser: def __init__(self, schema): self.dbUserDbSchema = DbSchema(schema) self.dbUserList = [] def add(self, user, email, update, access, fullname): if not user in self.dbUserList: record = DbRecord(self.dbUserDbSchema) record.row["USuser"] = user record.row["USemail"] = email record.row["USupdate"] = update record.row["USaccess"] = access record.row["USfullname"] = fullname self.dbUserList.append(record) def patch(self, jnl): for user in self.dbUserList: user.patch(jnl, "put") class DbDepot: def __init__(self, schema): self.dbDepotDbSchema = DbSchema(schema) self.dbDepotList = [] def add(self, name, type, depth, map): if name not in self.dbDepotList: record = DbRecord(self.dbDepotDbSchema) record.row["DPname"] = name record.row["DPtype"] = type record.row["DPextra"] = depth record.row["DPmap"] = map self.dbDepotList.append(record) def patch(self, jnl): for depot in self.dbDepotList: depot.patch(jnl, "put") class DbStream: def __init__(self, schema): self.dbStreamDbSchema = DbSchema(schema) self.dbStreamDict = {} def add(self, stream, parent, title, type, pdbReview): if stream not in self.dbStreamDict: record = DbRecord(self.dbStreamDbSchema) record.row["STstream"] = stream record.row["STparent"] = parent record.row["STtitle"] = title record.row["STtype"] = type record.row["STpreview"] = pdbReview self.dbStreamDict[stream] = record def update(self, stream, change, copychg, mergechg, highchg, hash, status): self.dbStreamDict[stream].row["STchange"] = change.replace("default","0") self.dbStreamDict[stream].row["STcopychg"] = copychg.replace("default","0") self.dbStreamDict[stream].row["STmergechg"] = mergechg.replace("default","0") self.dbStreamDict[stream].row["SThighchg"] = highchg.replace("default","0") self.dbStreamDict[stream].row["SThash"] = hash self.dbStreamDict[stream].row["STstatus"] = status def patch(self, jnl): for stream in self.dbStreamDict: self.dbStreamDict[stream].patch(jnl, "put") class DbDomain: def __init__(self, schema): self.dbDomainDbSchema = DbSchema(schema) self.dbDomainList = [] def add(self, name, type, mount, owner, update, access, options, desc): if name not in self.dbDomainList: record = DbRecord(self.dbDomainDbSchema) record.row["DOname"] = name record.row["DOtype"] = type record.row["DOmount"] = mount record.row["DOowner"] = owner record.row["DOupdate"] = update record.row["DOaccess"] = access record.row["DOoptions"] = options record.row["DOdesc"] = desc self.dbDomainList.append(record) def patch(self, jnl): for domain in self.dbDomainList: domain.patch(jnl, "put") class DbTemplate: def __init__(self, schema): self.dbTemplateDbSchema = DbSchema(schema) self.dbTemplateList = [] self.seq = 0 def add(self, name, change, parent, type, path, vfile, dfile, cmap): record = DbRecord(self.dbTemplateDbSchema) record.row["TVname"] = name record.row["TVchange"] = change record.row["TVseq"] = self.seq record.row["TVparent"] = parent record.row["TVtype"] = type record.row["TVpath"] = path record.row["TVvfile"] = vfile record.row["TVdfile"] = dfile if self.dbTemplateDbSchema.type[self.dbTemplateDbSchema.name.index("TVcmap")] == "int" and cmap == "": record.row["TVcmap"] = -1 else: record.row["TVcmap"] = cmap self.dbTemplateList.append(record) self.seq = self.seq + 1 def patch(self, jnl): for template in self.dbTemplateList: template.patch(jnl, "put") class DbView: def __init__(self, schema): self.dbViewDbSchema = DbSchema(schema) self.dbViewList = [] def add(self, name, seq, mapflag, vfile, dfile): record = DbRecord(self.dbViewDbSchema) record.row["VIname"] = name record.row["VIseq"] = seq record.row["VImapflag"] = mapflag record.row["VIvfile"] = vfile record.row["VIdfile"] = dfile self.dbViewList.append(record) def patch(self, jnl): for view in self.dbViewList: view.patch(jnl, "put") class DbInteged: def __init__(self, schema): self.dbIntegedDbSchema = DbSchema(schema) self.dbIntegedList = [] def add(self, tfile, ffile, sfrev, efrev, strev, etrev, how, change): record = DbRecord(self.dbIntegedDbSchema) record.row["INtfile"] = tfile record.row["INffile"] = ffile record.row["INsfrev"] = sfrev record.row["INefrev"] = efrev record.row["INstrev"] = strev record.row["INetrev"] = etrev record.row["INhow"] = how record.row["INchange"] = change self.dbIntegedList.append(record) def patch(self, jnl): for integed in self.dbIntegedList: integed.patch(jnl, "put") class DbRev: def __init__(self, schema): self.dbRevDbSchema = DbSchema(schema) self.dbRevList = [] def isIncluded(self, other): for rev in self.dbRevList: if (rev.row["REdfile"] == other.row["REdfile"] and rev.row["RErev"] == other.row["RErev"]): return(1) return(0) def add(self, dfile, rev, type, action, change, date, modtime, digest, size, traitlot, alazy, afile, arev, atype): record = DbRecord(self.dbRevDbSchema) record.row["REdfile"] = dfile record.row["RErev"] = rev record.row["REtype"] = type record.row["REaction"] = action record.row["REchange"] = change record.row["REdate"] = date record.row["REmodtime"] = modtime record.row["REdigest"] = digest record.row["REsize"] = size record.row["REtraitlot"] = traitlot record.row["REalazy"] = alazy record.row["REafile"] = afile record.row["REarev"] = arev record.row["REatype"] = atype if (self.isIncluded(record) == 0): self.dbRevList.append(record) def patch(self, jnl): for rev in self.dbRevList: rev.patch(jnl, "put") def getAction(self, dfile, revnum): for rev in self.dbRevList: if rev.row["REdfile"] == dfile and rev.row["RErev"] == revnum: return(rev.row["REaction"]) def getList(self): return(sorted(self.dbRevList, key = lambda rev: (rev.row["REdfile"], int(re.sub('^1\.', '', rev.row["REarev"]))), reverse=True)) class DbTraits: def __init__(self, schema): self.dbTraitsDbSchema = DbSchema(schema) self.traitList = [] def add(self, traitlot, name, type, value): record = DbRecord(self.dbTraitsDbSchema) record.row["TTtraitlot"] = traitlot record.row["TTname"] = name record.row["TTtype"] = type record.row["TTvalue"] = str(int(len(value) / 2)) + " " + value self.traitList.append(record) def patch(self, jnl): for trait in self.traitList: trait.patch(jnl, "put") class DbChange: def __init__(self, schema): self.dbChangeDbSchema = DbSchema(schema) self.dbChangeList = [] def add(self, change, key, client, user, date, status, desc, root): if not change in self.dbChangeList: record = DbRecord(self.dbChangeDbSchema) record.row["CHchange"] = change record.row["CHkey"] = key record.row["CHclient"] = client record.row["CHuser"] = user record.row["CHdate"] = date record.row["CHstatus"] = status record.row["CHdesc"] = desc record.row["CHroot"] = root self.dbChangeList.append(record) def patch(self, jnl): for change in self.dbChangeList: change.patch(jnl, "put") class DbDesc: def __init__(self, schema): self.dbDescDbSchema = DbSchema(schema) self.dbDescList = [] def add(self, key, desc): if not key in self.dbDescList: record = DbRecord(self.dbDescDbSchema) record.row["DEkey"] = key record.row["DEdesc"] = desc self.dbDescList.append(record) def patch(self, jnl): for desc in self.dbDescList: desc.patch(jnl, "put") class DbProtect: def __init__(self, schema): self.dbProtectDbSchema = DbSchema(schema) self.dbProtectList = [] def add(self, seq, group, user, host, perm, mapflag, dfile): if not seq in self.dbProtectList: record = DbRecord(self.dbProtectDbSchema) record.row["PRseq"] = seq record.row["PRgroup"] = group record.row["PRuser"] = user record.row["PRhost"] = host record.row["PRperm"] = perm record.row["PRmapflag"] = mapflag record.row["PRdfile"] = dfile self.dbProtectList.append(record) def patch(self, jnl): for protect in self.dbProtectList: protect.patch(jnl, "put") class IntegHistoryRebuild(): def __init__(self, logName): self.logPath = os.getcwd() if logName: if not os.path.dirname(logName): self.logName = os.path.abspath(self.logPath + "/" + logName) else: self.logName = logName else: self.logName = os.path.abspath(self.logPath + "/IntegHistory.log.gz") try: if mimetypes.guess_type(self.logName)[1] == 'gzip': self.log = gzip.open(self.logName, "rt") else: self.log = open(self.logName) except IOError: sys.exit("Error: can\'t open " + self.logName) self.jnlPatch = os.path.abspath(self.logPath + "/" + "jnl.patch") self.serverRoot = os.path.abspath(os.getcwd() + "/" + "IntegHistoryRebuildRoot") self.user = "perforce" self.client = "IntegHistoryRebuild" self.args = "" self.createServerRoot() p4Port = "rsh:p4d -r \""+ self.serverRoot + "\" " + " -L log -vserver=3 -i" p4 = P4() p4.port = p4Port try: p4.connect() self.dbConfig = DbConfig(p4.run_dbschema("db.config")[0]) self.dbCounters = DbCounters(p4.run_dbschema("db.counters")[0]) self.dbUser = DbUser(p4.run_dbschema("db.user")[0]) self.dbDepot = DbDepot(p4.run_dbschema("db.depot")[0]) self.dbStream = DbStream(p4.run_dbschema("db.stream")[0]) self.dbDomain = DbDomain(p4.run_dbschema("db.domain")[0]) self.dbTemplate = DbTemplate(p4.run_dbschema("db.template")[0]) self.dbView = DbView(p4.run_dbschema("db.view")[0]) self.dbInteged = DbInteged(p4.run_dbschema("db.integed")[0]) self.dbRev = DbRev(p4.run_dbschema("db.rev")[0]) self.dbTraits = DbTraits(p4.run_dbschema("db.traits")[0]) self.dbChange = DbChange(p4.run_dbschema("db.change")[0]) self.dbDesc = DbDesc(p4.run_dbschema("db.desc")[0]) self.dbProtect = DbProtect(p4.run_dbschema("db.protect")[0]) except P4Exception: sys.exit("Error: " + "".join(p4.errors)) p4.disconnect() self.info = {} def toEpoch(self, date): return(str(int(time.mktime(time.strptime(date,'%Y/%m/%d %H:%M:%S')) - time.timezone))) def parseInfo(self, output): for result in output: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: self.info[match.group(1)] = match.group(2) def parseSet(self, output): for result in output: match = re.match(r'(.*?)=(.*?) .*', result, re.DOTALL) if match: self.info[match.group(1)] = match.group(2) def parseProtects(self, output): for result in output: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: self.info[match.group(1)] = match.group(2) def parseConfigure(self, output): type = name = "" for result in output: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: if match.group(1) == "Type": name = match.group(2) elif match.group(1) == "Name": name = match.group(2) elif (match.group(1) == "Value" and not type == "default") and (name == "dm.integ.engine"): self.dbConfig.add(name, match.group(2)) def parseCounters(self, output): for result in output: match = re.match(r'(.*) = (.*)', result, re.DOTALL) if match: self.dbCounters.add(match.group(1), match.group(2)) def integHow(self, how, action): value = 0 if how == "merge from": value = 0 elif how == "merge into": value = 1 elif how == "branch from" and action != 2: value = 2 elif how == "branch into" and action != 2: value = 3 elif how == "copy from": value = 4 elif how == "copy into": value = 5 elif how == "ignored" and action != 2: # target is not a deleted dbRevision value = 6 elif how == "ignored by" and action != 2: # source is not a deleted dbRevision value = 7 elif how == "delete from": value = 8 elif how == "delete into": value = 9 elif how == "edit into": value = 10 elif how == "add into": value = 11 elif how == "edit from": value = 12 elif how == "add from": value = 13 elif how == "moved from": value = 14 elif how == "moved into": value = 15 elif how == "ignored": # target is a deleted dbRevision value = 16 elif how == "ignored by": # ? value = 17 elif how == "ignored": # ? value = 18 elif how == "ignored by": # source is a deleted dbRevision value = 19 elif how == "branch from": # deleted branch value = 20 elif how == "delete into": # deleted branch value = 21 return(value) def addInteged(self, dict): startToRev = dict["startToRev"].replace("#","").replace("none", "0") endToRev = dict["endToRev"].replace("#","").replace("none", "0") startFromRev = dict["startFromRev"].replace("#","").replace("none", "0") endFromRev = dict["endFromRev"].replace("#","").replace("none", "0") action = self.dbRev.getAction(dict["toFile"], endFromRev) self.dbInteged.add(dict["toFile"], dict["fromFile"], startFromRev, endFromRev, startToRev, endToRev, self.integHow(dict["how"], action), dict["change"]) def parseInteged(self, output): dict = {} for result in output: match = re.match(r'\.\.\. (toFile) (.*)', result, re.DOTALL) if match and dict: self.addInteged(dict) dict = {} dict[match.group(1)] = match.group(2) else: match = re.search('\.\.\. (.*?) (.*)', result) if match: dict[match.group(1)] = match.group(2) if dict: self.addInteged(dict) def fileType(self, type): value = 0 if re.search('.*text.*', type): value = 0x0000000 type = type.replace("text","") elif re.search('.*binary.*', type): value = 0x0010003 type = type.replace("binary","") elif re.search('.*unicode.*', type): value = 0x0080000 type = type.replace("unicode","") elif re.search('.*symlink.*', type): value = 0x0040000 type = type.replace("symlink","") elif re.search('.*apple.*', type): value = 0x000C0000 type = type.replace("apple","") elif re.search('.*resource.*', type): value = 0x00050000 type = type.replace("resource","") elif re.search('.*utf16.*', type): value = 0x1080000 type = type.replace("utf16","") elif re.search('.*utf8.*', type): value = 0x1040000 type = type.replace("utf8","") if re.search('.*x.*', type): value = value | 0x0020000 if re.search('.*ko.*', type): value = value | 0x0010 elif re.search('.*k.*', type): value = value | 0x0020 if re.search('.*\+.*l.*', type): value = value | 0x0040 if re.search('.*w.*', type): value = value | 0x0100000 if re.search('.*m.*', type): value = value | 0x0200000 if re.search('.*C.*|.*c.*', type): value = value | 0x0000003 if re.search('.*u.*', type): value = value | 0x0000001 if re.search('.*F.*', type): value = value | 0x0000001 if re.search('^l.*', type): value = value | 0x0000001 if re.search('.*D.*', type): value = value ^ 0x0000000 if re.search('.*X.*', type): value = value | 0x0000008 match = re.search('.*S(\d+).*|.*S.*', type) if match: value = value | 0x0080 if (match.group(1) == "2"): value = value | 0x0100 if (match.group(1) == "3"): value = value | 0x0200 if (match.group(1) == "4"): value = value | 0x0300 if (match.group(1) == "5"): value = value | 0x0400 if (match.group(1) == "6"): value = value | 0x0500 if (match.group(1) == "7"): value = value | 0x0600 if (match.group(1) == "8"): value = value | 0x0700 if (match.group(1) == "9"): value = value | 0x0800 if (match.group(1) == "10"): value = value | 0x0900 if (match.group(1) == "16"): value = value | 0x0A00 if (match.group(1) == "32"): value = value | 0x0B00 if (match.group(1) == "64"): value = value | 0x0C00 if (match.group(1) == "128"): value = value | 0x0D00 if (match.group(1) == "256"): value = value | 0x0E00 if (match.group(1) == "512"): value = value | 0x0F00 return(value) def actionType(self, action): value = 0 if action == "add": value = 0 elif action == "edit": value = 1 elif action == "delete": value = 2 elif action == "branch": value = 3 elif action == "integrate": value = 4 elif action == "import": value = 5 elif action == "purge": value = 6 elif action == "move/delete": value = 7 elif action == "move/add": value = 8 elif action == "archive": value = 9 return(value) def getRevStatus(self, lazy, task, charset): value = 0 if charset == "none": value = 0 elif charset == "utf8": value = 1 elif charset == "iso8859-1": value = 2 elif charset == "utf16-nobom": value = 3 elif charset == "shiftjis": value = 4 elif charset == "eucjp": value = 5 elif charset == "winansi": value = 6 elif charset == "cp850": value = 7 elif charset == "macosroman": value = 8 elif charset == "iso8859-15": value = 9 elif charset == "iso8859-5": value = 10 elif charset == "koi8-r": value = 11 elif charset == "cp1251": value = 12 elif charset == "utf16le": value = 13 elif charset == "utf16be": value = 14 elif charset == "utf16le-bom": value = 15 elif charset == "utf16be-bom": value = 16 elif charset == "utf16-bom": value = 17 elif charset == "utf8-bom": value = 18 elif charset == "utf32-nobom": value = 19 elif charset == "utf32le": value = 20 elif charset == "utf32be": value = 21 elif charset == "utf32le-bom": value = 22 elif charset == "utf32be-bom": value = 23 elif charset == "utf32": value = 24 elif charset == "UTF_8_UNCHECKED": value = 25 elif charset == "UTF_8_UNCHECKED_BOM": value = 26 elif charset == "cp949": value = 27 elif charset == "cp936": value = 28 elif charset == "cp950": value = 29 elif charset == "cp850": value = 30 elif charset == "cp858": value = 31 elif charset == "cp1253": value = 32 elif charset == "iso8859-7": value = 33 value = value << 24 if lazy == "1": value = value | 0x0001 if task == True: value = value | 0x0004 return(value) def addRev(self, dict, traitsdict): if "depotFile" in dict: if traitsdict: for traitLot in traitsdict: traitType = traitsdict[traitLot]["type"] traitName = traitsdict[traitLot]["name"] traitValue = binascii.b2a_hex(bytes(traitsdict[traitLot]["value"],'ascii')).decode('ascii').upper() self.dbTraits.add(traitLot, traitName, traitType, traitValue) traitValue = binascii.b2a_hex(bytes(traitsdict[traitLot]["refs"],'ascii')).decode('ascii').upper() self.dbTraits.add(traitLot, "refs", 0, traitValue) if "lbrFile" not in dict: dict["lbrFile"] = dict["depotFile"] if "lbrRev" not in dict: dict["lbrRev"] = "1." + dict["headChange"] if "lbrType" not in dict: dict["lbrType"] = dict["headType"] if not "isTask" in dict: dict["isTask"] = False self.dbRev.add(dict["depotFile"], dict["headRev"], self.fileType(dict["headType"]), self.actionType(dict["headAction"]), dict["headChange"], dict["headTime"], dict["headModTime"], dict["digest"], dict["fileSize"], dict["traitLot"], self.getRevStatus(dict["lbrIsLazy"], dict["isTask"], dict["headCharset"]), dict["lbrFile"], dict["lbrRev"], self.fileType(dict["lbrType"])) def parseFstat(self, output): dict = {} traitsdict = {} for result in output: match = re.match(r'\.\.\. depotFile (.*)', result, re.DOTALL) if match: self.addRev(dict, traitsdict) dict = {} traitsdict = {} dict["depotFile"] = match.group(1) dict["traitLot"] = 0 dict["digest"] = "00000000000000000000000000000000" dict["fileSize"] = "-1" dict["lbrIsLazy"] = 0 dict["headCharset"] = "none" else: match = re.match(r'\.\.\. attr(\d+?)-refs (.*)', result, re.DOTALL) if match: if match.group(1) in traitsdict and not "refs" in traitsdict[match.group(1)]: traitsdict[match.group(1)]["refs"] = match.group(2) else: match = re.match(r'\.\.\. attr(\d+?)-(\w+?) (.*)', result, re.DOTALL) if match: if not match.group(1) in traitsdict: dict["traitLot"] = match.group(1) traitsdict[match.group(1)] = {} traitsdict[match.group(1)]["type"] = 1 traitsdict[match.group(1)]["name"] = match.group(2) traitsdict[match.group(1)]["value"] = match.group(3) else: match = re.match(r'\.\.\. attrProp(\d+?)-(\w+?) (.*)', result, re.DOTALL) if match: if not match.group(1) in traitsdict: dict["traitLot"] = match.group(1) traitsdict[match.group(1)] = {} traitsdict[match.group(1)]["type"] = 2 traitsdict[match.group(1)]["name"] = match.group(2) traitsdict[match.group(1)]["value"] = match.group(3) else: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: if match.group(1) == "isTask": dict["isTask"] = True else: dict[match.group(1)] = match.group(2) if dict: self.addRev(dict, traitsdict) def getStatus(self, status, type): value = 0 if status == "pending" and type == "public": value = 0 elif status == "submitted" and type == "public": value = 1 elif status == "shelved" and type == "public": value = 2 elif status == "pending" and type == "restricted": value = 4 elif status == "submitted" and type == "restricted": value = 5 elif status == "shelved" and type == "restricted": value = 8 elif status == "shelved" and type == "promoted": value = 16 return(value) def parseChanges(self, output): dict = {} for result in output: match = re.match(r'\.\.\. (desc)(\n.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) else: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) if dict: if not "oldChange" in dict: dict["oldChange"] = dict["change"] self.dbChange.add(dict["change"], dict["oldChange"], dict["client"], dict["user"], dict["time"], self.getStatus(dict["status"], dict["changeType"]), dict["desc"][:31], dict["path"]) self.dbDesc.add(dict["oldChange"], dict["desc"]) def depotType(self, type): value = 0 if type == "local": value = 0 elif type == "remote": value = 1 elif type == "spec": value = 2 elif type == "stream": value = 3 elif type == "archive": value = 4 elif type == "unload": value = 5 return(value) def parseDepot(self, output): dict = {} for result in output: match = re.match(r'\.\.\. (Description)(\n.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) else: match = re.match(r'\.\.\. (StreamDepth) .*/(.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) else: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) if dict: if not "Owner" in dict: dict["Owner"] = "" if not "StreamDepth" in dict: dict["StreamDepth"] = "" self.dbDepot.add(dict["Depot"], self.depotType(dict["Type"]), dict["StreamDepth"], dict["Depot"] + "/...") self.dbDomain.add(dict["Depot"], "100", "", dict["Owner"], self.toEpoch(dict["Date"]), self.toEpoch(dict["Date"]), 0, dict["Description"]) def streamType(self, type): value = 0 if type == "mainline": value = 0 elif type == "release": value = 1 elif type == "development": value = 2 elif type == "virtual": value = 3 elif type == "task": value = 4 elif type == "task - unloaded": value = 5 return(value) def streamPathType(self, type): value = 0 if type == "share": value = 0 elif type == "isolate": value = 1 elif type == "import": value = 2 elif type == "exclude": value = 3 elif type == "Remapped": value = 4 elif type == "Ignored": value = 5 elif type == "exclude": value = 6 elif type == "import+": value = 7 return(value) def streamOptions(self, options): value = 0 if re.search('.*ownersubmit.*', options): value = value | 0x0001 if re.search('.* locked.*', options): value = value | 0x0002 if re.search('.* toparent.*', options): value = value | 0x0004 if re.search('.* fromparent.*', options): value = value | 0x0008 return(value) def parseStream(self, output): dict = {} mappings = {} for result in output: match = re.match(r'\.\.\. (\w+?)(\d+) (.*)', result, re.DOTALL) if match: key = match.group(1) index = int(match.group(2)) args = match.group(3) if not key in mappings: mappings[key] = {} match = re.match(r'(.*?) "(.*?)" "(.*)"', args, re.DOTALL) if match: mappings[key][index] = [match.group(1), match.group(2), match.group(3)] else: match = re.match(r'(.*?) "(.*?)" (.*)', args, re.DOTALL) if match: mappings[key][index] = [match.group(1), match.group(2), match.group(3)] else: match = re.match(r'(.*?) (.*?) "(.*)"', args, re.DOTALL) if match: mappings[key][index] = [match.group(1), match.group(2), match.group(3)] else: match = re.match(r'(.*?) (.*?) (.*)', args, re.DOTALL) if match: mappings[key][index] = [match.group(1), match.group(2), match.group(3)] else: match = re.match(r'(.*? .*)', args, re.DOTALL) if match: mappings[key][index] = match.group(1).split() else: mappings[key][index] = [args, ""] else: match = re.match(r'\.\.\. (Description)(\n.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) else: match = re.match(r'\.\.\. (\w+?) (.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) if dict: update = access = 0 if "Update" in dict: update = self.toEpoch(dict["Update"]) if "Access" in dict: access = self.toEpoch(dict["Access"]) description = dict["Description"] self.dbStream.add(dict["Stream"], dict["Parent"], dict["Name"], self.streamType(dict["Type"]), update) self.dbDomain.add(dict["Stream"], "115", "", dict["Owner"], update, access, self.streamOptions(dict["Options"]), description) for key in mappings: for seq in sorted(mappings[key]): cmap = "" if key == "Paths": vfile = dfile = "" list = mappings[key][seq] if len(list) > 1: vfile = list[1] if len(list) > 2: if list[2].find("@") == -1: dfile = list[2] else: dfile = list[2].split("@")[0] cmap = list[2].split("@")[1] self.dbTemplate.add(dict["Stream"], self.dbCounters.maxChangelist, dict["Parent"], self.streamType(dict["Type"]), self.streamPathType(list[0]), vfile, dfile, cmap) elif key == "Remapped": list = mappings[key][seq] vfile = list[1] dfile = list[0] self.dbTemplate.add(dict["Stream"], self.dbCounters.maxChangelist, dict["Parent"], self.streamType(dict["Type"]), self.streamPathType(key), vfile, dfile, cmap) elif key == "Ignored": list = mappings[key][seq] dfile = "..." + list[0] self.dbTemplate.add(dict["Stream"], self.dbCounters.maxChangelist, dict["Parent"], self.streamType(dict["Type"]), self.streamPathType(key), "", dfile, cmap) def parseIstat(self, output): dict = {} for result in output: match = re.match(r'\.\.\. (.*?) (.*)', result, re.DOTALL) if match: dict[match.group(1)] = match.group(2) if dict: self.dbStream.update(dict["stream"], dict["change"], dict["copyParent"], dict["mergeParent"], dict["mergeHighVal"], dict["branchHash"], dict["status"]) def parse(self): for line in self.log: list = eval(str(line)) cmd = list[0] output = list[1:] if cmd == "argument": self.args = " ".join(s for s in output) else: match = re.search('^p4.*? (\w+)', cmd) if match: if match.group(1) == "info": self.parseInfo(output) elif match.group(1) == "set": self.parseSet(output) elif match.group(1) == "configure": self.parseConfigure(output) elif match.group(1) == "integed": self.parseInteged(output) elif match.group(1) == "fstat": self.parseFstat(output) elif match.group(1) == "depot": self.parseDepot(output) elif match.group(1) == "stream": self.parseStream(output) elif match.group(1) == "istat": self.parseIstat(output) elif match.group(1) == "changes": self.parseChanges(output) elif match.group(1) == "counters": self.parseCounters(output) elif match.group(1) == "protects": self.parseProtects(output) self.log.close() now = self.toEpoch(time.strftime('%Y/%m/%d %H:%M:%S',time.localtime())) self.dbUser.add("super", "super@super" , now, now, "super") self.dbProtect.add(1, 0, "super", "*", 255, 0, "//...") self.dbConfig.add("lbr.verify.out", 0) if "permMax" in self.info and self.info["permMax"] == "super": self.dbProtect.add(0, 0, self.user, "*", 255, 0, "//...") else: self.dbProtect.add(0, 0, self.user, "*", 31, 0, "//...") if "unicode" in self.info: self.dbConfig.add("unicode", "1") self.dbConfig.add("server.filecharset", 1) if "integEngine" in self.info: self.dbConfig.add("dm.integ.engine", self.info["integEngine"]) match = re.search('(//.*?/)', self.args) self.dbUser.add(self.user, self.user + "@" + self.user, now, now, self.user) self.dbView.add(self.client, 0, 0, "//" + self.client + "/...", match.group(1) + "...") self.dbDomain.add(self.client, "99", self.serverRoot, self.user, now, now, 0, "Created by " + self.user + ".\n") def createJnlPatch(self): jnlFile = open(self.jnlPatch, "wt") self.dbConfig.patch(jnlFile) self.dbCounters.patch(jnlFile) self.dbDepot.patch(jnlFile) self.dbUser.patch(jnlFile) self.dbStream.patch(jnlFile) self.dbDomain.patch(jnlFile) self.dbTemplate.patch(jnlFile) self.dbView.patch(jnlFile) self.dbInteged.patch(jnlFile) self.dbRev.patch(jnlFile) self.dbTraits.patch(jnlFile) self.dbChange.patch(jnlFile) self.dbDesc.patch(jnlFile) self.dbProtect.patch(jnlFile) jnlFile.close() def getServerRoot(self): return(self.serverRoot) def createServerRoot(self): shutil.rmtree(self.serverRoot, ignore_errors = True) if not os.path.exists(self.serverRoot): os.mkdir(self.serverRoot) os.chdir(self.serverRoot) def createServer(self): caseFlag = "-C0" if self.info["caseHandling"] == "insensitive": caseFlag = "-C1" self.createServerRoot() shutil.move(self.jnlPatch, self.serverRoot) os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -jr jnl.patch > log 2>&1") os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -xu >> log 2>&1") os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -xx >> log 2>&1") os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -jr jnl.fix >> log 2>&1") os.system("p4d -r \"" + self.serverRoot + "\" " + caseFlag + " -jc >> log 2>&1") self.port = "rsh:p4d -r \""+ self.serverRoot + "\" " + caseFlag + " -L log -vserver=3 -i" p4 = P4() if p4.env('P4CONFIG'): p4config = os.path.abspath(self.serverRoot + "/" + p4.env('P4CONFIG')) configFile = open(p4config, "wt") print("P4PORT="+ self.port, file=configFile) print("P4USER=" + self.user, file=configFile) print("P4CLIENT=" + self.client, file=configFile) if "unicode" in self.info: p4charset = "utf8" if "P4CHARSET" in self.info: p4charset = self.info["P4CHARSET"] print("P4CHARSET=" + p4charset, file=configFile) configFile.close() def createFullFile(self, fullFile, digest, fileSize): fullDir = os.path.dirname(fullFile) if not os.path.exists(fullDir): os.makedirs(fullDir) full = open(fullFile, "wt") print(digest + " " + fileSize, file=full) full.close() def createGzipFile(self, gzipFile, digest, fileSize): gzipDir = os.path.dirname(gzipFile) if not os.path.exists(gzipDir): os.makedirs(gzipDir) gz = gzip.open(gzipFile, "wt") print(digest + " " + fileSize, file=gz) gz.close() def createRCSFile(self, rcsFile, lbrRev, date, digest, fileSize): rcsDir = os.path.dirname(rcsFile) if not os.path.exists(rcsDir): os.makedirs(rcsDir) rcs = open(rcsFile, "wt") print("head " + lbrRev + ";", file=rcs) print("access ;", file=rcs) print("symbols ;", file=rcs) print("locks ;comment @@;", file=rcs) print("", file=rcs) print("", file=rcs) print(lbrRev, file=rcs) print("date " + date + "; author p4; state Exp;", file=rcs) print("branches ;", file=rcs) print("next ;", file=rcs) print("", file=rcs) print("", file=rcs) print("desc", file=rcs) print("@@", file=rcs) print("", file=rcs) print("", file=rcs) print(lbrRev, file=rcs) print("log", file=rcs) print("@@", file=rcs) print("text", file=rcs) print("@" + digest + " " + fileSize, file=rcs) print("@", file=rcs) rcs.close() def updateRCSFile(self, rcsFile, lbrRev, date, digest, fileSize): if not os.path.exists(rcsFile): self.createRCSFile(rcsFile, lbrRev, date, digest, fileSize) else: rcs = open(rcsFile, "rt") rcstmp = open(rcsFile+".tmp", "wt") for line in rcs.read().splitlines(): match = re.search('^next ;', line) if match: print("next "+ lbrRev + ";", file=rcstmp) print("", file=rcstmp) print(lbrRev, file=rcstmp) print("date " + date + "; author p4; state Exp;", file=rcstmp) print("branches ;", file=rcstmp) print("next ;", file=rcstmp) else: print(line, file=rcstmp); print("", file=rcstmp) print("", file=rcstmp) print(lbrRev, file=rcstmp) print("log", file=rcstmp) print("@@", file=rcstmp) print("text", file=rcstmp) print("@d1 1", file=rcstmp) print("a1 1", file=rcstmp) print(digest + " " + fileSize, file=rcstmp) print("@", file=rcstmp) rcs.close() rcstmp.close() os.remove(rcsFile) os.rename(rcsFile+".tmp", rcsFile) def createServerDepotFiles(self): for rev in self.dbRev.getList(): if not int(rev.row["REalazy"]) & 0x0001 == 1 and not (rev.row["REaction"] == 2 or rev.row["REaction"] == 7): serverFileType = rev.row["REatype"] & 0x000F reafile = rev.row["REafile"] if self.info["caseHandling"] == "insensitive": reafile = reafile.lower() if serverFileType == 0: rcsFile = os.path.abspath(reafile.replace("//", self.serverRoot + "/") + ",v") self.updateRCSFile(rcsFile, rev.row["REarev"], time.strftime("%Y.%m.%d.%H.%M.%S", time.localtime(int(rev.row["REdate"]))), rev.row["REdigest"], rev.row["REsize"]) elif serverFileType == 1: fullFile = os.path.abspath(reafile.replace("//", self.serverRoot + "/") + ",d/" + rev.row["REarev"]) self.createFullFile(fullFile, rev.row["REdigest"], rev.row["REsize"]) elif serverFileType == 3: gzipFile = os.path.abspath(reafile.replace("//", self.serverRoot + "/") + ",d/" + rev.row["REarev"] + ".gz") self.createGzipFile(gzipFile, rev.row["REdigest"], rev.row["REsize"]) def filetolist(self, file): alist = [] if mimetypes.guess_type(file)[1] == 'gzip': f = gzip.open(file, "rt") else: f = open(file) for line in f: line = line.strip() match = re.search(r'\'... (serverVersion .*?\')', line) if match: line = match.group(1) line = re.sub(r', \'\.\.\. Map .*?\'', '', line) line = re.sub(r', \'\.\.\. clientFile .*?\'', '', line) line = re.sub(r', \'\.\.\. isMapped?\'', '', line) line = re.sub(r', \'\.\.\. haveRev .*?\'', '', line) line = re.sub(r', \'\.\.\. isSparse?\'', '', line) line = re.sub(r', \'\.\.\. isTask?\'', '', line) line = re.sub(r', \'\.\.\. action .*?\'', '', line) line = re.sub(r', \'\.\.\. actionOwner .*?\'', '', line) line = re.sub(r', \'\.\.\. resolved\'', '', line) line = re.sub(r', \'\.\.\. reresolvable\'', '', line) line = re.sub(r', \'\.\.\. change .*?\'', '', line) line = re.sub(r', \'\.\.\. Type .*?\'', '', line) line = re.sub(r', \'\.\.\. Date .*?\'', '', line) line = re.sub(r', \'\.\.\. time .*?\'', '', line) line = re.sub(r', \'\.\.\. headTime .*?\'', '', line) line = re.sub(r', \'\.\.\. headModTime .*?\'', '', line) if (not re.match(r'\[\'language\'', line) and not re.match(r'\[\'encoding\'', line) and not re.match(r'\[\'p4 set\'', line) and not re.match(r'\[\'p4 \-ztag configure show\'', line)): alist.append(line) f.close() return(alist) def validation(self): logNameRebuild = os.path.abspath(self.serverRoot + "/IntegHistory.log.gz") try: # first try to run IntegHistory as a module # therefore IntegHistory.py must be in the current directory history = IntegHistory.IntegHistory([self.args], logNameRebuild, "UTF-8") history.getData() except: # On Linux, the "IntegHistory.py" script can be in the user PATH process = Popen("IntegHistory.py " + self.args, shell = True, stdout = PIPE) (output, err) = process.communicate() exitCode = process.wait() if not os.path.exists(logNameRebuild): print(" ==> Validation failed: Cannot find the IntegHistory.py script") else: fromlines = self.filetolist(self.logName) tolines = self.filetolist(logNameRebuild) validationFilename = os.path.abspath(self.serverRoot + "/validation.log") validationFile = open(validationFilename, "w") errors = False for diff in difflib.unified_diff(fromlines, tolines, fromfile=self.logName, tofile=logNameRebuild, n=0): validationFile.write(diff.strip() + "\n") if not re.match(r'\-\-\-', diff) and not re.match(r'\+\+\+', diff): errors = True validationFile.close() if errors: print(" ==> problems detected during validation, review differences against original IntegHistory log in " + validationFilename) def main(): parser = argparse.ArgumentParser(prog='IntegHistoryRebuild', usage='%(prog)s [IntegHistory-output-filename]') parser.add_argument("logName", nargs="?", help="IntegHistory output filename", default="IntegHistory.log.gz") args = parser.parse_args() logName = args.logName p4 = P4() if not p4.env('P4CONFIG'): sys.exit("Error: P4CONFIG variable is needed and must be set!") history = IntegHistoryRebuild(logName) print("Reading server data from " + logName + "...") history.parse() print("Creating rebuilt server...") history.createJnlPatch() history.createServer() print("Creating dummy server depot files...") history.createServerDepotFiles() print("Validating rebuilt server...") history.validation() print("Completed!") print("Server root=" + history.getServerRoot()) print("P4CONFIG file=" + os.path.abspath(history.getServerRoot() + "/" + p4.env('P4CONFIG'))) if __name__ == "__main__": main()