""" Copyright (c) 2012-2014 Perforce Software, Inc. Provided for use as defined in the Perforce Consulting Services Agreement. To see expected sync bevhaviors for various forms of the 'p4 sync' command, run: /p4/common/test/test_cbd.sh -man """ # Support methods for CBD # imports import sys, types from P4 import P4, P4Exception import logging import logging.config import os import ntpath, posixpath import re # DEBUG uncomment these two lines for remote debugging # import pydevd; # pydevd.settrace('192.168.1.15') class Cbd: def __init__(self, logstyle): # constants self.key_config = 'cfgpkg' self.attr_key_config = "attr-" + self.key_config self.cbd_home = os.getenv("CBD_HOME") self.cbd_ws = "cbd_manager" self.logcfgfile = os.path.join(self.cbd_home, "log", "log.cfg") self.syncPaths = {} self.p4 = P4() # globals to read from config files self.p4user = None self.p4passwd = None # log global self.log = None # set up logging and read config data self.initLog(logstyle) def printRewrite(self, msg): self.log.debug("Rewrite: %s" % msg) print (msg) # set up logging def initLog(self, logstyle): logging.config.fileConfig(self.logcfgfile) self.log = logging.getLogger(logstyle) # make logger available externally def getLogger(self): return self.log def __del__(self): # disconnect if self.p4.connected(): self.p4.disconnect() # Connect to Perforce def initP4(self, p4port): ticket_file = os.getenv ('P4TICKETS', 'UNDEFINED_P4TICKETS_VALUE') self.p4.prog = 'CBD' self.p4.port = p4port self.p4.user = 'perforce' self.p4.ticket_file = ticket_file self.log.debug("Using P4USER=%s" % self.p4.user) self.log.debug("Using P4PORT=%s" % self.p4.port) self.log.debug("Using P4TICKETS=%s" % self.p4.ticket_file) try: self.p4.connect() return True except P4Exception: self.log.error("Unable to connect to Perforce at P4PORT=%s" % p4port) for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) return False # make the first letter of a string lowercase def firstLower(self, s): if len(s) == 0: return s else: return s[0].lower() + s[1:] def getdata(self, args): try: res = self.p4.run(args) return res except P4Exception: for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) def printdata(self, r): if not (isinstance(r, types.NoneType)): for i in range(len(r)): if(self.p4.tagged): dictP = r[i] for key in dictP.keys(): self.log.debug("%s :: %s" % (key, dictP[key])) else: self.log.debug(r[i]) # Parse broker arguments and store them in a dictionary for # easy access. def parseBrokerArgs(self): vals = dict() data = sys.stdin.readlines() for arg in data: arg.rstrip('\n') # self.log.debug("Read %s" % arg) m = re.match("^(.+?):\s+(.+)$", arg) k = m.group(1) v = m.group(2) vals[k] = v # self.log.debug("Arg %s = %s" % (k, v)) return vals def isWindowsClient(self, cspec): root = cspec["Root"] if '/' in root: return False else: return True # Return the 'Field:' field of the workspace. def getWsField(self, reqWs, reqField): self.log.debug("CALL getWsField (%s, %s) " % (reqWs, reqField)) try: cspec = self.p4.fetch_client (reqWs) self.log.debug("CSPEC= %s " % cspec) return (cspec[reqField]) except: self.log.debug("no cspec found") for e in self.p4.errors: # Display errors self.log.debug(e) return None def loadSyncPathsFromFile (self, reqWs, stream, vSpec): """ loadSyncPathsFromFile() """ self.log.debug("CALL loadSyncPathsFromFile(%s, %s)" % (reqWs, stream)) # Find the right version of the Stream Spec Template (.cbdsst) file: sstFileFinder = "%s/....cbdsst%s" % (stream, vSpec) try: self.log.debug ("Running: p4 files -m1 -e %s" % sstFileFinder) sstFileData = self.p4.run_files ('-m1', '-e', sstFileFinder) except P4Exception: for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) self.log.warning ("Could not access Stream Spec Template file.") # we are just going to throw a warning now and pass the command through # not having a CBD file in old versions should abide by old sync rules return # This will be something like: //Game/CBD_dev/Source/Game/Game.cbdsst#8 sstFileRev = "%s#%s" % (sstFileData[0]['depotFile'], sstFileData[0]['rev']) try: self.log.debug ("Running: p4 print -q %s" % sstFileRev) sstFileContentsData = self.p4.run_print ('-q', sstFileRev) except P4Exception: self.printRewrite("action: REJECT") self.printRewrite("message: ERROR: CBD Could not print stream spec template file [%s]. Sync not performed." % sstFileRev) sys.exit(1) # The p4.run_print() call returns an array, with sstFileContentsData[0] # containing metadata and sstFileContentsData[1] containing file contents. sstFileContents = sstFileContentsData[1] pathEntriesStarted = False # Extract Path import lines and their revision specifiers. for line in sstFileContents.split('\n'): if (re.match ('Paths\:', line)): pathEntriesStarted = True continue if (pathEntriesStarted == False): continue line = line.strip() if (re.match ('share ', line)): # shortPath is just what follows the 'share' token in the # 'Paths:' field entry of a stream spec. # sharePath is the fully-qualified form of the shortPath following # the 'share' token in values in the 'Paths:' field of the stream # spec. It is obtained by prefixing shortPath with the stream # name and '/'. So for 'share src/...' in the //Jam/MAIN # stream, sharePath would be "//Jam/MAIN/src/...". # PH - This shortPath isn't used? # shortPath = re.sub ('^share ', '', line) sharePath = "%s/%s" % (self.stream, re.sub ('^share ', '', line)) self.syncPaths [sharePath] = vSpec self.log.debug ("Added syncPaths[%s%s]." % (sharePath, vSpec)) if (re.match ('import ', line)): # If a revsion specifier was found on the import line, store it. revSpec = '#head' if (re.search('#', line)): revSpec = line revSpec = re.sub('^.*#', '#', line) if (re.search('@', line)): revSpec = line revSpec = re.sub('^.*@', '@', line) depotPath = re.sub ('^.*//', '//', line) depotPath = re.sub ('(#|@).*$', '', depotPath) self.syncPaths [depotPath] = revSpec self.log.debug ("Added syncPaths[%s%s]." % (depotPath, revSpec)) def loadSyncPathsFromKeys (self, reqWs, stream): """ loadSyncPathsFromKeys() This gets the default sync paths, those stored in 'keys' on the Peforce server, for the given stream. The sync paths are stored in the self.syncPaths class variable, a dictionary with keys being the path and values being the version spec for that path. """ self.log.debug("CALL loadSyncPathsFromKeys(%s, %s)" % (reqWs, stream)) shortStreamName = re.sub ('^\/\/', '', stream) shortStreamName = re.sub ('\/', '_', shortStreamName) pathKeySearchString = "cbd_stream_%s_path*" % shortStreamName vspecKeySearchString = "cbd_stream_%s_vspec*" % shortStreamName self.log.debug ("Searching for path keys: %s" % pathKeySearchString) self.log.debug ("Searching for vspec keys: %s" % vspecKeySearchString) # These p4.run_keys calls return arrays of keys, one returning an # array of path elements, and another an array of version # specifiers. # We assume that the number of values returned, which corresponds # to the number of lines of output of our 'p4 keys' command, is # is the same for both the path and vspec variants of our key # search command. It's the job of the StreamSpecUpdate trigger # to ensure this assumption is safe. pathList = self.p4.run_keys ("-e", pathKeySearchString) vspecList = self.p4.run_keys ("-e", vspecKeySearchString) i = 0 for p in pathList: # Add the version specifer for this path, which is in the # vspecList list with the same index as the path from # pathList. Add is at a two-part tuple consisting of the # path and the vspec. self.log.debug ("i=%s" % i) self.log.debug ("p=%s" % p['value']) self.log.debug ("v=%s" % vspecList[i]['value']) self.syncPaths[p['value']] = vspecList[i]['value'] self.log.debug ("Added syncPaths[%s%s]." % (p['value'], vspecList[i]['value'])) i = i + 1 def syncWs(self, reqWs, reqCmd, reqCwd, reqArgs): self.log.info("Syncing workspace %s with arguments %s" % (reqWs, ",".join(reqArgs))) try: stream = self.getWsField(reqWs, 'Stream') if stream == None: self.log.debug("Ignoring classic workspace %s." % reqWs) print ("action: PASS\n") return True wsRoot = self.getWsField(reqWs, 'Root') self.log.debug("Syncing with command: p4 %s" % reqCmd) self.stream = stream self.streamDepot = re.sub ('//', '', stream) self.streamDepot = re.sub ('/.*$', '', self.streamDepot) flagArgs = [] pathArgs = [] self.globalVSpec = '' # default to global path unless found otherwise treatAsGlobalPath = 1 for anArg in reqArgs: if anArg.startswith("-"): flagArgs.append(anArg) else: # This is a way to turn absolute paths, which are hard to translate into relative paths # It's simplistic, but should work for most cases, will need to check fringe cases # Are we using windows paths check for x:\ at start of WSRoot if (re.match('[a-zA-Z]:\\\\', wsRoot)): wsRoot = self.firstLower(wsRoot) # make the first letter, drive letter lowercase if(anArg.startswith(wsRoot)): self.log.debug ("Found absolute path in [%s], we will strip it off" % anArg) # some Root entries have trailing delimiter if (wsRoot.endswith('\\') or wsRoot.endswith('/')): wsRootLength = len(wsRoot) else: wsRootLength = len(wsRoot) + 1 # add +1 for final \/ delimiter # strip the root off the path anArg = anArg[wsRootLength:] # Are we just doing a global if anArg.startswith('...'): anArg = anArg[len('...'):] # "Global" version specifiers are those specified with no path, e.g. # 'p4 sync @5036'. In a stock Perforce, that applies to the enitre # scope of the workspace. In CBD, it determines which stream spec # template to use, which in turn drives which revisions of imported # files to sync. If multiple global specifiers are specifed, only # last one has meaning. if (anArg.startswith("@") or anArg.startswith("#")): if not self.globalVSpec == '': self.log.warn ("Multiple global version specifers defined. Ignoring earlier ones.") self.globalVSpec = anArg self.log.debug ("Global version specifier set to: [%s]." % self.globalVSpec) else: # Paths specified as //..., ///..., and # ////... are also treated as global. # If a path arg is "//...@vSpec", treat the version specifier as # global, just as if there were no path component. If the path # arg looks like //Lvl-1/...@vSpec or //Lvl-1/Lvl-2/...@vSPec, # treat it as global only if it refers to the stream depot associated # with the current workspace. # We now count #head as a vspec since we can do sub paths on virtual pathArg = anArg self.log.debug ("Looking for ^// and (...@.* or ...$) in [%s]." % pathArg) if (re.match ('\/\/', pathArg) and (re.search ('\.\.\.@', pathArg) or re.search('\.\.\.$', pathArg))): slashes = re.findall ('/', pathArg) self.log.debug ("Slash count for [%s] is %d." % (pathArg, len(slashes))) if (len(slashes) > 4): self.log.debug ("Deferring handling of narrow path arg [%s]." % pathArg) pathArgs.append(pathArg) # defer to not a globalpath self.log.debug ("Treating path arg [%s] as non global." % pathArg) treatAsGlobalPath = 0 else: self.log.debug ("Checking [%s] for global version specifiers." % pathArg) if (len(slashes) == 2): vSpec = "#head" if (re.search ('\.\.\.@', pathArg)): vSpec = re.sub('^.*\.\.\.@', '@', pathArg) if not self.globalVSpec == '': self.log.warn ("Multiple global version specifers defined. Ignoring earlier ones.") self.globalVSpec = vSpec self.log.debug ("Global version specifier set to: %s." % self.globalVSpec) if (len(slashes) == 3): lvl1 = re.sub ('//', '', pathArg) lvl1 = re.sub ('/.*$', '', lvl1) self.log.debug ("Comparing lvl1=[%s] with stream depot name [%s]." % (lvl1, self.streamDepot)) if (lvl1 == self.streamDepot): vSpec = "#head" if (re.search ('\.\.\.@', pathArg)): vSpec = re.sub('^.*\.\.\.@', '@', pathArg) if not self.globalVSpec == '': self.log.warn ("Multiple global version specifers defined. Ignoring earlier ones.") self.globalVSpec = vSpec self.log.debug ("Global version specifier set to: %s." % self.globalVSpec) if (len(slashes) == 4): if (re.search ('\.\.\.@', pathArg)): self.log.debug ("Has vSpec.") pathPart = re.sub ('\.\.\.@.*$', '', pathArg) pathPart = re.sub ('\/$', '', pathPart) else: self.log.debug ("No vSpec.") pathPart = re.sub ('\.\.\.$', '', pathArg) pathPart = re.sub ('\/$', '', pathPart) self.log.debug ("Comparing lvl1/lvl2=[%s] with stream name [%s]." % (pathPart, self.stream)) if (pathPart == self.stream): vSpec = "#head" if (re.search ('\.\.\.@', pathArg)): vSpec = re.sub('^.*\.\.\.@', '@', pathArg) if not self.globalVSpec == '': self.log.warn ("Multiple global version specifers defined. Ignoring earlier ones.") self.globalVSpec = vSpec self.log.debug ("Global version specifier set to: %s." % self.globalVSpec) else: self.log.debug ("Deferring handling of path arg [%s]." % pathArg) self.log.debug ("Treating path arg [%s] as non global." % pathArg) treatAsGlobalPath = 0 pathArgs.append(pathArg) else: self.log.debug ("Deferring handling of path arg [%s]." % pathArg) self.log.debug ("Treating path arg [%s] as non global." % pathArg) treatAsGlobalPath = 0 pathArgs.append(pathArg) # Populate the 'syncPaths' dictionary with path/vspec pairs, either from keys or a file. if self.globalVSpec == '': self.loadSyncPathsFromKeys (reqWs, stream) else: self.loadSyncPathsFromFile (reqWs, stream, self.globalVSpec) if (treatAsGlobalPath == 1): self.log.debug ("Handling rewrite for global sync.") self.printRewrite("action: REWRITE") self.printRewrite("command: %s" % reqCmd) # Write the flag args. for anArg in flagArgs: self.printRewrite("arg: %s" % anArg) notes = [] # Write that path/vspec args with CBD-defined vspecsif we have nay if self.syncPaths: for k in self.syncPaths.keys(): self.log.debug("K=%s" % k) self.printRewrite("arg: %s%s" % (k, self.syncPaths[k])) notes.append("p4 sync CBD Sync %s%s" % (k, self.syncPaths[k])) elif self.globalVSpec: self.printRewrite("arg: %s" % self.globalVSpec) # otherwise use any vspec we parsed notes.append("No CBD keys found") if (len (notes) > 0): print ("message: \"INFO: %s.\"" % notes) self.log.debug ("User Message: INFO: %s" % notes) if (treatAsGlobalPath == 0 and len(pathArgs) > 0): self.log.debug("Processing user-supplied path args: %s" % pathArgs) self.printRewrite("action: REWRITE") self.printRewrite("command: %s" % reqCmd) for anArg in flagArgs: self.printRewrite("arg: %s" % anArg) for pathArg in pathArgs: """ For each path in the user-supplied list of paths, first check for an explicit version specifier provided by the user. If one is found, use it, and provide a warning that CBD version specifiers are ignored for that path. If no explicit version specifier is provided by the user, determine the corresponding CBD version specifier (if any). Lastly, REWRITE each path as a separate 'p4 sync' command. """ notes = [] # For path arguments for which the user supplied an explicit revision specifer, # just use it. self.log.debug("Checking [%s] for an explicit revision specifier." % pathArg) if (re.search ('@|#', pathArg)): self.log.debug ("Explicit revision specifier detected. Using it.") self.printRewrite("arg: %s" % pathArg) notes.append("CBD version specifier ignored for [%s]." % pathArg) else: # In this block, we can safely assume pathArg does not contain a revision specifier. # Strip windows pathPart = re.sub ('\\\.\.\.$', '', pathArg) # strip linux pathPart = re.sub ('\/\.\.\.$', '', pathPart) pathPart = re.sub ('\/$', '', pathPart) self.log.debug ("No explicit vspec. Seeking CBD vspec for [%s] in [%s]." % (pathPart, pathArg)) pathFound = 0 for k in self.syncPaths.keys(): self.log.debug ("Comparing key [%s] to path [%s]." % (k, pathPart)) p1 = re.sub ('\/\.\.\.', '', k) # WE don't need to sub twice # p2 = re.sub ('\/\.\.\.', '', pathPart) p2 = pathPart self.log.debug ("Searching for [%s] in [%s]." % (p2, p1)) if (re.match (p2, p1)): self.log.debug ("Using CBD vspec [%s] for [%s] from narrower user path [%s]." % (self.syncPaths[k], k, pathArg)) self.printRewrite("arg: %s%s" % (pathArg, self.syncPaths[k])) pathFound = 1 else: self.log.debug ("Searching for [%s] in [%s]." % (p1, p2)) if (re.match (p1, p2)): self.log.debug ("Using CBD vspec [%s] for [%s] from broader user path [%s]." % (self.syncPaths[k], k, pathArg)) self.printRewrite("arg: %s%s" % (pathArg, self.syncPaths[k])) pathFound = 1 if (pathFound == 0): self.log.debug ("No explicit or CBD vspec found. Passing thru.") self.printRewrite("arg: %s" % pathArg) notes.append("Path [%s] is relative or out of CBD scope. Assuming #head." % pathArg) if (len (notes) > 0): print ("message: \"INFO: %s.\"" % notes) # Warning has been confusing people self.log.debug ("User Message: INFO: %s" % notes) self.log.debug ("Path processing complete.") except P4Exception: errors = [] for e in self.p4.errors: self.log.error("Errors: " + e) errors.append(e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) errors.append(w) print ("action: REJECT\nmessage: \"Error syncing workspace: %s\"" % ",".join(errors)) return False