Cbd.py #2

"""
Copyright (c) 2012-2014 Perforce Software, Inc.  Provided for use as defined in
the Perforce Consulting Services Agreement.

To see expected sync bevhaviors for various forms of the 'p4 sync' command,
run: /p4/common/test/test_cbd.sh -man

"""
# Support methods for CBD

# imports
import sys, types
from P4 import P4, P4Exception
import logging
import logging.config
import os
import ntpath, posixpath
import re
# DEBUG uncomment these two lines for remote debugging
# import pydevd;
# pydevd.settrace('192.168.1.15')

class Cbd:

    def __init__(self, logstyle):
        # constants
        self.key_config = 'cfgpkg'
        self.attr_key_config = "attr-" + self.key_config
        self.cbd_home = os.getenv("CBD_HOME")
        self.cbd_ws = "cbd_manager"
        self.logcfgfile = os.path.join(self.cbd_home, "log", "log.cfg")
        self.syncPaths = {}
        self.p4 = P4()

        # globals to read from config files
        self.p4user = None
        self.p4passwd = None

        # log global
        self.log = None

        # set up logging and read config data
        self.initLog(logstyle)

    def printRewrite(self, msg):
        self.log.debug("Rewrite: %s" % msg)
        print (msg)

    # set up logging
    def initLog(self, logstyle):
        logging.config.fileConfig(self.logcfgfile)
        self.log = logging.getLogger(logstyle)

    # make logger available externally
    def getLogger(self):
        return self.log

    def __del__(self):
        # disconnect
        if self.p4.connected():
            self.p4.disconnect()

    # Connect to Perforce
    def initP4(self, p4port):
        ticket_file = os.getenv ('P4TICKETS', 'UNDEFINED_P4TICKETS_VALUE')
        self.p4.prog = 'CBD'
        self.p4.port = p4port
        self.p4.user = 'perforce'
        self.p4.ticket_file = ticket_file
        self.log.debug("Using P4USER=%s" % self.p4.user)
        self.log.debug("Using P4PORT=%s" % self.p4.port)
        self.log.debug("Using P4TICKETS=%s" % self.p4.ticket_file)

        try:
            self.p4.connect()
            return True

        except P4Exception:
            self.log.error("Unable to connect to Perforce at P4PORT=%s" % p4port)
            for e in self.p4.errors:
                self.log.error("Errors: " + e)
            for w in self.p4.warnings:
                self.log.warn("Warnings: " + w)
            return False

    # make the first letter of a string lowercase
    def firstLower(self, s):
        if len(s) == 0:
            return s
        else:
            return s[0].lower() + s[1:]


    def getdata(self, args):
        try:
            res = self.p4.run(args)
            return res

        except P4Exception:
            for e in self.p4.errors:
                self.log.error("Errors: " + e)
            for w in self.p4.warnings:
                self.log.warn("Warnings: " + w)

    def printdata(self, r):
        if not (isinstance(r, types.NoneType)):
            for i in range(len(r)):
                if(self.p4.tagged):
                    dictP = r[i]
                    for key in dictP.keys():
                        self.log.debug("%s :: %s" % (key, dictP[key]))
                else:
                    self.log.debug(r[i])

    # Parse broker arguments and store them in a dictionary for
    # easy access.
    def parseBrokerArgs(self):

        vals = dict()
        data = sys.stdin.readlines()
        for arg in data:
            arg.rstrip('\n')
            # self.log.debug("Read %s" % arg)
            m = re.match("^(.+?):\s+(.+)$", arg)
            k = m.group(1)
            v = m.group(2)
            vals[k] = v
            # self.log.debug("Arg %s = %s" % (k, v))
        return vals

    def isWindowsClient(self, cspec):
        root = cspec["Root"]
        if '/' in root:
            return False
        else:
            return True

    # Return the 'Field:' field of the workspace.
    def getWsField(self, reqWs, reqField):
        self.log.debug("CALL getWsField (%s, %s) " % (reqWs, reqField))

        try:
            cspec = self.p4.fetch_client (reqWs)
            self.log.debug("CSPEC= %s " % cspec)
            return (cspec[reqField])

        except:
            self.log.debug("no cspec found")
            for e in self.p4.errors:  # Display errors
                self.log.debug(e)
            return None

    def loadSyncPathsFromFile (self, reqWs, stream, vSpec):
        """ loadSyncPathsFromFile()
        """
        self.log.debug("CALL loadSyncPathsFromFile(%s, %s)" % (reqWs, stream))

        # Find the right version of the Stream Spec Template (.cbdsst) file:
        sstFileFinder = "%s/....cbdsst%s" % (stream, vSpec)

        try:
            self.log.debug ("Running: p4 files -m1 -e %s" % sstFileFinder)
            sstFileData = self.p4.run_files ('-m1', '-e', sstFileFinder)

        except P4Exception:
            for e in self.p4.errors:
                self.log.error("Errors: " + e)
            for w in self.p4.warnings:
                self.log.warn("Warnings: " + w)

            self.log.warning ("Could not access Stream Spec Template file.")
            # we are just going to throw a warning now and pass the command through
            # not having a CBD file in old versions should abide by old sync rules
            return

        # This will be something like: //Game/CBD_dev/Source/Game/Game.cbdsst#8
        sstFileRev = "%s#%s" % (sstFileData[0]['depotFile'], sstFileData[0]['rev'])

        try:
            self.log.debug ("Running: p4 print -q %s" % sstFileRev)
            sstFileContentsData = self.p4.run_print ('-q', sstFileRev)

        except P4Exception:
            self.printRewrite("action: REJECT")
            self.printRewrite("message: ERROR: CBD Could not print stream spec template file [%s].  Sync not performed." % sstFileRev)
            sys.exit(1)

        # The p4.run_print() call returns an array, with sstFileContentsData[0]
        # containing metadata and sstFileContentsData[1] containing file contents.
        sstFileContents = sstFileContentsData[1]

        pathEntriesStarted = False

        # Extract Path import lines and their revision specifiers.
        for line in sstFileContents.split('\n'):
            if (re.match ('Paths\:', line)):
                pathEntriesStarted = True
                continue
            if (pathEntriesStarted == False):
                continue

            line = line.strip()

            if (re.match ('share ', line)):
                # shortPath is just what follows the 'share' token in the
                # 'Paths:' field entry of a stream spec.
                # sharePath is the fully-qualified form of the shortPath following
                # the 'share' token in values in the 'Paths:' field of the stream
                # spec.  It is obtained by prefixing shortPath with the stream
                # name and '/'.  So for 'share src/...' in the //Jam/MAIN
                # stream, sharePath would be "//Jam/MAIN/src/...".
                # PH - This shortPath isn't used?
                # shortPath = re.sub ('^share ', '', line)
                sharePath = "%s/%s" % (self.stream, re.sub ('^share ', '', line))
                self.syncPaths [sharePath] = vSpec
                self.log.debug ("Added syncPaths[%s%s]." % (sharePath, vSpec))

            if (re.match ('import ', line)):
                # If a revsion specifier was found on the import line, store it.
                revSpec = '#head'
                if (re.search('#', line)):
                    revSpec = line
                    revSpec = re.sub('^.*#', '#', line)
                if (re.search('@', line)):
                    revSpec = line
                    revSpec = re.sub('^.*@', '@', line)
                depotPath = re.sub ('^.*//', '//', line)
                depotPath = re.sub ('(#|@).*$', '', depotPath)
                self.syncPaths [depotPath] = revSpec
                self.log.debug ("Added syncPaths[%s%s]." % (depotPath, revSpec))

    def loadSyncPathsFromKeys (self, reqWs, stream):
        """ loadSyncPathsFromKeys()

         This gets the default sync paths, those stored in 'keys'
         on the Peforce server, for the given stream.

         The sync paths are stored in the self.syncPaths class
         variable, a dictionary with keys being the path and
         values being the version spec for that path.
        """
        self.log.debug("CALL loadSyncPathsFromKeys(%s, %s)" % (reqWs, stream))
        shortStreamName = re.sub ('^\/\/', '', stream)
        shortStreamName = re.sub ('\/', '_', shortStreamName)

        pathKeySearchString = "cbd_stream_%s_path*" % shortStreamName
        vspecKeySearchString = "cbd_stream_%s_vspec*" % shortStreamName

        self.log.debug ("Searching for path keys: %s" % pathKeySearchString)
        self.log.debug ("Searching for vspec keys: %s" % vspecKeySearchString)

        # These p4.run_keys calls return arrays of keys, one returning an
        # array of path elements, and another an array of version
        # specifiers.

        # We assume that the number of values returned, which corresponds
        # to the number of lines of output of our 'p4 keys' command, is
        # is the same for both the path and vspec variants of our key
        # search command.  It's the job of the StreamSpecUpdate trigger
        # to ensure this assumption is safe.
        pathList = self.p4.run_keys ("-e", pathKeySearchString)
        vspecList = self.p4.run_keys ("-e", vspecKeySearchString)

        i = 0
        for p in pathList:
            # Add the version specifer for this path, which is in the
            # vspecList list with the same index as the path from
            # pathList.  Add is at a two-part tuple consisting of the
            # path and the vspec.
            self.log.debug ("i=%s" % i)
            self.log.debug ("p=%s" % p['value'])
            self.log.debug ("v=%s" % vspecList[i]['value'])
            self.syncPaths[p['value']] = vspecList[i]['value']
            self.log.debug ("Added syncPaths[%s%s]." % (p['value'], vspecList[i]['value']))
            i = i + 1

    def syncWs(self, reqWs, reqCmd, reqCwd, reqArgs):
        self.log.info("Syncing workspace %s with arguments %s" % (reqWs, ",".join(reqArgs)))

        try:
            stream = self.getWsField(reqWs, 'Stream')
            if stream == None:
                self.log.debug("Ignoring classic workspace %s." % reqWs)
                print ("action: PASS\n")
                return True

            wsRoot = self.getWsField(reqWs, 'Root')
            self.log.debug("Syncing with command: p4 %s" % reqCmd)

            self.stream = stream
            self.streamDepot = re.sub ('//', '', stream)
            self.streamDepot = re.sub ('/.*$', '', self.streamDepot)

            flagArgs = []
            pathArgs = []
            self.globalVSpec = ''
            # default to global path unless found otherwise
            treatAsGlobalPath = 1

            for anArg in reqArgs:
                if anArg.startswith("-"):
                    flagArgs.append(anArg)
                else:
                    # This is a way to turn absolute paths, which are hard to translate into relative paths
                    # It's simplistic, but should work for most cases, will need to check fringe cases
                    # Are we using windows paths check for x:\ at start of WSRoot
                    if (re.match('[a-zA-Z]:\\\\', wsRoot)):
                        wsRoot = self.firstLower(wsRoot)  # make the first letter, drive letter lowercase
                    if(anArg.startswith(wsRoot)):
                        self.log.debug ("Found absolute path in [%s], we will strip it off" % anArg)
                        # some Root entries have trailing delimiter
                        if (wsRoot.endswith('\\') or wsRoot.endswith('/')):
                            wsRootLength = len(wsRoot)
                        else:
                            wsRootLength = len(wsRoot) + 1  # add +1 for final \/ delimiter
                        # strip the root off the path
                        anArg = anArg[wsRootLength:]
                        # Are we just doing a global
                        if anArg.startswith('...'):
                            anArg = anArg[len('...'):]

                    # "Global" version specifiers are those specified with no path, e.g.
                    # 'p4 sync @5036'.  In a stock Perforce, that applies to the enitre
                    # scope of the workspace.  In CBD, it determines which stream spec
                    # template to use, which in turn drives which revisions of imported
                    # files to sync.  If multiple global specifiers are specifed, only
                    # last one has meaning.
                    if (anArg.startswith("@") or anArg.startswith("#")):
                        if not self.globalVSpec == '':
                            self.log.warn ("Multiple global version specifers defined. Ignoring earlier ones.")
                        self.globalVSpec = anArg
                        self.log.debug ("Global version specifier set to: [%s]." % self.globalVSpec)
                    else:
                        # Paths specified as //..., //<StreamDepotName>/..., and
                        # //<StreamDepotName>/<StreamName>/... are also treated as global.
                        # If a path arg is "//...@vSpec", treat the version specifier as
                        # global, just as if there were no path component.  If the path
                        # arg looks like //Lvl-1/...@vSpec or //Lvl-1/Lvl-2/...@vSPec,
                        # treat it as global only if it refers to the stream depot associated
                        # with the current workspace.

                        # We now count #head as a vspec since we can do sub paths on virtual
                        pathArg = anArg

                        self.log.debug ("Looking for ^// and (...@.* or ...$) in [%s]." % pathArg)
                        if (re.match ('\/\/', pathArg) and (re.search ('\.\.\.@', pathArg) or re.search('\.\.\.$', pathArg))):
                            slashes = re.findall ('/', pathArg)
                            self.log.debug ("Slash count for [%s] is %d." % (pathArg, len(slashes)))

                            if (len(slashes) > 4):
                                self.log.debug ("Deferring handling of narrow path arg [%s]." % pathArg)
                                pathArgs.append(pathArg)
                                # defer to not a globalpath
                                self.log.debug ("Treating path arg [%s] as non global." % pathArg)
                                treatAsGlobalPath = 0

                            else:
                                self.log.debug ("Checking [%s] for global version specifiers." % pathArg)

                                if (len(slashes) == 2):
                                    vSpec = "#head"
                                    if (re.search ('\.\.\.@', pathArg)):
                                        vSpec = re.sub('^.*\.\.\.@', '@', pathArg)
                                    if not self.globalVSpec == '':
                                        self.log.warn ("Multiple global version specifers defined. Ignoring earlier ones.")
                                    self.globalVSpec = vSpec
                                    self.log.debug ("Global version specifier set to: %s." % self.globalVSpec)

                                if (len(slashes) == 3):
                                    lvl1 = re.sub ('//', '', pathArg)
                                    lvl1 = re.sub ('/.*$', '', lvl1)
                                    self.log.debug ("Comparing lvl1=[%s] with stream depot name [%s]." % (lvl1, self.streamDepot))

                                    if (lvl1 == self.streamDepot):
                                        vSpec = "#head"
                                        if (re.search ('\.\.\.@', pathArg)):
                                            vSpec = re.sub('^.*\.\.\.@', '@', pathArg)

                                        if not self.globalVSpec == '':
                                            self.log.warn ("Multiple global version specifers defined. Ignoring earlier ones.")
                                        self.globalVSpec = vSpec
                                        self.log.debug ("Global version specifier set to: %s." % self.globalVSpec)

                                if (len(slashes) == 4):
                                    if (re.search ('\.\.\.@', pathArg)):
                                        self.log.debug ("Has vSpec.")
                                        pathPart = re.sub ('\.\.\.@.*$', '', pathArg)
                                        pathPart = re.sub ('\/$', '', pathPart)
                                    else:
                                        self.log.debug ("No vSpec.")
                                        pathPart = re.sub ('\.\.\.$', '', pathArg)
                                        pathPart = re.sub ('\/$', '', pathPart)

                                    self.log.debug ("Comparing lvl1/lvl2=[%s] with stream name [%s]." % (pathPart, self.stream))
                                    if (pathPart == self.stream):
                                        vSpec = "#head"
                                        if (re.search ('\.\.\.@', pathArg)):
                                            vSpec = re.sub('^.*\.\.\.@', '@', pathArg)
                                        if not self.globalVSpec == '':
                                            self.log.warn ("Multiple global version specifers defined. Ignoring earlier ones.")
                                        self.globalVSpec = vSpec
                                        self.log.debug ("Global version specifier set to: %s." % self.globalVSpec)
                                    else:
                                        self.log.debug ("Deferring handling of path arg [%s]." % pathArg)
                                        self.log.debug ("Treating path arg [%s] as non global." % pathArg)
                                        treatAsGlobalPath = 0
                                        pathArgs.append(pathArg)

                        else:
                            self.log.debug ("Deferring handling of path arg [%s]." % pathArg)
                            self.log.debug ("Treating path arg [%s] as non global." % pathArg)
                            treatAsGlobalPath = 0
                            pathArgs.append(pathArg)

            # Populate the 'syncPaths' dictionary with path/vspec pairs, either from keys or a file.
            if self.globalVSpec == '':
                self.loadSyncPathsFromKeys (reqWs, stream)
            else:
                self.loadSyncPathsFromFile (reqWs, stream, self.globalVSpec)

            if (treatAsGlobalPath == 1):
                self.log.debug ("Handling rewrite for global sync.")
                self.printRewrite("action: REWRITE")
                self.printRewrite("command: %s" % reqCmd)

                # Write the flag args.
                for anArg in flagArgs:
                    self.printRewrite("arg: %s" % anArg)

                notes = []
                # Write that path/vspec args with CBD-defined vspecsif we have nay
                if self.syncPaths:
                    for k in self.syncPaths.keys():
                        self.log.debug("K=%s" % k)
                        self.printRewrite("arg: %s%s" % (k, self.syncPaths[k]))
                        notes.append("p4 sync CBD Sync %s%s" % (k, self.syncPaths[k]))
                elif self.globalVSpec:
                    self.printRewrite("arg: %s" % self.globalVSpec)  # otherwise use any vspec we parsed
                    notes.append("No CBD keys found")

                if (len (notes) > 0):
                    print ("message: \"INFO: %s.\"" % notes)
                    self.log.debug ("User Message: INFO: %s" % notes)

            if (treatAsGlobalPath == 0 and len(pathArgs) > 0):
                self.log.debug("Processing user-supplied path args: %s" % pathArgs)

                self.printRewrite("action: REWRITE")
                self.printRewrite("command: %s" % reqCmd)

                for anArg in flagArgs:
                    self.printRewrite("arg: %s" % anArg)

                for pathArg in pathArgs:
                    """
                    For each path in the user-supplied list of paths, first check for
                    an explicit version specifier provided by the user.  If one is
                    found, use it, and provide a warning that CBD version specifiers
                    are ignored for that path.

                    If no explicit version specifier is provided by the user, determine
                    the corresponding CBD version specifier (if any).

                    Lastly, REWRITE each path as a separate 'p4 sync' command.

                    """

                    notes = []

                    # For path arguments for which the user supplied an explicit revision specifer,
                    # just use it.
                    self.log.debug("Checking [%s] for an explicit revision specifier." % pathArg)
                    if (re.search ('@|#', pathArg)):
                        self.log.debug ("Explicit revision specifier detected.  Using it.")
                        self.printRewrite("arg: %s" % pathArg)
                        notes.append("CBD version specifier ignored for [%s]." % pathArg)

                    else:
                        # In this block, we can safely assume pathArg does not contain a revision specifier.
                        # Strip windows
                        pathPart = re.sub ('\\\.\.\.$', '', pathArg)
                        # strip linux
                        pathPart = re.sub ('\/\.\.\.$', '', pathPart)
                        pathPart = re.sub ('\/$', '', pathPart)
                        self.log.debug ("No explicit vspec.  Seeking CBD vspec for [%s] in [%s]." % (pathPart, pathArg))

                        pathFound = 0

                        for k in self.syncPaths.keys():
                            self.log.debug ("Comparing key [%s] to path [%s]." % (k, pathPart))
                            p1 = re.sub ('\/\.\.\.', '', k)
                            # WE don't need to sub twice
                            # p2 = re.sub ('\/\.\.\.', '', pathPart)
                            p2 = pathPart

                            self.log.debug ("Searching for [%s] in [%s]." % (p2, p1))
                            if (re.match (p2, p1)):
                                self.log.debug ("Using CBD vspec [%s] for [%s] from narrower user path [%s]." % (self.syncPaths[k], k, pathArg))
                                self.printRewrite("arg: %s%s" % (pathArg, self.syncPaths[k]))
                                pathFound = 1

                            else:
                                self.log.debug ("Searching for [%s] in [%s]." % (p1, p2))
                                if (re.match (p1, p2)):
                                    self.log.debug ("Using CBD vspec [%s] for [%s] from broader user path [%s]." % (self.syncPaths[k], k, pathArg))
                                    self.printRewrite("arg: %s%s" % (pathArg, self.syncPaths[k]))
                                    pathFound = 1

                            if (pathFound == 0):
                                self.log.debug ("No explicit or CBD vspec found.  Passing thru.")
                                self.printRewrite("arg: %s" % pathArg)
                                notes.append("Path [%s] is relative or out of CBD scope.  Assuming #head." % pathArg)

                    if (len (notes) > 0):
                        print ("message: \"INFO: %s.\"" % notes)  # Warning has been confusing people
                        self.log.debug ("User Message: INFO: %s" % notes)

            self.log.debug ("Path processing complete.")

        except P4Exception:
            errors = []
            for e in self.p4.errors:
                self.log.error("Errors: " + e)
                errors.append(e)
            for w in self.p4.warnings:
                self.log.warn("Warnings: " + w)
                errors.append(w)
            print ("action: REJECT\nmessage: \"Error syncing workspace: %s\"" % ",".join(errors))
            return False