SSTemplateUpdate.py #1

#!/p4/common/python/bin/python3
# -*- coding: utf-8 -*-
"""@SSTemplateUpdate.py
|------------------------------------------------------------------------------|
| Copyright (c) 2008-2014 Perforce Software, Inc.  Provided for use as defined |
| in the Perforce Consulting Services Agreement.                               |
|------------------------------------------------------------------------------|
SSTemplateUpdate.py - Stream Spec Template Update, a part of the CBD system.

Environment Dependencies:
   This script assumes a complete Perforce environment including a valid,
   non-expiring ticket in the P4TICKETS file.  This script is called by
   the Perforce server, so the environment should be reliable.

"""

# Python 2.7/3.3 compatibility.
from __future__ import print_function

import sys

# Python 2.7/3.3 compatibility.
if sys.version_info[0] >= 3:
   from configparser import ConfigParser
else:
   from ConfigParser import ConfigParser

import argparse
import textwrap
import os
import re
from datetime import datetime
import logging
from P4 import P4, P4Exception

P4INSTANCE = os.getenv ('P4INSTANCE', '1')
P4PORT = os.getenv ('P4PORT', 'UNDEFINED_P4PORT_VALUE')
P4USER = os.getenv ('P4USER', 'UNDEFINED_P4USER_VALUE')
P4CLIENT = os.getenv ('P4CLIENT', 'UNDEFINED_P4CLIENT_VALUE')
del os.environ ['P4CONFIG']

DEFAULT_LOG_FILE = '/p4/%s/logs/SSTemplateUpdate.log' % P4INSTANCE
DEFAULT_VERBOSITY = 'INFO'

LOGGER_NAME = 'SSTemplateUpdateTrigger'
DEFAULTS_SECTION = 'Defaults'

VERSION = '1.0.0'

class Main:
   """ SSTemplateUpdate
   """

   def __init__(self, *argv):
      """ Initialization.  Process command line argument and initialize logging.
      """
      parser = argparse.ArgumentParser(
         formatter_class=argparse.RawDescriptionHelpFormatter,
         description=textwrap.dedent('''\
            NAME
         
            SSTemplateUpdate.py
            
            VERSION
            
            1.0.0
            
            DESCRIPTION
            
            Stream Spec template update.
            
            TRIGGER CONFIGURATION
               This script is intended to be configured as a Perforce server trigger.
               The entry in the 'Triggers:' table looks like the following: 

               SSTemplateUpdate change-commit //GreatZeus/CBD_dev/....cbdsst "/p4/common/bin/cbd/triggers/SSTemplateUpdate.py %changelist%"
            
            EXIT CODES
            
            Zero indicates normal completion.  Non-zero indicates an error.
            
            '''),
         epilog="Copyright (c) 2008-2014 Perforce Software, Inc.  Provided for use as defined in the Perforce Consulting Services Agreement."
      )
      
      parser.add_argument('changelist', help="Changelist containing an update to an versioned stream spec template (*.cbdsst) file.")
      parser.add_argument('-n', '--NoOp', action='store_true', help="Take no actions that affect data (\"No Operation\").")
      parser.add_argument('-L', '--log', default=DEFAULT_LOG_FILE, help="Default: " + DEFAULT_LOG_FILE)
      parser.add_argument('-v', '--verbosity', 
         nargs='?', 
         const="INFO",
         default=DEFAULT_VERBOSITY,
         choices=('DEBUG', 'WARNING', 'INFO', 'ERROR', 'FATAL') ,
         help="Output verbosity level. Default is: " + DEFAULT_VERBOSITY)
      
      self.myOptions = parser.parse_args()
      
      self.log = logging.getLogger(LOGGER_NAME)
      self.log.setLevel(self.myOptions.verbosity)
      #self.logHandler = logging.FileHandler(self.myOptions.log)
      self.logHandler = logging.FileHandler(self.myOptions.log, mode='w') #Use 'w' for debugging, 'a' for live production usage.
      # df = datestamp formatter; bf= basic formatter.
      df = logging.Formatter('%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S')
      bf = logging.Formatter('%(levelname)s: %(message)s')
      self.logHandler.setFormatter(bf)
      self.log.addHandler (self.logHandler)

      self.log.debug ("Command Line Options: %s\n" % self.myOptions)
      
      if (self.myOptions.NoOp):
         self.log.info ("Running in NO-OP mode.")

   # Connect to Perforce
   def initP4(self):
      """Connect to Perforce."""
      self.p4 = P4()
      self.p4.prog = LOGGER_NAME
      self.p4.version = VERSION
      self.p4.port = P4PORT
      self.p4.user = P4USER
      self.p4.client = P4CLIENT
      self.p4.api_level = 74
      ###self.p4.handler = self.logHandler

      try:
         self.p4.connect()

      except P4Exception:
         self.log.fatal("Unable to connect to Perforce at P4PORT=%s" % p4port)
         for e in self.p4.errors:
            self.log.error("Errors: " + e)
         for w in self.p4.warnings:
            self.log.warn("Warnings: " + w)
         return False

      return True

   def get_sst_files(self):
      """Find all *.cbdsst files submitted in the given changelist.
         Set set.workspace, self.sst_files[].
         Return sst_file_count, the number of *.cbdsst files associated with the changelist.
      """

      sst_file_count = 0

      if (not re.match ('\d+$', self.myOptions.changelist)):
         self.log.fatal ("The value supplied as a changelist number [%s] must be purely numeric." % self.myOptions.changelist)
         return 0

      self.log.info ("Processing changelist %s." % self.myOptions.changelist)
      self.initP4()

      try:
         cData = self.p4.run_describe('-s', self.myOptions.changelist)

      except P4Exception:
         self.log.fatal("Failed to describe changlist %s." % self.myOptions.changelist)

         for e in self.p4.errors:
            self.log.error("Errors: " + e)
         for w in self.p4.warnings:
            self.log.warn("Warnings: " + w)

         return 0

      self.log.debug ("Data: %s" % cData)

      self.sst_files = []
      index = -1
      sst_file_count = 0

      # Sausage here!  Take the 'path' field, which will look something
      # like //some/path/then/maybe/several/sub/folders/*, and distill
      # it down to just //streamDepot/streamName
      self.stream = cData[0]['path']
      self.stream = re.sub ('//', '', self.stream)
      self.stream = re.sub ('/', '__Q__', self.stream, count=1)
      self.stream = re.sub ('/.*$', '', self.stream)
      self.stream = re.sub ('__Q__', '/', self.stream, count=1)
      self.stream = "//%s" % self.stream

      self.log.debug ("Stream: %s" % self.stream)

      isTaskStream = 0

      try:
         if cData[0]['depotFile']:
            isTaskStream = 0

      except KeyError:
         isTaskStream = 1

      if not isTaskStream:
         for file in cData[0]['depotFile']:
            index = index + 1

            if (re.search ('\.cbdsst$', file)):
               action = cData[0]['action'][index]
               rev = cData[0]['rev'][index]

               # Ignore actions other than add, move/add, branch, edit, and integrate.
               # The delete and move/delete actions are ignored, as well as the
               # purged and archived.
               if (not re.search ('add|branch|edit|integrate', action)):
                  self.log.warn ("Ignored '%s' action on %s#%s." % (action, file, rev))
                  continue

               self.sst_files.append(file)
               sst_file_count = sst_file_count + 1

      return sst_file_count

   def update_stream_spec_and_keys (self, file):
      """Update the stream spec on the live server from the template.
         Paths:
         share Source/...
         import GreatZeus/wp_framework/... //wp_framework/...@5036

       """

      self.log.info ("Updating live stream spec from %s." % file)

      sData = self.p4.fetch_stream (self.stream)
      self.log.debug ("SDATA1: %s" % sData)

      if (re.search ('\[\s*NO_CBD\s*\]', sData['Description'], re.IGNORECASE)):
         self.log.warn ("Found '[NO_CBD]' tag in stream spec.  Not updating stream %s." % self.stream)
         return False

      sstFileContents = self.p4.run_print ('-q', file)
      oldDesc = sstFileContents[1]
      newDesc = ''

      # Parse 'Description:' field value from the *.cbdsst file, substituting the
      # stream name tag.
      for line in oldDesc.split('\n'):
         if (re.match ('^\s*#', line) or re.match('^Description\:', line)):
            continue
         if (re.match ('Paths\:', line)):
            break
         line = re.sub ('__EDITME_STREAM__', self.stream, line)
         line.strip()

         if not newDesc:
            newDesc = line
         else:
            newDesc = newDesc + '\n' + line

      newDesc = re.sub ('^\t', '', newDesc)
      newDesc = re.sub ('\n\t', '\n', newDesc)

      sData['Description'] = newDesc

      # The paths() array includes data from the 'Paths:' field of the stream
      # spec, augmented by revision specifiers extracted from the stream spec
      # template file.
      paths = list()

      # The newPaths() list is similar to paths, but excludes the revsion
      # specifiers, as they're not valid for P4D 2013.2 and lower servers.
      # It is in a form suitalbe for feeding directly to the 'Paths' field
      # of the stream spec using the P4Python API.
      newPaths = list()

      # This is a count/index for both the paths() and newPaths() arrays,
      # as they both have the name number of elements.
      pathsCount = 0

      # Parse import path entries form the 'Paths:' field in the *.cbdsst file.
      # If we find revision specifiers on the depot paths, strip them off
      # before feeding them to the server, but also preserve the revision
      # specifiers for writing  to the 'p4 keys' store.
      pathEntriesStarted = False

      for line in oldDesc.split('\n'):
         if (re.match ('Paths\:', line)):
            pathEntriesStarted = True
            continue
         if (pathEntriesStarted == False):
            continue

         line = line.strip()

         if (re.match ('share ', line)):
            # shortPath is just what follows the 'share' token in the
            # 'Paths:' field entry of a stream spec.
            # sharePath is the fully-qualified form of the shortPath following
            # the 'share' token in values in the 'Paths:' field of the stream
            # spec.  It is obtained by prefixing shortPath with the stream
            # name and '/'.  So for 'share src/...' in the //Jam/MAIN
            # stream, sharePath would be "//Jam/MAIN/src/...".
            shortPath = re.sub ('^share ', '', line)
            sharePath = "%s/%s" % (self.stream, re.sub ('^share ', '', line))
            paths.append (('share', sharePath))
            newPaths.append ("share %s" % shortPath)
            self.log.debug ("PATH DATA %s %s" % paths[pathsCount])
         if (re.match ('import ', line)):
            # If a revsion specifier was found on the import line, store it.
            revSpec = '#head'
            if (re.search('#', line)):
               revSpec = line
               revSpec = re.sub('^.*#', '#', line)
            if (re.search('@', line)):
               revSpec = line
               revSpec = re.sub('^.*@', '@', line)
            localPath = re.sub ('import ', '', line)
            localPath = re.sub (' //.*$', '', localPath)
            depotPath = re.sub ('^.*//', '//', line)
            depotPath = re.sub ('(#|@).*$', '', depotPath)
            paths.append (('import', localPath, depotPath, revSpec))
            # The newPaths() list excludes the revisionspecifier, since P4D servers 2013.2 and
            # lower cannot handle it.
            newPaths.append ("import %s %s" % (localPath, depotPath))
            self.log.debug ("PATH DATA %s L:[%s] D:[%s] R:[%s]" % paths[pathsCount])

         pathsCount = pathsCount + 1

      self.log.debug ("== Path Entries ==")
      for pathEntry in paths:
         if (pathEntry[0] == 'share'):
            self.log.debug ("PATH ENTRY: share %s" % pathEntry[1])
         if (pathEntry[0] == 'import'):
            self.log.debug ("PATH ENTRY: import %s %s%s" % (pathEntry[1], pathEntry[2], pathEntry[3]))

      sData['Paths'] = newPaths
      self.log.debug ("SDATA2: %s" % sData)

      if (not self.myOptions.NoOp):
         self.log.debug ("Uploading modified stream spec to live server.")

         try:
            self.p4.save_stream(sData)
            self.log.debug ("Saved stream spec %s" % self.stream)

         except P4Exception:
            self.log.fatal("Failed to save stream spec to live server.")

            for e in self.p4.errors:
               self.log.error("Errors: " + e)
            for w in self.p4.warnings:
               self.log.warn("Warnings: " + w)

         #return False

      else:
         self.log.debug ("NO-OP: Would upload modified stream spec to live server.")

      # Next, update the 'p4 keys' on the server.
      # First, generate a list of existing keys for this stream to remove.
      keyNameBase = 'cbd_stream_%s' % self.stream
      keyNameBase = re.sub ('//', '', keyNameBase)
      keyNameBase = re.sub ('/', '_', keyNameBase)
      pathKeySearch = keyNameBase
      vSpecKeySearch = keyNameBase
      pathKeySearch = '%s_path*' % pathKeySearch
      vSpecKeySearch = '%s_vspec*' % vSpecKeySearch
      pathKeysData = self.p4.run_keys('-e', pathKeySearch)
      vSpecKeysData = self.p4.run_keys('-e', vSpecKeySearch)

      for keyData in pathKeysData:
         if (not self.myOptions.NoOp):
            self.log.debug ("Running: p4 keys -d %s" % keyData['key'])

            try:
               ### self.p4.run_key('-d', keyData['key'])
               self.p4.run_key(keyData['key'])

            except P4Exception:
               self.log.fatal("Failed to delete key [%s] from server." % keyData['key'])

               for e in self.p4.errors:
                  self.log.error("Errors: " + e)
               for w in self.p4.warnings:
                  self.log.warn("Warnings: " + w)
         else:
            self.log.debug ("NO-OP: Would run p4 keys -d %s" % keyData['key'])

      for keyData in vSpecKeysData:
         if (not self.myOptions.NoOp):
            self.log.debug ("Running: p4 keys -d %s" % keyData['key'])

            try:
               ### self.p4.run_key('-d', keyData['key'])
               self.p4.run_key(keyData['key'])

            except P4Exception:
               self.log.fatal("Failed to delete key [%s] from server." % keyData['key'])

               for e in self.p4.errors:
                  self.log.error("Errors: " + e)
               for w in self.p4.warnings:
                  self.log.warn("Warnings: " + w)

         else:
            self.log.debug ("NO-OP: Would run p4 keys -d %s" % keyData['key'])

      # Finally, generate the new keys.
      i = 0
      for pathEntry in paths:
         pathKeyName = "%s_path%d" % (keyNameBase, i)
         vSpecKeyName = "%s_vspec%d" % (keyNameBase, i)
         if (pathEntry[0] == 'share'):
            pathKeyValue = pathEntry[1]
            vSpecKeyValue = '#head'
         if (pathEntry[0] == 'import'):
            pathKeyValue = pathEntry[2]
            vSpecKeyValue = pathEntry[3]

         if (not self.myOptions.NoOp):
            self.log.debug ("Running: p4 key %s %s" % (pathKeyName, pathKeyValue))

            try:
               self.p4.run_key(pathKeyName, pathKeyValue)

            except P4Exception:
               self.log.fatal("Failed to create path key [%s] with value [%s]." % (pathKeyName, pathKeyValue))

               for e in self.p4.errors:
                  self.log.error("Errors: " + e)
               for w in self.p4.warnings:
                  self.log.warn("Warnings: " + w)

            self.log.debug ("Running p4 key %s %s" % (vSpecKeyName, vSpecKeyValue))

            try:
               self.p4.run_key(vSpecKeyName, vSpecKeyValue)

            except P4Exception:
               self.log.fatal("Failed to create vspec key [%s] with value [%s]." % (vSpecKeyName, vSpecKeyValue))

               for e in self.p4.errors:
                  self.log.error("Errors: " + e)
               for w in self.p4.warnings:
                  self.log.warn("Warnings: " + w)


         else:
            self.log.debug ("NO-OP: Would run p4 key %s %s" % (pathKeyName, pathKeyValue))
            self.log.debug ("NO-OP: Would run p4 key %s %s" % (vSpecKeyName, vSpecKeyValue))

         i = i + 1

      return True

   def update_modified_stream_specs (self):
      """Update stream specs for the given changelist."""

      if (self.get_sst_files()):
         for file in self.sst_files:
            self.update_stream_spec_and_keys (file)
      else:
         return False

if __name__ == '__main__':
   """ Main Program
   """
   main = Main(*sys.argv[1:])
   Main.update_modified_stream_specs(main)