#!/p4/common/python/bin/python3 # -*- coding: utf-8 -*- """@SSTemplateUpdate.py |------------------------------------------------------------------------------| | Copyright (c) 2008-2014 Perforce Software, Inc. Provided for use as defined | | in the Perforce Consulting Services Agreement. | |------------------------------------------------------------------------------| SSTemplateUpdate.py - Stream Spec Template Update, a part of the CBD system. Environment Dependencies: This script assumes a complete Perforce environment including a valid, non-expiring ticket in the P4TICKETS file. This script is called by the Perforce server, so the environment should be reliable. """ # Python 2.7/3.3 compatibility. from __future__ import print_function import sys # Python 2.7/3.3 compatibility. if sys.version_info[0] >= 3: from configparser import ConfigParser else: from ConfigParser import ConfigParser import argparse import textwrap import os import re from datetime import datetime import logging from P4 import P4, P4Exception P4INSTANCE = os.getenv ('P4INSTANCE', '1') P4PORT = os.getenv ('P4PORT', 'UNDEFINED_P4PORT_VALUE') P4USER = os.getenv ('P4USER', 'UNDEFINED_P4USER_VALUE') P4CLIENT = os.getenv ('P4CLIENT', 'UNDEFINED_P4CLIENT_VALUE') del os.environ ['P4CONFIG'] DEFAULT_LOG_FILE = '/p4/%s/logs/SSTemplateUpdate.log' % P4INSTANCE DEFAULT_VERBOSITY = 'INFO' LOGGER_NAME = 'SSTemplateUpdateTrigger' DEFAULTS_SECTION = 'Defaults' VERSION = '1.0.0' class Main: """ SSTemplateUpdate """ def __init__(self, *argv): """ Initialization. Process command line argument and initialize logging. """ parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent('''\ NAME SSTemplateUpdate.py VERSION 1.0.0 DESCRIPTION Stream Spec template update. TRIGGER CONFIGURATION This script is intended to be configured as a Perforce server trigger. The entry in the 'Triggers:' table looks like the following: SSTemplateUpdate change-commit //GreatZeus/CBD_dev/....cbdsst "/p4/common/bin/cbd/triggers/SSTemplateUpdate.py %changelist%" EXIT CODES Zero indicates normal completion. Non-zero indicates an error. '''), epilog="Copyright (c) 2008-2014 Perforce Software, Inc. Provided for use as defined in the Perforce Consulting Services Agreement." ) parser.add_argument('changelist', help="Changelist containing an update to an versioned stream spec template (*.cbdsst) file.") parser.add_argument('-n', '--NoOp', action='store_true', help="Take no actions that affect data (\"No Operation\").") parser.add_argument('-L', '--log', default=DEFAULT_LOG_FILE, help="Default: " + DEFAULT_LOG_FILE) parser.add_argument('-v', '--verbosity', nargs='?', const="INFO", default=DEFAULT_VERBOSITY, choices=('DEBUG', 'WARNING', 'INFO', 'ERROR', 'FATAL') , help="Output verbosity level. Default is: " + DEFAULT_VERBOSITY) self.myOptions = parser.parse_args() self.log = logging.getLogger(LOGGER_NAME) self.log.setLevel(self.myOptions.verbosity) #self.logHandler = logging.FileHandler(self.myOptions.log) self.logHandler = logging.FileHandler(self.myOptions.log, mode='w') #Use 'w' for debugging, 'a' for live production usage. # df = datestamp formatter; bf= basic formatter. df = logging.Formatter('%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S') bf = logging.Formatter('%(levelname)s: %(message)s') self.logHandler.setFormatter(bf) self.log.addHandler (self.logHandler) self.log.debug ("Command Line Options: %s\n" % self.myOptions) if (self.myOptions.NoOp): self.log.info ("Running in NO-OP mode.") # Connect to Perforce def initP4(self): """Connect to Perforce.""" self.p4 = P4() self.p4.prog = LOGGER_NAME self.p4.version = VERSION self.p4.port = P4PORT self.p4.user = P4USER self.p4.client = P4CLIENT self.p4.api_level = 74 ###self.p4.handler = self.logHandler try: self.p4.connect() except P4Exception: self.log.fatal("Unable to connect to Perforce at P4PORT=%s" % p4port) for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) return False return True def get_sst_files(self): """Find all *.cbdsst files submitted in the given changelist. Set set.workspace, self.sst_files[]. Return sst_file_count, the number of *.cbdsst files associated with the changelist. """ sst_file_count = 0 if (not re.match ('\d+$', self.myOptions.changelist)): self.log.fatal ("The value supplied as a changelist number [%s] must be purely numeric." % self.myOptions.changelist) return 0 self.log.info ("Processing changelist %s." % self.myOptions.changelist) self.initP4() try: cData = self.p4.run_describe('-s', self.myOptions.changelist) except P4Exception: self.log.fatal("Failed to describe changlist %s." % self.myOptions.changelist) for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) return 0 self.log.debug ("Data: %s" % cData) self.sst_files = [] index = -1 sst_file_count = 0 # Sausage here! Take the 'path' field, which will look something # like //some/path/then/maybe/several/sub/folders/*, and distill # it down to just //streamDepot/streamName self.stream = cData[0]['path'] self.stream = re.sub ('//', '', self.stream) self.stream = re.sub ('/', '__Q__', self.stream, count=1) self.stream = re.sub ('/.*$', '', self.stream) self.stream = re.sub ('__Q__', '/', self.stream, count=1) self.stream = "//%s" % self.stream self.log.debug ("Stream: %s" % self.stream) isTaskStream = 0 try: if cData[0]['depotFile']: isTaskStream = 0 except KeyError: isTaskStream = 1 if not isTaskStream: for file in cData[0]['depotFile']: index = index + 1 if (re.search ('\.cbdsst$', file)): action = cData[0]['action'][index] rev = cData[0]['rev'][index] # Ignore actions other than add, move/add, branch, edit, and integrate. # The delete and move/delete actions are ignored, as well as the # purged and archived. if (not re.search ('add|branch|edit|integrate', action)): self.log.warn ("Ignored '%s' action on %s#%s." % (action, file, rev)) continue self.sst_files.append(file) sst_file_count = sst_file_count + 1 return sst_file_count def update_stream_spec_and_keys (self, file): """Update the stream spec on the live server from the template. Paths: share Source/... import GreatZeus/wp_framework/... //wp_framework/...@5036 """ self.log.info ("Updating live stream spec from %s." % file) sData = self.p4.fetch_stream (self.stream) self.log.debug ("SDATA1: %s" % sData) if (re.search ('\[\s*NO_CBD\s*\]', sData['Description'], re.IGNORECASE)): self.log.warn ("Found '[NO_CBD]' tag in stream spec. Not updating stream %s." % self.stream) return False sstFileContents = self.p4.run_print ('-q', file) oldDesc = sstFileContents[1] newDesc = '' # Parse 'Description:' field value from the *.cbdsst file, substituting the # stream name tag. for line in oldDesc.split('\n'): if (re.match ('^\s*#', line) or re.match('^Description\:', line)): continue if (re.match ('Paths\:', line)): break line = re.sub ('__EDITME_STREAM__', self.stream, line) line.strip() if not newDesc: newDesc = line else: newDesc = newDesc + '\n' + line newDesc = re.sub ('^\t', '', newDesc) newDesc = re.sub ('\n\t', '\n', newDesc) sData['Description'] = newDesc # The paths() array includes data from the 'Paths:' field of the stream # spec, augmented by revision specifiers extracted from the stream spec # template file. paths = list() # The newPaths() list is similar to paths, but excludes the revsion # specifiers, as they're not valid for P4D 2013.2 and lower servers. # It is in a form suitalbe for feeding directly to the 'Paths' field # of the stream spec using the P4Python API. newPaths = list() # This is a count/index for both the paths() and newPaths() arrays, # as they both have the name number of elements. pathsCount = 0 # Parse import path entries form the 'Paths:' field in the *.cbdsst file. # If we find revision specifiers on the depot paths, strip them off # before feeding them to the server, but also preserve the revision # specifiers for writing to the 'p4 keys' store. pathEntriesStarted = False for line in oldDesc.split('\n'): if (re.match ('Paths\:', line)): pathEntriesStarted = True continue if (pathEntriesStarted == False): continue line = line.strip() if (re.match ('share ', line)): # shortPath is just what follows the 'share' token in the # 'Paths:' field entry of a stream spec. # sharePath is the fully-qualified form of the shortPath following # the 'share' token in values in the 'Paths:' field of the stream # spec. It is obtained by prefixing shortPath with the stream # name and '/'. So for 'share src/...' in the //Jam/MAIN # stream, sharePath would be "//Jam/MAIN/src/...". shortPath = re.sub ('^share ', '', line) sharePath = "%s/%s" % (self.stream, re.sub ('^share ', '', line)) paths.append (('share', sharePath)) newPaths.append ("share %s" % shortPath) self.log.debug ("PATH DATA %s %s" % paths[pathsCount]) if (re.match ('import ', line)): # If a revsion specifier was found on the import line, store it. revSpec = '#head' if (re.search('#', line)): revSpec = line revSpec = re.sub('^.*#', '#', line) if (re.search('@', line)): revSpec = line revSpec = re.sub('^.*@', '@', line) localPath = re.sub ('import ', '', line) localPath = re.sub (' //.*$', '', localPath) depotPath = re.sub ('^.*//', '//', line) depotPath = re.sub ('(#|@).*$', '', depotPath) paths.append (('import', localPath, depotPath, revSpec)) # The newPaths() list excludes the revisionspecifier, since P4D servers 2013.2 and # lower cannot handle it. newPaths.append ("import %s %s" % (localPath, depotPath)) self.log.debug ("PATH DATA %s L:[%s] D:[%s] R:[%s]" % paths[pathsCount]) pathsCount = pathsCount + 1 self.log.debug ("== Path Entries ==") for pathEntry in paths: if (pathEntry[0] == 'share'): self.log.debug ("PATH ENTRY: share %s" % pathEntry[1]) if (pathEntry[0] == 'import'): self.log.debug ("PATH ENTRY: import %s %s%s" % (pathEntry[1], pathEntry[2], pathEntry[3])) sData['Paths'] = newPaths self.log.debug ("SDATA2: %s" % sData) if (not self.myOptions.NoOp): self.log.debug ("Uploading modified stream spec to live server.") try: self.p4.save_stream(sData) self.log.debug ("Saved stream spec %s" % self.stream) except P4Exception: self.log.fatal("Failed to save stream spec to live server.") for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) #return False else: self.log.debug ("NO-OP: Would upload modified stream spec to live server.") # Next, update the 'p4 keys' on the server. # First, generate a list of existing keys for this stream to remove. keyNameBase = 'cbd_stream_%s' % self.stream keyNameBase = re.sub ('//', '', keyNameBase) keyNameBase = re.sub ('/', '_', keyNameBase) pathKeySearch = keyNameBase vSpecKeySearch = keyNameBase pathKeySearch = '%s_path*' % pathKeySearch vSpecKeySearch = '%s_vspec*' % vSpecKeySearch pathKeysData = self.p4.run_keys('-e', pathKeySearch) vSpecKeysData = self.p4.run_keys('-e', vSpecKeySearch) for keyData in pathKeysData: if (not self.myOptions.NoOp): self.log.debug ("Running: p4 keys -d %s" % keyData['key']) try: ### self.p4.run_key('-d', keyData['key']) self.p4.run_key(keyData['key']) except P4Exception: self.log.fatal("Failed to delete key [%s] from server." % keyData['key']) for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) else: self.log.debug ("NO-OP: Would run p4 keys -d %s" % keyData['key']) for keyData in vSpecKeysData: if (not self.myOptions.NoOp): self.log.debug ("Running: p4 keys -d %s" % keyData['key']) try: ### self.p4.run_key('-d', keyData['key']) self.p4.run_key(keyData['key']) except P4Exception: self.log.fatal("Failed to delete key [%s] from server." % keyData['key']) for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) else: self.log.debug ("NO-OP: Would run p4 keys -d %s" % keyData['key']) # Finally, generate the new keys. i = 0 for pathEntry in paths: pathKeyName = "%s_path%d" % (keyNameBase, i) vSpecKeyName = "%s_vspec%d" % (keyNameBase, i) if (pathEntry[0] == 'share'): pathKeyValue = pathEntry[1] vSpecKeyValue = '#head' if (pathEntry[0] == 'import'): pathKeyValue = pathEntry[2] vSpecKeyValue = pathEntry[3] if (not self.myOptions.NoOp): self.log.debug ("Running: p4 key %s %s" % (pathKeyName, pathKeyValue)) try: self.p4.run_key(pathKeyName, pathKeyValue) except P4Exception: self.log.fatal("Failed to create path key [%s] with value [%s]." % (pathKeyName, pathKeyValue)) for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) self.log.debug ("Running p4 key %s %s" % (vSpecKeyName, vSpecKeyValue)) try: self.p4.run_key(vSpecKeyName, vSpecKeyValue) except P4Exception: self.log.fatal("Failed to create vspec key [%s] with value [%s]." % (vSpecKeyName, vSpecKeyValue)) for e in self.p4.errors: self.log.error("Errors: " + e) for w in self.p4.warnings: self.log.warn("Warnings: " + w) else: self.log.debug ("NO-OP: Would run p4 key %s %s" % (pathKeyName, pathKeyValue)) self.log.debug ("NO-OP: Would run p4 key %s %s" % (vSpecKeyName, vSpecKeyValue)) i = i + 1 return True def update_modified_stream_specs (self): """Update stream specs for the given changelist.""" if (self.get_sst_files()): for file in self.sst_files: self.update_stream_spec_and_keys (file) else: return False if __name__ == '__main__': """ Main Program """ main = Main(*sys.argv[1:]) Main.update_modified_stream_specs(main)
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#16 | 19419 | C. Thomas Tyler | Fixed typo in logged output. | ||
#15 | 19337 | C. Thomas Tyler |
Added support for StreamDepth > 1. Enhanced test cases for StreamDepth > 1. |
||
#14 | 19253 | C. Thomas Tyler | Routine merge-down from main to dev. | ||
#13 | 15274 | C. Thomas Tyler |
Merge Down of cbd to dev from main using: p4 merge -b perforce_software-cbd-dev |
||
#12 | 15157 | C. Thomas Tyler |
Fixed CBD environment-deprivation issue for CBD trigger. Enhanced stream spec parsing. |
||
#11 | 14978 | C. Thomas Tyler | Enhanced error auditing. | ||
#10 | 14855 | C. Thomas Tyler |
Removed '-n' (NoOp) mode, as it is no longer viable as a pre-commit triggers. Updated test suite to use pre-commit 'submit-content' type triggers. |
||
#9 | 14854 | C. Thomas Tyler |
Removed ability to opt out with 'NO_CBD' tag in the changelist description. This feature has not been used, and is not compatible with the new pre-commit mode of operation, nor with other changes in progress to enable a bi-directional flow. |
||
#8 | 14853 | C. Thomas Tyler |
This change introduces a 'p4 login -s' login status check. If it fails, the changelist submit attempt is rejected, with an appropriate message for the user. Added user message and change submit rejection if the Perforce connection check fails. This is unlikely as we're called by the Perforce server. Added user message and change submit rejection if the 'p4 describe' on the user's changelist fails for whatever reason. |
||
#7 | 14852 | C. Thomas Tyler |
Converted SSTemplateUpdate to work as a trigger of type 'change-content' rather than 'change-commit', where it has the new responsibility of rejecting the submit of changelists attempting to update *.cbdsst when the generated stream specs are rejected by the Helix server (or otherwise fail to update). The CBD keys are updated if and only the stream spec is accepted by the live server. This change removes a common failure mode where *.cbdsst file could be submitted containing bad content (e.g. syntax errors in the 'Paths:' field), causing the the corresponding keys not to be updated. This change introduces a failure mode where the keys could be updated, yet the changelist containing the *.cbdsst file fails to be submitted, e.g. due to server crash. Because the server processes changes first before calling the trigger, the submit is likely to succeed, since things like protections and required resolves are already checked by the time this script is called. We do NOT rollback stream spec or keys updates in event of such failure. The assumption is that the issue with the bogus submit will be addressed quickly. Several tests performed manually: * Submit of valid *.cbdsst file. Succeeds. * Submit of bogus *.cbdsst file. Rejected as expected, with server error message deliverd with user to help debug it. * Deletion of jam.cbdsst. Works as expected. * Recovery of deleted jam.cbdsst. Works as expected. * Rename of jam.cbdsst to foo (a non-*.cbdsst file). Allows change to go thru, no keys processing or stream spec update. * Failed submit of *.cbdsst due to protections change. Works as expected; handled entirley by p4d. * Attempted submit of unmodified *.cbdsst in a workspact with SubmitUnchnaged not set. Works as expected; handled entirley by p4d. TO DO: * Add several test cases covering this functionality to the test regression test suite. * Update deployment documentation. * Consider whether we should attempt to rollback keys and stream spec updates in unlikely event the 'p4 submit' fails. Such a failure is unlikely because, by the time this script is called, the content s of files are already on the server, and 'p4d' has already done various checks such as access checks and resolve-needed checks. Worst case scenario is bad though: stream specs could get updated without a submit, leading to lack of notificaiton. (This situation is no worse, and overall much better, than the earlier version.) |
||
#6 | 14839 | C. Thomas Tyler | Updated to latest CBD from dev box. | ||
#5 | 14198 | C. Thomas Tyler |
Got Workshop up to date with latest version of CBD developed elsewhere. Added CbdDev.py to illustrate enabling testing a newer version on a live server, as a supplement to the test suite. This comes with supporting scripts wssync.dev.(sh,py) Added cmd_trig_by_auth.pl, a technology sample script. Not used presently. |
||
#4 | 13833 | C. Thomas Tyler |
Merge-down from main. Updated Stream Spec Update Trigger (SSTemplateUpdate.py.) * Refined import+ handling. * Switched logging to 'append' mode. * Increased default logging verbosity INFO -> DEBUG. * Added stream spec data to debug output. |
||
#3 | 13773 | C. Thomas Tyler |
Added support for handling import+ entries. Fixed bug where excess keys were not cleaned up, e.g. if a stream spec was modified to delete an import entry. Corrected text in debug output. |
||
#2 | 11363 | C. Thomas Tyler |
SSTemplateUpdate.py: * Added exception handling for case where an exception is thrown in the re.match() block when checking to see if the changelist description has the NO_CBD tag. * Generiziced some comments. * Started to add logic to avoid stripping the 'import @' entry if the revision specifier is a changelist nubmer, since import@ as of 2014.1 now supports this. But then adjusted to avoid writing @change numbers in the revision specifiers to avoid potential user confusion when change specifiers are made in more than one place (i.e. in the *.cbdsst file and the live server stream spec). There is still the possibility users might add @change values to stream specs directly. Do we need a stream spec validator to prevent users from adding @change to import lines, and if possible messaging the user and letting them know they need to update the *.cbdsst file instead if they try adding @change to import entries on the live stream spec? |
||
#1 | 11355 | C. Thomas Tyler | Added CBD triggers. |