############################ # Python script to send weekly merge reports to all engineers # Paul C. Pharr # May 25, 2001 # ############################ # What is MergeReport # # MergeReport is a Python program intended to be run at a regular interval to generate # a series of useful reports to assist engineers and managers in keeping current with # ongoing integrations between multiple active code branches in a Perforce depot. # # This tool assumes that most branches will have a "preferred direction" for integrating # changes. For example, if you have a "Mainline" branch and a "New feature development" # branch derived from it, it's likely that while you don't necessarily want all mainline # changes integrated into the development branch, you will ultimately want the # development branch integrated back to mainline (e.g. once development is complete), # Similarly, a release branch with some last minute bug fixes needs to be integrated back to # the Mainline branch from which it was derived. # # Once you have identified branches which have a policy of all changes being integrated back # to the source branch, you can set this tool up to watch all such branches and generate # reports which indicate: # # What integrations each engineer needs to do for all branches? # # What are the changelists for each branch which depend on # no prior revisions, but have a lot of other changelists # dependent on them? # # Each engineer gets a report that indicates (in part): # ############################################################## # #Zero dependency merges for branch MainlineToVectorWorks1001: # This branch is actively being merged. You should # merge these changes at the earliest opportunity. # # #13108 (1 unmerged files) on 2002/09/11 #14028 (1 unmerged files) on 2002/10/25 #14054 (2 unmerged files) on 2002/10/28 #14096 (21 unmerged files) on 2002/10/30 # # # Branch MainlineToVectorWorks1001: # # Changelist 14748 on 2002/11/27: # 2 earlier revisions on which this changelist may depend # 2 later revisions which may depend on this changelist # # # //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks10.0.1/AppSource/Source/Core/Source/Builtins.cpp#2 # //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks10.0.1/AppSource/Source/Core/Source/Builtins.h#2 # //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks10.0.1/AppSource/Source/Core/Source/GSWin.cpp#2 # //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks10.0.1/AppSource/Source/Core/Source/GSWin.h#2 # //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks10.0.1/AppSource/Source/Core/Source/MMenus.Definitions2.cpp#7 # //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks10.0.1/AppSource/Source/Plug-Ins/Shipping/Plug-in Support/Plug-in Support.cpp#4 # 2 earlier revisions (2: 14233 karim) (3: 14650 karim) # 2 later revisions (5: 14751 dave) (6: 14779 lyndsey) # ############################################################## # # The top listing is of zero dependency changelists - those that can't # depend on unmerged code to themselves be mergable. # # The 14748 listing indicates that the file Plug-in Support.cpp has two # unmerged earlier revisions (which could cause it to # be impossible to merge this revision) and two later revisions # (which could be waiting on this one before they can be merged). # # The manager gets a report which lists all pending merges in a priority ranking: # ############################################################## # #13864 depends on 0 and has 19 dependencies on it (abdel on 2002/10/18) (1 files) #14103 depends on 0 and has 9 dependencies on it (dave on 2002/10/31) (1 files) #14096 depends on 0 and has 5 dependencies on it (paul on 2002/10/30) (21 files) #14554 depends on 0 and has 2 dependencies on it (alex on 2002/11/19) (1 files) #14485 depends on 0 and has 1 dependencies on it (abdel on 2002/11/15) (1 files) #14486 depends on 0 and has 1 dependencies on it (abdel on 2002/11/15) (1 files) #14489 depends on 0 and has 1 dependencies on it (abdel on 2002/11/15) (1 files) #13108 depends on 0 and has 0 dependencies on it (paul on 2002/09/11) (1 files) #13854 depends on 0 and has 0 dependencies on it (karim on 2002/10/17) (3 files) #14657 depends on 0 and has 0 dependencies on it (daved on 2002/11/24) (136 files) #14776 depends on 0 and has 0 dependencies on it (andrewb on 2002/12/01) (1 files) #14804 depends on 0 and has 0 dependencies on it (stevej on 2002/12/03) (1 files) #13893 depends on 1 and has 22 dependencies on it (ayodeji on 2002/10/21) (2 files) #14119 depends on 1 and has 16 dependencies on it (ayodeji on 2002/10/31) (1 files) #14139 depends on 1 and has 13 dependencies on it (karim on 2002/11/01) (4 files) #14217 depends on 1 and has 2 dependencies on it (karim on 2002/11/06) (1 files) #14605 depends on 1 and has 2 dependencies on it (ejkerr on 2002/11/20) (6 files) #14710 depends on 1 and has 1 dependencies on it (daved on 2002/11/26) (2 files) #14097 depends on 1 and has 0 dependencies on it (paul on 2002/10/30) (1 files) #14153 depends on 1 and has 0 dependencies on it (karim on 2002/11/01) (1 files) # ############################################################## # # This report lists first those changelists that have no files with any possible # dependencies on any unmerged prior revisions. These changelists can likely be # merged easily with no complications. They are also sorted by the number of later # revisions which depend on this changelist being merged so they can subsequently # be merged with no unmerged prior revisions. # # The top of this list is always the list of changelists for the given branch which # are simultaneously those which are least likely to be difficult to merge and most # likely to make it easier for other changelists to be merged in the future. # # We run this script on a daily basis so our engineers always have current information # on their most important integration work. # # ############################ # # This Python file assumes tabs at 4 spaces per tab # It is tested with Python 2.1, but will likely work # without modification with later versions. # # This was written to run under Windows. While it will likely # be straightforward to port it to some other environment, it # has never been tested elsewhere. # # I am by no means a python expert. I write for ease of # implementation and maintainability. # ############################ # # SETUP # # To setup this script, follow the instructions in the coments at the beginning of the script # and fill in appropriate values for the variables listed. # ############################ # TODO # # The perforce integrate preview will generate errors in some situations which should not be errors(such # as when a file in the destination branch is marked exclusive and is already opened. # This can temporarily make the merge report omit some pending merges. # # The engineer's merge report needs to be better organized & formatted to improve usefulness. # # import sys import os # add the local libraries to the python path so we can include our own code gCurrentScriptDirectory = os.path.join(os.getcwd(), sys.path[0]) gPythonLibraryRoot = os.path.normpath(gCurrentScriptDirectory+'/../Libraries/') sys.path.append(gPythonLibraryRoot) import MailUtilities import P4Utilities import Logging import string import re import time ############################ ## Constants ############################ kForwardMerge = 1 kReverseMerge = 0 kProdEngineers = 1 kDontProdEngineers = 0 kNormalIntegration = 0 kDeletedInSourceBranch = 1 kDeletedInDestinationBranch = 2 kUnrelatedFiles = 3 ############################ ## Configuration README ############################ # # This tool primarily uses the file hierarchy and branch configuration # in Perforce to do its job, but there is some local tool configuration # which is necessary. # # First, The MailUtilities library module needs to have gMailDomain and gMailhost # customized to reference your local SMTP server and the domain names for your # local domain. # # Then, the branch array and perforce and email configuration sections of this file # need to be customized for your site. # # after that is done, you can execute this tool using # # python mergereport.py -debug # # and the primary administrator will be sent all generated reports for all # engineers. Check these to make sure they are reasonably correct, and then # you can hook up the command # # python mergereport.py # # to some form of cron job so it gets executed either daily or weekly. # # Please send questions or comments to # # Paul C. Pharr # pharr@nemetschek.net # ############################ ## Perforce branch configuration ############################ # # gBranchList is the main configuration necessary. Each active branch for which merge reports are to be # generated needs to be represented here. Each branch requires the following: # # Branch name - The name of the perforce branch which will be used to generate this report. This tool # requires branchspecs to be defined for the branches to be reported. # # Direction flag - The script will assume that all unmerged revisions need to be merged in one direction # across the branch spec. Depending on how the branch is defined, this will be either forward or reverse. # # Policy string - a string defining the merge policy. This is not yet used effectively in all reports, but # serves as an effective comment nonetheless. # # Administrators - List of email addresses for the branch administrators. They get summary report emails # describing all merges queued for their branches in addition to personalized merge reports. # # if 1: gBranchList = ( ("VW901ToMainline", kForwardMerge, "All revisions should be merged into mainline ASAP. ALL VERSIONS MUST ALSO be merged to VectorWorks9.5.0", kProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Mark Farnan', MailUtilities.MakeLocalAddress('mfarnan'))]), ("MainlineTo3DDevelopment", kReverseMerge, "All revisions should be merged into mainline after they are stable.", kProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))]), ("MainlineToVectorWorks950", kReverseMerge, "All revisions should be merged into mainline after they are stable.", kProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))]), ("MainlineToVectorWorks951", kReverseMerge, "All revisions should be merged into mainline after they are stable.", kProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))]), ("MainlineTo10ObjectBrowser", kReverseMerge, "See Karim for merge schedule information.", kProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))]), ("MainlineToGradientFills", kReverseMerge, "See Victor for merge schedule information.", kProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))]), ("MainlineToAlturaRemoval", kReverseMerge, "See Dave Bowman for merge schedule information.", kDontProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))]), ("MainlineToVectorWorks1000", kReverseMerge, "All revisions should be merged into mainline after they are stable.", kProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))]), ("MainlineToVectorWorks1001", kReverseMerge, "All revisions should be merged into mainline after they are stable.", kProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))]), ) else: gBranchList = ( ("MainlineToVectorWorks1050", kReverseMerge, "All revisions should be merged into mainline after they are stable.", kProdEngineers, [('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')),('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))]), ) ############################ ## Perforce account configuration ############################ # gPerforceAdminUser - Name of user (with sufficient permissions) under which # reports will be generated # gPerforceAdminClinet - Name of client (with sufficiently broad depot mappings) # under which reports will be generated gPerforceAdminUser = "Paul" gPerforceAdminPassword = "" gPerforceAdminClinet = "ServerScriptsUtility" ############################ ## email configuration ############################ # gMailDomain and gMailhost in the MailUtilities library module must be customized for your site # the second element of these tuples can be generated using MailUtilities.MakeLocalAddress for # ease of maintenance, or it can be a hardcoded email address in a string # gPrimaryAdministrator - a tuple containing the name and email address of # the primary administrator of this system. Receives debug & error reports # and merge reports for users whose user spec (and consequently email address) # have been deleted from Perforce. gPrimaryAdministrator = ('Paul Pharr', MailUtilities.MakeLocalAddress('pharr')) # gPrimaryAdministrator - a list containing additional admins for some global reports gAdditionalGlobalAdmins = [('Darick DeHart', MailUtilities.MakeLocalAddress('darick'))] # gReportReturnAddress - set this to the email address of the account from which reports # will be sent. Your SMTP server may require this account to actually exist. gReportReturnAddress = MailUtilities.MakeLocalAddress('EngineeringReports') ############################ ## report configuration ############################ # # Set to 1 to turn report on, 0 to turn report off # # # These reports are sent for each branch separately # # send a report containing minor errors encountered in generating this report # (for troubleshooting) gSendErrorReport = 1 # send a report containing each revision needing to be merged sorted by engineer # (relatively raw data) gSendSummaryReport = 1 # send a report containing each revision needing to be merged sorted by number of other # revisions dependent on it (relatively raw data) gSendGridlockReport = 1 # send a report containing a prioritixed listing of changelists which need to be merged # this is very useful high level data. gSendTopChangelistReport = 1 # # These reports are sent as a single report for all branches # # send a report containing a copy of every engineer's individual report (for troubleshooting) gSendAllEngineersReport = 1 #location to which reports will be saved locally (for web integration) Leave empty to disable saving reports. #gLocalSavePath = "e:\MergeReports" gLocalSavePath = r"\\nnafiles\inetpub$\MergeReports" # filename generation strings gEngineerFilename = "engineer_%s.htm" gAdminFilename = "admin_%s.htm" ############################ ## Troubleshooting ############################ # # # The basis of everything this script does is the following Perforce command. If you are getting # no output from the script, run this manually and see if Perforce is reporting changes # to be integrated. You may find that the -r (reverse) flag is necessary depending on how your branch # is specified. # # p4 -c ServerScripts integ -n [-r] -b //... # # # Also, if you suspect there is some problem with your mail configuration which is preventing the # email functionality from working, use gLocalSavePath to set the location where you want the # reports saved to the file system. # # If all else fails, contact me: # # Paul Pharr # Nemetschek North America # pharr@nemetschek.net # # # ############################ ## END Configuration section ############################ gToBeMerged = [] gErrorReport = "" # in debug mode, all messages are sent to gPrimaryAdministrator rather than individual engineers # use this for system maintenance without peppering engineers with merge reports (set by runtime command line flag) gDebugMode = 0 # set this to 0 to just send the summary reports (set by runtime command line flag) gSendIndividualReports = 1 # if this is turned off, then reports will be saved according to gLocalSavePath, # but there will be no email notifications gSendEmailNotifications = 1 def CallPerforce(command): return P4Utilities.CallP4(command, P4Utilities.kReturnFile, 'r', gPerforceAdminClinet, gPerforceAdminUser, gPerforceAdminPassword) def GetPerforceUserEmail(theUser): userReport = CallPerforce("users " + theUser).read() #example userReport #paul (Paul Pharr) accessed 2001/08/17 m = re.match(r"^.*<([^>]*)>.*\(([^)]*)\).*$", userReport) if m: address = m.group(1) fullName = m.group(2) return (fullName, address) else: return None def getRevisionInfo(theFile, theVersion): # on about May 20, 2003 This script started failing on EngineeringNT when processing the AlturaRemoval project #(which had lots of pending files, but nothing extreme). It would always fail with history coming back false #in this function, but it did not always happen on the same file. This function would simply fail intermittently. #After adding this loop, I found that sometimes all iterations of the loop would fail whereas sometimes it would #regain its composure halfway through its multiple attempts. # # Logging out and back into the server cured the problem. - PCP global gErrorReport attempts = 0 while attempts < 10: attempts += 1; history = CallPerforce("filelog -m 1 \""+ theFile + "#" + str(theVersion)+"\"").readlines() if history: m = re.match(r"^... #(\d+) change (\d+).*on (\S+) by (\S+)@.*$", history[1]) if m: revision = m.group(1) changelist = m.group(2) date = m.group(3) engineer = m.group(4) return (engineer.lower(), date, int(changelist), int(revision)) else: thisError = 'DID NOT GET HISTORY (attempt %d)' % (attempts) + theFile + '#' + str(theVersion) gErrorReport += thisError + '\n\n' sys.stdout.write(thisError + '\n\n') time.sleep(2) return None def displayVersionInfo(branchSpec, theFile, history, thisRev, operation, flagMissing): # once we know the file and revision which needs to be merged, we need to find out some more # info to let us intelligently assign this file - namely the engineer & changelist global gErrorReport h = history[thisRev] weDependOn = history[0:thisRev] dependOnUs = history[thisRev+1:] #sys.stdout.write(str(h) + str(theFile) + '\n\n') if operation == "sync/delete" and flagMissing == "": #this is the case where there are no edits to the file in the destination branch specialStatus = kDeletedInSourceBranch elif operation == "can't delete" and flagMissing == " without -d flag": #this is the case where there are edits to the file in the destination branch specialStatus = kDeletedInSourceBranch elif operation == "can't branch" and flagMissing == " without -d flag": specialStatus = kDeletedInDestinationBranch elif operation == "can't integrate" and flagMissing == " without -i flag": specialStatus = kUnrelatedFiles elif operation == "integrate" and flagMissing == "": specialStatus = kNormalIntegration elif operation == "branch/sync" and flagMissing == "": specialStatus = kNormalIntegration elif operation == "sync/integrate" and flagMissing == "": specialStatus = kNormalIntegration else: gErrorReport += 'displayVersionInfo could not interpret (%s:%s)\n' % (operation, flagMissing) return # engineer, path to file, file revision, changelist number, date, branchspec, list of changes we depend on, list of changes which depend on us, boolean indicating that the file is deleted in destination gToBeMerged.append((h[0], theFile, h[3], h[2], h[1], branchSpec, weDependOn, dependOnUs, specialStatus)) def parseMergeStatus(branchSpec, mergeDirection): global gErrorReport if mergeDirection == kForwardMerge: directionFlag = '' else: directionFlag = '-r' resultLines = CallPerforce("integ -n -b " + branchSpec + " " + directionFlag + " //...").readlines() for line in resultLines: badInfo = "Bad Information Returned from Perforce" try: #sys.stdout.write(line) # example of a single revision #//depot/Engineering/IwTangentField.cpp#2 - integrate from //depot/Engineering/IwTangentField.cpp#2 #//depot/Engineering/IwTangentField.cpp#2 - branch/sync from //depot/Engineering/IwTangentField.cpp#2 # example of a revision range #//depot/Engineering/IwTess.cpp#2 - integrate from //depot/Engineering/IwTess.cpp#2,#4 #//depot/Engineering/IwTess.cpp#2 - branch/sync from //depot/Engineering/IwTess.cpp#2,#4 # example of a delete #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Plug-Ins/Shipping/ExtendedProperties/ExtendedProperties.rsr#1 - delete from //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks10.5.0/AppSource/Source/Plug-Ins/Shipping/ExtendedProperties/ExtendedProperties.rsr#2,#3 # this is a fragile part of this script - if perforce returns some unexpected operation string, # the script stops working. Need to check errors frequently until this is improved #actually, these strings are different depending on whether the destination file is synced #to the head, an older revision, or no revision at all. It adds sync if a sync must be performed. operationStrings = "integrate|branch/sync|sync/integrate|sync/delete|can't branch|can't integrate|can't delete" #(depotfile, firstRev, lastRev) = re.match(r'^.* from (\S+)#(\d+)(?:()|,#(\d+)).*', line).groups() m = re.match(r"^(.*)(?:#\d+)? - (" + operationStrings + ") from (.*)#(\d+),#(\d+)(.*?)$", line) if m != None: # this case handles a range of revisions which need to be merged if m.lastindex == 6: #sys.stdout.write(m.group(1) + ' ' + m.group(3) + ' ' + m.group(4) + ' ' + m.group(5) + ' ' + m.group(6) + '\n') mergeDestinationPath = m.group(1) operation = m.group(2) mergeSourcePath = m.group(3) firstrev = int(m.group(4)) lastrev = int(m.group(5)) flagMissing = m.group(6) versionlist = range(firstrev, lastrev+1) #sys.stderr.write(mergeDestinationPath + '\n') #sys.stderr.write( str(versionlist) + '\n') #copy the list to avoid loop problems fullRange = versionlist[:] # now we specifically check each candidate revision to see if it really needs to be merged. # if not, we remove it from the list for candidateRev in fullRange: integResultLines = CallPerforce("-s integ -n -b " + branchSpec + " " + directionFlag + " -s " + mergeSourcePath + "#" + str(candidateRev) + ",#" + str(candidateRev)).readlines() integResult = integResultLines[0] #sys.stderr.write(str(candidateRev) + ": " + integResult + '\n') m = re.match(r"(.*) - all revision\(s\) already integrated.*", integResult) if m != None: #sys.stderr.write('removing revision\n') versionlist.remove(candidateRev) if (0): #for line2 in CallPerforce("integrated " + mergeSourcePath).readlines(): #typical syntax #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1 - branch from //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks9.0.1/Source/Components/LayoutManager/Dialog.Win.cpp#1 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#2 - ignored //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks9.0.1/Source/Components/LayoutManager/Dialog.Win.cpp#2 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1,#2 - branch into //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks9.5.0/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#4 - merge from //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks9.5.0/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#2 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#5 - merge from //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks9.5.0/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#3,#4 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1,#5 - branch into //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks9.5.1/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#8 - edit into //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks9.5.1/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#2 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#9 - merge into //depot/Engineering/VectorWorks/ReleaseBranches/VectorWorks9.5.1/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#3 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1 - branch into //depot/Engineering/VectorWorks/TaskBranches/VW10/3DDevelopment/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1,#5 - branch into //depot/Engineering/VectorWorks/TaskBranches/VW10/AlturaRemoval/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1,#5 - branch into //depot/Engineering/VectorWorks/TaskBranches/VW10/GradientFills/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#7 - merge into //depot/Engineering/VectorWorks/TaskBranches/VW10/GradientFills/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#2 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#12 - copy from //depot/Engineering/VectorWorks/TaskBranches/VW10/GradientFills/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#3,#14 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#6,#11 - edit into //depot/Engineering/VectorWorks/TaskBranches/VW10/GradientFills/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#14 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1,#4 - branch into //depot/Engineering/VectorWorks/TaskBranches/VW10/ObjectBrowser/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#1 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#11 - merge from //depot/Engineering/VectorWorks/TaskBranches/VW10/ObjectBrowser/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#2,#5 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#5 - merge into //depot/Engineering/VectorWorks/TaskBranches/VW10/ObjectBrowser/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#3 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#6,#9 - merge into //depot/Engineering/VectorWorks/TaskBranches/VW10/ObjectBrowser/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#4 #//depot/Engineering/VectorWorks/Mainline/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#10 - merge into //depot/Engineering/VectorWorks/TaskBranches/VW10/ObjectBrowser/AppSource/Source/Components/LayoutManager/Dialog.Win.cpp#5 proceed = 0 #check for a single revision number m2 = re.match(r"^(.*)#(\d+) - (.*) (//.*)#.*$", line2) if m2 != None: integrateSource = m2.group(1) startRevision = int(m2.group(2)) endRevision = startRevision action = m2.group(3) secondaryFile = m2.group(4) proceed = 1 else: #check for a range of revision numbers m2 = re.match(r"^(.*)#(\d+),#(\d+) - (.*) (//.*)#.*$", line2) if m2 != None: integrateSource = m2.group(1) startRevision = int(m2.group(2)) endRevision = int(m2.group(3)) action = m2.group(4) secondaryFile = m2.group(5) proceed = 1 else: gErrorReport += 'DID NOT GET INTEGRATED LIST ' + mergeSourcePath + '\n\n' if proceed: # this code assumes that p4 integrate accurately identifies the range of revisions # which likely need to be merged. Its responsibility is to remove those integrations # that we are certain do not. It could be more sophisticated and possibly remove more # but the returns are not work the risk #sys.stderr.write('got range %d to %d for action "%s"\n' % (startRevision, endRevision, action)) #sys.stderr.write(secondaryFile + ' ' + mergeDestinationPath + '\n') # we only want to consider integrations to the same destination as our current merge which # are either branch, merge or edit into the destination. We specifically want to avoid removing those # revisions which result from merges or edits from the destinationm as those revisions may contain changes # which need to be merged back - PCP if secondaryFile == mergeDestinationPath and action in ["branch into", "merge into", "edit into"]: for removeRevision in range(startRevision, endRevision+1): if removeRevision in versionlist: versionlist.remove(removeRevision) #sys.stderr.write('removing version ' + str(removeRevision) +\ # ' for file ' + integrateSource + '\n') fullHistory = [] # first we go through the list and figure out all possible dependencies of this revision for thisVersion in versionlist: revisionInfo = getRevisionInfo(mergeSourcePath, thisVersion) if revisionInfo != None: fullHistory.append(revisionInfo) else: raise badInfo #sys.stdout.write(mergeSourcePath + '\n ' + str(fullHistory) + '\n') for i in range(len(fullHistory)): displayVersionInfo(branchSpec, mergeSourcePath, fullHistory, i, operation, flagMissing) #sys.stdout.write(mergeSourcePath + ' ' + str(firstrev) + ' ' + str(lastrev) + '\n') else: gErrorReport += 'DID NOT MATCH LINE - last index was ' + m.lastindex + '\n\n' else: # this case handles a single revision which needs to be merged m = re.match(r"^.* - (" + operationStrings + ") from (.*)#(\d+)(.*?)$", line) if m != None: operation = m.group(1) mergeSourcePath = m.group(2) firstrev = int(m.group(3)) flagMissing = m.group(4) #sys.stdout.write(mergeSourcePath + ' ' + str(firstrev) + ' ' + str(flagMissing) + '\n') revisionInfo = getRevisionInfo(mergeSourcePath, firstrev) if revisionInfo != None: # do a log of the unmerged revision displayVersionInfo(branchSpec, mergeSourcePath, [revisionInfo], 0, operation, flagMissing) else: raise badInfo else: gErrorReport += 'DID NOT MATCH LINE:' + line + '\n' except badInfo: sys.stdout.write('skipping file because of badInfo exception!\n') pass def CompareItems(a, b): # sort by engineer, then branch, then changelist, then file if a[0]>b[0]: return 1 elif a[0]b[5]: return 1 elif a[5]b[3]: return 1 elif a[3]b[1]: return 1 elif a[1]len(b[7]): return -1 else: return 0 def CompareByChangelist(a, b): # sort by changelist if a[3]>b[3]: return 1 elif a[3]b[3]: return 1 elif a[3]b[4]: return -1 elif a[0]>b[0]: return 1 elif a[0]b[3]: return 1 elif a[3]b[0]: return 1 elif a[0] timeLimit: os.remove(filePath) def DeleteOldReports(timeLimit): if gLocalSavePath != "": os.path.walk(gLocalSavePath, DeleteOldReport, timeLimit) kEngineerReport = 1 kAdminReport = 2 def SaveReport(reportName, reportKind, reportBody): if gLocalSavePath != "": if reportKind == kEngineerReport: fullName = gEngineerFilename % (reportName) elif reportKind == kAdminReport: fullName = gAdminFilename % (reportName) else: sys.stderr.write("unknown reportKind\n") fullPath = os.path.normpath(os.path.join(gLocalSavePath, fullName)) sys.stdout.write("Saving report %s\n" % (fullPath)) thisFile = open(fullPath, 'w') thisFile.write("
\n")
		thisFile.write(reportBody)
		thisFile.write("
\n") thisFile.close() def SendReport(recipients, subject, body): if gSendEmailNotifications: result = MailUtilities.SendMail(gReportReturnAddress, recipients, subject, body) if result != MailUtilities.kErrorNoError: Logging.Write('Mail "%s" failed for recipient(s) %s' % (subject, str(recipients))) def DoMergeReport(branchSpec, mergeDirection, mergeComment, sendProdReport, reportAddresses): global gErrorReport global gToBeMerged gErrorReport = "" mergeSummaryReport = "" mergeGridlockReport = "" topChangelistReport = "" gToBeMerged = [] if mergeDirection == kForwardMerge: directionString = 'forward' else: directionString = 'reverse' parseMergeStatus(branchSpec, mergeDirection) if len(gToBeMerged) > 0: # ============================================================================================================================ # do the main merge report of files which need to be merged in this branch # ============================================================================================================================ gToBeMerged.sort() totalCount = 0 engineerCount = 0 lastEngineer = "nobody" for item in gToBeMerged: engineer = item[0] if lastEngineer != engineer: if lastEngineer != "nobody": mergeSummaryReport += 'TOTAL for ' + lastEngineer + ': ' + str(engineerCount) + '\n\n' engineerCount = 0 lastEngineer = engineer mergeSummaryReport += engineer + ' ' + item[1] + '#' + str(item[2]) + ' (changelist ' + str(item[3]) + ' on ' + item[4] + ')\n' engineerCount += 1 totalCount += 1 mergeSummaryReport += 'TOTAL for ' + lastEngineer + ': ' + str(engineerCount) + '\n\n' mergeSummaryReport += 'TOTAL for all engineers: ' + str(totalCount) + '\n\n\n\n\n' # ============================================================================================================================ # now we generate a report of the revisions causing the most merge gridlock # ============================================================================================================================ gToBeMerged.sort(CompareGridlockItems) for item in gToBeMerged: engineer = item[0] mergeGridlockReport += str(len(item[7])) + ' ' + engineer + ' ' + item[1] + '#' + str(item[2]) + '\n' # ============================================================================================================================ # now we generate a report of the changelists in most need of merging # ============================================================================================================================ changelistReport = ConvertToChangelists(gToBeMerged) topChangelistReport += "\n\n There are " + str(len(changelistReport)) + " changelists with unmerged revisions.\n\n" topChangelistReport += "\n\nPrioritized Listing - sorted by files we depend on, then by files dependent on us\n\n" changelistReport.sort(CompareProdReportItems) for item in changelistReport: topChangelistReport += TopMergeReport(item) topChangelistReport += "\n\n\nWeighted Chrolological Listing - sorted by files we depend on, then changelist\n\n" changelistReport.sort(CompareProdReportItems2) for item in changelistReport: topChangelistReport += TopMergeReport(item) topChangelistReport += "\n\n\nStraight Chrolological Listing - sorted by changelist\n\n" changelistReport.sort() for item in changelistReport: topChangelistReport += TopMergeReport(item) # ============================================================================================================================ # send all the reports # ============================================================================================================================ if gDebugMode: reportAddresses = [gPrimaryAdministrator] if gErrorReport != "" and gSendErrorReport: finalReport = 'Errors encountered generating the merge report for ' + branchSpec + '\n\n' +\ gErrorReport finalSubject = 'MERGE REPORT: Errors for ' + branchSpec SendReport([gPrimaryAdministrator], finalSubject, finalReport) SaveReport(branchSpec + "Errors", kAdminReport, finalReport) if mergeSummaryReport != "" and gSendSummaryReport: finalReport = 'Merge report for branch ' + branchSpec + ' as of ' + time.ctime() + ' (merging in ' + directionString + ' direction)\n' +\ mergeComment + '\n' +\ '===================================================================================\n\n\n' +\ mergeSummaryReport finalSubject = 'MERGE REPORT: Summary for ' + branchSpec SendReport(reportAddresses, finalSubject, finalReport) SaveReport(branchSpec + "Summary", kAdminReport, finalReport) if mergeGridlockReport != "" and gSendGridlockReport: finalReport = 'Merge gridlock report for branch ' + branchSpec + ' as of ' + time.ctime() + ' (merging in ' + directionString + ' direction)\n' +\ '===================================================================================\n\n\n' +\ mergeGridlockReport finalSubject = 'MERGE REPORT: Gridlock report for ' + branchSpec SendReport(reportAddresses, finalSubject, finalReport) SaveReport(branchSpec + "Gridlock", kAdminReport, finalReport) if topChangelistReport != "" and gSendTopChangelistReport: finalReport = 'Changelist dependency report for branch ' + branchSpec + ' as of ' + time.ctime() + ' (merging in ' + directionString + ' direction)\n' +\ '===================================================================================\n\n\n' +\ topChangelistReport finalSubject = 'MERGE REPORT: Top changelists for ' + branchSpec SendReport(reportAddresses, finalSubject, finalReport) SaveReport(branchSpec + "TopChangelists", kAdminReport, finalReport) def SendEngineerReport(engineer, report): theAddress = GetPerforceUserEmail(engineer) theSubject = 'MERGE REPORT: Perforce merge report for ' + engineer Logging.Write(str(theAddress)) if gDebugMode: recipient = [gPrimaryAdministrator] fullReport = "When debug mode is off, report goes to: " + str(theAddress) + "\n" elif theAddress != None: recipient = [theAddress] fullReport = "" else: recipient = [gPrimaryAdministrator] theSubject += " (original recipient unavailable)" fullReport = "Original recipient unavailable in Perforce: " + str(theAddress) + "\n" fullReport += "Merge report for " + engineer + " as of " + time.ctime() + "\n" +\ "For information about this report, see\n " +\ "http://engineering.nemetschek.net/articles.php?action=view&article_id=63\n\n\n" + report SendReport(recipient, theSubject, fullReport) SaveReport(engineer, kEngineerReport, fullReport) def createDependenciesList(list): stringList = "" for thisItem in list: stringList += "(" + str(thisItem[3]) + ": " + str(thisItem[2]) + " " + thisItem[0] + ") " return stringList def DependsReport(weDependOnTotal, dependsOnUsTotal): result = ' ' + str(weDependOnTotal) + ' earlier revisions on which this changelist may depend\n' result += ' ' + str(dependsOnUsTotal) + ' later revisions which may depend on this changelist\n\n' return result def CollateEngineers(masterList): byEngineerList = [] engineerList = None lastEngineer = "nobody" masterList.sort(CompareItems) for item in masterList: engineer = item[0] if lastEngineer != engineer: if engineerList != None: byEngineerList.append(engineerList) engineerList = [] lastEngineer = engineer engineerList.append(item) # this fixes a bug where the last engineer in the list was dropped if engineerList != None: byEngineerList.append(engineerList) return byEngineerList def AppendReport(reports, engineer, body): if reports.has_key(engineer): reports[engineer] = reports[engineer] + body else: reports[engineer] = body def DoAllMergeReports(): Logging.ToFile('MergeReportLog.txt') Logging.TimeStampNextWrite() Logging.AutoNewLine(1) reports = {} masterList = [] for b in gBranchList: if gDebugMode: sys.stdout.write("Analyzing branch %s\n" % (b[0])) DoMergeReport(b[0], b[1], b[2], b[3], b[4]) masterList.append((gToBeMerged, b[3])) if gSendIndividualReports: for listRecord in masterList: fullList = listRecord[0] doProd = listRecord[1] if len(fullList) > 0: # each separate list in the masterList is for a separate branch thisBranch = fullList[0][5] sys.stdout.write('processing header for branch %s\n' % (thisBranch)) #print fullList #sys.stdout.write('\n\n') byEngineer = CollateEngineers(fullList) #print byEngineer for engineerList in byEngineer: thisReport = "Zero dependency merges for branch " + thisBranch + ":\n" if doProd: thisReport += " This branch is actively being merged. You should\n" +\ " merge these changes at the earliest opportunity.\n\n" else: thisReport += " This branch is not currently being merged. You should\n" +\ " obtain a manager's approval before merging these changelists.\n\n" engineer = engineerList[0][0] changelistReport = ConvertToChangelists(engineerList) sys.stdout.write('processing list for engineer %s\n' % (engineer)) changelistReport.sort(CompareProdReportItems2) for item in changelistReport: if (item[3] == 0): thisReport += TopMergeReport(item) thisReport += "\n\n" AppendReport(reports, engineer, thisReport) # now report individual changelists for listRecord in masterList: fullList = listRecord[0] if len(fullList) > 0: fullList.sort(CompareItems) totalCount = 0 engineerCount = 0 lastEngineer = "nobody" changeListReport = "" lastChangelist = 0 for item in fullList: engineer = item[0] branch = item[5] changelist = item[3] specialCircumstances = item[8] if lastChangelist != changelist: if changeListReport != "": thisReport += DependsReport(weDependOnTotal, dependsOnUsTotal) thisReport += changeListReport if lastEngineer != engineer: if lastEngineer != "nobody": thisReport += 'TOTAL for ' + lastEngineer + ': ' + str(engineerCount) + '\n\n' AppendReport(reports, lastEngineer, thisReport) thisReport = "" thisReport += 'Merges for ' + engineer + ':\n' engineerCount = 0 lastEngineer = engineer lastBranch = "none" lastChangelist = 0 if lastBranch != branch: thisReport += '\n\n Branch ' + branch + ':\n' lastBranch = branch lastChangelist = 0 if lastChangelist != changelist: thisReport += '\n\n Changelist ' + str(item[3]) + ' on ' + item[4] + ':\n' lastChangelist = changelist dependsOnUsTotal = 0 weDependOnTotal = 0 changeListReport = "" changeListReport += ' ' + item[1] + '#' + str(item[2]) + '\n' ssString = GetSpecialCircumstancesDescription(specialCircumstances) if ssString != "": changeListReport += " " + ssString + "\n" dependsOnUs = item[7] weDependOn = item[6] dependsOnUsCount = len(dependsOnUs) weDependOnCount = len(weDependOn) dependsOnUsTotal += dependsOnUsCount weDependOnTotal += weDependOnCount if weDependOnCount > 0: changeListReport += ' ' + str(weDependOnCount) + " earlier revision" if weDependOnCount > 1: changeListReport += 's' if weDependOnCount < 10: changeListReport += " " + createDependenciesList(weDependOn) + "\n" else: changeListReport += " (too many to display).\n" if dependsOnUsCount > 0: changeListReport += ' ' + str(dependsOnUsCount) + " later revision" if dependsOnUsCount > 1: changeListReport += 's' if dependsOnUsCount < 10: changeListReport += " " + createDependenciesList(dependsOnUs) + "\n" else: changeListReport += " (too many to display).\n" engineerCount += 1 totalCount += 1 # now wrap up the final reports for the last item if lastChangelist != 0: thisReport += DependsReport(weDependOnTotal, dependsOnUsTotal) thisReport += changeListReport if lastEngineer != "nobody": thisReport += 'TOTAL for ' + lastEngineer + ': ' + str(engineerCount) + '\n\n' AppendReport(reports, lastEngineer, thisReport) masterReport = "" for engineer in reports.keys(): SendEngineerReport(engineer, reports[engineer]) masterReport += reports[engineer] if gDebugMode: reportAddresses = [gPrimaryAdministrator] else: reportAddresses = [gPrimaryAdministrator] reportAddresses.extend([gAdditionalGlobalAdmins]) if masterReport != "" and gSendAllEngineersReport: finalSubject = 'MERGE REPORT: All Engineers Report' SendReport(reportAddresses, finalSubject, masterReport) #SaveReport("AllEngineers", kAdminReport, masterReport) # delete reports older than 60 minutes (assumes that this script completes in less than 60 minutes DeleteOldReports(60*60) for arg in sys.argv: if arg == "-debug": sys.stdout.write("Debug mode on\n") gDebugMode = 1 gLocalSavePath = "c:\MergeReportsDebug" elif arg == "-reports": sys.stdout.write("Report mode on\n") gSendIndividualReports = 0 elif arg == "-nomail": sys.stdout.write("no mail will be sent\n") gSendEmailNotifications = 0 elif arg == "-?": sys.stdout.write("USAGE: MergeReport [-reports] [-nomail] [-debug] [-?]\n") sys.exit() if not os.path.isdir(gLocalSavePath): os.makedirs(gLocalSavePath) DoAllMergeReports()