#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright (c) 2011-2014 Sven Erik Knop/Robert Cowham, Perforce Software Ltd # # ======================================== # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the # distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE # SOFTWARE, INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # User contributed content on the Perforce Public Depot is not supported by Perforce, # although it may be supported by its author. This applies to all contributions # even those submitted by Perforce employees. # ======================================== # P4Transfer.py # # This python script will transfer Perforce changelists with all contents # between independent servers when no remote depots are possible. # # This script transfers changes in one direction - from a source server to a target server. # There is an alternative called PerforceExchange.py which transfers changes # in both directions. # # Usage: # python2 P4Transfer.py [options] # # The -h/--help option describes all options. # # The script requires a config file, normally called transfer.cfg, # that provides the Perforce connection information for both servers. # The config file has three sections: [general], [source] and [target]. # See DEFAULT_CONFIG for details. An initial example can be generated, e.g. # # P4Transfer.py --sample_config > transfer.cfg # # The script also needs a directory in which it can place the mapped files. # This directory has to be the root of both servers' workspaces (this will be verified). # # $Id: //guest/robert_cowham/perforce/utils/python/p4transfer/P4Transfer.py#1 $ from __future__ import print_function import sys, re from collections import OrderedDict from P4 import P4, P4Exception, Resolver, Map, OutputHandler def python3(): return sys.version_info[0] >= 3 # Although this should work with Python 3, it doesn't currently handle Windows Perforce servers # with filenames containing charaters such as umlauts etc: åäö if python3(): from configparser import ConfigParser else: from ConfigParser import ConfigParser import argparse import os.path from datetime import datetime import logging import re import time import platform import logutils CONFIG = 'transfer.cfg' GENERAL_SECTION = 'general' SOURCE_SECTION = 'source' TARGET_SECTION = 'target' LOGGER_NAME = "P4Transfer" # This is for writing to sample config file - OrderedDict used to preserve order of lines. DEFAULT_CONFIG = OrderedDict({ GENERAL_SECTION: OrderedDict([ ("# counter_name: Unique counter on target server to use for recording source changes processed. No spaces.", None), ("# Name sensibly if you have multiple instances transferring into the same target p4 repository.", None), ("# The counter value represents the last transferred change number - script will start from next change.", None), ("# If not set, or 0 then transfer will start from first change.", None), ("counter_name", "p4transfer_counter"), ("# instance_name: Name of the instance of P4Transfer - for emails etc. Spaces allowed.", None), ("instance_name", "Perforce Transfer from XYZ"), ("# For notification - testing purposes only", None), ("mail_form_url", ""), ("# The mail_* parameters must all be valid (non-blank) to receive email updates during processing. ", None), ("# mail_to: One or more valid email addresses - comma separated for multiple values", None), ("# E.g. somebody@example.com,somebody-else@example.com", None), ("mail_to", ""), ("# mail_from: Email address of sender of emails, E.g. p4transfer@example.com", None), ("mail_from", ""), ("# mail_server: The SMTP server to connect to for email sending, E.g. smtpserver.example.com", None), ("mail_server", ""), ("# sleep_on_error_interval: How long (in minutes) to sleep when error is encountered in the script", None), ("sleep_on_error_interval", "60"), ("# poll_interval: How long (in minutes) to wait between polling source server for new changes", None), ("poll_interval", "60"), ("# The following *_interval values result in reports, but only if mail_* values are specified", None), ("# report_interval: Interval (in minutes) between regular update emails being sent", None), ("report_interval", "30"), ("# error_report_interval: Interval (in minutes) between error emails being sent e.g. connection error", None), ("# Usually some value less than report_interval. Useful if transfer being run with --repeat option.", None), ("error_report_interval", "15"), ("# summary_report_interval: Interval (in minutes) between summary emails being sent e.g. changes processed", None), ("# Typically some value such as 1 week (10080 = 7 * 24 *60). Useful if transfer being run with --repeat option.", None), ("summary_report_interval", "10080"), ("# sync_progress_size_interval: Size in bytes controlling when syncs are reported to log file. ", None), ("# Useful for keeping an eye on progress for large syncs over slow links. ", None), ("sync_progress_size_interval", "500000000")]), SOURCE_SECTION: OrderedDict([ ("# P4PORT to connect to, e.g. some-server:1666", None), ("p4port", ""), ("# P4USER to use", None), ("p4user", ""), ("# P4CLIENT to use, e.g. p4-transfer-client", None), ("p4client", ""), ("# P4PASSWD for the user - valid password. If blank then no login performed.", None), ("p4passwd", "")]), TARGET_SECTION: OrderedDict([ ("# P4PORT to connect to, e.g. some-server:1666", None), ("p4port", ""), ("# P4USER to use", None), ("p4user", ""), ("# P4CLIENT to use, e.g. p4-transfer-client", None), ("p4client", ""), ("# P4PASSWD for the user - valid password. If blank then no login performed.", None), ("p4passwd", "")]), }) def p4time(unixtime): "Convert time to Perforce format time" return time.strftime("%Y/%m/%d:%H:%M:%S", time.localtime(unixtime)) def printSampleConfig(): "Print defaults from above dictionary for saving as a base file" config = ConfigParser(allow_no_value=True) for sec in DEFAULT_CONFIG.keys(): config.add_section(sec) for k in DEFAULT_CONFIG[sec].keys(): config.set(sec, k, DEFAULT_CONFIG[sec][k]) print("") print("# Save this output to a file to e.g. transfer.cfg and edit it for your configuration") print("") config.write(sys.stdout) def fmtsize(num): for x in ['bytes','KB','MB','GB','TB']: if num < 1024.0: return "%3.1f %s" % (num, x) num /= 1024.0 class ChangeRevision: "Represents a change - created from P4API supplied information and thus encoding" def __init__(self, r, a, t, d): self.rev = r self.action = a self.type = t self.depotFile = d self.localFile = None def setIntegrationInfo(self, integ): self.integration = integ def setLocalFile(self, localFile): self.localFile = localFile localFile = localFile.replace("%40", "@") localFile = localFile.replace("%23", "#") localFile = localFile.replace("%2A", "*") localFile = localFile.replace("%25", "%") localFile = localFile.replace("/", os.sep) self.fixedLocalFile = localFile def __repr__(self): return 'rev = {rev} action = {action} type = {type} depotFile = {depotfile}' .format( rev=self.rev, action=self.action, type=self.type, depotfile=self.depotFile, ) class ReportProgress(object): "Report overall progress" def __init__(self, p4, changes, logger): self.logger = logger self.filesToSync = 0 self.changesToSync = len(changes) self.sizeToSync = 0 self.filesSynced = 0 self.changesSynced = 0 self.sizeSynced = 0 self.previousSizeSynced = 0 self.sync_progress_size_interval = None # Set to integer value to get reports for chg in changes: sizes = p4.run('sizes', '-s', '@%s,%s' % (chg['change'], chg['change'])) self.sizeToSync += int(sizes[0]['fileSize']) self.filesToSync += int(sizes[0]['fileCount']) self.logger.info("Syncing %d changes, files %d, size %s" % (self.changesToSync, self.filesToSync, fmtsize(self.sizeToSync))) def SetSyncProgressSizeInterval(self, interval): "Set appropriate" if interval: self.sync_progress_size_interval = int(interval) def ReportChangeSync(self): self.changesSynced += 1 def ReportFileSync(self, fileSize): self.filesSynced += 1 self.sizeSynced += fileSize if not self.sync_progress_size_interval: return if self.sizeSynced > self.previousSizeSynced + self.sync_progress_size_interval: self.previousSizeSynced = self.sizeSynced syncPercent = 100 * float(self.filesSynced) / float(self.filesToSync) sizePercent = 100 * float(self.sizeSynced) / float(self.sizeToSync) self.logger.info("Synced %d/%d changes, files %d/%d (%2.1f %%), size %s/%s (%2.1f %%)" % ( \ self.changesSynced, self.changesToSync, self.filesSynced, self.filesToSync, syncPercent, fmtsize(self.sizeSynced), fmtsize(self.sizeToSync), sizePercent)) class P4Base(object): "Processes a config" section = None P4PORT = None P4CLIENT = None P4USER = None P4PASSWD = None counter = 0 def __init__(self, section, options): self.section = section self.options = options self.logger = logging.getLogger(LOGGER_NAME) self.p4 = None def __str__(self): return '[section = {} P4PORT = {} P4CLIENT = {} P4USER = {} P4PASSWD = {}]'.format( \ self.section, self.P4PORT, self.P4CLIENT, self.P4USER, self.P4PASSWD, ) def connect(self, progname): self.p4 = P4() self.p4.port = self.P4PORT self.p4.client = self.P4CLIENT self.p4.user = self.P4USER self.p4.prog = progname self.p4.exception_level = P4.RAISE_ERROR self.p4.connect() if not self.P4PASSWD == None: self.p4.password = self.P4PASSWD self.p4.run_login() clientspec = self.p4.fetch_client(self.p4.client) self.root = clientspec._root self.p4.cwd = self.root self.clientmap = Map(clientspec._view) ctr = Map('//"'+clientspec._client+'/..." "' + clientspec._root + '/..."') self.localmap = Map.join(self.clientmap, ctr) self.depotmap = self.localmap.reverse() def p4cmd(self, *args): "Execute p4 cmd while logging arguments and results" self.logger.debug(args) output = self.p4.run(args) self.logger.debug(output) return output def disconnect(self): if self.p4: self.p4.disconnect() def checkWarnings(self, where): if self.p4 and self.p4.warnings: self.logger.warning('warning in {} : {}'.format(where, str(self.p4.warnings))) def resetWorkspace(self): self.p4cmd('sync', '...#none') class P4Source(P4Base): "Functionality for reading from source Perforce repository" def missingChanges(self, counter): changes = self.p4cmd('changes', '-l', '...@{rev},#head'.format(rev=counter + 1)) changes.reverse() if self.options.maximum: changes = changes[:self.options.maximum] return changes def getChange(self, change): """Expects change number as a string""" class SyncOutput(OutputHandler): "Log sync progress" def __init__(self, progress): OutputHandler.__init__(self) self.progress = progress def outputStat(self, stat): if 'fileSize' in stat: self.progress.ReportFileSync(int(stat['fileSize'])) return OutputHandler.HANDLED def outputInfo(self, info): return OutputHandler.HANDLED def outputMessage(self, msg): return OutputHandler.HANDLED self.progress.ReportChangeSync() callback = SyncOutput(self.progress) self.p4.run('sync', '...@{},{}'.format(change, change), handler=callback) change = self.p4cmd('describe', change)[0] files = [] for (n, rev) in enumerate(change['rev']): localFile = self.localmap.translate(change['depotFile'][n]) if len(localFile) > 0: chRev = ChangeRevision(rev, change['action'][n], change['type'][n], change['depotFile'][n]) files.append(chRev) chRev.setLocalFile(localFile) if chRev.action in ('branch', 'integrate', 'add', 'delete'): filelog = self.p4.run_filelog('-m1', '{}#{}'.format(chRev.depotFile, chRev.rev)) if filelog: self.logger.debug('filelog', filelog) depotFile = filelog[0] revision = depotFile.revisions[0] if len(revision.integrations) > 0: for integ in revision.integrations: if 'from' in integ.how or integ.how == "ignored": chRev.setIntegrationInfo(integ) integ.localFile = self.localmap.translate(integ.file) break else: self.logger.error(u"Failed to retrieve filelog for {}#{}".format(chRev.depotFile, chRev.rev)) if chRev.action == 'move/add': depotFile = self.p4.run_filelog('-m1', '{}#{}'.format(chRev.depotFile, chRev.rev))[0] self.logger.debug('filelog', depotFile) revision = depotFile.revisions[0] integration = revision.integrations[0] chRev.setIntegrationInfo(integration) integration.localFile = self.localmap.translate(integration.file) return files class P4Target(P4Base): "Functionality for transferring changes to target Perforce repository" def replicateChange(self, files, change, sourcePort): """This is the heart of it all. Replicate all changes according to their description""" for f in files: self.logger.debug(f) if not self.options.preview: if f.action == 'edit': self.p4cmd('sync', '-k', f.localFile) self.p4cmd('edit', '-t', f.type, f.localFile) self.checkWarnings('edit') elif f.action == 'add': if 'integration' in f.__dict__: self.replicateBranch(f, True) # dirty branch else: self.p4cmd('add', '-ft', f.type, f.fixedLocalFile) self.checkWarnings('add') elif f.action == 'delete': if 'integration' in f.__dict__: self.replicateIntegration(f) self.checkWarnings('integrate (delete)') else: self.p4cmd('delete', '-v', f.localFile) self.checkWarnings('delete') elif f.action == 'purge': # special case. Type of file is +S, and source.sync removed the file # create a temporary file, it will be overwritten again later dummy = open(f.localFile, 'w') dummy.write('purged file') dummy.close() self.p4cmd('sync', '-k', f.localFile) self.p4cmd('edit', '-t', f.type, f.localFile) if self.p4.warnings: self.p4cmd('add', '-tf', f.type, f.fixedLocalFile) self.checkWarnings('purge -add') elif f.action == 'branch': self.replicateBranch(f, False) self.checkWarnings('branch') elif f.action == 'integrate': self.replicateIntegration(f) self.checkWarnings('integrate') elif f.action == 'move/add': self.move(f) newChangeId = None opened = self.p4cmd('opened') if len(opened) > 0: description = change['desc'] \ + ''' Transferred from p4://%s@%s''' % (sourcePort, change['change']) if self.options.nokeywords: self.removeKeywords(opened) result = self.p4cmd('submit', '-d', description) # the submit information can be followed by refreshFile lines # need to go backwards to find submittedChange a = -1 while 'submittedChange' not in result[a]: a -= 1 newChangeId = result[a]['submittedChange'] self.updateChange(change, newChangeId) self.reverifyRevisions(result) self.logger.info("source = {} : target = {}".format(change['change'], newChangeId)) return newChangeId def updateChange(self, change, newChangeId): # need to update the user and time stamp newChange = self.p4.fetch_change(newChangeId) newChange._user = change['user'] # date in change is in epoch time, we need it in canonical form newDate = datetime.utcfromtimestamp(int(change['time'])).strftime("%Y/%m/%d %H:%M:%S") newChange._date = newDate self.p4.save_change(newChange, '-f') def reverifyRevisions(self, result): revisionsToVerify = ["{file}#{rev},{rev}".format(file=x['refreshFile'], rev=x['refreshRev']) for x in result if 'refreshFile' in x] if revisionsToVerify: self.p4cmd('verify', '-qv', revisionsToVerify) def removeKeywords(self, opened): for openFile in opened: if self.hasKeyword(openFile["type"]): fileType = self.removeKeyword(openFile["type"]) self.p4.run_reopen('-t', fileType, openFile["depotFile"]) self.logger.debug("Changed type from {} to {} for {}". format(openFile["type"], fileType, openFile["depotFile"])) KTEXT = re.compile("(.*)\+([^k]*)k([^k]*)") def hasKeyword(self, fileType): return(fileType in ["ktext", "kxtext"] or self.KTEXT.match(fileType)) def removeKeyword(self, fileType): if fileType == "ktext": newType = "text" elif fileType == "kxtext": newType = "xtext" else: m = self.KTEXT.match(fileType) newType = m.group(1) + "+" + m.group(2) + m.group(3) return newType def replicateBranch(self, file, dirty): # An integration where source has been obliterated will not have integrations has_integration = False try: i = file.integration has_integration = True except: pass if has_integration and not self.options.ignore and file.integration.localFile: if file.integration.how == 'add from': # determine the filelog of the file in the target database # this is not so easy since filelog will return nothing for a deleted file # so we need to find the depotFile for the localFile first df = self.depotmap.translate(file.localFile) f = self.p4.run_filelog(df) self.logger.debug('filelog', f) if len(f) > 0 and len(f[0].revisions) >= 2: # in 2011.1 we can ignore into deleted files, so we need to make sure # we catch a real version i = 0 while f[0].revisions[i].action == 'delete': i += 1 rev = f[0].revisions[i] # this is the revision just before the delete self.p4cmd('sync', '-f', '%s#%d' % (rev.depotFile, rev.rev)) self.p4cmd('add', '-f', file.fixedLocalFile) else: # something fishy going on. Just add the file self.p4cmd('add', '-ft', file.type, file.fixedLocalFile) else: self.p4cmd('integrate', '-v', file.integration.localFile, file.localFile) if dirty: self.p4cmd('edit', file.localFile) else: self.p4cmd('add', '-ft', file.type, file.fixedLocalFile) def replicateIntegration(self, file): if not self.options.ignore and file.integration.localFile: if file.integration.how == 'edit from': with open(file.localFile) as f: content = f.read() self.p4cmd('sync', '-f', file.localFile) # to avoid tamper checking self.p4cmd('integrate', file.integration.localFile, file.localFile) class MyResolver(Resolver): "Local resolver to accept edits on merge" def __init__(self, content): self.content = content def resolve(self, mergeData): with open(mergeData.result_path, 'w') as f: f.write(self.content) return 'ae' self.p4.run_resolve(resolver=MyResolver(content)) else: self.p4cmd('sync', '-f', file.localFile) # to avoid tamper checking self.p4cmd('integrate', file.integration.localFile, file.localFile) if file.integration.how == 'copy from': self.p4cmd('resolve', '-at') elif file.integration.how == 'ignored': self.p4cmd('resolve', '-ay') elif file.integration.how in ('delete', 'delete from'): self.p4cmd('resolve', '-at') elif file.integration.how == 'merge from': # self.p4cmd('edit(file.localFile) # to overcome tamper check self.p4cmd('resolve', '-am') else: self.logger.error('Cannot deal with {}'.format(file.integration)) else: if file.integration.how in ('delete', 'delete from'): self.p4cmd('delete', '-v', file.localFile) else: self.p4cmd('sync', '-k', file.localFile) self.p4cmd('edit', file.localFile) def move(self, file): source = file.integration.localFile self.p4cmd('sync', '-f', source) self.p4cmd('edit', source) self.p4cmd('move', '-k', source, file.localFile) def getCounter(self): "Returns value of counter as integer" result = self.p4cmd('counter', self.options.counter_name) if result and 'counter' in result[0]: return int(result[0]['value']) return 0 def setCounter(self, value): "Set's the counter to specified value" self.p4cmd('counter', self.options.counter_name, str(value)) class P4Transfer(object): "Main transfer class" def __init__(self, *args): parser = argparse.ArgumentParser( description="P4Transfer", epilog="Copyright (C) 2012-14 Sven Erik Knop/Robert Cowham, Perforce Software Ltd" ) parser.add_argument('-n', '--preview', action='store_true', help="Preview only, no transfer") parser.add_argument('-c', '--config', default=CONFIG, help="Default is " + CONFIG) parser.add_argument('-m', '--maximum', default=None, type=int, help="Maximum number of changes to transfer") parser.add_argument('-k', '--nokeywords', action='store_true', help="Do not expand keywords and remove +k from filetype") parser.add_argument('-p', '--preflight', action='store_true', help="Run a sanity check first to ensure target is empty") parser.add_argument('-r', '--repeat', action='store_true', help="Repeat transfer in a loop - for continuous transfer") parser.add_argument('-s', '--stoponerror', action='store_true', help="Stop on any error even if --repeat has been specified") parser.add_argument('--sample_config', action='store_true', help="Print an example config file and exit") parser.add_argument('-i', '--ignore', action='store_true', help="Treat integrations as adds and edits") self.options = parser.parse_args(list(args)) self.options.sync_progress_size_interval = None self.logger = logutils.getLogger(LOGGER_NAME) self.previous_target_change_counter = 0 # Current value def getOption(self, section, option_name, default=None): result = default try: result = self.parser.get(section, option_name) except: pass return result def getIntOption(self, section, option_name, default=None): result = default strval = self.getOption(section, option_name, default) if strval: try: result = int(eval(strval)) except: pass return result def readConfig(self): self.parser = ConfigParser() self.options.parser = self.parser # for later use try: self.parser.readfp(open(self.options.config)) except Exception as e: raise Exception('Could not read %s: %s' % (self.options.config, str(e))) self.options.counter_name = self.getOption(GENERAL_SECTION, "counter_name") if not self.options.counter_name: raise Exception("Option counter_name in the [general] section must be specified") self.options.instance_name = self.getOption(GENERAL_SECTION, "instance_name", self.options.counter_name) self.options.mail_form_url = self.getOption(GENERAL_SECTION, "mail_form_url") self.options.mail_to = self.getOption(GENERAL_SECTION, "mail_to") self.options.mail_from = self.getOption(GENERAL_SECTION, "mail_from") self.options.mail_server = self.getOption(GENERAL_SECTION, "mail_server") self.options.sleep_on_error_interval = self.getIntOption(GENERAL_SECTION, "sleep_on_error_interval", 60) self.options.poll_interval = self.getIntOption(GENERAL_SECTION, "poll_interval", 60) self.options.report_interval = self.getIntOption(GENERAL_SECTION, "report_interval", 30) self.options.error_report_interval = self.getIntOption(GENERAL_SECTION, "error_report_interval", 30) self.options.summary_report_interval = self.getIntOption(GENERAL_SECTION, "summary_report_interval", 10080) self.options.sync_progress_size_interval = self.getIntOption(GENERAL_SECTION, "sync_progress_size_interval") self.source = P4Source(SOURCE_SECTION, self.options) self.target = P4Target(TARGET_SECTION, self.options) self.readSection(self.source) self.readSection(self.target) def readSection(self, p4config): if self.parser.has_section(p4config.section): self.readOptions(p4config) else: raise Exception('Config file needs section %s' % p4config.section) def readOptions(self, p4config): self.readOption('P4CLIENT', p4config) self.readOption('P4USER', p4config) self.readOption('P4PORT', p4config) self.readOption('P4PASSWD', p4config, optional=True) def readOption(self, option, p4config, optional=False): if self.parser.has_option(p4config.section, option): p4config.__dict__[option] = self.parser.get(p4config.section, option) elif not optional: raise Exception('Required option %s not found in section %s' % (option, p4config.section)) def replicate_changes(self): "Perform a replication loop" self.source.connect('source replicate') self.target.connect('target replicate') changes = self.source.missingChanges(self.target.getCounter()) self.logger.info("Transferring %d changes" % len(changes)) if len(changes) > 0: self.save_previous_target_change_counter() self.source.progress = ReportProgress(self.source.p4, changes, self.logger) self.source.progress.SetSyncProgressSizeInterval(self.options.sync_progress_size_interval) for change in changes: msg = 'Processing : {} "{}"'.format(change['change'], change['desc'].strip()) self.logger.info(msg) files = self.source.getChange(change['change']) resultedChange = self.target.replicateChange(files, change, self.source.p4.port) self.target.setCounter(change['change']) # Tidy up the workspaces after successful transfer self.source.p4cmd("sync", "#0") self.target.p4cmd("sync", "#0") self.source.disconnect() self.target.disconnect() return len(changes) def log_exception(self, e): "Log exceptions appropriately" etext = str(e) if re.search("WSAETIMEDOUT", etext, re.MULTILINE) or re.search("WSAECONNREFUSED", etext, re.MULTILINE): self.logger.error(etext) else: self.logger.exception(e) def save_previous_target_change_counter(self): "Save the latest change transferred to the target" chg = self.target.p4cmd('changes', '-m1', '-ssubmitted', '//{client}/...'.format(client=self.target.P4CLIENT)) if chg: self.previous_target_change_counter = int(chg[0]['change']) + 1 def send_summary_email(self, time_last_summary_sent, change_last_summary_sent): "Send an email summarising changes transferred" time_str = p4time(time_last_summary_sent) self.target.connect('target replicate') # Combine changes reported by time or since last changelist transferred changes = self.target.p4cmd('changes', '-l', '//{client}/...@{rev},#head'.format( client=self.target.P4CLIENT, rev=time_str)) chgnums = [chg['change'] for chg in changes] counter_changes = self.target.p4cmd('changes', '-l', '//{client}/...@{rev},#head'.format( client=self.target.P4CLIENT, rev=change_last_summary_sent)) for chg in counter_changes: if chg['change'] not in chgnums: changes.append(chg) changes.reverse() lines = [] lines.append(["Date", "Time", "Changelist", "File Revisions", "Size (bytes)", "Size"]) total_changes = 0 total_rev_count = 0 total_file_sizes = 0 for chg in changes: sizes = self.target.p4cmd('sizes', '-s', '@%s,%s' % (chg['change'], chg['change'])) lines.append([time.strftime("%Y/%m/%d", time.localtime(int(chg['time']))), time.strftime("%H:%M:%S", time.localtime(int(chg['time']))), chg['change'], sizes[0]['fileCount'], sizes[0]['fileSize'], fmtsize(int(sizes[0]['fileSize']))]) total_changes += 1 total_rev_count += int(sizes[0]['fileCount']) total_file_sizes += int(sizes[0]['fileSize']) lines.append([]) lines.append(['Totals', '', str(total_changes), str(total_rev_count), str(total_file_sizes), fmtsize(total_file_sizes)]) report = "Changes transferred since %s\n%s" % (time_str, "\n".join(["\t".join(line) for line in lines])) self.logger.debug("Transfer summary report:\n%s" % report) self.logger.info("Sending Transfer summary report") self.logger.notify("Transfer summary report", report, include_output=False) self.save_previous_target_change_counter() self.target.disconnect() # # This is the central method # It provides the replication process # Algorithm: # Read the config file # Connect to server1 and server # Determine if counter is there def replicate(self): """Central method that performs the replication between server1 and server2""" if self.options.sample_config: printSampleConfig() return 0 try: self.logger.debug("Reading config file") self.readConfig() self.source.connect('source replicate') self.target.connect('target replicate') self.logger.debug("connected to source and target") if not self.source.root == self.target.root: raise Exception('server1 and server2 workspace root directories must be the same') except Exception as e: self.log_exception(e) logging.shutdown() return 1 # self.source.resetWorkspace() if self.options.preflight: print("Running pre-flight check first ...") targetFiles = self.target.p4cmd('fstat', '-T clientFile', '...') sourceFiles = self.source.p4cmd('fstat', '-T clientFile', '...') for f in targetFiles: if f in sourceFiles: depotFile = self.target.p4cmd('fstat', f['clientFile'])[0] raise Exception("Failed pre-flight check, file '{}' in source and target".format(depotFile['depotFile'])) print("Finished pre-flight check ...") time_last_summary_sent = time.time() change_last_summary_sent = 0 self.logger.debug("Time last summary sent: %s" % p4time(time_last_summary_sent)) time_last_error_occurred = 0 error_encountered = False # Flag to indicate error encountered which may require reporting error_notified = False finished = False while not finished: try: self.readConfig() # Read every time to allow user to change them self.logger.setReportingOptions(instance_name=self.options.instance_name, mail_form_url=self.options.mail_form_url, mail_to=self.options.mail_to, mail_from=self.options.mail_from, mail_server=self.options.mail_server, report_interval=self.options.report_interval) self.logger.debug(self.source.options) self.logger.debug(self.target.options) self.source.disconnect() self.target.disconnect() num_changes = self.replicate_changes() if num_changes > 0: self.logger.info("Transferred %d changes successfully" % num_changes) if change_last_summary_sent == 0: change_last_summary_sent = self.previous_target_change_counter if not self.options.repeat: finished = True else: if error_encountered: self.logger.info("Logging - reset error interval") self.logger.notify("Cleared error", "Previous error has now been cleared") error_encountered = False error_notified = False if time.time() - time_last_summary_sent > self.options.summary_report_interval * 60: time_last_summary_sent = time.time() self.send_summary_email(time_last_summary_sent, change_last_summary_sent) self.logger.info("Sleeping for %d minutes" % self.options.poll_interval) time.sleep(self.options.poll_interval * 60) except Exception as e: self.log_exception(e) if self.options.stoponerror: self.logger.notify("Error", "Exception encountered and --stoponerror specified") logging.shutdown() return 1 else: # Decide whether to report an error if not error_encountered: error_encountered = True time_last_error_occurred = time.time() elif not error_notified: if time.time() - time_last_error_occurred > self.options.error_report_interval * 60: error_notified = True self.logger.info("Logging - Notifying recurring error") self.logger.notify("Recurring error", "Multiple errors seen") self.logger.info("Sleeping on error for %d minutes" % self.options.sleep_on_error_interval) time.sleep(self.options.sleep_on_error_interval * 60) self.logger.notify("Changes transferred", "Completed successfully") logging.shutdown() return 0 if __name__ == '__main__': result = 0 try: prog = P4Transfer(*sys.argv[1:]) result = prog.replicate() except Exception as e: print(str(e)) result = 1 sys.exit(result)
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#1 | 9642 | Robert Cowham | Files with new name | ||
//guest/perforce_software/p4transfer/P4Transfer.py | |||||
#3 | 9641 | Robert Cowham |
Latest changes by Robert. Added new options: --repeat for continuous operation --sample-config to produce sample config -Improved logging and notification options (via emails if configured) -Retries in case of error. |
||
#2 | 9473 | Sven Erik Knop |
Added the ability to remove +k from the target Currently tested for add, need to test for edit and integrate as well invoked by using option -k or --nokeywords |
||
#1 | 9170 | Sven Erik Knop |
Branched PerforceTransfer from private area to perforce_software This tool will now get back its original name P4Transfer. |
||
//guest/sven_erik_knop/P4Pythonlib/scripts/PerforceTransfer.py | |||||
#17 | 8554 | Sven Erik Knop | Added debug output for failed filelog retrieval. | ||
#16 | 8463 | Sven Erik Knop |
Fixed further problem with files that have an illegal file name containing @,#,* or %. Now it is possible to re-edit the file again as well. Added test case to prove the point. |
||
#15 | 8461 | Sven Erik Knop |
Fixed adding files with illegal chars like '@'. Also added test case. |
||
#14 | 8432 | Sven Erik Knop | Added pre-flight checks (-p) to avoid overwriting existing files. | ||
#13 | 8430 | Sven Erik Knop |
Added maximum option for changes to limit the number of changes transferred in each run. Should be useful for testing. Mind that "p4 changes" starts at the latest changes, so if there are millions of changes to transfer it will still take a long time to load all of the changes into memory first. |
||
#12 | 8429 | Sven Erik Knop | Added logging | ||
#11 | 8428 | Sven Erik Knop |
Transferred changes are now adjusted: - the transfer user is replaced with the original user - the submit date is reset to the original date Any +k files are re-verified to assure that they have the correct checksum. Still missing: - Logging |
||
#10 | 8425 | Sven Erik Knop |
Make PerforceTransfer unidirectional from source to target. Adjusted test cases accordingly. Still missing: Update change user and timestamp to the source user and timestamp Reverify ktext files affected by the change update. Add proper logging |
||
#9 | 8232 | Sven Erik Knop | Better safe than sorry: quotes around the path of the localMap entries. | ||
#8 | 8231 | Sven Erik Knop |
Removed all traces of p4.run_where and replaced them with local map.translate. Hopefully this will improve the performance of PerforceTransfer. |
||
#7 | 8216 | Sven Erik Knop |
Added test cases for integration from outside transfer scope. Fixed bug for integrated deletes from the outside. |
||
#6 | 8215 | Sven Erik Knop |
Upgraded test to include merge w/ edit Fixed a bug in PerforceTransfer.py avoiding a tamper check error. |
||
#5 | 8212 | Sven Erik Knop |
Added integrate-delete test case Solved integrate-delete problem in PerforceTransfer |
||
#4 | 8211 | Sven Erik Knop |
Additional test cases for integrate Fixed a bug with "ignore", can now be replicated. |
||
#3 | 8210 | Sven Erik Knop |
Fixed a bug in PerforceTransfer where an add followed by an integ to another branch would break the add. Also added the beginning of a test framework to catch those kind of problems in the future. Currently the test framework only checks add, edit, delete and simple integrates. |
||
#2 | 8209 | Sven Erik Knop |
Change formatting to tabs Made Python3 compatible Fixed a small bug in integrate |
||
#1 | 7986 | Sven Erik Knop | Changed P4Transfer to PerforceTransfer to conform with naming convention. | ||
//guest/sven_erik_knop/P4Pythonlib/scripts/P4Transfer.py | |||||
#10 | 7973 | Sven Erik Knop |
Enable re-adding of files for 2010.2+ servers. The problem was that the server now adds integration records for re-added files, which made P4Transfer believe this was a dirty branch instead of an add. Now we check if the "how" is "add from", indicating a re-add. |
||
#9 | 7971 | Sven Erik Knop |
Updated P4Transfer to deal with merge w/ edit integrations. All types of integrations should now be supported. Also updated the documentation. |
||
#8 | 7966 | Sven Erik Knop |
Changed master and local to server1 and server2. Also added first draft of a documentation that should serve pretty much as the blog post I intend to write on this tool. |
||
#7 | 7965 | Sven Erik Knop | Updated the shebang to avoid hardcoding the Python version. | ||
#6 | 7964 | Sven Erik Knop | Changed type to kxtext by popular demand. | ||
#5 | 7963 | Sven Erik Knop | Fixed the tamper problem. | ||
#4 | 7962 | Sven Erik Knop |
Updated P4Transfer with the ability to deal with +k types and merged files from integration. The result of the latter is an 'edit from' to avoid a tamper check problem. This is a hack for now until I can find a better way around it, but the repercussions should be low. |
||
#3 | 7961 | Sven Erik Knop |
Enable preview (-n) again. Not sure how it got lost. |
||
#2 | 7960 | Sven Erik Knop | Updated Copyright date and changed to ktext. | ||
#1 | 7959 | Sven Erik Knop |
P4Transfer release 1.0. Documentation to follow. |