#!/usr/bin/env python # -*- coding: -*- # # P4Bucket # #******************************************************************************* # Copyright (c) 2009, Sven Erik Knp[, Perforce Software, Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #******************************************************************************* # $Id: //guest/sven_erik_knop/p4bucket/perforcebucket.py#1 $ # import sys import platform import socket import string import re import os import ConfigParser import P4 import gzip import shutil import datetime # Adjust this if necessary # All parameters and buckets are stored in this file # Do not modify the content of this file by hand, use "init", "create", "edit" and "delete" for that purpose CONFIG_FILE = "p4bucket.conf" LOG_FILE = "p4bucket.log" SECTION_P4CONFIG = "P4 CONFIG" SECTION_BUCKETS = "BUCKETS" MAX_FILESIZE_CHECK = 512L P4BUCKET_MESSAGE = "# P4BUCKET: This file has been archived.\n" class UsageException(BaseException): def __init__( self, explanation ): self.explanation = explanation def __str__( self ): return "Usage error : %s" % self.explanation class P4Bucket: def croak_usage( self, message ): print >> sys.stderr, """ Usage: p4bucket command [options]: init - initialize the connection parameters create -r - create a bucket with root edit -r - change the root of bucket to delete - deletes the bucket (must be empty) list [name] [-a] - list bucket information option -a lists contents archive -b [-n] [opts] file[revRange] ... - archives the files restore -b [-n] file[revRange] ... - restores the files opts are -f forces archiving of head revisions -s snaps lazy copies before archiving -m size minimum size for archiving files -n do not archive or restore, instead report the files -k keep specify the number of revisions to keep (default 1) -h no host check (for testing only) -f and -k are mutually exclusive options. """ print >> sys.stderr, message def __init__( self, configFile ): self.config = ConfigParser.RawConfigParser() found = self.config.read( CONFIG_FILE ) if not (CONFIG_FILE in found): # config file does not exist yet self.config.add_section(SECTION_P4CONFIG) self.config.add_section(SECTION_BUCKETS) self.p4 = P4.P4() self.logFileName = LOG_FILE def verify_P4_config( self, check ): """Sets self.p4 and verifies that we can connect and that we are running on the same host as the server""" for name, value in self.config.items(SECTION_P4CONFIG): if name.lower() == "p4port": self.p4.port = value elif name.lower() == "p4user": self.p4.user = value elif name.lower() == "p4client": self.p4.client = value elif name.lower() == "log": self.logFileName = value else: print "verify_P4_config: Option %s unknown" % name try: self.log = None self.log = open(self.logFileName, "a+") # this will throw an exception if it cannot connect print "Attempting to connect to Perforce using : %s/%s@%s ..." % \ (self.p4.user, self.p4.client, self.p4.port), self.p4.connect() print "succeeded\n" serverInfo = self.p4.run_info()[0] serverhost = serverInfo['serverAddress'].split(':')[0] serverip = socket.gethostbyname(serverhost) if check: # verify that this script runs on the same host as the Perforce server hostname = platform.uname()[1] hostip = socket.gethostbyname(hostname) if serverip != "127.0.0.1" and serverip != hostip: raise Exception("Cannot run this script from a host different to the server host") # save the server root for later self.serverRoot = serverInfo["serverRoot"] # load the depots here and save them for later as well self.depotMap = P4.Map() for depot in self.p4.run_depots(): if depot["type"] != "remote": map = depot["map"] if map[0:-4].find("/") == -1: map = os.path.normpath(self.serverRoot + "/" + map) self.depotMap.insert("//%s/..." % depot["name"], map) except P4.P4Exception, e: print "Perforce error %s\n" % str(e) sys.exit(1) except IOError, e: print "Could not open log file %s\n" % str(e) sys.exit(1) def save_configuration( self ): with open(CONFIG_FILE, "wb") as config_file: self.config.write(config_file) def run( self, command, args ): try: check = ('-h' not in args) if not check: args.remove('-h') if command != "init": self.verify_P4_config(check) method = "run_" + command if hasattr(self, method): getattr(self, method)(args) else: self.croak_usage("Unknown command : %s" % command) except UsageException, e: self.croak_usage(e) finally: if self.p4.connected(): self.p4.disconnect() if command != "init" and self.log and not self.log.closed: self.log.close() def run_help( self, args ): self.croak_usage("Help") def run_init( self, args ): """ command : init """ self.get_choice( "P4PORT", self.p4.port ) self.get_choice( "P4USER", self.p4.user ) self.get_choice( "P4CLIENT", self.p4.client ) self.get_choice( "LOG", LOG_FILE ) self.save_configuration() def get_choice( self, name, default ): if self.config.has_option(SECTION_P4CONFIG, name): answer = self.config.get(SECTION_P4CONFIG, name) else: answer = default prompt = "Provide a value for %s [%s] : " % (name, answer) answer = raw_input(prompt) if answer != '': self.config.set(SECTION_P4CONFIG, name, answer) elif not self.config.has_option(SECTION_P4CONFIG, name): self.config.set(SECTION_P4CONFIG, name, default) def run_create( self, args ): """ command : create """ if ( len(args) < 3 ): raise UsageException("Missing option -r ") name = args.pop(0) if name[0] == '-': raise UsageException("Bucket name cannot contain '-' : %s" % name) if name in self.config.options(SECTION_BUCKETS): raise UsageException("Bucket name '%s' already exists" % name) if (args[0] != "-r"): raise UsageException("Missing option -r , found '%s' instead" % args[0]) root = args[1] self.config.set(SECTION_BUCKETS, name, root) self.save_configuration() def run_edit( self, args ): """ command : edit """ if ( len(args) < 3 ): raise UsageException("Missing option -r ") name = args.pop(0) if name[0] == '-': raise UsageException("Bucket name cannot contain '-' : %s" % name) if not self.config.has_option( SECTION_BUCKETS, name ): raise UsageException( "Bucket name '%s' does not exist" % name ) if (args[0] != "-r"): raise UsageException( "Missing option -r , found '%s' instead" % args[0] ) root = args[1] self.config.set(SECTION_BUCKETS, name, root) self.save_configuration() def run_delete( self, args ): """ command : delete """ if ( len(args) < 1 ): raise UsageException("Missing bucket name") name = args.pop(0) if not self.config.has_option( SECTION_BUCKETS, name ): raise UsageException( "Bucket name '%s' does not exist" % name ) bucketRoot = self.config.get( SECTION_BUCKETS, name ) # check if there are any files stored in here for root, dirs, files in os.walk(bucketRoot, topdown=False): for file in files: raise UsageException("Cannot delete bucket %s - not empty: %s" % (name, os.path.join(root, file))) for root, dirs, files in os.walk(bucketRoot, topdown=False): for directory in dirs: os.rmdir(os.path.join(root, directory) ) self.config.remove_option( SECTION_BUCKETS, name ) self.save_configuration() def run_list( self, args ): """ command : list """ name = "" contents = False while ( len(args) > 0 ): arg = args.pop(0) if (arg == '-a'): contents = True else: name = arg if name: if not self.config.has_option( SECTION_BUCKETS, name ): raise UsageException( "Bucket name '%s' does not exist" % name ) root = self.config.get( SECTION_BUCKETS, name ) self.print_bucket_and_root( name, root ) if contents: self.run_list_contents ( name, root ) else: for (name, root) in self.config.items(SECTION_BUCKETS): self.print_bucket_and_root( name, root ) if contents: self.run_list_contents ( name, root ) def print_bucket_and_root( self, name, root): output = "%s : %s" % ( name, root ) print output print "=" * len(output) def run_list_contents( self, name, startRoot ): for root, dirs, files in os.walk(startRoot, topdown=False): for file in files: print os.path.join(root, file) print def get_fstats( self, filepattern ): fstats = [] candidates = [] with self.p4.at_exception_level(self.p4.RAISE_ERROR): for filerev in self.p4.run_files("-a", filepattern): if filerev["action"] not in ["delete","move/delete","archive", "purge"] \ and "binary" in filerev["type"] \ and not "D" in filerev["type"] \ and not "S" in filerev["type"]: candidates.append(filerev["depotFile"] + "#" + filerev["rev"]) for w in self.p4.warnings: print w # fstat -Oacz # a : show attributes # c : show librarian information # z : show lazy copies originating from this file # l : show digest and fileSize if len(candidates) > 0: fstats = self.p4.run_fstat("-Oaczl", candidates) return fstats def calculate_size( self, s ): matchers = [] matchers.append( re.compile("(\d+)$") ) matchers.append( re.compile("(\d+)[kK]$") ) matchers.append( re.compile("(\d+)[mM]$") ) matchers.append( re.compile("(\d+)[gG]$") ) matchers.append( re.compile("(\d+)[tT]$") ) matchers.append( re.compile("(\d+)[pP]$") ) factor = 1L for matcher in matchers: g = matcher.match(s) if g: return factor * long(g.group(1)) else: factor *= 1024 raise UsageException("Illegal size specifier %s" % s) def run_archive( self, args ): # verify the bucket and connection # find all files in the args list # for each file: # ensure it is not already archived - otherwise ignore # ensure it is a binary file, that is, stored with type +F or +C - otherwise ignore # identify depot file location and verify file exists - otherwise throw wobbly # # move file to bucket location # create attributes for archived file """command archive""" # check for option switches name = None snap = False force = False keep = 1 size = None counter = 0 report = False reportFileNumber = 0 reportFileSizes = 0L fileArgs = [] while ( len(args) > 0 ): a = args.pop(0) if ( a == '-s' ): snap = True elif ( a == '-f' ): force = True elif ( a == '-n' ): report = True elif ( a == '-m' ): if ( len(args) < 1 ): raise UsageException("Missing size") s = args.pop(0) size = self.calculate_size( s ) elif ( a == '-k' ): if ( len(args) < 1 ): raise UsageException("Missing number of revisions to keep") k = args.pop(0) keep = int(k) if ( force ): raise UsageException("Force (-f) and keep (-k) cannot both be active") elif ( a == '-b' ): if ( len(args) < 1 ): raise UsageException("Missing bucket name") name = args.pop(0) else: fileArgs.append(a) if not name: raise UsageException("Missing option -b bucketname") if not self.config.has_option( SECTION_BUCKETS, name ): raise UsageException( "Bucket name '%s' does not exist" % name ) bucketRoot = self.config.get( SECTION_BUCKETS, name ) for filepattern in fileArgs: # first list all files by using "p4 files -a", # then check for non-deleted revisions and correct binary types fstats = self.get_fstats( filepattern ) try: for afile in fstats: if not "attr-archiveBucket" in afile: depotFile = afile["depotFile"] thisRev = depotFile + "#" + afile["headRev"] filelog = self.p4.run_filelog(depotFile)[0] describe = self.p4.run_describe('-s', afile['headChange'])[0] ch = describe['oldChange'] if 'oldChange' in describe else afile['headChange'] if afile["lbrFile"] != depotFile or afile["lbrRev"] != "1.%s" % ch : # this file is a lazy copy # lets trail through the filelog and find out what the source was found = False for r in filelog.revisions: if r.rev == int(afile["headRev"]): for integs in r.integrations: # this file was created through a branch, so it must # have action "branch from" if integs.how in ("branch from", "moved from", "copy from"): found = True print "File %s is a lazy copy of %s#%s with lbrFile %s" % \ (thisRev, integs.file, integs.erev, afile["lbrFile"]) if not found: # this can happen if the original file was obliterated # or after a perfmerge print "File %s does not match changelist lbrFile %s rev %s, archiving anyway" % \ (thisRev, afile["lbrFile"], afile["lbrRev"]) else: continue # find out if this is the head revision of the file # if it is the head revision, then write a warning and bail # unless force is active headRev = filelog.revisions[0] if ( headRev.rev == int(afile["headRev"]) and not force ): print "Revision %s is the head revision and cannot be archived without -f option" % thisRev continue elif ( headRev.rev < int( afile["headRev"] ) + keep and not force ): print "Revision %s is higher than #head - keep" % thisRev continue compressed = self.lbrTypeCompressed( afile["lbrType"] ) fileName = self.getFileName( afile["lbrRev"], compressed ) libFile = self.getLibrarianFile( afile["lbrFile"], fileName) targetPath = os.path.normpath( bucketRoot + "/" + afile["lbrFile"] + ",d" ) targetFile = os.path.join(targetPath, os.path.basename(libFile)) # check if there are any lazy copies # if we have lazy copies, either bail with an error message, or # if snap is defined, snap this file, then archive the source if "lazyCopyFile" in afile: if snap: for (ld, lr) in zip(afile["lazyCopyFile"], afile["lazyCopyRev"]): lazyRev = ld + lr + "," + lr self.p4.run_snap(lazyRev) print >> self.log, "Snapped lazy copy for %s" % lazyRev else: print "There are lazy copies for %s, will not archive without -s option" % thisRev for (ld, lr) in zip(afile["lazyCopyFile"], afile["lazyCopyRev"]): print "\t%s%s" % (ld,lr) continue # check if libFile is already archived # if the afile size is less than MAX_FILESIZE_CHECK bytes, # check the afile, uncompressed if necessary fileSize = os.path.getsize(libFile) if fileSize < MAX_FILESIZE_CHECK: if compressed: f = gzip.GzipFile(libFile, "rb") else: f = open(libFile, "rb") fcontent = f.readlines() f.close() # check the first line whether it matches the standard p4bucket message if P4BUCKET_MESSAGE in fcontent: print "File %s already archived, but attributes are missing." % thisRev continue # if option -m is used and file size is less than specified, ignore if size and fileSize < size: print "File %s ignored, file size %s less than %s" % (thisRev, os.path.getsize(libFile), size) continue # create directory with os.makedirs(targetpath) if not os.path.isdir( targetPath ): os.makedirs( targetPath ) # check if the target file exists already, # bail in that case, we do not want to overwrite an archived file # under any circumstances if os.path.isfile( targetFile ): print "Target file %s for %s already exists, will not overwrite!" % ( targetFile, thisRev ) continue if report: print "Archive file %s : %s -> %s" % (thisRev, libFile, targetFile) reportFileNumber += 1 reportFileSizes += fileSize else: # move the afile shutil.move(libFile, targetFile) # replace it with placeholder (compressed if necessary) if compressed: f = gzip.GzipFile(libFile, "wb") else: f = open(libFile, "wb") newMessage = P4BUCKET_MESSAGE newMessage += "User=%s\n" % self.p4.user newMessage += "Date=%s\n" % datetime.datetime.now().ctime() newMessage += "Digest=%s\n" % afile["digest"] newMessage += "Bucket=%s\n" % name f.write(newMessage) f.close() # update the attributes self.p4.run_attribute("-f", "-n", "archiveUser", "-v", self.p4.user, thisRev) self.p4.run_attribute("-f", "-n", "archiveDate", "-v", datetime.datetime.now().ctime(), thisRev) self.p4.run_attribute("-f", "-n", "archiveDigest", "-v", afile["digest"], thisRev) self.p4.run_attribute("-f", "-n", "archiveBucket", "-v", name, thisRev) # reverify to set the correct digest for the replaced afile # need the form file#rev,#rev, otherwise all previous revs get verified as well self.p4.run_verify("-v", thisRev + "," + afile["headRev"]) print >> self.log, "Archived file %s : %s -> %s" % (thisRev, libFile, targetFile) self.log.flush() counter += 1 except KeyError as e: print >> self.log, "KeyError %s for %s" % ( e, afile ) print "\n%s file(s) archived" % counter if report: print print "This was reporting mode. Use the command without -n to archive files." print print "Total files that would be archived : %s" % reportFileNumber print "Total space these files consume : %s (%6f GB)" % (reportFileSizes, reportFileSizes / (1024.0*1024*1024)) def run_restore( self, args ): # verify the bucket and connection # find all files in the args list # for each file: # ensure it is archived - otherwise ignore # identify depot file location and verify file exists - otherwise throw wobbly # # move file back to original position # remove attributes # verify -v the file # verify -v all lazy copies """command restore""" name = None counter = 0 report = False reportFileNumber = 0 reportFileSizes = 0 fileArgs = [] while ( len(args) > 0 ): a = args.pop(0) if ( a == '-b' ): if ( len(args) < 1 ): raise UsageException("Missing bucket name") name = args.pop(0) elif ( a == '-n' ): report = True else: fileArgs.append(a) if not name: raise UsageException("Missing option -b bucketname") if not self.config.has_option( SECTION_BUCKETS, name ): raise UsageException( "Bucket name '%s' does not exist" % name ) bucketRoot = self.config.get( SECTION_BUCKETS, name ) for filepattern in fileArgs: fstats = self.get_fstats( filepattern ) for afile in fstats: if "attr-archiveBucket" in afile: bucket = afile["attr-archiveBucket"] user = afile["attr-archiveUser"] date = afile["attr-archiveDate"] digest = afile["attr-archiveDigest"] depotFile = afile["depotFile"] thisRev = depotFile + "#" + afile["headRev"] if bucket != name: print "Cannot restore %s from bucket %s, it is stored in bucket %s" % (afile["depotFile"], name, bucket) continue compressed = self.lbrTypeCompressed( afile["lbrType"] ) fileName = self.getFileName( afile["lbrRev"], compressed ) libDirectory = self.getLibrarianDirectory( afile["lbrFile"]) sourceFile = os.path.normpath( bucketRoot + "/" + afile["lbrFile"] + ",d" + "/" + fileName) targetFile = os.path.join(libDirectory, os.path.basename(sourceFile)) if report: print "Restore file %s : %s -> %s" % (thisRev, sourceFile, targetFile) reportFileNumber += 1 reportFileSizes += os.path.getsize(sourceFile) else: # move the archived file back to its position shutil.move(sourceFile, targetFile) # delete the attributes self.p4.run_attribute("-f", "-n", "archiveUser", thisRev) self.p4.run_attribute("-f", "-n", "archiveDate", thisRev) self.p4.run_attribute("-f", "-n", "archiveDigest", thisRev) self.p4.run_attribute("-f", "-n", "archiveBucket", thisRev) # clean up the digest, also verify that the digest is correct verifyResult = self.p4.run_verify("-v", thisRev + "," + afile["headRev"])[0] if verifyResult['digest'] != digest: print "*** WARNING *** digest for %s is incorrect. Should be %s, but verify reports %s" % \ (thisRev, digest, verifyResult['digest']) # fix the digest of the lazy copies if "lazyCopyFile" in afile: for (ld, lr) in zip(afile["lazyCopyFile"], afile["lazyCopyRev"]): self.p4.run_verify("-v", ld + lr + "," + lr) print >> self.log, "Restored file %s : %s -> %s" % (thisRev, sourceFile, targetFile) self.log.flush() counter += 1 print "\n%s files restored" % counter if report: print "Total files that would be restored : %s" % reportFileNumber print "Total space these files consume : %s (%6f GB)" % (reportFileSizes, reportFileSizes / (1024.0*1024*1024)) def getFileName( self, lbrFile, compressed ): result = lbrFile if compressed: result += ".gz" return result def getLibrarianFile( self, lbrFile, fileName): path = self.getLibrarianDirectory( lbrFile ) afile = path + os.sep + fileName if not os.path.isfile( afile ): raise Exception("Cannot find file %s" % afile ) return afile def getLibrarianDirectory( self, lbrFile): path = self.depotMap.translate(lbrFile) if path: path += ",d" path = os.path.normpath(path) if not os.path.isdir(path): raise Exception("Cannot find directory %s for lbrFile %s" % (path, lbrFile)) return path else: raise Exception("Cannot translate %s into directory" % lbrFile) def lbrTypeCompressed( self, lbrType ): if (lbrType == "ubinary") or (lbrType == "uxbinary") or ("F" in lbrType): return False return True if __name__ == "__main__": bucket = P4Bucket( CONFIG_FILE ) if len(sys.argv) > 1: bucket.run( sys.argv[1], sys.argv[2:] ) # sys.argv[0] is the name of the script else: bucket.croak_usage("No command found")