# incremental_obliterate.py - run obliterate in small incremental chunks # Copyright robert@vaccaperna.co.uk from __future__ import print_function """ Obliterate a Perforce path in an incremental fashion. All the usual warnings about how dangerous this command is!!! If you have largeish db.rev/integed tables (e.g. >10Gb) an obliterate can: - take large amounts of time and cause server thrashing depending on memory - end up taking a global lock on various tables which prevents other people working. In this script we group things to gether and obliterate in reverse order of changelists. Performance example - for 3k changelists, this took 30mins instead of hours and hours on a small server! Also, it could be run during normal working hours (not peak load) as it executes lots of smaller commands and doesn't lock the database. """ import P4 import time import sys import argparse # Max no of files to obliterate together - can be modified MAX_FILES_PER_OBLITERATE = 5000 class ObliterateException(Exception): pass def message(*args): """Output to stdout with flush - useful for forked jobs""" print(" ".join([str(arg) for arg in args])) sys.stdout.flush() class HMS: """Output time (duration) in Hours:Minutes:Seconds""" def __init__(self, secs): secs = int(secs) self.hours = secs / 3600 secs = secs % 3600 self.minutes = secs / 60 secs = secs % 60 self.seconds = secs def __repr__(self): return "%02d:%02d:%02d" % ( self.hours, self.minutes, self.seconds) class Status: """Utility class to output progress - performing rate calculations as appropriate""" def __init__(self, description, total): self.description = description self.total = total self.count = 0 self.start_time = time.time() self.tick_start = time.time() def tick(self, count): """Called every action to output progress""" self.count += count if self.count == 0: return elapsed = time.time() - self.tick_start self.tick_start = time.time() rate = (time.time() - self.start_time) / self.count time_togo = (self.total - self.count) * rate delta = HMS(time_togo) msg = "%s (%s) - so far/to go: %d / %d, %s / %s" % ( HMS(elapsed), self.description, self.count, self.total - self.count, str(HMS(time.time() - self.start_time)), str(delta)) message(msg) sys.stdout.flush() class ChangeInfo: def __init__(self, changeNo, fileCount): self.changeNo = changeNo self.fileCount = fileCount class Obliterator: def __init__(self, *args): parser = argparse.ArgumentParser( description="incremental_obliterate", epilog="Copyright (C) 2011-15 Robert Cowham" ) parser.add_argument('-a', '--skip-archive', action='store_true', default=False, dest='skip_archive', help="Skip archive file deletion") parser.add_argument('-b', '--branch-only', action='store_true', default=False, dest='branch_only', help="Obliterate only files which are branched and where #head==#1 (after the fact sparse branching)") parser.add_argument('-e', '--skip-have', action='store_true', default=False, dest='skip_have', help="Skip deleting from db.have - equivalent of -h flag") parser.add_argument('-s', '--batch-size', type=int, default=MAX_FILES_PER_OBLITERATE, dest='batch_size', help="Size of batch") parser.add_argument('-y', '--yes', action='store_true', default=False, help="Actually perform the obliterate instead of just reporting") parser.add_argument('-p', '--port', help="P4PORT (otherwise uses standard environment settings)") parser.add_argument('-u', '--user', help="P4USER (otherwise uses standard environment settings)") parser.add_argument('depot_path', help="Perforce depot path to obliterate (required)") self.options = parser.parse_args(list(args)) if not self.options.depot_path or not self.options.depot_path.startswith("//"): raise ObliterateException("Path is required and must be a Perforce depot path, starting with '//'") self.p4 = P4.P4() if self.options.port: self.p4.port = self.options.port if self.options.user: self.p4.user = self.options.user message('Options:', self.options) message("Connecting to %s as user %s" % (self.p4.port, self.p4.user)) self.p4.connect() def filesInChange(self, chgno): """Count how many files in the changelist""" desc = self.p4.run_describe("-s", chgno) return len(desc[0]['depotFile']) def runObliterate(self, path, chgGroup): """Actually run a specific obliterate command on a group of changelists""" if len(chgGroup) == 0: return cmd = ["obliterate"] if self.options.skip_archive: cmd.append('-a') if self.options.branch_only: cmd.append('-b') if self.options.skip_have: cmd.append('-h') if self.options.yes: cmd.append('-y') cmd.append("%s@%s,@%s" % (path, chgGroup[-1], chgGroup[0])) result = self.p4.run(cmd) message("@%s,@%s %s" % (chgGroup[-1], chgGroup[0], result[-1])) for chg in chgGroup: self.p4.run_change("-f", "-d", chg) def obliteratePath(self, path): """Obliterate all files in the path""" changes = self.p4.run_changes(path) message("Obliterating %d changes for path %s" % (len(changes), path)) revCount = 0 chgCounts = [] for c in changes: chgCounts.append(ChangeInfo(c['change'], self.filesInChange(c['change']))) revCount += chgCounts[-1].fileCount chgStatus = Status("changes", len(changes)) revStatus = Status("revisions", revCount) fcount = 0 chgGroup = [] for ci in chgCounts: if fcount + ci.fileCount > self.options.batch_size and len(chgGroup) > 0: self.runObliterate(path, chgGroup) chgStatus.tick(len(chgGroup)) revStatus.tick(fcount) chgGroup = [] fcount = 0 chgGroup.append(ci.changeNo) fcount += ci.fileCount if len(chgGroup) > 0: # don't forget final group if any self.runObliterate(path, chgGroup) chgStatus.tick(len(chgGroup)) revStatus.tick(fcount) def doIt(self): """Perform the obliterate""" self.obliteratePath(self.options.depot_path) if not self.options.yes: message("This was in report mode only - please use -y/--yes to perform the obliterate action") def main(): try: obl = Obliterator(*sys.argv[1:]) except ObliterateException as e: print(str(e)) sys.exit(1) obl.doIt() if __name__ == "__main__": main()