# do_obl.py - obliterate chunks of Perforce database
# Copyright robert.cowham@squaremilesystems.com
"""
Obliterate chunks of Perforce database
All the usual warnings about how dangerous this is!!!
If you have largeish db.rev/integed tables (e.g. >10Gb) an obliterate
can take very large amounts of time and cause server thrashing etc.
In this script we group things to gether and obliterate in reverse order of changelists.
Performance tests - for 3k changelists, this took 30mins instead of hours and hours!
Also, it could be run during normal working hours (not peak load) as it executes lots of smaller
commands and doesn't lock the database.
"""
import P4
import time
import sys
# Max no of files to obliterate together
MAX_FILES_PER_OBLITERATE = 5000
def message(*args):
"Output to stdout with flush - useful for forked jobs"
print " ".join(args)
sys.stdout.flush()
p4 = P4.P4()
p4.connect()
message("Connecting to %s" % p4.port)
class HMS:
"Output time (duration) in Hours:Minutes:Seconds"
def __init__(self, secs):
secs = int(secs)
self.hours = secs / 3600
secs = secs % 3600
self.minutes = secs / 60
secs = secs % 60
self.seconds = secs
def __repr__(self):
return "%02d:%02d:%02d" % (
self.hours, self.minutes, self.seconds)
class Status:
" Utility class to output progress - performing rate calculations as appropriate"
def __init__(self, total):
self.total = total
self.count = 0
self.start_time = time.time()
self.tick_start = time.time()
def tick(self, count):
"Called every action to output progress"
self.count += count
if self.count == 0:
return
elapsed = time.time() - self.tick_start
self.tick_start = time.time()
rate = (time.time() - self.start_time) / self.count
time_togo = (self.total - self.count) * rate
delta = HMS(time_togo)
msg = "%s - so far/to go: %d / %d, %s / %s" % (
str(HMS(elapsed)),
self.count, self.total - self.count,
str(HMS(time.time() - self.start_time)), str(delta))
message(msg)
sys.stdout.flush()
def files_in_change(chgno):
"Count how many files in the changelist"
desc = p4.run_describe("-s", chgno)
return len(desc[0]['depotFile'])
def run_obl(path, grp):
"Actually run a specific obliterate command"
if len(grp) == 0:
return
cmd = ["obliterate", "-y", "%s@%s,@%s" % (path, grp[-1], grp[0])]
result = p4.run(cmd)
message("@%s,@%s %s" % (grp[-1], grp[0], result[-1]))
for c in grp:
p4.run_change("-f", "-d", c)
def obl_path(path):
"Obliterate all files in the path"
changes = p4.run_changes(path)
message("Obliterating %d changes for path %s" % (len(changes), path))
status = Status(len(changes))
counts = [] # array of tuples ('changeno', 'filecount')
for c in changes:
counts.append((c['change'], files_in_change(c['change'])))
fcount = 0
grp = []
for chg in counts:
if fcount + chg[1] > MAX_FILES_PER_OBLITERATE and len(grp) > 0:
run_obl(path, grp)
status.tick(len(grp))
grp = []
fcount = 0
grp.append(chg[0])
fcount += chg[1]
if len(grp) > 0: # don't forget final group if any
run_obl(path, grp)
status.tick(len(grp))
if __name__ == "__main__":
for path in (sys.argv[1:]):
obl_path(path)