#!/bin/env python3
'''
Perforce Baseline and Branch Import Tool
Primary Import Tool
Copyright (c) 2012 Perforce Software, Inc.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE
SOFTWARE, INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
THIS SOFTWARE IS NOT OFFICIALLY SUPPORTED.
Please send any feedback to consulting@perforce.com.
'''
import sys
import os
import re
import platform
from optparse import OptionParser
from subprocess import *
import shutil
import stat
import tarfile
import zipfile
mode=""
verbosity="3"
p4cfg_file="test"
bbicfg_file=""
removejunk=False
detectcase=0
if platform.system() == "Windows":
p4="p4.exe"
else:
p4="p4"
valid_commands = ("BRANCH", "UPDATE", "SAFEUPDATE", "COPY_MERGE", "RECORD_AS_MERGED", "RENAME")
valid_options = ("EMPTYDIRS", "SYMLINKS", "BRANCHSPEC", "NODELETE", "ALLOWWILDCARDS", "RMTOP", "FULLCOPY")
def parseoptions():
global mode
global verbosity
global p4cfg_file
global bbicfg_file
global removejunk
global detectcase
usage = "usage: %prog [options] P4CONFIG_FILE BBI_CONFIG_FILE"
parser = OptionParser(usage)
parser.add_option("-m", "--mode", dest="test",
help="Test mode. Shows what would be done, but takes no action.")
parser.add_option("-r", "--remove", dest="rjd",
help="Remove junk dirs. Removes known old SCM files and directories.")
parser.add_option("-c", "--case", dest="detectcase",
help="When set to 1, detects and reports potential case sensitivity issues. At level 2, fails when these problems are detected.")
parser.add_option("-v", "--verbosity", dest="level",
help="Specify the verbosity level, from 1-3. 1 is quiet mode;"
" only error and test output is displayed. 2 is normal verbosity; some messages"
" displayed. 3 is debug-level verbosity. The default for this program is 3.")
(options, args) = parser.parse_args()
if len(args) < 2:
parser.error("Run p4bbi.py -h for help.")
if not os.path.isfile(args[0]):
log("ERROR", "P4CONFIG file: '{0}' not found.".format(args[0]))
else:
p4cfg_file = args[0]
if not os.path.isfile(args[1]):
log("ERROR", "p4bbi config file: '{0}' not found.".format(args[1]))
else:
bbicfg_file=args[1]
if options.test:
mode = "test"
print("Running in test mode. No updates will be made to the server.")
if options.rjd:
removejunk = True
log("INFO", "Junk directories will be removed from new Perforce server.")
if options.detectcase:
try:
if 0 < int(options.detectcase) < 3:
detectcase = int(options.detectcase)
log("INFO", "Case sensitivity detection set to {0}".format(detectcase))
else:
sys.exit(1)
except:
log("ERROR", "Case sensitivity mode must be an integer value of 1 or 2.")
if options.level:
try:
if 0 < int(options.level) < 4:
verbosity = options.level
print("Verbosity level set to: {0}.".format(options.level))
else:
sys.exit(1)
except:
log("ERROR", "Verbosity level must be an integer value of 1, 2 or 3.")
def loadp4config():
global P4USER
global P4PORT
global P4CLIENT
global P4CHARSET
global P4COMMANDCHARSET
P4CHARSET="none"
P4COMMANDCHARSET="none"
cfgfile = open(p4cfg_file, "r")
for line in cfgfile.readlines():
if re.search(r"^P4USER=", line):
P4USER = re.match(r"^P4USER=(.*)", line).group(1)
continue
if re.search(r"^P4PORT=", line):
P4PORT = re.match(r"^P4PORT=(.*)", line).group(1)
continue
if re.search(r"^P4CLIENT=", line):
P4CLIENT = re.match(r"^P4CLIENT=(.*)", line).group(1)
continue
if re.search(r"^P4CHARSET=", line):
P4CHARSET = re.match(r"^P4CHARSET=(.*)", line).group(1)
continue
if re.search(r"^P4COMMANDCHARSET=", line):
P4COMMANDCHARSET = re.match(r"^P4COMMANDCHARSET=(.*)", line).group(1)
continue
cfgfile.close()
def loadbbiconfig():
baselines = {}
commands = []
cfgfile = open(bbicfg_file, "r")
referenced_baseline_found = False
log("DEBUG", "Loading info from {0}.".format(bbicfg_file))
for line in cfgfile.readlines():
if (re.search("^#", line) or re.search("^\s*$", line)):
continue
if re.search("^BASELINE", line):
(name,path) = re.match("^BASELINE\|(.*)\|(.*)", line).groups()
if not name:
cfgfile.close()
log("ERROR", "Error in Baseline definition for {0}.".format(line))
elif not (os.path.isdir(path) or os.path.isfile(path) or path.startswith("CMD:")):
cfgfile.close()
log("ERROR", "Baseline path {0} not found.".format(path))
else:
log("DEBUG", "Verified path: {0}".format(path))
log("DEBUG", "Baseline {0} loaded.".format(name))
baselines[name] = path
else:
try:
cmdline_match = re.match("(.*)\|(.*)\|(.*)\|(.*)\|(.*)", line)
if cmdline_match == None:
cmdline_match = re.match("(.*)\|(.*)\|(.*)\|(.*)", line)
cmdline_items = cmdline_match.groups()
if len(cmdline_items) == 5:
options = cmdline_items[4]
options_list = options.split(",")
for o in options_list:
parts = o.split("=")
if o not in valid_options and parts[0] not in valid_options:
log("ERROR", "Invalid option {0} specified in {1}".format(o, line))
if o == 'SYMLINKS' and platform.system() == "Windows":
log("ERROR", "Invalid option {0} specified: SYMLINKS not applicable on Windows".format(o))
if parts[0] == 'BRANCHSPEC' and len(parts) != 2:
log("ERROR", "Invalid option {0} specified: BRANCHSPEC option must include a branch spec name".format(o))
if cmdline_items[0] in valid_commands:
commands.append(cmdline_items)
log("DEBUG", "Command line: {0} loaded.".format(cmdline_items))
else:
log("ERROR", "Invalid command in line:\n {0}".format(line))
except:
log("ERROR", "Invalid line in config file:\n {0}".format(line))
if cmdline_items[0] == "UPDATE":
referenced_baseline_found = False
for key in iter(baselines):
if cmdline_items[1] == key:
referenced_baseline_found = True
if not referenced_baseline_found:
cfgfile.close()
log("ERROR", "Baseline {0} referenced, but not defined.".format("referenced_baseline"))
cfgfile.close()
if baselines == {}:
log("ERROR", "No baselines found in config file.")
return baselines, commands
def log(msglevel="DEBUG", message=""):
if msglevel == "TEST":
print("Running in test mode. Command run would have been:\n", message)
elif msglevel == "ERROR":
print(message)
sys.exit(1)
elif (verbosity == "3"):
print(message)
elif (verbosity == "2" and msglevel == "INFO"):
print(message)
# run OS cmd
def runcmd(cmd):
try:
log("DEBUG", cmd);
pipe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, universal_newlines=True)
stdout, stderr = pipe.communicate()
log("DEBUG", stdout)
if pipe.returncode != 0:
log("INFO", "{0} generated the following output: {1}".format(cmd, stdout))
log("ERROR", "{0} generated the following error: {1}".format(cmd, stderr))
else:
return stdout
except OSError as err:
log("ERROR", "Execution failed: {0}".format(err))
def runp4cmd(cmd, instr=None):
try:
pipe = Popen(p4 + cmd, shell=True, stdin=PIPE, stdout=PIPE, universal_newlines=True)
stdout, stderr = pipe.communicate(instr)
#log("INFO", stdout)
if pipe.returncode != 0:
log("ERROR", "{0}{1} generated the following error: {2}".format(p4, cmd, stderr))
else:
return stdout
except OSError as err:
log("ERROR", "Execution failed: {0}".format(err))
def p4submit(desc, target):
opened = runp4cmd("opened -c default")
if re.search(r"not opened on this client\.", opened) or len(opened.strip()) < 1:
log("DEBUG", "No changes to submit")
# store last changelist and description in a counter
cnt = runp4cmd("counter -i bbi_cnt").strip()
cname = "bbi_{0}".format(cnt)
lastchange = runp4cmd("changes -m1 {0}".format(target))
change = re.match(r"Change (\d+) on \S+ by", lastchange).group(1)
cvalue = "{0}={1}".format(change, desc)
runp4cmd("counter -f {0} \"{1}\"".format(cname, cvalue))
else:
runp4cmd('submit -d "{0}"'.format(desc))
def getclientpath(p4dir,part=2):
result = runp4cmd("where {0}".format(p4dir))
mappings = result.split()
if platform.system() == "Windows":
localdir = mappings[part].replace("\\...", "")
else:
localdir = mappings[part].replace("/...", "")
return localdir
def processcommands( baselines={}, commands=[]):
"""
Each command contains four components as follows:
UPDATE|BaselineName|P4Dir|Description
SAFEUPDATE|BaselineName|P4Dir|Description
BRANCH|SourcePath|TargetPath|Description
COPY_MERGE|SourcePath|TargetPath|Description
RECORD_AS_MERGED|SourcePath|TargetPath|Description
RENAME|SourcePath|TargetPath|Description
"""
for cmd in commands:
desc = cmd[3].replace("$N", "\n")
options = None
if len(cmd) == 5:
options = cmd[4]
if cmd[0] == "UPDATE":
if mode == "test":
log("TEST", "p4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, False, options))
p4localdir = getclientpath(cmd[2])
log("DEBUG", "p4 local dir is: {0}".format(p4localdir))
log("DEBUG", "Baseline dir is: {0}".format(baselines[cmd[1]]))
else:
log("DEBUG", "Running: p4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, False, options))
p4update(cmd[0], baselines[cmd[1]], cmd[2], desc,False, options)
else:
if cmd[0] == "SAFEUPDATE":
if mode == "test":
log("TEST", "p4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, True, options))
p4localdir = getclientpath(cmd[2])
log("DEBUG", "p4 local dir is: {0}".format(p4localdir))
log("DEBUG", "Baseline dir is: {0}".format(baselines[cmd[1]]))
else:
log("DEBUG", "Running: p4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, True, options))
p4update(cmd[0], baselines[cmd[1]], cmd[2], desc,True, options)
else:
if mode == "test":
log("TEST", "p4branch({0}, {1}, {2}, {3}, {4})".format(cmd[0], cmd[1], cmd[2], desc, options))
else:
log("DEBUG", "Running: p4branch({0}, {1}, {2}, {3}, {4})".format(cmd[0], cmd[1], cmd[2], desc, options))
p4branch(cmd[0], cmd[1], cmd[2], desc, options)
def p4update(cmd, baseline, p4dir, desc,safe, options = None):
options_list = []
if options != None:
options_list = options.split(",")
p4localdir = getclientpath(p4dir)
if os.path.isdir(p4localdir):
try:
for root, dirs, names in os.walk(p4localdir):
for name in names:
os.chmod(os.path.join(root,name), stat.S_IWRITE)
shutil.rmtree(p4localdir)
except OSError as e:
log("ERROR", "Could not remove {0}.\nThe error was:\n{1}".format(p4localdir, e))
if baseline.startswith("CMD:"):
savedPath = os.getcwd()
os.makedirs(p4localdir)
os.chdir(p4localdir)
runcmd(baseline[4:])
os.chdir(savedPath)
elif re.search(r"\.tar\.gz", baseline.lower()) or re.search(r"\.tar", baseline.lower()):
try:
tar = tarfile.open(baseline)
if detectcase > 0:
files_all_lower = []
for member in tar.getnames():
fname_lower = member.lower()
if fname_lower in files_all_lower:
if detectcase == 1:
log("INFO", "Potential case sensitivity conflict: {0}".format(member))
else:
log("ERROR", "Potential case sensitivity conflict: {0}".format(member))
else:
files_all_lower.append(fname_lower)
tar.extractall(path=p4localdir)
tar.close()
except:
log("ERROR", "Invalid or unsupported tar file format for baseline: {0}\n\t{1}\n\t{2}".format(baseline, sys.exc_info()[0], sys.exc_info()[1]))
elif re.search("\.zip", baseline.lower()):
try:
zip = zipfile.ZipFile(baseline)
if detectcase > 0:
files_all_lower = []
for member in zip.namelist():
fname_lower = member.lower()
if fname_lower in files_all_lower:
if detectcase == 1:
log("INFO", "Potential case sensitivity conflict: {0}".format(member))
else:
log("ERROR", "Potential case sensitivity conflict: {0}".format(member))
else:
files_all_lower.append(fname_lower)
os.makedirs(p4localdir)
for member in zip.namelist():
if member.endswith("/"):
os.makedirs(os.path.join(p4localdir, member))
else:
zip.extract(member, p4localdir)
zip.close()
except:
log("ERROR", "Invalid or unsupported zip file format for baseline: {0}\n\t{1}\n\t{2}".format(baseline, sys.exc_info()[0], sys.exc_info()[1]))
else:
try:
preserve_links = False
if 'SYMLINKS' in options_list:
preserve_links = True
shutil.copytree(baseline, p4localdir, symlinks=preserve_links)
except OSError as err:
log("ERROR", "Error copying baseline to local directory. {0}".format(err))
# optionally populate empty directories
if 'EMPTYDIRS' in options_list:
log("DEBUG", "Populating empty directories")
for root, dirs, names in os.walk(p4localdir):
if len(dirs)==0 and len(names)==0:
placeholder = open(os.path.join(root, "placeholder.txt"), "w")
placeholder.write("Placeholder\n")
placeholder.close()
# optionally remove top level dir
if 'RMTOP' in options_list:
log("DEBUG", "Removing top level baseline directory")
entries = os.listdir(p4localdir)
if len(entries) > 1:
log("ERROR", "RMTOP option can only be used if the top level directory contains one folder.")
ignore_dir = os.path.join(p4localdir, entries[0])
p4_temp_dir = "{0}.tmp".format(p4localdir)
os.rename(ignore_dir, p4_temp_dir)
os.rmdir(p4localdir)
os.rename(p4_temp_dir, p4localdir)
files = open("files.txt", "w")
files_all_lower = []
for root, dirs, names in os.walk(p4localdir):
if 'SYMLINKS' in options_list:
for dirname in dirs:
adir = os.path.join(root, dirname)
if os.path.islink(adir):
files.write(adir + "\n")
if detectcase > 0:
fname_lower = adir.lower()
if fname_lower in files_all_lower:
if detectcase == 1:
log("INFO", "Potential case sensitivity conflict: {0}".format(adir))
else:
log("ERROR", "Potential case sensitivity conflict: {0}".format(adir))
else:
files_all_lower.append(fname_lower)
for name in names:
if safe:
# check to see if fstat -F "headAction=delete" //<file> returns anything
fileToChk = getclientpath(os.path.join(root, name),0)
deleteChk = runp4cmd("fstat -F \"headAction=delete\" {0}".format(fileToChk))
#log("INFO","deleteChk({0}) = {1}".format(fileToChk,deleteChk))
if re.search("headAction delete",deleteChk):
log("INFO", "File deleted at head revision in target, not included in update: {0}".format(os.path.join(root, name)))
else:
files.write(os.path.join(root, name) + "\n")
else:
files.write(os.path.join(root, name) + "\n")
if detectcase > 0:
fname_lower = os.path.join(root,name).lower()
if fname_lower in files_all_lower:
if detectcase == 1:
log("INFO", "Potential case sensitivity conflict: {0}".format(os.path.join(root,name)))
else:
log("ERROR", "Potential case sensitivity conflict: {0}".format(os.path.join(root,name)))
else:
files_all_lower.append(fname_lower)
files.close()
runp4cmd("sync -k {0}".format(p4dir))
if 'ALLOWWILDCARDS' in options_list:
runp4cmd("-x files.txt add -f")
else:
runp4cmd("-x files.txt add")
os.remove("files.txt")
runp4cmd("-d {0} diff -se ... | {1} -x - edit".format(p4localdir, p4))
if 'NODELETE' not in options_list:
runp4cmd("-d {0} diff -sd ... | {1} -x - delete".format(p4localdir, p4))
p4submit(desc, p4dir)
def p4branch(cmd, srcdir, targdir, desc, options=None):
options_list = []
branchspec = None
if options != None:
options_list = options.split(",")
for o in options_list:
m = re.match(r"BRANCHSPEC=(.+)", o)
if m != None:
branchspec = m.group(1)
if cmd == "BRANCH":
result = runp4cmd("files {0}".format(targdir))
if not re.search("- no such file(s).", result):
runp4cmd("integ -v {0} {1}".format(srcdir, targdir))
else:
log("ERROR", "{0} target specified in BRANCH command already exists.".format(targdir))
if branchspec != None:
log("DEBUG", "Creating branch spec {0}".format(branchspec))
bform = runp4cmd("branch -o {0}".format(branchspec))
bform = re.sub(r"\n\s+Created by \w+\.", "\n\t{0}".format(desc), bform)
bindex = bform.find("\nView:")
bform = bform[:bindex]
bform = "{0}\nView:\n\t\"{1}\" \"{2}\"\n".format(bform,srcdir,targdir)
runp4cmd("branch -i", bform)
elif cmd == "COPY_MERGE":
runp4cmd("integ -d -i -t {0} {1}".format(srcdir, targdir))
runp4cmd("resolve -at")
elif cmd == "RECORD_AS_MERGED":
runp4cmd("integ -i -t {0} {1}".format(srcdir, targdir))
runp4cmd("resolve -ay")
elif cmd == "RENAME":
result = runp4cmd("files {0}".format(targdir))
if not re.search("- no such file(s).", result):
runp4cmd("sync -f {0}".format(srcdir))
runp4cmd("edit {0}".format(srcdir))
runp4cmd("move {0} {1}".format(srcdir, targdir))
else:
log("ERROR", "{0} target specified in RENAME command already exists.".format(targdir))
else:
log("ERROR", "Invalid command: {0}".format(cmd))
p4submit(desc,targdir)
if cmd == "COPY_MERGE" and 'FULLCOPY' in options_list:
'''
Need to find content/type diffs or files only on src or target.
'''
log("DEBUG", "Starting diff driven merge for {0}".format(targdir))
diff2out = runp4cmd("diff2 -q -dw -t {0} {1}".format(srcdir, targdir))
if re.search(r"no differing files\.$", diff2out, re.M) or len(diff2out.strip()) < 1:
log("DEBUG", "Diff driven merge found nothing to do for {0}".format(targdir))
else:
for difffile in diff2out.split("\n"):
diffmatch_diffs = re.match(r"^==== ([^#]+)#\d+ \([^)]+\) - ([^#]+)#\d+", difffile)
diffmatch_right = re.match(r"^==== <none> - ([^#]+)#\d+", difffile)
diffmatch_left = re.match(r"^==== ([^#]+)#\d+ - <none>", difffile)
if diffmatch_diffs != None:
(sfile, tfile) = diffmatch_diffs.groups()
runp4cmd("integ -f -d -i -t {0} {1}".format(sfile, tfile))
elif diffmatch_right != None:
tfile = diffmatch_right.group(1)
runp4cmd("delete {0}".format(tfile))
elif diffmatch_left != None:
sfile = diffmatch_left.group(1)
tfile = targdir[:len(targdir)-4] + sfile[len(srcdir)-4:]
runp4cmd("integ -f -d -i -t {0} {1}".format(sfile, tfile))
runp4cmd("resolve -at")
p4submit("DIFF_DRIVEN_MERGE {0}".format(desc),targdir)
def removejunkdirs():
runp4cmd("obliterate -y //....hgignore") # Removes Mercurial junk
runp4cmd("obliterate -y //....hgtags") # Removes Mercurial junk
runp4cmd("obliterate -y //.../.hg/...") # Removes Mercurial junk
runp4cmd("obliterate -y //.../.git/...") # Removes Git junk
runp4cmd("obliterate -y //.../CVS/...") # Removes CVS junk
runp4cmd("obliterate -y //....cvsignore") # Removes CVS junk
runp4cmd("obliterate -y //.../.svn/...") # Removes SVN junk
if __name__ == "__main__":
parseoptions()
loadp4config()
p4 = "{0} -p {1} -c {2} -u {3} ".format(p4, P4PORT, P4CLIENT, P4USER)
if P4CHARSET != "none":
p4 = p4 + "-C {0} ".format(P4CHARSET)
if P4COMMANDCHARSET != "none":
p4 = p4 + "-Q {0} ".format(P4COMMANDCHARSET)
print(p4)
baselines, commands = loadbbiconfig()
processcommands( baselines, commands)
if removejunk:
removejunkdirs()
if mode == "test":
log("TEST", "Test complete.")
else:
log("INFO", "Import complete.")
sys.exit(0)