p4bbi.py #1

#!/bin/env python3

'''
Perforce Baseline and Branch Import Tool
Primary Import Tool

Copyright (c) 2012 Perforce Software, Inc.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

1.  Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.

2.  Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in the
    documentation and/or other materials provided with the
    distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE
SOFTWARE, INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

THIS SOFTWARE IS NOT OFFICIALLY SUPPORTED.

Please send any feedback to consulting@perforce.com.

'''

import sys
import os
import re
import platform
from optparse import OptionParser
from subprocess import *
import shutil
import stat
import tarfile
import zipfile

mode=""
verbosity="3"
p4cfg_file="test"
bbicfg_file=""
removejunk=False
detectcase=0
if platform.system() == "Windows": 
    p4="p4.exe"
else:
    p4="p4"
valid_commands = ("BRANCH", "UPDATE", "SAFEUPDATE", "COPY_MERGE", "RECORD_AS_MERGED", "RENAME")
valid_options = ("EMPTYDIRS", "SYMLINKS", "BRANCHSPEC", "NODELETE", "ALLOWWILDCARDS", "RMTOP", "FULLCOPY")

def parseoptions():
    global mode
    global verbosity
    global p4cfg_file
    global bbicfg_file
    global removejunk
    global detectcase
    
    usage = "usage: %prog [options] P4CONFIG_FILE BBI_CONFIG_FILE"
    parser = OptionParser(usage)
    parser.add_option("-m", "--mode", dest="test",
                      help="Test mode. Shows what would be done, but takes no action.")
    parser.add_option("-r", "--remove", dest="rjd",
                      help="Remove junk dirs. Removes known old SCM files and directories.")                      
    parser.add_option("-c", "--case", dest="detectcase",
                      help="When set to 1, detects and reports potential case sensitivity issues.  At level 2, fails when these problems are detected.")                      
    parser.add_option("-v", "--verbosity", dest="level",
                      help="Specify the verbosity level, from 1-3. 1 is quiet mode;"
                      " only error and test output is displayed. 2 is normal verbosity; some messages"
                      " displayed. 3 is debug-level verbosity. The default for this program is 3.")
    
    (options, args) = parser.parse_args()
    
    if len(args) < 2:
        parser.error("Run p4bbi.py -h for help.")
    if not os.path.isfile(args[0]):
        log("ERROR", "P4CONFIG file: '{0}' not found.".format(args[0]))
    else:
        p4cfg_file = args[0]
    if not os.path.isfile(args[1]):
        log("ERROR", "p4bbi config file: '{0}' not found.".format(args[1]))
    else:
        bbicfg_file=args[1]
    if options.test:
        mode = "test"
        print("Running in test mode. No updates will be made to the server.")
    if options.rjd:
        removejunk = True
        log("INFO", "Junk directories will be removed from new Perforce server.")
    if options.detectcase:
        try:
            if 0 < int(options.detectcase) < 3:
                detectcase = int(options.detectcase)
                log("INFO", "Case sensitivity detection set to {0}".format(detectcase))
            else:
                sys.exit(1)
        except:
            log("ERROR", "Case sensitivity mode must be an integer value of 1 or 2.")
    if options.level:
        try:
            if 0 < int(options.level) < 4:
                verbosity = options.level
                print("Verbosity level set to: {0}.".format(options.level))
            else:
                sys.exit(1)
        except:
            log("ERROR", "Verbosity level must be an integer value of 1, 2 or 3.")
            
def loadp4config():
    global P4USER
    global P4PORT
    global P4CLIENT
    global P4CHARSET
    global P4COMMANDCHARSET
    
    P4CHARSET="none"
    P4COMMANDCHARSET="none"
    
    cfgfile = open(p4cfg_file, "r")
    
    for line in cfgfile.readlines():
        if re.search(r"^P4USER=", line):
            P4USER = re.match(r"^P4USER=(.*)", line).group(1)
            continue
        if re.search(r"^P4PORT=", line):
            P4PORT = re.match(r"^P4PORT=(.*)", line).group(1)
            continue
        if re.search(r"^P4CLIENT=", line):
            P4CLIENT = re.match(r"^P4CLIENT=(.*)", line).group(1)
            continue
        if re.search(r"^P4CHARSET=", line):
            P4CHARSET = re.match(r"^P4CHARSET=(.*)", line).group(1)
            continue
        if re.search(r"^P4COMMANDCHARSET=", line):
            P4COMMANDCHARSET = re.match(r"^P4COMMANDCHARSET=(.*)", line).group(1)
            continue
    
    cfgfile.close()

def loadbbiconfig():
    baselines = {}
    commands = []
    cfgfile = open(bbicfg_file, "r")
    referenced_baseline_found = False

    log("DEBUG", "Loading info from {0}.".format(bbicfg_file))
    
    for line in cfgfile.readlines():
        if (re.search("^#", line) or re.search("^\s*$", line)):
            continue
        if re.search("^BASELINE", line):
            (name,path) = re.match("^BASELINE\|(.*)\|(.*)", line).groups()
            if not name:
                cfgfile.close()
                log("ERROR", "Error in Baseline definition for {0}.".format(line))           
            elif not (os.path.isdir(path) or os.path.isfile(path) or path.startswith("CMD:")):
                cfgfile.close()
                log("ERROR", "Baseline path {0} not found.".format(path))
            else:
                log("DEBUG", "Verified path: {0}".format(path))
                log("DEBUG", "Baseline {0} loaded.".format(name))
                baselines[name] = path
        else:
            try:
                cmdline_match = re.match("(.*)\|(.*)\|(.*)\|(.*)\|(.*)", line)
                if cmdline_match == None:
                    cmdline_match = re.match("(.*)\|(.*)\|(.*)\|(.*)", line)
                cmdline_items = cmdline_match.groups()
                if len(cmdline_items) == 5:
                    options = cmdline_items[4]
                    options_list = options.split(",")
                    for o in options_list:
                        parts = o.split("=")
                        if o not in valid_options and parts[0] not in valid_options:
                            log("ERROR", "Invalid option {0} specified in {1}".format(o, line))
                        if o == 'SYMLINKS' and platform.system() == "Windows":
                            log("ERROR", "Invalid option {0} specified: SYMLINKS not applicable on Windows".format(o))
                        if parts[0] == 'BRANCHSPEC' and len(parts) != 2:
                            log("ERROR", "Invalid option {0} specified: BRANCHSPEC option must include a branch spec name".format(o))
                if cmdline_items[0] in valid_commands:
                    commands.append(cmdline_items)
                    log("DEBUG", "Command line: {0} loaded.".format(cmdline_items))
                else:
                    log("ERROR", "Invalid command in line:\n {0}".format(line))
            except:
                log("ERROR", "Invalid line in config file:\n {0}".format(line))
            if cmdline_items[0] == "UPDATE":
                referenced_baseline_found = False                
                for key in iter(baselines):
                    if cmdline_items[1] == key:
                        referenced_baseline_found = True
                if not referenced_baseline_found:
                    cfgfile.close()
                    log("ERROR", "Baseline {0} referenced, but not defined.".format("referenced_baseline"))
    cfgfile.close()

    if baselines == {}:
        log("ERROR", "No baselines found in config file.")
   
    return baselines, commands
    
def log(msglevel="DEBUG", message=""):
    if msglevel == "TEST":
        print("Running in test mode. Command run would have been:\n", message)
    elif msglevel == "ERROR":
        print(message)
        sys.exit(1)
    elif (verbosity == "3"):
        print(message)
    elif (verbosity == "2" and msglevel == "INFO"):
        print(message)

# run OS cmd
def runcmd(cmd):
    try:
        log("DEBUG", cmd);
        pipe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, universal_newlines=True)
        stdout, stderr = pipe.communicate()
        log("DEBUG", stdout)
        if pipe.returncode != 0:
            log("INFO", "{0} generated the following output: {1}".format(cmd, stdout))
            log("ERROR", "{0} generated the following error: {1}".format(cmd, stderr))
        else:
            return stdout
    except OSError as err:
        log("ERROR", "Execution failed: {0}".format(err))

def runp4cmd(cmd, instr=None):
    try:
        pipe = Popen(p4 + cmd, shell=True, stdin=PIPE, stdout=PIPE, universal_newlines=True)
        stdout, stderr = pipe.communicate(instr)
        #log("INFO", stdout)
        if pipe.returncode != 0:
            log("ERROR", "{0}{1} generated the following error: {2}".format(p4, cmd, stderr))
        else:
            return stdout
    except OSError as err:
        log("ERROR", "Execution failed: {0}".format(err))

def p4submit(desc, target):
    opened = runp4cmd("opened -c default")
    if re.search(r"not opened on this client\.", opened) or len(opened.strip()) < 1:
        log("DEBUG", "No changes to submit")
        # store last changelist and description in a counter
        cnt = runp4cmd("counter -i bbi_cnt").strip()
        cname = "bbi_{0}".format(cnt)
        lastchange = runp4cmd("changes -m1 {0}".format(target))
        change = re.match(r"Change (\d+) on \S+ by", lastchange).group(1)
        cvalue = "{0}={1}".format(change, desc)
        runp4cmd("counter -f {0} \"{1}\"".format(cname, cvalue))
    else:
        runp4cmd('submit -d "{0}"'.format(desc))

def getclientpath(p4dir,part=2):
    result = runp4cmd("where {0}".format(p4dir))
    mappings = result.split()
    if platform.system() == "Windows":
        localdir = mappings[part].replace("\\...", "")
    else:
        localdir = mappings[part].replace("/...", "")    
    return localdir

def processcommands( baselines={}, commands=[]):
    """
    Each command contains four components as follows:
    UPDATE|BaselineName|P4Dir|Description
    SAFEUPDATE|BaselineName|P4Dir|Description
    BRANCH|SourcePath|TargetPath|Description
    COPY_MERGE|SourcePath|TargetPath|Description
    RECORD_AS_MERGED|SourcePath|TargetPath|Description
    RENAME|SourcePath|TargetPath|Description
    """
    for cmd in commands:
        desc = cmd[3].replace("$N", "\n")
        options = None
        if len(cmd) == 5:
            options = cmd[4]
        if cmd[0] == "UPDATE":
            if mode == "test":
                log("TEST", "p4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, False, options))
                p4localdir = getclientpath(cmd[2])
                log("DEBUG", "p4 local dir is: {0}".format(p4localdir))
                log("DEBUG", "Baseline dir is: {0}".format(baselines[cmd[1]]))
            else:
                log("DEBUG", "Running: p4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, False, options))
                p4update(cmd[0], baselines[cmd[1]], cmd[2], desc,False, options)
        else:
            if cmd[0] == "SAFEUPDATE":
                if mode == "test":
                    log("TEST", "p4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, True, options))
                    p4localdir = getclientpath(cmd[2])
                    log("DEBUG", "p4 local dir is: {0}".format(p4localdir))
                    log("DEBUG", "Baseline dir is: {0}".format(baselines[cmd[1]]))
                else:
                    log("DEBUG", "Running: p4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, True, options))
                    p4update(cmd[0], baselines[cmd[1]], cmd[2], desc,True, options)
            else:
                if mode == "test":
                    log("TEST", "p4branch({0}, {1}, {2}, {3}, {4})".format(cmd[0], cmd[1], cmd[2], desc, options))
                else:
                    log("DEBUG", "Running: p4branch({0}, {1}, {2}, {3}, {4})".format(cmd[0], cmd[1], cmd[2], desc, options))
                    p4branch(cmd[0], cmd[1], cmd[2], desc, options)

def p4update(cmd, baseline, p4dir, desc,safe, options = None):
    options_list = []
    if options != None:
        options_list = options.split(",")

    p4localdir = getclientpath(p4dir)
    if os.path.isdir(p4localdir):
        try:
            for root, dirs, names in os.walk(p4localdir):
                for name in names:
                    os.chmod(os.path.join(root,name), stat.S_IWRITE)
            shutil.rmtree(p4localdir)
        except OSError as e:
            log("ERROR", "Could not remove {0}.\nThe error was:\n{1}".format(p4localdir, e))

    if baseline.startswith("CMD:"):
        savedPath = os.getcwd()
        os.makedirs(p4localdir)
        os.chdir(p4localdir)
        runcmd(baseline[4:])
        os.chdir(savedPath)
    elif re.search(r"\.tar\.gz", baseline.lower()) or re.search(r"\.tar", baseline.lower()):
        try:
            tar = tarfile.open(baseline)
            if detectcase > 0:
                files_all_lower = []
                for member in tar.getnames():
                    fname_lower = member.lower()
                    if fname_lower in files_all_lower:
                        if detectcase == 1:
                            log("INFO", "Potential case sensitivity conflict: {0}".format(member))
                        else:
                            log("ERROR", "Potential case sensitivity conflict: {0}".format(member))
                    else:
                        files_all_lower.append(fname_lower)
            tar.extractall(path=p4localdir)
            tar.close()
        except:
            log("ERROR", "Invalid or unsupported tar file format for baseline: {0}\n\t{1}\n\t{2}".format(baseline, sys.exc_info()[0], sys.exc_info()[1]))
    elif re.search("\.zip", baseline.lower()):
        try:
            zip = zipfile.ZipFile(baseline)
            if detectcase > 0:
                files_all_lower = []
                for member in zip.namelist():
                    fname_lower = member.lower()
                    if fname_lower in files_all_lower:
                        if detectcase == 1:
                            log("INFO", "Potential case sensitivity conflict: {0}".format(member))
                        else:
                            log("ERROR", "Potential case sensitivity conflict: {0}".format(member))
                    else:
                        files_all_lower.append(fname_lower)
            os.makedirs(p4localdir)
            for member in zip.namelist():
                if member.endswith("/"):
                    os.makedirs(os.path.join(p4localdir, member))
                else:
                    zip.extract(member, p4localdir)
            zip.close()
        except:
            log("ERROR", "Invalid or unsupported zip file format for baseline: {0}\n\t{1}\n\t{2}".format(baseline, sys.exc_info()[0], sys.exc_info()[1]))
    else:
        try:
            preserve_links = False
            if 'SYMLINKS' in options_list:
                preserve_links = True
            shutil.copytree(baseline, p4localdir, symlinks=preserve_links)
        except OSError as err:
            log("ERROR", "Error copying baseline to local directory. {0}".format(err))

    # optionally populate empty directories
    if 'EMPTYDIRS' in options_list:
        log("DEBUG", "Populating empty directories")
        for root, dirs, names in os.walk(p4localdir):
            if len(dirs)==0 and len(names)==0:
                placeholder = open(os.path.join(root, "placeholder.txt"), "w")
                placeholder.write("Placeholder\n")
                placeholder.close()

    # optionally remove top level dir
    if 'RMTOP' in options_list:
        log("DEBUG", "Removing top level baseline directory")
        entries = os.listdir(p4localdir)
        if len(entries) > 1:
            log("ERROR", "RMTOP option can only be used if the top level directory contains one folder.")
        ignore_dir = os.path.join(p4localdir, entries[0])
        p4_temp_dir = "{0}.tmp".format(p4localdir)
        os.rename(ignore_dir, p4_temp_dir)
        os.rmdir(p4localdir)
        os.rename(p4_temp_dir, p4localdir)

    files = open("files.txt", "w")
    files_all_lower = []
    for root, dirs, names in os.walk(p4localdir):
        if 'SYMLINKS' in options_list:
            for dirname in dirs:
                adir = os.path.join(root, dirname)
                if os.path.islink(adir):
                    files.write(adir + "\n")
                    if detectcase > 0:
                        fname_lower = adir.lower()
                        if fname_lower in files_all_lower:
                            if detectcase == 1:
                                log("INFO", "Potential case sensitivity conflict: {0}".format(adir))
                            else:
                                log("ERROR", "Potential case sensitivity conflict: {0}".format(adir))
                        else:
                            files_all_lower.append(fname_lower)
        for name in names:
            if safe:                                      
                # check to see if fstat -F "headAction=delete" //<file> returns anything
                fileToChk = getclientpath(os.path.join(root, name),0)
                deleteChk = runp4cmd("fstat -F \"headAction=delete\" {0}".format(fileToChk))
                #log("INFO","deleteChk({0}) = {1}".format(fileToChk,deleteChk))
                if re.search("headAction delete",deleteChk):
                    log("INFO", "File deleted at head revision in target, not included in update: {0}".format(os.path.join(root, name)))
                else:
                    files.write(os.path.join(root, name) + "\n")
            else:
                files.write(os.path.join(root, name) + "\n")
            if detectcase > 0:
                fname_lower = os.path.join(root,name).lower()
                if fname_lower in files_all_lower:
                    if detectcase == 1:
                        log("INFO", "Potential case sensitivity conflict: {0}".format(os.path.join(root,name)))
                    else:
                        log("ERROR", "Potential case sensitivity conflict: {0}".format(os.path.join(root,name)))
                else:
                    files_all_lower.append(fname_lower)
    files.close()
   
    runp4cmd("sync -k {0}".format(p4dir))
    if 'ALLOWWILDCARDS' in options_list:
        runp4cmd("-x files.txt add -f")
    else:
        runp4cmd("-x files.txt add")
    os.remove("files.txt")
    runp4cmd("-d {0} diff -se ... | {1} -x - edit".format(p4localdir, p4))
    if 'NODELETE' not in options_list:
        runp4cmd("-d {0} diff -sd ... | {1} -x - delete".format(p4localdir, p4))
    p4submit(desc, p4dir)
    
def p4branch(cmd, srcdir, targdir, desc, options=None):
    options_list = []
    branchspec = None
    if options != None:
        options_list = options.split(",")
        for o in options_list:
            m = re.match(r"BRANCHSPEC=(.+)", o)
            if m != None:
                branchspec = m.group(1)

    if cmd == "BRANCH":
        result = runp4cmd("files {0}".format(targdir))
        if not re.search("- no such file(s).", result):
            runp4cmd("integ -v {0} {1}".format(srcdir, targdir))
        else:
            log("ERROR", "{0} target specified in BRANCH command already exists.".format(targdir))
        if branchspec != None:
            log("DEBUG", "Creating branch spec {0}".format(branchspec))
            bform = runp4cmd("branch -o {0}".format(branchspec))
            bform = re.sub(r"\n\s+Created by \w+\.", "\n\t{0}".format(desc), bform)
            bindex = bform.find("\nView:")
            bform = bform[:bindex]
            bform = "{0}\nView:\n\t\"{1}\" \"{2}\"\n".format(bform,srcdir,targdir)
            runp4cmd("branch -i", bform)
    elif cmd == "COPY_MERGE":
        runp4cmd("integ -d -i -t {0} {1}".format(srcdir, targdir))
        runp4cmd("resolve -at")
    elif cmd == "RECORD_AS_MERGED":
        runp4cmd("integ -i -t {0} {1}".format(srcdir, targdir))
        runp4cmd("resolve -ay")
    elif cmd == "RENAME":
        result = runp4cmd("files {0}".format(targdir))
        if not re.search("- no such file(s).", result):
            runp4cmd("sync -f {0}".format(srcdir))
            runp4cmd("edit {0}".format(srcdir))
            runp4cmd("move {0} {1}".format(srcdir, targdir))
        else:
            log("ERROR", "{0} target specified in RENAME command already exists.".format(targdir))
    else:
        log("ERROR", "Invalid command: {0}".format(cmd))

    p4submit(desc,targdir)
    if cmd == "COPY_MERGE" and 'FULLCOPY' in options_list:
        '''
        Need to find content/type diffs or files only on src or target.
        '''
        log("DEBUG", "Starting diff driven merge for {0}".format(targdir))
        diff2out = runp4cmd("diff2 -q -dw -t {0} {1}".format(srcdir, targdir))
        if re.search(r"no differing files\.$", diff2out, re.M) or len(diff2out.strip()) < 1:
            log("DEBUG", "Diff driven merge found nothing to do for {0}".format(targdir))
        else:
            for difffile in diff2out.split("\n"):
                diffmatch_diffs = re.match(r"^==== ([^#]+)#\d+ \([^)]+\) - ([^#]+)#\d+", difffile)
                diffmatch_right = re.match(r"^==== <none> - ([^#]+)#\d+", difffile)
                diffmatch_left = re.match(r"^==== ([^#]+)#\d+ - <none>", difffile)
                if diffmatch_diffs != None:
                    (sfile, tfile) = diffmatch_diffs.groups()
                    runp4cmd("integ -f -d -i -t {0} {1}".format(sfile, tfile))
                elif diffmatch_right != None:
                    tfile = diffmatch_right.group(1)
                    runp4cmd("delete {0}".format(tfile))
                elif diffmatch_left != None:
                    sfile = diffmatch_left.group(1)
                    tfile = targdir[:len(targdir)-4] + sfile[len(srcdir)-4:]
                    runp4cmd("integ -f -d -i -t {0} {1}".format(sfile, tfile))
            runp4cmd("resolve -at")
            p4submit("DIFF_DRIVEN_MERGE {0}".format(desc),targdir)

def removejunkdirs():
    runp4cmd("obliterate -y //....hgignore") # Removes Mercurial junk
    runp4cmd("obliterate -y //....hgtags") # Removes Mercurial junk
    runp4cmd("obliterate -y //.../.hg/...") # Removes Mercurial junk
    runp4cmd("obliterate -y //.../.git/...") # Removes Git junk
    runp4cmd("obliterate -y //.../CVS/...") # Removes CVS junk
    runp4cmd("obliterate -y //....cvsignore") # Removes CVS junk
    runp4cmd("obliterate -y //.../.svn/...") # Removes SVN junk
        
if __name__ == "__main__":
    parseoptions()
    loadp4config()

    p4 = "{0} -p {1} -c {2} -u {3} ".format(p4, P4PORT, P4CLIENT, P4USER)
    if P4CHARSET != "none":
        p4 = p4 + "-C {0} ".format(P4CHARSET)
    if P4COMMANDCHARSET != "none":
        p4 = p4 + "-Q {0} ".format(P4COMMANDCHARSET)
    print(p4)        
    baselines, commands = loadbbiconfig()
    processcommands( baselines, commands)
    if removejunk:
        removejunkdirs()
    if mode == "test":
        log("TEST", "Test complete.")
    else:
        log("INFO", "Import complete.")
    sys.exit(0)