p4bbi_validate.py #1

#!/bin/env python3

'''
Perforce Baseline and Branch Import Tool
Configuration Validator

Copyright (c) 2012 Perforce Software, Inc.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

1.  Redistributions of source code must retain the above copyright
    notice, this list of conditions and the following disclaimer.

2.  Redistributions in binary form must reproduce the above copyright
    notice, this list of conditions and the following disclaimer in the
    documentation and/or other materials provided with the
    distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE
SOFTWARE, INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

THIS SOFTWARE IS NOT OFFICIALLY SUPPORTED.

Please send any feedback to consulting@perforce.com.

'''

import sys
import os
import re
import platform
from optparse import OptionParser
from subprocess import *
import shutil
import stat
import tarfile
import zipfile
import hashlib
import tempfile

P4USER="p4admin"
P4PORT="perforce:1666"
P4CLIENT="bbi_import"
mode="1"
verbosity="2"
p4cfg_file="test"
bbicfg_file=""
removejunk=False
if platform.system() == "Windows": 
    p4="p4.exe"
else:
    p4="p4"
valid_commands = ("BRANCH", "UPDATE", "SAFEUPDATE", "COPY_MERGE", "RECORD_AS_MERGED", "RENAME")
valid_options = ("EMPTYDIRS", "SYMLINKS", "BRANCHSPEC", "NODELETE", "ALLOWWILDCARDS", "RMTOP", "FULLCOPY")

def parseoptions():
    global mode
    global verbosity
    global p4cfg_file
    global bbicfg_file
    global removejunk
    
    usage = "usage: %prog [options] P4CONFIG_FILE BBI_CONFIG_FILE"
    parser = OptionParser(usage)
    parser.add_option("-m", "--mode", dest="mode",
                      help="Mode: 1 = check file metadata, 2 = check file contents.")
    parser.add_option("-r", "--remove", dest="rjd",
                      help="Remove junk dirs. Removes known old SCM files and directories.")                      
    parser.add_option("-v", "--verbosity", dest="level",
                      help="Specify the verbosity level, from 1-3. 1 is quiet mode;"
                      " only error and test output is displayed. 2 is normal verbosity; some messages"
                      " displayed. 3 is debug-level verbosity. The default for this program is 3.")
    
    (options, args) = parser.parse_args()
    
    if len(args) < 2:
        parser.error("Run p4bbi_validate.py -h for help.")
    if not os.path.isfile(args[0]):
        log("ERROR", "P4CONFIG file: '{0}' not found.".format(args[0]))
    else:
        p4cfg_file = args[0]
    if not os.path.isfile(args[1]):
        log("ERROR", "p4bbi config file: '{0}' not found.".format(args[1]))
    else:
        bbicfg_file=args[1]
    if options.mode:
        try:
            if 0 < int(options.mode) < 3:
                mode = options.mode
                print("Mode level set to: {0}.".format(options.mode))
            else:
                sys.exit(1)
        except:
            log("ERROR", "Mode level must be an integer value of 1 or 2.")
    if options.rjd:
        removejunk = True
        log("INFO", "Junk directories were removed from new Perforce server.")
    if options.level:
        try:
            if 0 < int(options.level) < 4:
                verbosity = options.level
                print("Verbosity level set to: {0}.".format(options.level))
            else:
                sys.exit(1)
        except:
            log("ERROR", "Verbosity level must be an integer value of 1, 2 or 3.")
            
def loadp4config():
    global P4USER
    global P4PORT
    global P4CLIENT
    
    cfgfile = open(p4cfg_file, "r")
    
    for line in cfgfile.readlines():
        if re.search(r"^P4USER=", line):
            P4USER = re.match(r"^P4USER=(.*)", line).group(1)
            continue
        if re.search(r"^P4PORT=", line):
            P4PORT = re.match(r"^P4PORT=(.*)", line).group(1)
            continue
        if re.search(r"^P4CLIENT=", line):
            P4CLIENT = re.match(r"^P4CLIENT=(.*)", line).group(1)
            continue
    
    cfgfile.close()

def loadbbiconfig():
    baselines = {}
    commands = []
    cfgfile = open(bbicfg_file, "r")
    referenced_baseline_found = False

    log("DEBUG", "Loading info from {0}.".format(bbicfg_file))
    
    for line in cfgfile.readlines():
        if (re.search("^#", line) or re.search("^\s*$", line)):
            continue
        if re.search("^BASELINE", line):
            (name,path) = re.match("^BASELINE\|(.*)\|(.*)", line).groups()
            if not name:
                cfgfile.close()
                log("ERROR", "Error in Baseline definition for {0}.".format(line))           
            elif not (os.path.isdir(path) or os.path.isfile(path) or path.startswith("CMD:")):
                cfgfile.close()
                log("ERROR", "Baseline path {0} not found.".format(path))
            else:
                log("DEBUG", "Verified path: {0}".format(path))
                log("DEBUG", "Baseline {0} loaded.".format(name))
                baselines[name] = path
        else:
            try:
                cmdline_match = re.match("(.*)\|(.*)\|(.*)\|(.*)\|(.*)", line)
                if cmdline_match == None:
                    cmdline_match = re.match("(.*)\|(.*)\|(.*)\|(.*)", line)
                cmdline_items = cmdline_match.groups()
                if len(cmdline_items) == 5:
                    options = cmdline_items[4]
                    options_list = options.split(",")
                    for o in options_list:
                        parts = o.split("=")
                        if o not in valid_options and parts[0] not in valid_options:
                            log("ERROR", "Invalid option {0} specified in {1}".format(o, line))
                        if o == 'SYMLINKS' and platform.system() == "Windows":
                            log("ERROR", "Invalid option {0} specified: SYMLINKS not applicable on Windows".format(o))
                        if parts[0] == 'BRANCHSPEC' and len(parts) != 2:
                            log("ERROR", "Invalid option {0} specified: BRANCHSPEC option must include a branch spec name".format(o))
                if cmdline_items[0] in valid_commands:
                    commands.append(cmdline_items)
                    log("DEBUG", "Command line: {0} loaded.".format(cmdline_items))
                else:
                    log("ERROR", "Invalid command in line:\n {0}".format(line))
            except:
                log("ERROR", "Invalid line in config file:\n {0}".format(line))
            if cmdline_items[0] == "UPDATE":
                referenced_baseline_found = False                
                for key in iter(baselines):
                    if cmdline_items[1] == key:
                        referenced_baseline_found = True
                if not referenced_baseline_found:
                    cfgfile.close()
                    log("ERROR", "Baseline {0} referenced, but not defined.".format(referenced_baseline))
    cfgfile.close()

    if baselines == {}:
        log("ERROR", "No baselines found in config file.")
   
    return baselines, commands
    
def md5_for_file(fname, block_size=2**20):
    m = hashlib.md5()
    f = open(fname, "r")
    while True:
        data = f.read(block_size)
        if not data:
            break
        m.update(data)
    f.close()
    return m.hexdigest()

def log(msglevel="DEBUG", message=""):
    if msglevel == "ERROR":
        print(message)
        sys.exit(1)
    elif (verbosity == "3"):
        print(message)
    elif (verbosity == "2" and msglevel == "INFO"):
        print(message)

# delete tree
def rmtree(treedir):
    try:
        for root, dirs, names in os.walk(treedir):
            for name in names:
                os.chmod(os.path.join(root,name), stat.S_IWRITE)
        shutil.rmtree(treedir)
    except OSError as e:
        log("ERROR", "Could not remove {0}.\nThe error was:\n{1}".format(treedir, e))

# run OS cmd
def runcmd(cmd):
    try:
        log("DEBUG", cmd);
        pipe = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, universal_newlines=True)
        stdout, stderr = pipe.communicate()
        log("DEBUG", stdout)
        if pipe.returncode != 0:
            log("INFO", "{0} generated the following output: {1}".format(cmd, stdout))
            log("ERROR", "{0} generated the following error: {1}".format(cmd, stderr))
        else:
            return stdout
    except OSError as err:
        log("ERROR", "Execution failed: {0}".format(err))

def runp4cmd(cmd):
    try:
        pipe = Popen(p4 + cmd, shell=True, stdin=PIPE, stdout=PIPE, universal_newlines=True)
        stdout, stderr = pipe.communicate()
        log("DEBUG", stdout)
        if pipe.returncode != 0:
            log("ERROR", "{0}{1} generated the following error: {2}".format(p4, cmd, stderr))
        else:
            return stdout
    except OSError as err:
        log("ERROR", "Execution failed: {0}".format(err))

def getclientpath(p4dir):
    result = runp4cmd("where {0}".format(p4dir))
    mappings = result.split()
    if platform.system() == "Windows":
	    localdir = mappings[2].replace("\\...", "")
    else:
        localdir = mappings[2].replace("/...", "")    
    return localdir

def processcommands( baselines={}, commands=[]):
    """
    Each command contains four components as follows:
    UPDATE|BaselineName|P4Dir|Description
    SAFEUPDATE|BaselineName|P4Dir|Description
    BRANCH|SourcePath|TargetPath|Description
    COPY_MERGE|SourcePath|TargetPath|Description
    RECORD_AS_MERGED|SourcePath|TargetPath|Description
    RENAME|SourcePath|TargetPath|Description
    """
    for cmd in commands:
        desc = cmd[3].replace("$N", "\n")
        options = None
        if len(cmd) == 5:
            options = cmd[4]
        if cmd[0] == "UPDATE":
            log("DEBUG", "\np4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, False, options))
            p4update(cmd[0], baselines[cmd[1]], cmd[2], desc, False, options)
        else:
        	if cmd[0] == "SAFEUPDATE":
        		log("DEBUG", "\np4update({0}, {1}, {2}, {3}, {4}, {5})".format(cmd[0], baselines[cmd[1]], cmd[2], desc, True, options))
        		p4update(cmd[0], baselines[cmd[1]], cmd[2], desc, True, options)
        	else:
                	log("DEBUG", "\np4branch({0}, {1}, {2}, {3}, {4})".format(cmd[0], cmd[1], cmd[2], desc, options))
                	p4branch(cmd[0], cmd[1], cmd[2], desc, options)

def get_changelist(p4dir, desc):
    changelists = runp4cmd("changes -l {0}".format(p4dir))
    changes = re.findall(r"Change (\d+) on", changelists)
    matches = re.split(r"Change \d+ on", changelists)
    ii = 0
    change = ''
    for match in matches:
        if len(match.strip()) < 1:
            continue
        lines = match.splitlines()[2:]
        cdesc = "\n".join(lines)
        cdesc = cdesc.strip()
        if cdesc == desc:
            change = changes[ii]
            break
        ii = ii+1

    if len(change) < 1:
        cnt = int(runp4cmd("counter bbi_cnt").strip())
        for ii in range(1, cnt+1):
            cvalue = runp4cmd("counter bbi_{0}".format(ii))
            (achange,adesc) = re.match(r"(\d+)=(.+)", cvalue).groups()
            if adesc == desc:
                change = achange
                break

    return change

def p4update(cmd, baseline, p4dir, desc, safe, options = None):
    options_list = []
    if options != None:
        options_list = options.split(",")
    
    topdir = baseline
    orig_topdir = False
    if baseline.startswith("CMD:"):
        topdir = tempfile.mkdtemp(dir=os.getcwd())
        savedPath = os.getcwd()
        os.chdir(topdir)
        runcmd(baseline[4:])
        os.chdir(savedPath)
    elif re.search(r"\.tar\.gz", baseline.lower()) or re.search(r"\.tar", baseline.lower()):
        try:
            topdir = tempfile.mkdtemp(dir=os.getcwd())
            tar = tarfile.open(baseline)
            tar.extractall(path=topdir)
            tar.close()
        except:
            log("ERROR", "Invalid or unsupported tar file format for baseline: {0}".format(baseline))
    elif re.search("\.zip", baseline.lower()):
        try:
            topdir = tempfile.mkdtemp(dir=os.getcwd())
            zip = zipfile.ZipFile(baseline)
            for member in zip.namelist():
                if member.endswith("/"):
                    os.makedirs(os.path.join(topdir, member))
                else:
                    zip.extract(member, topdir)
            zip.close()
        except:
            log("ERROR", "Invalid or unsupported zip file format for baseline: {0}".format(baseline))        
    else:
        orig_topdir = True

    # optionally remove top level dir
    if 'RMTOP' in options_list:
        if orig_topdir:
            topdir = tempfile.mkdtemp(dir=os.getcwd())
            preserve_links = False
            if 'SYMLINKS' in options_list:
                preserve_links = True
            shutil.copytree(baseline, topdir, symlinks=preserve_links)
        log("DEBUG", "Checking removing top level baseline directory")
        entries = os.listdir(topdir)
        if len(entries) > 1:
            log("ERROR", "RMTOP option can only be used if the top level directory contains one folder.")
        ignore_dir = os.path.join(topdir, entries[0])
        p4_temp_dir = "{0}.tmp".format(topdir)
        os.rename(ignore_dir, p4_temp_dir)
        os.rmdir(topdir)
        os.rename(p4_temp_dir, topdir)

    # get file name (and md5) for each file in baseline
    fmap = {}
    # optionally check for empty directories
    if 'EMPTYDIRS' in options_list:
        log("DEBUG", "Checking for empty directories")
        for root, dirs, names in os.walk(topdir):
            if len(dirs)==0 and len(names)==0:
                fname = os.path.join(root, "placeholder.txt")[len(topdir)+1:]
                fname = fname.replace("\\","/")
                if 'ALLOWWILDCARDS' in options_list:
                    fname = fname.replace("%","%25")
                    fname = fname.replace("*","%2A")
                    fname = fname.replace("#","%23")
                    fname = fname.replace("@","%40")
                elif re.match(r"%|\*|#|@", fname):
                    continue
                if mode == "1":
                    fmap[fname] = 0
                else:
                    placeholder_fh, placeholder_name = tempfile.mkstemp()
                    placeholder = os.fdopen(placeholder_fh, "w")
                    placeholder.write("Placeholder\n")
                    placeholder.close()
                    fmap[fname] = md5_for_file(placeholder_name).lower()
                    os.unlink(placeholder_name)
    preserve_links = True
    if 'SYMLINKS' in options_list:
        preserve_links = False
    for root, dirs, names in os.walk(topdir, followlinks=preserve_links):
        if 'SYMLINKS' in options_list:
            for dirname in dirs:
                adir = os.path.join(root, dirname)
                if os.path.islink(adir):
                    fname = adir[len(topdir)+1:]
                    fname = fname.replace("\\","/")
                    if 'ALLOWWILDCARDS' in options_list:
                        fname = fname.replace("%","%25")
                        fname = fname.replace("*","%2A")
                        fname = fname.replace("#","%23")
                        fname = fname.replace("@","%40")
                    elif re.match(r"%|\*|#|@", fname):
                        continue
                    if mode == "1":
                        fmap[fname] = 0
                    else:
                        fmap[fname] = os.readlink(adir).strip()
        for name in names:
            thefile = os.path.join(root,name)
            fname = thefile[len(topdir)+1:]
            fname = fname.replace("\\","/")
            if 'ALLOWWILDCARDS' in options_list:
                fname = fname.replace("%","%25")
                fname = fname.replace("*","%2A")
                fname = fname.replace("#","%23")
                fname = fname.replace("@","%40")
            elif re.match(r"%|\*|#|@", fname):
                continue
            if mode == "1":
                fmap[fname] = 0
            else:
                if 'SYMLINKS' in options_list and os.path.islink(thefile):
                    fmap[fname] = os.readlink(thefile).strip()
                else:
                    fmap[fname] = md5_for_file(thefile).lower()

    if baseline.startswith("CMD:"):
        rmtree(topdir)
    elif re.search(r"\.tar\.gz", baseline.lower()) or re.search(r"\.tar", baseline.lower()):
        rmtree(topdir)
    elif re.search("\.zip", baseline.lower()):
        rmtree(topdir)
    elif (orig_topdir and 'RMTOP' in options_list):
        rmtree(topdir)

    if removejunk:
        fmap_rjd = {}
        for key in iter(fmap):
            notjunk = True
            if re.search(r"\.hgignore$", key):
                notjunk = False
            if re.search(r"\.hgtags$", key):
                notjunk = False
            if re.search(r"\.cvsignore$", key):
                notjunk = False
            if re.search(r"/\.hg/", key) or key.find(".hg/") == 0:
                notjunk = False
            if re.search(r"/\.git/", key) or key.find(".git/") == 0:
                notjunk = False
            if re.search(r"/\.svn/", key) or key.find(".svn/") == 0:
                notjunk = False
            if re.search(r"/CVS/", key) or key.find("CVS/") == 0:
                notjunk = False
            if notjunk:
                fmap_rjd[key] = fmap[key]
        fmap = fmap_rjd

    # find changelist for this command
    change = get_changelist(p4dir, desc) 
    if len(change) < 1:
        log("ERROR", "Unable to find changelist that produced this command.  Searched for description {0}".format(desc))

    # get file list from Perforce
    pmap = {}
    if mode == "1":
        fstat = runp4cmd("fstat -T \"depotFile,headAction\" {0}@{1}".format(p4dir, change))
        index = fstat.find("... depotFile")
        while index != -1:
            endindex = fstat.find("... depotFile", index+1)
            chunk = ''
            if endindex != -1:
                chunk = fstat[index:endindex].strip()
            else:
                chunk = fstat[index:].strip()
            pname = ''
            paction = ''
            for c in chunk.splitlines():
                m = re.match(r"^\.\.\. depotFile (.+?)$", c)
                if m != None:
                    pname = m.group(1)
                    pname = pname[len(p4dir)-3:]
                m = re.match(r"^\.\.\. headAction (.+?)$", c)
                if m != None:
                    paction = m.group(1)
            if len(paction) > 0 and paction.find("delete") == -1:
                pmap[pname] = 0
            index = endindex
    else:
        fstat = runp4cmd("fstat -Ol -T \"depotFile,digest,headAction,headType\" {0}@{1}".format(p4dir, change))
        index = fstat.find("... depotFile")
        while index != -1:
            endindex = fstat.find("... depotFile", index+1)
            chunk = ''
            if endindex != -1:
                chunk = fstat[index:endindex].strip()
            else:
                chunk = fstat[index:].strip()
            pname = ''
            depotname = ''
            pdigest = ''
            paction = ''
            ptype = ''
            for c in chunk.splitlines():
                m = re.match(r"^\.\.\. depotFile (.+?)$", c)
                if m != None:
                    pname = m.group(1)
                    depotname = pname
                    pname = pname[len(p4dir)-3:]
                m = re.match(r"^\.\.\. digest (.+?)$", c)
                if m != None:
                    pdigest = m.group(1)
                m = re.match(r"^\.\.\. headAction (.+?)$", c)
                if m != None:
                    paction = m.group(1)
                m = re.match(r"^\.\.\. headType (.+?)$", c)
                if m != None:
                    ptype = m.group(1)
            if len(paction) > 0 and paction.find("delete") == -1:
                if 'SYMLINKS' in options_list and ptype == 'symlink':
                    pmap[pname] = runp4cmd("print -q \"{0}\"".format(depotname)).strip()
                else:
                    pmap[pname] = pdigest.lower()
            index = endindex

    # find diffs
    set_fmap = set(iter(fmap))
    set_pmap = set(iter(pmap))
    fmap_only = set_fmap - set_pmap
    for d in fmap_only:
        log("INFO", "\tDIFF Only in baseline {0}: {1} ({2})".format(baseline, d, change))
    pmap_only = set_pmap - set_fmap
    if 'NODELETE' not in options_list:
        for d in pmap_only:
            log("INFO", "\tDIFF Only in {0}: {1} ({2})".format(p4dir, d, change))
    if mode != "1":
        common = set_fmap & set_pmap
        for key in common:
            if fmap[key] != pmap[key]:
                log("INFO", "\tDIFF Digests differ for {0}".format(key))
    if 'NODELETE' in options_list:
        changedesc = runp4cmd("describe -s {0}".format(change))
        if re.search(r"#\d+ delete$", changedesc, re.M):
            log("INFO", "\tDIFF Files were deleted in changelist {0}".format(change))
    
def p4branch(cmd, srcdir, targdir, desc, options=None):
    # find changelist for this command
    change = get_changelist(targdir, desc) 
    if len(change) < 1:
        log("ERROR", "Unable to find changelist that produced this command.  Searched for description {0}".format(desc))

    options_list = []
    branchspec = None
    if options != None:
        options_list = options.split(",")
        for o in options_list:
            m = re.match(r"BRANCHSPEC=(.+)", o)
            if m != None:
                branchspec = m.group(1)

    if cmd == "BRANCH":
        # Note that only metadata validation is done for branches.
        srcfiles = runp4cmd("files {0}@{1}".format(srcdir, change)).strip().split("\n")
        tgtfiles = runp4cmd("files {0}@{1}".format(targdir, change)).strip().split("\n")
        for tgtfile in tgtfiles:
            (depotpath,action,ftype) = re.match(r"^([^#]+)#\d+ - (\S+) change \d+ \((\S+)\)", tgtfile).groups()
            if action != "branch":
                log("INFO", "\tDIFF Invalid branch action for {0}".format(tgtfile))
                continue 
            flog = runp4cmd("filelog -m1 {0}@{1}".format(depotpath, change)).strip().split("\n")
            if len(flog) != 3:
                log("INFO", "\tDIFF Invalid filelog for {0}".format(tgtfile))
                continue 
            (action, srcfile) = re.match(r"^\.\.\. \.\.\. ([^/]+) ([^#]+)#", flog[2]).groups()
            if action != "branch from":
                log("INFO", "\tDIFF Invalid branch action for {0}".format(srcfile))
                continue 
            found = False
            expected_line = "{0}#".format(srcfile)
            for possible in srcfiles:
                if possible.find(expected_line) == 0:
                    srcfiles.remove(possible)
                    found = True
                    break
            if not found:
                log("INFO", "\tDIFF Could not find branch pair for {0}".format(tgtfile))
        for remaining in srcfiles:
            log("INFO", "\tDIFF Could not find branch pair for {0}".format(remaining))
        if branchspec != None:
            bform = runp4cmd("branch -o {0}".format(branchspec))
            if bform.find(desc) == -1:
                log("INFO", "\tDIFF Unable to find branch spec {0} with description {1}".format(branchspec, desc))
    elif cmd == "COPY_MERGE":
        # Note that only metadata validation is done for this operation.
        srcfiles = runp4cmd("files {0}@{1}".format(srcdir, change)).strip().split("\n")
        tgtfiles = runp4cmd("files {0}@{1}".format(targdir, change)).strip().split("\n")
        for tgtfile in tgtfiles:
            (depotpath,action,ftype) = re.match(r"^([^#]+)#\d+ - (\S+) change \d+ \((\S+)\)", tgtfile).groups()
            if action != "integrate":
                # because an ignore merge can still introduce new files, 
                # we only check the ones that were integrated
                continue 
            flog = runp4cmd("filelog -m1 {0}@{1}".format(depotpath, change)).strip().split("\n")
            if len(flog) != 3:
                log("INFO", "\tDIFF Invalid filelog for {0}".format(tgtfile))
                continue 
            (action, srcfile) = re.match(r"^\.\.\. \.\.\. ([^/]+) ([^#]+)#", flog[2]).groups()
            if action != "copy from":
                log("INFO", "\tDIFF Invalid resolve action for {0}".format(srcfile))
                continue 
            found = False
            expected_line = "{0}#".format(srcfile)
            for possible in srcfiles:
                if possible.find(expected_line) == 0:
                    found = True
                    break
            if not found:
                log("INFO", "\tDIFF Could not find integ pair for {0}".format(tgtfile))
        if 'FULLCOPY' in options_list:
            log("DEBUG", "Checking diff driven merge for COPY_MERGE")
            ddmchange = get_changelist(targdir, "DIFF_DRIVEN_MERGE {0}".format(desc)) 
            if len(ddmchange) < 1:
                log("ERROR", "Unable to find changelist for diff driven merge for {0}".format(targdir))
            diff2out = runp4cmd("diff2 -q -dw -t {0}@{1} {2}@{3}".format(srcdir, ddmchange, targdir, ddmchange))
            if re.search(r"no differing files\.$", diff2out, re.M) or len(diff2out.strip()) < 1:
                log("DEBUG", "Diff driven merge: no diffs for {0}".format(targdir))
            else:
                for difffile in diff2out.split("\n"):
                    log("INFO", "\tDIFF {0} ({1})".format(difffile, ddmchange))
    elif cmd == "RECORD_AS_MERGED":
        # Note that only metadata validation is done for this operation.
        srcfiles = runp4cmd("files {0}@{1}".format(srcdir, change)).strip().split("\n")
        tgtfiles = runp4cmd("files {0}@{1}".format(targdir, change)).strip().split("\n")
        for tgtfile in tgtfiles:
            (depotpath,action,ftype) = re.match(r"^([^#]+)#\d+ - (\S+) change \d+ \((\S+)\)", tgtfile).groups()
            if action != "integrate":
                # because an ignore merge can still introduce new files, 
                # we only check the ones that were integrated
                continue 
            flog = runp4cmd("filelog -m1 {0}@{1}".format(depotpath, change)).strip().split("\n")
            if len(flog) != 3:
                log("INFO", "\tDIFF Invalid filelog for {0}".format(tgtfile))
                continue 
            (action, srcfile) = re.match(r"^\.\.\. \.\.\. ([^/]+) ([^#]+)#", flog[2]).groups()
            if action != "ignored":
                log("INFO", "\tDIFF Invalid resolve action for {0}".format(srcfile))
                continue 
            found = False
            expected_line = "{0}#".format(srcfile)
            for possible in srcfiles:
                if possible.find(expected_line) == 0:
                    found = True
                    break
            if not found:
                log("INFO", "\tDIFF Could not find integ pair for {0}".format(tgtfile))
    elif cmd == "RENAME":
        # Note that only metadata validation is done for renames.
        srcfiles = runp4cmd("files {0}@{1}".format(srcdir, change)).strip().split("\n")
        tgtfiles = runp4cmd("files {0}@{1}".format(targdir, change)).strip().split("\n")
        for tgtfile in tgtfiles:
            (depotpath,action,ftype) = re.match(r"^([^#]+)#\d+ - (\S+) change \d+ \((\S+)\)", tgtfile).groups()
            if action != "move/add":
                log("INFO", "\tDIFF Invalid rename action for {0}".format(tgtfile))
                continue 
            flog = runp4cmd("filelog -m1 {0}@{1}".format(depotpath, change)).strip().split("\n")
            if len(flog) != 3:
                log("INFO", "\tDIFF Invalid filelog for {0}".format(tgtfile))
                continue 
            (action, srcfile, srcver) = re.match(r"^\.\.\. \.\.\. ([^/]+) ([^#]+)#(\d+)", flog[2]).groups()
            srcver_i = int(srcver) + 1
            if action != "moved from":
                log("INFO", "\tDIFF Invalid rename action for {0}".format(srcfile))
                continue 
            expected_line = "{0}#{1} - move/delete change {2} ({3})".format(srcfile,srcver_i,change,ftype)
            if expected_line in srcfiles:
                srcfiles.remove(expected_line)
            else:
                log("INFO", "\tDIFF Could not find rename pair for {0}: expected {1}".format(tgtfile,expected_line))
        for remaining in srcfiles:
            log("INFO", "\tDIFF Could not find rename pair for {0}".format(remaining))
    else:
        log("ERROR", "Invalid command: {0}".format(cmd))


if __name__ == "__main__":
    parseoptions()
    loadp4config()
    p4 = "{0} -p {1} -c {2} -u {3} ".format(p4, P4PORT, P4CLIENT, P4USER)
    baselines, commands = loadbbiconfig()
    processcommands( baselines, commands)
    log("INFO", "\n\nValidations complete.")
    sys.exit(0)