#! /usr/bin/env python3.3 """A subset of repo_compare_to_depot.py that compares a single Perforce changelist against its corresponding Git commit. Intended for huge histories that would spew gigabytes of useless noise if run through repo_compare.py or repo_compare_to_depot.py """ import argparse from collections import defaultdict import logging import os import pygit2 import sys import tempfile import P4 import p4gf_branch import p4gf_const import p4gf_context from p4gf_filemode import FileModeStr from p4gf_l10n import NTR, _ import p4gf_log from p4gf_object_type import ObjectType from p4gf_p2g_print_handler import PrintHandler import p4gf_p4filetype from p4gf_progress_reporter import Determinate import p4gf_util import p4gf_version_3 # Some day we'll support Python 3.4 and get its shiny # new Enum support. Until then, here have a fake # replacement. try: from enum import Enum except ImportError: class Enum: """Gee golly I wish we had Python 3.4 Enum.""" def __init__(self): pass LOG = logging.getLogger("p4gf_first_push_stdout") class RepoCompareSingle: """Class that knows how to compare Perforce changelists against their Git commit counterparts.""" def __init__(self, ctx, args): self.ctx = ctx self.max_gwt_char_ct = 10 self.line_format = None self.how_ct = defaultdict(int) # Our Git repo that contains the Git # half of what we compare. try: repo_dir = pygit2.discover_repository(args.git_dir) except KeyError: raise RuntimeError("Not a Git directory: {}".format(args.git_dir)) #pylint: disable=protected-access repo_dir_abspath = os.path.abspath(repo_dir) ctx._repo = pygit2.Repository(repo_dir_abspath) # A temporary repo for calculating blob sha1 # from Perforce file revisions. self.temp_repo_dir = tempfile.TemporaryDirectory( prefix="p4gf_repo_cmp_") self.temp_repo = pygit2.init_repository(self.temp_repo_dir.name) os.chdir(self.temp_repo_dir.name) def compare_change_num(self, change_num): """Compare one Perforce changelist with its corresponding Git commit. """ ot = ObjectType.change_num_to_commit(self.ctx, change_num) if not ot: key_pattern = (p4gf_const.P4GF_P4KEY_INDEX_OT .format( repo_name = self.ctx.config.repo_name , change_num = change_num , branch_id = "*")) raise RuntimeError("No Git commit for @{change_num}." "\nNo 'p4 keys -e {key_pattern}" .format( change_num=change_num , key_pattern = key_pattern )) branch = self.ctx.branch_dict().get(ot.branch_id) if not branch: raise RuntimeError("No branch view defined for {}" .format(ot.branch_id)) self._compare_ot(ot) def _compare_ot(self, ot): """Compare a single changelist against a Git commit, using a specific branch_id. """ gwt_to_p4file = self._p4_files(ot) gwt_to_git = self._git_ls_tree(ot.sha1) gwt_superset = set(gwt_to_p4file.keys()) | set(gwt_to_git.keys()) self.max_gwt_char_ct = max(len(gwt) for gwt in gwt_superset) # Make the progress reporter more useful. with Determinate(len(gwt_to_p4file)): for gwt in sorted(gwt_superset): diff_how_set = self._compare_file( gwt = gwt , p4file = gwt_to_p4file.get(gwt) , git = gwt_to_git.get(gwt) ) self._report(gwt, diff_how_set) for dh in diff_how_set: self.how_ct[dh] += 1 def _compare_file(self, gwt, p4file, git): """Compare a file's 'p4 files' and 'git-ls-tree' results. Return a set of mismatches. """ if not p4file: return set([DiffHow.FILE_MISSING_P4]) if not git: return set([DiffHow.FILE_MISSING_GIT]) result = set() if not _mode_match( git_mode_str = git.mode , p4filetype = p4file['type'] , gwt_path = gwt ): result.add(DiffHow.FILE_MODE) if not self._sha1_match(p4file, git): result.add(DiffHow.FILE_SHA1) return result def _sha1_match(self, p4file, git): """Calculate a blob sha1 for a Perforce file revision and compare to what Git has for the same blob. """ printhandler = PrintHandler(self.ctx) printhandler.repo = self.temp_repo printhandler.change_set = set() depot_file_rev = p4gf_util.to_path_rev( p4file['depotFile'] , p4file['rev']) with p4gf_util.raw_encoding(self.ctx.p4) \ , self.ctx.p4.using_handler(printhandler) \ , self.ctx.p4.at_exception_level(P4.P4.RAISE_ALL): self.ctx.p4run('print', '-k', depot_file_rev) printhandler.flush() # PrintHandler.revs is a RevList, which # lacks easy access to "just give me the one # p4file I know you have." So iterate and break. p4_sha1 = None for p4file in printhandler.revs: p4_sha1 = p4file.sha1 break if p4_sha1 is None: LOG.error("No revision 'p4 print'ed for {}" .format(depot_file_rev)) return False return p4_sha1 == git.sha1 def _git_ls_tree(self, commit_sha1): """Run 'git ls-tree' to get a list of files in Git at a this commit. Returns a list of pygit2.TreeEntry objects. """ gwt_to_git = {} for ls_tree in p4gf_util.git_ls_tree_r( self.ctx.repo , commit_sha1 ): LOG.debug3("Git: {gwt} {ls}" .format(gwt=ls_tree.gwt_path, ls=ls_tree)) # Directories never exist in Perforce. if ls_tree.mode == FileModeStr.DIRECTORY: continue gwt_to_git[ls_tree.gwt_path] = ls_tree return gwt_to_git def _p4_files(self, ot): """Run 'p4 files' to get a list of files in Perforce at this change on this branch view. """ gwt_to_p4file = {} with self.ctx.switched_to_branch(self.ctx.branch_dict()[ot.branch_id]): p4files = self.ctx.p4run( 'files' , '-e' , self.ctx.client_view_path(ot.change_num) ) for p4file in p4files: if not isinstance(p4file, dict): LOG.debug("Skipping non-dict response to 'p4 files':\n{}" .format(p4file)) continue if "depotFile" not in p4file: LOG.debug("Skipping non-depotFile dict response to 'p4 files':\n{}" .format(p4file)) continue gwt_path = self.ctx.depot_to_gwt_path(p4file['depotFile']) gwt_to_p4file[gwt_path] = p4file LOG.debug3("P4: {gwt} {p4}".format(gwt=gwt_path, p4=p4file)) return gwt_to_p4file def _line_format(self): """Return the one-line format string we use.""" if not self.line_format: self.line_format = "{gwt:<" + str(self.max_gwt_char_ct) + "} {how}" return self.line_format def _report(self, gwt, diff_how_set): """Explain how Git and Perforce differ.""" level = logging.ERROR # No difference? Report only if logging at # --verbose/INFO level or noisier. if not diff_how_set: level = logging.INFO diff_how_str = "=" else: diff_how_str = " ".join(diff_how_set)#DiffHow.to_string(diff_how_set) LOG.log(level, self._line_format().format( gwt = gwt , how = diff_how_str )) def report_summary(self): """How many of each?""" labels = [dh for dh in DiffHow.ALL if dh in self.how_ct] values = [self.how_ct[dh] for dh in labels] if not labels: LOG.info("No mismatches.") return lines = p4gf_util.tabular( labels = labels , int_values = values ) LOG.warning("\n".join(lines)) # end class RepoCompareSingle # -- enum DiffHow ------------------------------------------------------------ class DiffHow(Enum): """How Perforce and Git can differ.""" FILE_MISSING_GIT = 'file.missing.git' FILE_MISSING_P4 = 'file.missing.p4' FILE_SHA1 = 'file.sha1' FILE_MODE = 'file.mode' # Canonical order so things stay lined up ALL = [ FILE_MISSING_GIT , FILE_MISSING_P4 , FILE_SHA1 , FILE_MODE ] @staticmethod def to_string(diff_how_set): """Return string suitable for report.""" l = [dh for dh in DiffHow.ALL if dh in diff_how_set] return " ".join(l) # end enum DiffHow # -- module-wide ------------------------------------------------------------- def _mode_match(git_mode_str, p4filetype, gwt_path): """Match up +x and symlink.""" p4_base_mods = p4gf_p4filetype.to_base_mods(p4filetype) p4_is_symlink = p4_base_mods[0] == "symlink" p4_is_executable = "x" in p4_base_mods if git_mode_str == FileModeStr.PLAIN : # "100644" return not p4_is_executable and not p4_is_symlink if git_mode_str == FileModeStr.EXECUTABLE : # "100755" return p4_is_executable and not p4_is_symlink if git_mode_str == FileModeStr.SYMLINK : # "120000" return not p4_is_executable and p4_is_symlink LOG.error("Unexpected Git file mode {} {}".format(git_mode_str, gwt_path)) return False # -- Begin 15.1 copypasta ---------------------------------------------------- # # Copied and pasted from 15.1 code so that this script can stand alone in # older installations. def p4gf_util_create_arg_parser( desc = None , * , epilog = None , usage = None , help_custom = None , help_file = None , add_p4_args = False , add_log_args = False , add_debug_arg = False , add_profiler_arg = False ): """Creates and returns an instance of ArgumentParser configured with the options common to all Git Fusion commands. The caller may further customize the parser prior to calling parse_args(). Supply your own user-visible text: :param desc: Description of the command being invoked, appears at top of argparser-generated help string. Not used if help_custom supplied. :param epilog: More helpful text that appears at bottom of argparser-generated help string. Not used if help_custom supplied. :param usage: Custom "usage" string that appears when incorrect options specified. Also appears as part of argparser-generated help text, unless help_custom specified. :param help_custom: Custom help text. Displayed upon --help/-h. If you're reading this from a text file, use help_file. :param help_file: Name of text file with custom help. Add common parameters: Include common command-line arguments with common descriptions. :param add_p4_args: --p4port, -p P4PORT of server. --p4user, -u P4USER of user. See also apply_p4_args(args, p4). :param add_log_args: --verbose, -v Write additional diagnostics to standard output. --quiet, -q Write nothing but errors to standard output. See also apply_log_args(args, [logger]). :param add_debug_arg: --debug Write debug diagnostics to standard output. :param add_profiler_arg: --profiler Run with Python performance profiler. You must add your own calls to p4gf_profiler.start_cprofiler() and stop_cprofiler() if set. Not optional: Always includes, and handles, -h, --help show this help message and exit -V displays version information and exits """ class VersionAction(argparse.Action): """Custom argparse action to display version to stdout. Stdout instead of stderr, which seems to be the default in argparse. """ def __call__(self, parser, namespace, values, option_string=None): print(p4gf_version_3.as_string(include_checksum=True)) sys.exit(0) class HelpAction(argparse.Action): """Dump help and exit.""" def __call__(self, parser, namespace, values, option_string=None): if help_file: help_txt = p4gf_util.read_bin_file(help_file) print(help_txt) else: print(help_custom) sys.exit(0) # argparse wraps the description and epilog text by default, but # could customize using formatter_class parser = argparse.ArgumentParser( description = desc , epilog = epilog , usage = usage , add_help = not (help_custom or help_file) ) parser.add_argument("-V", action=VersionAction, nargs=0, help=_('displays version information and exits')) # We normally get -h and --help for free: prints programmatically # generated help then exit. Bypass and supply our own help dumper if # custom help provided. if help_custom or help_file: parser.add_argument('-h', '--help' , nargs = 0 , action = HelpAction) if add_p4_args: parser.add_argument('--p4port', '-p', metavar='P4PORT' , help=_('P4PORT of server')) parser.add_argument('--p4user', '-u', metavar='P4USER' , help=_('P4USER of user')) if add_log_args: parser.add_argument('--verbose', '-v', action=NTR("store_true") , help=_('Write additional diagnostics to standard output.')) parser.add_argument('--quiet', '-q', action=NTR("store_true") , help=_('Write nothing but errors to standard output.')) if add_debug_arg: parser.add_argument('--debug', nargs='?', default=argparse.SUPPRESS , help=_('Write debug diagnostics to standard output.')) if add_profiler_arg: parser.add_argument('--profiler', action=NTR('store_true') , help=_('Run with Python performance profiler.')) return parser def p4gf_util_apply_log_args( args , logger = None , for_stdout = True ): """Honor any optional --verbose/--quiet command line arguments by setting the logger's level to verbose = INFO = WARNING quiet = ERROR If both set, --quiet beats --verbose. :param logger: Which logger to configure. If none specified, use the root logger. But this is rarely what you want. You want your own specific logger for your command line script, with messages designed for standard out, not just generic debugging dumps. :param for_stdout: Configure logger for stdout? This is common (why pass --verbose if you don't expect that verbosity to dump to console?). """ l = logger if logger else logging.getLogger() if 'quiet' in args and args.quiet: l.setLevel(logging.ERROR) elif 'debug' in args: # None = value set by passing --debug when # defualt=argparse.SUPPRESS # True = value set by action=NTR('store_true'), # not used in create_arg_parser() but some # older/custom parsers still might. if args.debug in [None, True, 1, '1', 'debug', 'DEBUG']: l.setLevel(logging.DEBUG) elif args.debug in [2, '2', 'debug2', 'DEBUG2']: l.setLevel(logging.DEBUG2) elif args.debug in [3, '3', 'debug3', 'DEBUG3']: l.setLevel(logging.DEBUG3) elif 'verbose' in args and args.verbose: l.setLevel(logging.INFO) else: l.setLevel(logging.WARNING) if for_stdout: handler = logging.StreamHandler(sys.stdout) handler.setFormatter(logging.Formatter("%(message)s")) l.addHandler(handler) # -- End 15.1 copypasta ------------------------------------------------------ def parse_argv(): """Process command line args.""" help_custom = \ """ usage: repo_compare_changelist.py [options] --repo-name [--git-dir GIT_DIR] change-num [change-num ...] Compare one or more individual Perforce changelists against their corresponding Git commits. Detects only these mismatches: {} --repo-name name of existing Git Fusion repo change-num changelist number(s) to compare --git-dir GIT_DIR Git repo to compare against Perforce other options: -h, --help show this help message and exit -V displays version information and exits --p4port,-p P4PORT P4PORT of server --p4user/-u P4USER P4USER of user --verbose, -v Write additional diagnostics to standard output. --quiet, -q Write nothing but errors to standard output. --debug [DEBUG] Write debug diagnostics to standard output. """.format("\n ".join(DiffHow.ALL)) parser = p4gf_util_create_arg_parser( help_custom = help_custom , add_p4_args = True , add_log_args = True , add_debug_arg = True ) parser.add_argument( '--repo-name' , required=True , help=_('name of existing Git Fusion repo')) parser.add_argument( '--git-dir' , default='.' , help=_('Git repo to compare against Perforce')) parser.add_argument( 'change_num', metavar="change-num" , nargs='+' , help=_('Changelist number(s) to compare.')) args = parser.parse_args() p4gf_util_apply_log_args(args, logger=LOG) LOG.debug("args={}".format(args)) return args def main(): """Do the thing.""" args = parse_argv() ctx = p4gf_context.create_context(args.repo_name) ctx.config.p4user = args.p4user ctx.config.p4port = args.p4port ctx.connect_cli(log=LOG) p4gf_branch.init_case_handling(ctx.p4) rc = RepoCompareSingle(ctx, args) for change_num in args.change_num: rc.compare_change_num(change_num) rc.report_summary() if rc.how_ct: sys.exit(1) if __name__ == "__main__": p4gf_log.run_with_exception_logger(main, write_to_stderr=True)