#! /usr/bin/env python3.3 """Code for checking a 'git push' before writing anything to Perforce.""" import functools import logging import re import sys import P4 import p4gf_branch from p4gf_case_conflict_checker import CaseConflictChecker import p4gf_config # no p4gf_copy_to_p4 avoid circular import import p4gf_const from p4gf_filemode import FileModeStr import p4gf_g2p_job as G2PJob import p4gf_git from p4gf_lfs_row import LFSRow from p4gf_l10n import _, NTR import p4gf_path_convert from p4gf_profiler import Timer import p4gf_progress_reporter as ProgressReporter import p4gf_protect import p4gf_pygit2 import p4gf_util LOG = logging.getLogger(__name__) # Assume the cached objects will not be mapped into the repository, and # skip permission checks on those paths since they're submitted as the # git-fusion-user anyway. P4GF_DEPOT_OBJECTS_RE = re.compile('//' + p4gf_const.P4GF_DEPOT + '/objects/') P4GF_DEPOT_BRANCHES_RE = re.compile('//' + p4gf_const.P4GF_DEPOT + '/branches/') # timer names CHECK_PROTECTS = NTR('Check Protects') CHECK_OVERLAP = NTR('Check Overlap') class PreflightChecker: """A class that knows how to check a 'git push' for things that should not be copied into Perforce. Things you might not expect from a preflight checker: - user lookup: PreflightChecker has to see if the author has permission to write files, so PreflightChecker needs a way to look up a single Git commit's author. Could move to a separate class, passed in as part of __init__. - branch definition: A side effect of rolling through commits is that we have to fully define depot branches to hold those commits, and branch views to map the Git work tree to those depot branches. Could move elsewhere. Not sure if we want to require this as something done BEFORE preflight, or done DURING preflight. Extracted and decoupled from "classic push" p4gf_copy_to_p4.G2P and reused by "fast push" p4gf_fast_push.FastPush. If you change this for classic push, test your changes against fast push, too. Such is the price of maintaining two parallel code paths. """ def __init__(self, * , ctx , g2p_user , assigner , gsreview_coll = None , already_copied_commit_runner = None , finish_branch_definition = None ): self.ctx = ctx self.g2p_user = g2p_user self.assigner = assigner self.gsreview_coll = gsreview_coll # ------------- # Explicitly listing each callback into G2P so I can # find the API between these two snotballs. # These are all "an instace (of G2P) that implements X: # already_copied_commit(self, commit_sha1, branch_id) self.already_copied_commit_runner = already_copied_commit_runner # finish_branch_definition(self, commit, branch) self._finish_branch_definition = finish_branch_definition # ------------- # End callbacks into G2P # set() of depot names of type 'stream' self.stream_depots = set() # PreReceiveTuple whose commits we're checking. # Used when permitting Swarm review merges, but # prohibiting all other merge commits. # # OK to leave None if you don't care about # Swarm reviews. self._current_prt = None # p4gf_fastexport_marks.Marks instance # of all known git-fast-export marks. self.fast_export_marks = None self._current_branch = None self._curr_fe_commit = None # Current git-fast-export 'commit'. # Behavior control for fast push: what to pass to # Context.switched_to_branch() for set_client. # # When set to False, the Context.p4 client is # untouched, which can save a huge amount of time # in check_commit_for_branch(): 45 seconds drops # to 9 seconds, but works only if you promist to make # no calls to Perforce through that ctx.p4 connection. # self.set_client_on_branch_switch = True # list of fully populated Branch instances that overlap # other fully populated branches. # # See _overlapping_branch_list() for calc and cache, # call _invalidate_branch_cache() if you change any # branch definitions after most recent call to # _overlapping_branch_list(). # self._cached_overlapping_branch_list = None # list[] of FP Branch instances # LFSRow text pointers that are part of this push. self.lfs_row_list = [] # # -- callbacks into G2P -------------------------------------------------- # Optional. NOP if not running with a full G2P. # def _already_copied_commit(self, commit_sha1, branch_id): """Do we already have a Perforce changelist for this commit, this branch_id? """ if not self.already_copied_commit_runner: return False return self.already_copied_commit_runner.already_copied_commit( commit_sha1, branch_id) def finish_branch_definition(self, commit, branch): """Inflate a partially-created Branch with a proper depot branch to hold its files, and a branch view to map that depot branch to the Git work tree. """ assert self._finish_branch_definition self._finish_branch_definition.finish_branch_definition(commit, branch) self._invalidate_branch_cache() # # -- end callbacks into G2P ---------------------------------------------- # def check_prt_and_commits(self, prt, commits, marks): """Run a preflight check on a set of commits from Git. :param prt: pre-receive tuple :param commits: FastExport.commits list :type marks: :class:`p4gf_fastexport_marks.Marks` :param marks: all known fast-export marks Raises a PreflightException if anything is out of sorts. """ self._current_prt = prt self.fast_export_marks = marks self.fast_export_marks.set_head(prt.ref) self.check_p4gf_user_write_permission() self.check_commits(commits) def check_p4gf_user_write_permission(self): """Ensure git-fusion-user has permissions to write to depot.""" gf_client_map = P4.Map() gf_client_map.insert("//...", "//client/...") utp = p4gf_protect.UserToProtect(self.ctx.p4) prot = utp.user_to_protect(p4gf_const.P4GF_USER) gf_write_filter = prot.map_for_perm(p4gf_protect.WRITE) gf_write_filter = P4.Map.join(gf_write_filter, gf_client_map) if not gf_write_filter.includes('//{depot}/...'.format(depot=p4gf_const.P4GF_DEPOT)): raise RuntimeError(_('permission denied')) def _find_locked_by(self): """Return a dict of depot_path => user of any locked files.""" fstat_flags = NTR('otherLock | otherOpen0 & headType=*+l') any_locked_files = {} # depot_path : user for branch_chunk in self.ctx.iter_writable_branch_chunks(): # Skip any newly defined branches: they're new, won't contain any # files yet, and won't get a view definition until later at per- # commit preflight time. bvl = [b for b in branch_chunk if b.view_lines] if not bvl: continue with self.ctx.switched_to_union(bvl): r = self.ctx.p4run('fstat', '-F', fstat_flags, '-m1', '//{}/...'.format(self.ctx.p4.client), log_warnings=logging.DEBUG) # Collect a dictionary of the locked files from the writable union of branch views for lf in r: user = lf['otherOpen'][0] if 'otherOpen' in lf else NTR('') any_locked_files[lf['depotFile']] = user return any_locked_files def check_commits(self, commits): """Ensure the entire sequence of commits will (likely) go through without any errors related to permissions or locks. Raises an exception if anything goes wrong. Arguments: commits -- commits from FastExport class """ LOG.info('Checking Perforce permissions and locks') self.ctx.checkpoint("copy_to_p4._preflight_check") # Stop if files are opened in our repo client # We expect this to be none, since we have the view lock opened = self.ctx.p4.run(['opened', '-m1']) if opened: raise PreflightException(_('There are files opened by Git Fusion for this repo.')) # fetch the repo setting only, without cascading to global config is_read_only = self.ctx.repo_config.getboolean(p4gf_config.SECTION_REPO, p4gf_config.KEY_READ_ONLY, fallback=False) if is_read_only: raise PreflightException(_("Push to repo {repo_name} prohibited.") .format(repo_name=self.ctx.config.repo_name)) # get a list of stream depots for later checks for read-only paths depots = self.ctx.p4.run(['depots']) self.stream_depots = set([d['name'] for d in depots if d['type'] == 'stream']) any_locked_files = self._find_locked_by() LOG.debug("any_locked_files {0}".format(any_locked_files)) case_conflict_checker = None if not self.ctx.server_is_case_sensitive: case_conflict_checker = CaseConflictChecker(self.ctx) case_conflict_checker.read_perforce_paths() ui_name = self._curr_ref_ui_name() if ui_name: progress_msg = _('Checking commits for {ref}...').format(ref=ui_name) else: progress_msg = _('Checking commits...') with ProgressReporter.Determinate(len(commits)): for commit in commits: ProgressReporter.increment(progress_msg) self.g2p_user.get_author_pusher_owner(commit) rev = commit['sha1'] if not self.assigner.is_assigned(commit['sha1']): continue self.check_commit(commit) for branch_id in self.assigner.branch_id_list(rev): self.check_commit_for_branch( commit , branch_id , any_locked_files , case_conflict_checker ) if case_conflict_checker: cc_text = case_conflict_checker.conflict_text() if cc_text: raise PreflightException(cc_text) def check_commit(self, commit): """Prior to copying a commit, perform a set of checks to ensure the commit will (likely) go through successfully. This includes: * verifying permission to commit for author p4 user * screening for merge commits * screening for submodules * checking valid filenames * checking write permissions for each file """ # pylint: disable=too-many-branches if LOG.isEnabledFor(logging.DEBUG): LOG.debug('check_commit() Checking mark={} sha1={} file-ct={} -- {}' .format( commit['mark'] , p4gf_util.abbrev(commit['sha1']) , len(commit['files']) , repr(commit['data'])[:20].splitlines()[0])) if not commit['author_p4user']: raise PreflightException(_("User '{user}' not permitted to commit") .format(user=commit['author']['email'].strip('<>'))) if 'merge' in commit: ref_is_review = (self.gsreview_coll and self.gsreview_coll.ref_in_review_list(self._current_prt.ref)) if not ref_is_review and not self.ctx.merge_commits: raise PreflightException(_('Merge commits are not enabled for this repo.')) if (not ref_is_review and not self.ctx.branch_creation and self.assigner.have_anonymous_branches): msg = _('Git branch creation is prohibited for this repo.') p4_branch_names_non_lw = [b.git_branch_name for b in self.ctx.branch_dict().values() if b.git_branch_name and not b.is_lightweight] if len(p4_branch_names_non_lw) > 1: msg += _('\nThis repo has more than one named branch.' '\nTry altering the push order - ' 'pushing branches with merge ancestors first.') raise PreflightException(msg) if LOG.isEnabledFor(logging.DEBUG): for parent_mark in commit['merge']: parent_sha1 = self.fast_export_marks.get_commit(parent_mark)[:7] LOG.debug("check_commit() merge mark={} sha1={}" .format(parent_mark, parent_sha1)) if not self.ctx.submodules and 'files' in commit: for f in commit['files']: if f.get('mode') == '160000': if 'first_commit' in commit and not self._path_added(f.get('path'), commit): LOG.debug2('check_commit() passed {} in {}'.format( f.get('path'), p4gf_util.abbrev(commit['sha1']))) continue raise PreflightException( _('Git submodules not permitted: path={path} commit={commit_sha1}') .format(path=f.get('path'), commit_sha1=p4gf_util.abbrev(commit['sha1']))) for f in commit['files']: LOG.debug3("check_commit : commit files: " + _log_fe_file(f)) err = check_valid_filename(f['path'], self.ctx) if err: raise PreflightException(err) if self.ctx.is_lfs_enabled: self._check_lfs(commit, f) # Warn user about any jobs that appear to not exist jobs = G2PJob.lookup_jobs(self.ctx, G2PJob.extract_jobs(commit['data'])) if jobs: for job_id in jobs: r = self.ctx.p4run('jobs', '-e', 'job={}'.format(job_id)) if not r: _print_error(_("Job '{job_id}' doesn't exist").format(job_id=job_id)) # Create pending changes for any Git-Swarm reviews def check_commit_for_branch( self , commit , branch_id , any_locked_files , case_conflict_checker ): """ Prior to copying a commit, perform a set of checks for a specific branch to ensure the commit will (likely) go through successfully. """ rev = commit['sha1'] if LOG.isEnabledFor(logging.DEBUG): LOG.debug("check_commit_for_branch() " "Checking branch={} mark={} sha1={} file-ct={} -- {}" .format( branch_id , commit['mark'] , p4gf_util.abbrev(rev) , len(commit['files']) , repr(commit['data'])[:20].splitlines()[0])) if self._already_copied_commit(rev, branch_id): return # following checks assume client has been set for branch self.ensure_branch_preflight(commit, branch_id) with self.ctx.switched_to_branch( self._current_branch , set_client=self.set_client_on_branch_switch ): if case_conflict_checker: case_conflict_checker.read_fast_export_commit( commit, self._current_branch) # Empty commits require root-level .p4gf_placeholder to be mapped # in the current branch view. if not commit['files'] and not self._is_placeholder_mapped(): raise PreflightException( _("Empty commit {sha1} not permitted. Git Fusion branch views" " must include root to permit empty commits.") .format(sha1=p4gf_util.abbrev(rev))) with Timer(CHECK_PROTECTS): self._check_protects(commit['author_p4user'], commit['files']) with Timer(CHECK_OVERLAP): self._check_overlap(commit) # fetch the branch setting only, without cascading to repo/global config if self._current_branch.is_read_only: raise PreflightException(_("Push to branch {branch} prohibited.") .format(branch=self._current_branch.git_branch_name)) self._check_stream_writable(commit) self._check_stream_in_classic(commit) LOG.debug('checking locked files under //{}/...'.format(self.ctx.p4.client)) if any_locked_files: # Convert the git commit paths to depotPaths files_in_commit = [self.ctx.gwt_path(f['path']).to_depot() for f in commit['files']] LOG.debug("files_in_commit {0}".format(files_in_commit)) for f in files_in_commit: if f in any_locked_files: # Collect the names (and clients) of users with locked files. # Report back to the pusher so they can take appropriate action. msg = _('{file} - locked by {user}').format(file=f, user=any_locked_files[f]) LOG.info(msg) raise PreflightException(msg) # +++ Spend time extracting Jobs and P4Changelist owner # here if we actually do need to call # the preflight-commit hook. if self.ctx.preflight_hook.is_callable(): jobs = G2PJob.extract_jobs(commit['data']) jobs2 = G2PJob.lookup_jobs(self.ctx, jobs) self.ctx.preflight_hook( ctx = self.ctx , fe_commit = commit , branch_id = branch_id , jobs = jobs2 ) def ensure_branch_preflight(self, commit, branch_id): """If not already switched to and synced to the correct branch for the given commit, do so. If this is a new lightweight branch, perform whatever creation we can do at preflight time. We don't have commits/marks for any not-yet-submitted parent commits, so the depot_branch_info will often lack a correct parent or fully populated basis. * depot tree, along with a branch-info file * branch mapping, along with entry in p4gf_config2 (if not anonymous) Return requested branch """ log = LOG.getChild('ensure_branch_preflight') branch = self.ctx.branch_dict().get(branch_id) # branch should never be None here. p4gf_branch_id.Assigner() must # create Branch objects for each assignment. if self._current_branch \ and self._current_branch.branch_id == branch_id: log.debug("sha={} want branch_id={} curr branch_id={} NOP" .format( commit['sha1'][:7] , branch_id[:7] , self._current_branch.branch_id[:7])) log.debug("staying on branch {}" .format(self.ctx.branch_dict().get(branch_id))) return branch cbid = self._current_branch.branch_id if self._current_branch else 'None' log.debug("sha={} want branch_id={} curr branch_id={} switch" .format(commit['sha1'][:7], branch_id[:7], cbid[:7])) if not branch.view_lines: self.finish_branch_definition(commit, branch) elif branch.view_p4map: # if this is a stream branch, check for mutation of the stream's # view by comparing with the original view saved in p4gf_config2 if branch.original_view_lines: original_view_lines = '\n'.join(branch.original_view_lines) view_lines = p4gf_path_convert.convert_view_to_no_client_name(branch.view_lines) if not view_lines == original_view_lines: raise PreflightException( _('Unable to push. Stream view changed from:\n' '{old_view}\nto:\n{new_view}') .format(old_view=original_view_lines, new_view=view_lines)) # Find existing depot branch for branch view's LHS. lhs = branch.view_p4map.lhs() branch.depot_branch = self.ctx.depot_branch_info_index() \ .find_depot_path(lhs[0]) log.debug("switching to branch {}".format(branch)) # By now we should have a branch and a branch.view_lines. # First remove current branch's files from workspace # Client spec is set to normdir self._current_branch = branch return branch def _is_placeholder_mapped(self): """Does this branch map our placeholder file? Returns non-False if mapped, None or empty string if not. """ return self.ctx.gwt_path( p4gf_const.P4GF_EMPTY_CHANGELIST_PLACEHOLDER).to_depot() def _check_protects(self, p4user, blobs): """check if author is authorized to submit files.""" pc = ProtectsChecker(self.ctx, p4user, self.ctx.authenticated_p4user, self.ctx.foruser) pc.filter_paths(blobs) if pc.has_error(): raise PreflightException(pc.error_message()) def _check_overlap(self, fe_commit): """If any of the files in this commit intersect any fully populated branch (other than the current branch), then reject this commit. Shared/common/overlapping paths in branch views must be read-only from Git. Otherwise you end up with a Git push of commit on one Git branch inserting changes into other Git branches behind Git's back. To modify shared paths, either do so from Perforce, or create a Git Fusion repo with no more than one branch that maps that shared path. """ # +++ Avoid O(b branches * r rev) checks when # overlap is impossible because current branch # overlaps no other branch. if self._current_branch not in self._overlapping_branch_list(): return for fe_file in fe_commit['files']: gwt_path = fe_file['path'] depot_path = self.ctx.gwt_to_depot_path(gwt_path) for branch in self._overlapping_branch_list(): if branch == self._current_branch: continue if not branch.intersects_depot_path(depot_path): continue LOG.debug("_check_overlap() branch {br1} <> {br2}" " gwt={gwt:<40} {dp}\n{view}" .format( br1 = p4gf_util.abbrev(self._current_branch.branch_id) , br2 = p4gf_util.abbrev(branch.branch_id) , gwt = gwt_path , dp = depot_path , view = "\n".join(branch.view_p4map.as_array()) )) if self._current_branch.is_new_fp_from_push or branch.is_new_fp_from_push: current_branch_name = self._current_branch.git_branch_name if self._current_branch.is_new_fp_from_push: current_branch_name += '(new)' other_branch_name = branch.git_branch_name if branch.is_new_fp_from_push: other_branch_name += '(new)' human_msg = (_( "Perforce: Cannot commit {sha1} '{gwt_path}' to '{depot_path}'.\n" " You are attempting to push and create a new fully populated branch\n" " with paths which overlap another branch. Contact your admin\n" " to configure non-conflicting destination branch paths.\n" " Branches: '{b1}', '{b2}'") .format( sha1 = p4gf_util.abbrev(fe_commit['sha1']) , gwt_path = gwt_path , depot_path = depot_path , b1 = current_branch_name , b2 = other_branch_name )) else: human_msg = (_( "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'." " Paths that overlap multiple Git Fusion branches are read-only." " Branches: '{b1}', '{b2}'") .format( sha1 = p4gf_util.abbrev(fe_commit['sha1']) , gwt_path = gwt_path , depot_path = depot_path , b1 = self._current_branch.branch_id , b2 = branch.branch_id )) raise PreflightException(human_msg) def _check_stream_writable(self, fe_commit): """If this is a stream branch, check that all files in the commit are writable. If any of the files is not writable then reject this commit. """ if not self._current_branch.stream_name: return prefix = self._current_branch.writable_stream_name + '/' for fe_file in fe_commit['files']: gwt_path = fe_file['path'] depot_path = self.ctx.gwt_path(gwt_path).to_depot() if depot_path.startswith(prefix): continue human_msg = (_( "Cannot commit {sha1} '{gwt_path}' to '{depot_path}'." " Paths not in stream '{stream}' are read-only for branch '{b}'.") .format( sha1 = p4gf_util.abbrev(fe_commit['sha1']) , gwt_path = gwt_path , depot_path = depot_path , stream = self._current_branch.writable_stream_name , b = self._current_branch.branch_id )) raise PreflightException(human_msg) def _check_stream_in_classic(self, fe_commit): """If this is a classic branch, check that none of the files in the commit are in stream depots and thus not writable. If any of the files is not writable then reject this commit. """ if self._current_branch.stream_name: return depot_re = re.compile(r'^//([^/]+)/([^/]+)/.*$') for fe_file in fe_commit['files']: gwt_path = fe_file['path'] depot_path = self.ctx.gwt_path(gwt_path).to_depot() m = depot_re.match(depot_path) if m: depot = m.group(1) if depot in self.stream_depots: stream = '//{}/{}'.format(m.group(1), m.group(2)) human_msg = ( _("Cannot commit {sha1} '{gwt_path}' to '{depot_path}'." " Paths in stream '{stream}' are read-only for branch '{b}'.") .format( sha1 = p4gf_util.abbrev(fe_commit['sha1']) , gwt_path = gwt_path , depot_path = depot_path , stream = stream , b = self._current_branch.branch_id )) raise PreflightException(human_msg) def _path_added(self, path, fecommit): """Return True if the named path was introduced in the HEAD commit. :param self: this object :param path: repo path to be evaluated. :param fecommit: commit object from fast-export parser. """ # Because git-fast-export includes the entire tree in its output, # regardless of whether the requested commit is the first in the # branch or not, we need to check the repo itself to be certain if # this path was truly introduced in this commit, or simply existed # in the tree prior to the "first" commit. commit = self.ctx.repo.get(fecommit['sha1']) if commit is None: # empty repository? LOG.debug2("_path_added() commit {} is missing".format(fecommit['sha1'])) return True for parent in commit.parents: if p4gf_git.exists_in_tree(self.ctx.repo, path, parent.tree): LOG.debug2("_path_added() {} exists in parent tree {}".format( path, p4gf_util.abbrev(p4gf_pygit2.object_to_sha1(parent)))) return False return True def _invalidate_branch_cache(self): """We have changed our branch_dict (or more likely finish_branch_definition()ed a branch within that dict) in a way that invalidates any cached calculations that consumed the branch dict. """ self._cached_overlapping_branch_list = None def _overlapping_branch_list(self): """Return a list of fully populated branches that overlap other fully populated branches. Caches the result because we check every file revision path for overlap, and for huge repos with thousands of non-overlapping LW branches, just iterating through the branch list starts to waste measurable CPU time. """ if self._cached_overlapping_branch_list is not None: return self._cached_overlapping_branch_list have_overlap = set() for outer in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()): outer_lhs = P4.Map() outer_lhs.insert(outer.view_p4map.lhs()) for inner in p4gf_branch.iter_fp_non_deleted(self.ctx.branch_dict()): if outer == inner: continue overlap = P4.Map.join(outer_lhs, inner.view_p4map) # Any non-exclusionary lines shared between branches? for line in overlap.as_array(): if line.startswith('-') or line.startswith('"-'): continue # Yep. Non-exclusionary line implies overlap have_overlap.add(outer) have_overlap.add(inner) break self._cached_overlapping_branch_list = have_overlap return self._cached_overlapping_branch_list def _curr_ref_ui_name(self): """Return a string suitable for use in progress messages as a short hint to what we're processing. If we have a PreReceiveTuple with a Git branch/tag name, return that name (minus any refs/heads/ prefix). If not, return None. """ if not (self._current_prt and self._current_prt.ref): return None s = self._current_prt.ref prefixes = ["refs/heads/", "refs/tags/"] for p in prefixes: if s.startswith(p): return s[len(p):] return s def _check_lfs(self, fe_commit, fe_file): """If gfe_file is under Git LFS control, require that its large file content exist somewhere, either in our upload cache (it's new!) or in depot de-dupe storage (already got it). """ # Deleted files carry no LFS pointer. if "sha1" not in fe_file: return # Symlinks and non-files carry no LFS pointer. if fe_file.get("mode") not in [ FileModeStr.PLAIN , FileModeStr.EXECUTABLE ]: return # Files not under Git LFS control should not carry LFS # pointer information. While legal and permissible, # this is usually a mistake (misconfigured Git client) # and something most users want caught before the push # gets into Helix. is_tracked = self.ctx.lfs_tracker.is_tracked_git( commit_sha1 = fe_commit["sha1"] , gwt_path = fe_file["path"]) LOG.debug3("_check_lfs() tracked {lfs} commit {commit_sha1} gwt {gwt}" .format( commit_sha1 = p4gf_util.abbrev(fe_commit["sha1"]) , lfs = 1 if is_tracked else 0 , gwt = fe_file["path"] )) if not is_tracked: lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file) if lfs_row: raise PreflightException( _("Push of Git LFS text pointer not tracked by LFS:" "\ncommit {commit_sha1} path {gwt_path}") .format( commit_sha1 = p4gf_util.abbrev(fe_commit["sha1"]) , gwt_path = fe_file["path"] )) return # Files under Git LFS control should carry LFS pointer # information, but sometimes might not, and that's # okay. lfs_row = LFSRow.from_gfe(self.ctx, fe_commit, fe_file) if not lfs_row: return # But if they DO carry LFS pointer information, that # pointer needs to point to a valid LFS large file # either already in Perforce or recently uploaded. if not lfs_row.large_file_source: LOG.error("LFS text pointer missing content.") LOG.error("LFS commit {}".format(p4gf_util.abbrev(fe_commit["sha1"]))) LOG.error("LFS lfs oid {}".format(lfs_row.large_file_oid)) LOG.error("LFS ptr {blob_sha1} {blob_mode} {gwt}" .format( blob_sha1 = p4gf_util.abbrev(fe_file["sha1"]) , blob_mode = p4gf_util.mode_str(fe_file["mode"]) , gwt = fe_file["path"])) LOG.error("LFS upload {}".format(lfs_row.to_lfsfs().cache_path(self.ctx))) LOG.error("LFS de-dupe {}".format(lfs_row.to_lfsfs().depot_path(self.ctx))) raise PreflightException(_("Push of Git LFS text pointer missing content:" "\ncommit {commit_sha1} path {gwt_path}") .format( commit_sha1 = p4gf_util.abbrev(fe_commit["sha1"]) , gwt_path = fe_file["path"] )) # We have an acceptable LFS text pointer. # Remember it for later. self.lfs_row_list.append(lfs_row) # end class PreflightChecker # ---------------------------------------------------------------------------- class ProtectsChecker: """Handle filtering a list of paths against view and protections.""" def __init__(self, ctx, author, pusher, foruser): """Init P4.Map objects for author, pusher, view and combination.""" self.ctx = ctx self.author = author self.pusher = pusher self.foruser = foruser self.ignore_author_perms = ctx.repo_config.getboolean(p4gf_config.SECTION_GIT_TO_PERFORCE, p4gf_config.KEY_IGNORE_AUTHOR_PERMS) self.view_map = None self.write_protect_author = None self.write_protect_pusher = None self.write_protect_foruser = None self.write_protect_fusion = None self.write_filter = None self.init_view() self.init_write_filter() self.author_denied = [] self.pusher_denied = [] self.foruser_denied = [] self.fusion_denied = [] self.unmapped = [] def init_view(self): """Init view map for client.""" self.view_map = self.ctx.clientmap def init_write_filter(self): """Init write filter.""" self.write_protect_author = self.ctx.user_to_protect(self.author).map_for_perm( p4gf_protect.WRITE) if self.author != self.pusher or self.ignore_author_perms: # If the author and pusher differ, _or_ if we are ignoring # author permissions, then we must also included the pusher # protections, as well as set write_protect_pusher. self.write_protect_pusher = self.ctx.user_to_protect(self.pusher).map_for_perm( p4gf_protect.WRITE) if self.foruser: self.write_protect_foruser = self.ctx.user_to_protect(self.foruser).map_for_perm( p4gf_protect.WRITE) # Ensure that git-fusion-user can write to the depot self.write_protect_fusion = self.ctx.user_to_protect( p4gf_const.P4GF_USER).map_for_perm(p4gf_protect.WRITE) self.write_filter = functools.reduce(P4.Map.join, [ vmap for vmap in [ self.write_protect_author, self.write_protect_pusher, self.write_protect_foruser, self.write_protect_fusion, self.view_map, # view_map has rhs, must come last in join ] if vmap]) def filter_paths(self, blobs): """Run list of paths through filter and set list of paths that don't pass.""" # check against one map for read, one for write # if check fails, figure out if it was the view map or the protects # that caused the problem and report accordingly self.author_denied = [] self.pusher_denied = [] self.foruser_denied = [] self.fusion_denied = [] self.unmapped = [] c2d = P4.Map.RIGHT2LEFT LOG.debug('filter_paths() write_filter: %s', self.write_filter) for blob in blobs: gwt_path = self.ctx.gwt_path(blob['path']) topath_c = gwt_path.to_client() topath_d = gwt_path.to_depot() LOG.debug('filter_paths() topath_d: %s', topath_d) # for all actions, need to check write access for dest path result = " " # zum loggen if topath_d and P4GF_DEPOT_OBJECTS_RE.match(topath_d): LOG.debug('filter_paths() topath_d in //.git-fusion/objects') continue # do not require user write access to //.git-fusion/branches if topath_d and P4GF_DEPOT_BRANCHES_RE.match(topath_d): LOG.debug('filter_paths() topath_d in //.git-fusion/branches') continue if not self.write_filter.includes(topath_c, c2d): if not self.view_map.includes(topath_c, c2d): self.unmapped.append(topath_c) result = NTR('unmapped') elif not (self.ignore_author_perms or self.write_protect_author.includes(topath_d)): self.author_denied.append(topath_c) result = NTR('author denied') elif (self.write_protect_pusher and not self.write_protect_pusher.includes(topath_d)): self.pusher_denied.append(topath_c) result = NTR('pusher denied') elif (self.write_protect_foruser and not self.write_protect_foruser.includes(topath_d)): self.foruser_denied.append(topath_c) result = NTR('foruser denied') elif not self.write_protect_fusion.includes(topath_d): self.fusion_denied.append(topath_c) result = NTR('Git Fusion denied') else: result = "?" LOG.error('filter_paths() {:<13} {}, {}, {}' .format(result, blob['path'], topath_d, topath_c)) elif LOG.isEnabledFor(logging.DEBUG): LOG.debug('filter_paths() topath_c in write_filter: %s', topath_c) def has_error(self): """Return True if any paths not passed by filters.""" return len(self.unmapped) or len(self.author_denied) \ or len(self.pusher_denied) or len(self.foruser_denied) \ or len(self.fusion_denied) def error_message(self): """Return message indicating what's blocking the push.""" if len(self.unmapped): return _('file(s) not in client view') if len(self.author_denied): restricted_user = self.author elif len(self.pusher_denied): restricted_user = self.pusher elif len(self.foruser_denied): restricted_user = self.foruser elif len(self.fusion_denied): restricted_user = p4gf_const.P4GF_USER else: restricted_user = _('') return _("user '{user}' not authorized to submit file(s) in git commit").format( user=restricted_user) # end ProtectsChecker # ---------------------------------------------------------------------------- class PreflightException(Exception): """This exception is raised when a push was rejected during preflight checks. """ pass # -- module-wide ------------------------------------------------------------- def _log_fe_file(fe_file): """Return loggable string for a single fe_commit['files'] element.""" mode = ' ' if 'mode' in fe_file: mode = fe_file['mode'] sha1 = ' ' if 'sha1' in fe_file: sha1 = p4gf_util.abbrev(fe_file['sha1']) return NTR('{mode} {action} {sha1} {path}') \ .format( mode = mode , action = fe_file['action'] , sha1 = sha1 , path = fe_file['path']) def _print_error(msg): """Print the given message to the error stream, as well as to the log.""" sys.stderr.write(msg + '\n') LOG.error(msg) def is_p4d_printable(c): """Check if c will be rejected by P4D as non-printable. P4D rejects "non-printable" characters with ErrorId MsgDm::IdNonPrint = { ErrorOf( ES_DM, 6, E_FAILED, EV_USAGE, 1 ) "Non-printable characters not allowed in '%id%'." } ; Where "character" here means C "char" aka "byte" and non-printable means 0x00-0x1f or 0x7f """ if ord(c) < 0x20: return False if ord(c) == 0x7F: return False return True def check_valid_filename(name, ctx): """Test the given name for illegal characters. Return None if okay, otherwise an error message. Illegal characters and sequences include: [...] """ for idx, c in enumerate(name): if not is_p4d_printable(c): fullname = name[:idx] + "x{ch:02X}".format(ch=ord(c)) + name[idx:] return _("Perforce: Non-printable characters not allowed in Perforce: " "character x{ch:02X} in filepath: {filename}").format( filename=fullname, ch=ord(c)) if '...' in name: return _("Perforce: bad filename (...): '{filename}'").format(filename=name) if 'P4D/NT' in ctx.server_version: if ':' in name: return _("Perforce: unsupported filename on windows: {filename}").format(filename=name) # This should usually be en_US.UTF-8 which also needs to be defined # on the os encoding = sys.getfilesystemencoding() try: name.encode(encoding, "strict") except UnicodeEncodeError: return _("Perforce: Cannot convert filename to '{encoding}': {filename}").format( encoding=encoding, filename=name) return None