CheckCaseTrigger.py #1

# Version 2.5.0
#
# 2.4.0 changes (performance):
#   Several of the internal path-prefix helpers (getDirList, getFileDirList,
#   and the fallback path in searchCache) rebuilt each ancestor-directory
#   prefix from scratch via list-slicing + "/".join() at every path depth,
#   and getFileDirList additionally recomputed an invariant (the file's
#   parent directory) at every depth level despite it not changing. None of
#   this affected correctness, but it adds up on very large changelists --
#   e.g. Unreal Engine 5 "One File Per Actor" workflows routinely produce
#   changelists of 100K-400K+ files, and this trigger runs on every submit.
#   These helpers now build each prefix incrementally instead. filterRenames
#   was also changed from an O(N*M) list-membership scan (and O(N) list
#   .remove() per match) to a single O(N) pass using a set, with an early
#   exit when a changelist contains no deletes at all (the common case for
#   bulk-add changelists). All of the above are pure refactors -- verified
#   against the previous implementation across randomized and edge-case
#   inputs -- with no change in trigger behavior. Additionally,
#   getFileDirList() now returns a set instead of a dict, since its values
#   were write-only (never read by any caller); this required a matching
#   update to TestCheckCaseTrigger.py's testBasic(), which asserted directly
#   on the old dict shape.

# tag::includeManual[]
"""
CaseCheckTrigger.py

This trigger ensures users are not adding new files (directly or by branching) which only
differ in case (either filename or a directory element of their path) from existing depot paths.
It is useful for both case-sensitive and case-insensitive servers, although most used for the former.

Example 1: Typical usage from the Helix Core server triggers table:

    Triggers:
        CheckCaseTrigger change-submit //... "/usr/bin/python3 /p4/common/bin/triggers/CheckCaseTrigger.py %changelist% myuser=%user%"

SAMPLE OUTPUT:
    Submit validation failed -- fix problems then use 'p4 submit -c 1234'.
    'CheckCaseTrigger' validation failed:

    Your submission has been rejected because the following files
    are inconsistent in their use of case with respect to existing
    directories:

    Your file:         '//depot/dir/test'
    existing file/dir: '//depot/DIR'

CASE CONFLICTS WITH DELETED FILES:
    By default (as of version 2.3.0), this trigger treats a depot path as
    "existing" for case-comparison purposes even if its head revision action
    is 'delete'. This closes a gap where, e.g., '//depot/foo.txt' is deleted
    in one changelist and '//depot/Foo.txt' is added in a later changelist:
    the case conflict against the (deleted) history of foo.txt is now caught,
    because the earlier submitted revisions of foo.txt still exist in the
    depot and can still cause problems for case-insensitive clients (see
    'p4 help protect' / SDP documentation on case sensitivity for details).

    This is deliberately NOT the same thing as a same-changelist case-only
    rename (delete old-case path + add/branch/move-add new-case path in one
    submit) -- that pattern is still recognized and allowed automatically;
    see filterRenames() below. The stricter check above only applies across
    separate changelists.

    Sites that need to phase in this stricter behavior (e.g. because
    existing depot history already contains such conflicts from before this
    fix was deployed) can temporarily restore the old, more permissive
    behavior with 'includedeletedpaths=no' on the trigger command line.
    This is intended as a rollout aid, not a long-term setting, since it
    re-opens the original gap.

BYPASS LOGIC:
    By default, this trigger can be bypassed by any user by adding the
    token BYPASS_CASE_CHECK to the changelist description.  Specify
    'allowbypass=no' on the command line to disable the ability to
    bypass this trigger

    As an exception, if the user is 'git-fusion-user', the case check
    is always bypassed if 'myuser' is defined.

    NOTE: With the stricter deleted-path checking described above, this
    bypass is the intended, auditable way to perform a deliberate case-only
    rename across two separate changelists (e.g. delete '//depot/foo.txt' in
    one submit, then later add '//depot/Foo.txt'). Because it requires an
    explicit token in the changelist description, it leaves a record of who
    chose to override the check and when, rather than requiring a blanket,
    site-wide relaxation of the rule.

DEPENDENCIES:
    This trigger requires P4Triggers.py and the P4Python API.

SEE ALSO:
    See 'p4 help triggers'.

"""
# end::includeManual[]

from __future__ import print_function

import logging
import os
import platform
import re
import subprocess
import sys
import P4
import P4Triggers

# Method canonical
# IN:  string, utf8 compatible
# OUT: unicode string, all lower case
def canonical(aString):
    return aString.lower()

def getDepot(path):
    return path[2:].split("/")[0]

class CheckCaseTrigger(P4Triggers.P4Trigger):
# tag::includeManual[]
    """CaseCheckTrigger is a subclass of P4Trigger. Use this trigger to ensure
       that your depot does not contain two filenames or directories that only
       differ in case.
       Having files with different case spelling will cause problems in mixed
       environments where both case-insensitive clients like Windows and case-
       sensitive clients like UNIX access the same server.
    """
# end::includeManual[]

    def __init__(self, *args, **kwargs):
        kwargs['charset'] = 'none'
        kwargs['api_level'] = 71

        self.allowBypass=AllowBypass
        self.includeDeletedPaths=IncludeDeletedPaths

        fileFilter = None
        if 'filefilter' in kwargs:
            fileFilter = kwargs['filefilter']
            del kwargs['filefilter']

        P4Triggers.P4Trigger.__init__(self, **kwargs)
        self.parse_args(__doc__, args)

        # Ensure that -ztag global option is used.
        self.p4.tagged = True

        # need to reset the args in case a p4config file overwrote them
        for (k, v) in kwargs.items():
            if k != "log":
                try:
                    setattr(self.p4, k, v)
                except:
                    self.logger.error("error setting p4 property: '%s' to '%s'" % (k, v))

        self.map = None
        if fileFilter:
            try:
                with open(fileFilter) as f:
                    self.map = P4.Map()
                    for line in f:
                        self.map.insert(line.strip())
            except IOError:
                self.logger.error("Could not open filter file %s" % fileFilter)

        self.depotCache = {}
        self.masterCache = {}
        self.maxWildcards = 9
        self.loggingEnabled = self.logger.isEnabledFor(logging.DEBUG)
        self.caseSensitive = platform.system() == "Linux" # Default - will be checked later

    def add_parse_args(self, parser):
        """Specific args for this trigger - also calls super class to add common trigger args"""
        parser.add_argument('change', help="Change to validate - %%change%% argument from triggers entry.")
        parser.add_argument('-m', '--max-errors', default=10, help="Max no of errors before aborting submit. Default 10.")
        super(CheckCaseTrigger, self).add_parse_args(parser)

    def setUp(self):
        info = self.p4.run_info()[0]
        if "unicode" in info and info["unicode"] == "enabled":
            self.p4.charset = "utf8"
        self.p4.exception_level = 1 # ignore WARNINGS like "no such file"
        self.p4.prog = "CheckCaseTrigger"

        if self.allowBypass:
            self.USER_MESSAGE="""

    Your changelist submit attempt has been rejected because
    one or more file paths opened for add vary only by case
    from existing files/directory paths.  Creating file/folders
    that vary only in case from existing paths causes inconsistent
    behavior across platforms with different case handling behaviors
    (e.g. Windows,  Linux/UNIX,  Mac OSX).  Thus,  adding case-only
    variations of existing paths is strongly discouraged.

    If you are certain the files to be added will only be accessed
    from workspaces on case-sensitive platforms (like UNIX/Linux),
    then this trigger can be bypassed by adding the token
    BYPASS_CASE_CHECK to the changelist description and attempting
    the submit again.

    Alternately,  you can revert any files opened for add in your
    changelists that vary only in case from existing files,  or
    move them to new names that don't conflict with existing files.

    Offending files:
    """
        else:
            self.USER_MESSAGE="""

    Your changelist submit attempt has been rejected because
    one or more file paths opened for add vary only by case
    from existing files/directory paths.  Creating file/folders
    that vary only in case from existing paths causes inconsistent
    behavior across platforms with different case handling behaviors
    (e.g. Windows,  Linux/UNIX,  Mac OSX). Thus, adding case-only
    variations of existing paths is disallowed.

    To move forward, you can revert any files opened for add in your
    changelists that vary only in case from existing files,  or
    move them to new names that don't conflict with existing files.

    Offending files:
    """

        self.BADFILE_FORMAT="""
      Your file:         '%s'
      existing file/dir: '%s'
      """

    def validate(self):
        """Here the fun begins. This method overrides P4Trigger.validate()"""
        badlist = {}
        info = self.p4.run_info()
        if "caseHandling" in info[0]:
            self.caseSensitive = "insensitive" != info[0]["caseHandling"]
            self.logger.debug("validate: p4d caseSensitive %s", self.caseSensitive)

        files = self.change.files
        if self.loggingEnabled:
            self.logger.debug("validate: Files to submit: %s", files)

        self.filterRenames(files)

        # Determine valid file list
        validFiles = []

        for file in files:
            action = file.revisions[0].action
            if self.map and self.map.includes(file.depotFile):
                continue
            if not action in ("add", "branch", "move/add"):
                continue

            path = file.depotFile[2:]
            if self.loggingEnabled:
                self.logger.debug("validate: path = %s", path)

            validFiles.append(file.depotFile)
            if self.loggingEnabled:
                self.logger.debug("validate: file.depotFile = %s", file.depotFile)
        if self.loggingEnabled:
            self.logger.debug("validate: validFiles = %s", validFiles)

        # Build cache for each unique depot.
        self.buildCache(validFiles)
        if self.loggingEnabled:
            self.logger.debug("validate: masterCache_1 = %s", self.masterCache)

        # Look for files in cache. This includes looking for directories in file path along the way.
        self.searchCache(validFiles, badlist)
        if self.loggingEnabled:
            self.logger.debug("validate: badlist = %s", badlist)
            self.logger.debug("validate: masterCache_2 = %s", self.masterCache)

        if len(badlist) > 0:
            self.report(badlist)
        return len(badlist) == 0

    # This method returns a list of all dirs between root and lowest level in the filelist
    # Can then run "p4 dirs -i a/* a/b/*" against this list to find any other potential conflicts at each level
    # IN: filelist
    # OUT: dirlist for dirs command
    def getDirList(self, fileList):
        # Files:
        #   //D/a/f.txt
        #   //D/a/b/c/f.txt
        # Output:
        #   //D
        #   //D/a
        #   //D/a/b
        #   //D/a/b/c
        # We don't need to go any deeper than max path of files in list
        #
        # PERF NOTE: builds each ancestor prefix incrementally (appending one
        # path segment at a time) rather than re-slicing parts[:i] and calling
        # "/".join() from scratch at every depth level i. On case-sensitive
        # servers this also avoids computing the identical "/".join(parts[:i])
        # twice per level (once for the key, once for the value) as the prior
        # implementation did. Verified to produce identical output to the
        # previous slicing-based implementation.
        dirList = {}
        for f in fileList:
            cf = canonical(f)
            parts = f[2:].split('/')    # Original case
            cparts = cf[2:].split('/')  # Lower case
            prefix = ""
            cprefix = ""
            for i in range(1, len(cparts)): # Process up to the parent dir of the file
                seg = parts[i - 1]
                cseg = cparts[i - 1]
                prefix = seg if i == 1 else prefix + "/" + seg
                cprefix = cseg if i == 1 else cprefix + "/" + cseg
                p = prefix if self.caseSensitive else cprefix
                if not p in dirList:
                    dirList[p] = prefix
        return dirList

    # This method returns a list of dirs containing files in the filelist (includes intermediat dirs to allow for
    # dir and filename collision
    # Can then run "p4 files -i a/b/c/* a/b/d/*" against this list to find any other potential conflicts at each level
    # IN: filelist
    # OUT: dirlist (as a set) for files command
    #
    # PERF/TYPE NOTE (2.4.0): this used to return a dict whose values were
    # always just the file's own parent directory, recomputed (uselessly)
    # at every depth level. The only caller (buildCache) only ever iterates
    # 'for d in fileDirList', i.e. only the keys were ever used -- the values
    # were write-only and never read anywhere. This now returns a plain set
    # of directory paths instead, which is both cheaper (no value string to
    # build at all) and a more honest representation of what this data
    # actually is: a set of directories to query, not a mapping.
    #
    # NOTE: this is a return-type change from the previous dict-returning
    # implementation. buildCache() only ever does 'for d in fileDirList' and
    # 'if not fileDirList', both of which work identically for a set, so
    # trigger behavior is unaffected. TestCheckCaseTrigger.py's testBasic()
    # asserted directly on this method's dict shape (including values) and
    # has been updated to match; see that file for details.
    def getFileDirList(self, fileList):
        # Files:
        #   //D/a/c.txt
        #   //D/a/b/c/d.txt
        # Output:
        #   //D/a
        #   //D/a/b
        #   //D/a/b/C - sensitive
        #   //D/a/b/c - insensitive
        # We don't need to go any deeper than max path of files in list
        dirSet = set()
        for f in fileList:
            parts = f[2:].split('/')
            if self.caseSensitive:
                prefix = ""
                for i in range(1, len(parts)): # Process up to the parent dir of the file
                    seg = parts[i - 1]
                    prefix = seg if i == 1 else prefix + "/" + seg
                    dirSet.add(prefix)
            else:
                cf = canonical(f)
                cparts = cf[2:].split('/')
                cprefix = ""
                for i in range(1, len(cparts)): # Process up to the parent dir of the file
                    cseg = cparts[i - 1]
                    cprefix = cseg if i == 1 else cprefix + "/" + cseg
                    dirSet.add(cprefix)
        return dirSet

    # Builds a global cache to use for mismatch searches.
    # IN: depots to use
    #     fileList to parse
    # OUT: None
    def buildCache(self, fileList):
        if self.loggingEnabled:
            self.logger.debug("buildCache: fileList = %s", fileList)

        depots = self.p4.run_depots()
        for d in depots:
            dname = d["name"]
            cd = canonical(dname)
            if self.caseSensitive:
                self.depotCache[dname] = dname
            else:
                self.depotCache[cd] = dname

        dirList = self.getDirList(fileList)
        if self.loggingEnabled:
            self.logger.debug("buildCache: dirList = %s", dirList)
        # Note depots will exist in the list but we need to ensure correct case is used
        if not self.caseSensitive:
            for d in self.depotCache:
                if not d in dirList:
                    dirList[d] = d
        if self.caseSensitive:
            dirParams = ["//" + d + "/*" for d in dirList]
        else:
            dirParams = ["//" + dirList[d] + "/*" for d in dirList]
        cdirs = {}
        if dirParams:
            for d in self.p4.run_dirs(*dirParams):
                d = d["dir"]  # result is in tagged mode, single entry "dir"=>directory name
                cd = d.lower()
                self.masterCache[cd] = d
                if self.caseSensitive:
                    if not d in dirList and getDepot(cd) in self.depotCache:
                        cdirs[cd] = d
                else:
                    if cd != d and not d in dirList:
                        cdirs[cd] = d

        # If necessary, repeat the dirs command on case sensitive systems with any extra dirs found from
        # previous call
        if self.caseSensitive and len(cdirs) > 0:
            dirParams = [d + "/*" for d in cdirs.keys()]
            for d in self.p4.run_dirs(*dirParams):
                d = d["dir"]  # result is in tagged mode, single entry "dir"=>directory name
                cd = d.lower()
                self.masterCache[cd] = d

        fileDirList = self.getFileDirList(fileList)
        if not fileDirList:
            return
        fileParams = ["//" + d + "/*" for d in fileDirList]
        for f in self.p4.run_files(*fileParams):
            # NOTE: Prior versions of this trigger unconditionally skipped any
            # file whose head revision action is 'delete' when building this
            # cache. That meant a path could be deleted in one changelist and
            # a case-differing path added in a later changelist without ever
            # being caught, even though the deleted file's earlier revisions
            # still exist in the depot and can still cause problems for
            # case-insensitive clients. As of 2.3.0 we include deleted paths
            # by default so this history is caught too; a same-changelist
            # case-only rename (delete + add/branch/move-add together) is
            # still allowed via filterRenames() below, and a deliberate
            # cross-changelist rename can use BYPASS_CASE_CHECK. Sites that
            # need to temporarily restore the old behavior (e.g. to phase in
            # this stricter check against existing depot history) can pass
            # 'includedeletedpaths=no' on the trigger command line.
            if self.includeDeletedPaths or not "delete" in f["action"]:
                f = f["depotFile"]
                cf = f.lower()
                if not cf in self.masterCache:
                    self.masterCache[cf] = f

    # Method filterRenames:
    # Removes files opened for add, branch, or move/add that are paired in
    # this SAME changelist with a delete of a case-differing path. This
    # represents an intentional, atomic case-only rename/fix (e.g. deleting
    # '//depot/foo.txt' and adding '//depot/Foo.txt' in one submit) and
    # should not be blocked, regardless of which of the three actions was
    # used to add the new path back.
    #
    # This is distinct from -- and does not affect -- the deleted-path
    # history check in buildCache(), which only looks at PREVIOUSLY
    # COMMITTED changelists, since files in the changelist currently being
    # submitted are not yet reflected by 'p4 files' at change-submit time.
    def filterRenames(self, files):
        # PERF NOTE: the previous implementation checked membership against
        # 'deletes' while it was still a list, which is O(len(deletes)) per
        # check, and called files.remove(f) once per match, which is itself
        # O(len(files)) per call. For a large changelist that mixes many
        # deletes with many adds (a common shape for bulk reorganizations --
        # exactly the case this broader add/branch/move-add check targets),
        # that combination can get expensive. 'deletes' is now a set for O(1)
        # membership checks, and 'files' is rebuilt in a single O(N) pass
        # instead of being mutated one .remove() call at a time. There is
        # also an early exit when the changelist contains no deletes at all
        # (the common case for pure-add changelists), which skips the
        # filtering pass entirely.
        deletes = {x.depotFile.lower() for x in files if x.revisions[0].action == 'delete'}
        if not deletes:
            return
        keep = []
        for x in files:
            if x.revisions[0].action in ('add', 'branch', 'move/add') and x.depotFile.lower() in deletes:
                continue  # part of an in-changelist case-only rename; not a conflict
            keep.append(x)
        files[:] = keep

    def report(self, badfiles):
        msg = self.USER_MESSAGE
        for (n, (file, mismatch)) in enumerate(badfiles.items()):
            if n >= self.options.max_errors:
                break
            msg += self.BADFILE_FORMAT % (file, mismatch)

        self.message(msg)

    def run(self):
        """Runs trigger"""
        try:
            self.logger.debug("CheckCaseTrigger firing")
            self.setupP4()
            return self.parseChange(self.options.change)
        except Exception:
            return self.reportException()

    # This method searches a global cache to find case mismatches for changelist files.
    # File subdirectories and file itself are added to cache if no mismatches are found.
    # IN: List of changelist files for which we want verify there are no case mismatches.
    #     Mismatch dictionary that records mismatches.
    # OUT: May modify mismatches parameter
    def searchCache(self, cfiles, mismatches):

        if self.loggingEnabled:
            self.logger.debug("searchCache: changelist files = %s, mismatches = %s", cfiles, mismatches)
            self.logger.debug("searchCache: masterCache = %s", self.masterCache)
        for f in cfiles:
            if self.loggingEnabled:
                self.logger.debug("searchCache: f = %s", f)
            mismatch = ""
            # Check depot
            depot = getDepot(f)
            cdepot = canonical(depot)
            # Search on depots - require depot to be in cache
            if self.caseSensitive:
                if not depot in self.depotCache:
                    mismatch = self.depotCache[cdepot]
                    if self.loggingEnabled:
                        self.logger.debug("depot not found: %s", depot)
                    mismatches[f] = mismatch
                    continue
            else:
                if depot != self.depotCache[cdepot]:
                    mismatch = self.depotCache[cdepot]
                    if self.loggingEnabled:
                        self.logger.debug("mismatch2: sd = %s, f = %s, m = %s", cdepot, depot, mismatch)
                    mismatches[f] = mismatch
                    continue

            # Look for file and continue if it's in the cache. It's already been added.
            cf = canonical(f)
            if cf in self.masterCache:
                if self.loggingEnabled:
                    self.logger.debug("searchCache: found %s: %s", f, self.masterCache[cf])
                if f != self.masterCache[cf]:
                    mismatch = self.masterCache[cf]
                    if self.loggingEnabled:
                        self.logger.debug("mismatch3: cf = %s, f = %s, m = %s", cf, f, mismatch)
                    mismatches[f] = mismatch
                continue

            # Need to check for mismatch of file path components.
            # If none, add components to cache.
            #
            # PERF NOTE: this is the path taken for any file not already in
            # masterCache -- i.e. essentially every file in a changelist
            # dominated by brand-new adds (the common shape for large Unreal
            # Engine "One File Per Actor" changelists). Prefixes are built
            # incrementally (appending one segment at a time) instead of
            # re-slicing parts[:i]/cparts[:i] and calling "/".join() from
            # scratch at every depth level. Verified to produce identical
            # cp/p values to the previous slicing-based implementation.
            parts = f[2:].split('/')
            cparts = cf[2:].split('/')
            cprefix = ""
            prefix = ""
            for i in range(1, len(cparts) + 1):
                cseg = cparts[i - 1]
                seg = parts[i - 1]
                cprefix = cseg if i == 1 else cprefix + "/" + cseg
                prefix = seg if i == 1 else prefix + "/" + seg
                cp = "//" + cprefix
                p = "//" + prefix
                if not cp in self.masterCache: # Save in cache
                    self.masterCache[cp] = p
                    if self.loggingEnabled:
                        self.logger.debug("adding to cache: %s", self.masterCache[cp])
                else:
                    m = self.masterCache[cp]
                    if m != p:
                        mismatch = m
                        if self.loggingEnabled:
                            self.logger.debug("mismatch4: cp = %s, p = %s, mismatch = %s", cp, p, mismatch)
                        break

            if mismatch:
                mismatches[f] = mismatch
            else:
                self.masterCache[cf] = f

if __name__ == "__main__":
    # Generate new args - parsing out port=123 style way of specifying
    # parameters intended for p4 properties
    kwargs = {}
    args = []
    for arg in sys.argv[1:]:
        p = arg.split("=", 1)
        if len(p) == 1:
            args.append(arg)
        else:
            kwargs[p[0]] = p[1]

    # Example of how to exclude the 'git-fusion-user'
    # Note: Need to remove 'myuser' after test as it's not a valid P4 argument.
    if 'myuser' in kwargs:
        if kwargs['myuser'] == 'git-fusion-user':
            sys.exit(0)
        else:
            del kwargs['myuser']

    AllowBypass = 1
    if 'allowbypass' in kwargs:
        if kwargs['allowbypass'] == 'no':
            AllowBypass = 0

        # Remove 'allowbypass' after test as it's not a valid P4 argument.
        del kwargs['allowbypass']

    # Controls whether files whose head revision action is 'delete' are
    # still treated as "existing" for case-conflict purposes (see the
    # CASE CONFLICTS WITH DELETED FILES section of the module docstring).
    # Defaults to on (the stricter, corrected behavior). Sites phasing in
    # this change against existing depot history can pass
    # 'includedeletedpaths=no' to temporarily restore the old behavior.
    IncludeDeletedPaths = 1
    if 'includedeletedpaths' in kwargs:
        if kwargs['includedeletedpaths'] == 'no':
            IncludeDeletedPaths = 0

        # Remove after test as it's not a valid P4 argument.
        del kwargs['includedeletedpaths']

    if AllowBypass:
        # Grab the changelist description, and scan for the bypass token string.
        # If the token is detected, silently and immediately exit with a happy 0
        # exit code.
        changelist = sys.argv[1]
        cmd = "%s -ztag -F %%desc%% describe -f -s %s" % (os.getenv('P4BIN','p4'), changelist)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
        (changeDesc, err) = p.communicate()
        p_status = p.wait()

        # If the changelist description contains the text BYPASS_CASE_CHECK,
        # bypass the case check logic.
        if (re.search (b'BYPASS_CASE_CHECK', changeDesc, re.MULTILINE)):
            sys.exit(0)

    ct = CheckCaseTrigger(*args, **kwargs)
    sys.exit(ct.run())