IntegHistory.py #1

#!/usr/bin/env python

################################################################################
#
# Copyright (c) 2019, Perforce Software, Inc.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# DATE
#   
#   $Date: 2020/03/25 $
#
# SYNOPSIS
# 
#   IntegHistory.py [-v] [-o] [-p p4port] [-u p4user] [-c p4client] depotFile(s)[revRange]
#   * depotFile must be in depot syntax
#   * depotFile with spaces requires quote
#   * multiple depotFiles allowed
#   * wildcards and revision specifiers are supported
#
# DESCRIPTION
#
#   This script collects the full integration history of files.
#   The result is stored in a gzipped file (IntegHistory.log.gz) that can
#   be used by IntegHistoryRebuild.py to rebuild a partial Helix server.
#
# REQUIREMENT
#
# Command line p4 2013.2 or greater (check "p4 -V")
#
################################################################################

from __future__ import print_function
from collections import OrderedDict, deque
import subprocess
import argparse
import ast
import sys
import re
import gzip
import shutil
import os
import random
import string

python3 = sys.version_info[0] >= 3
minP4Version = 2013.2

class IntegHistory:
      def __init__(self, depotFiles, logName, verbose = False, obfuscate = False):
          self.depotFiles = depotFiles
          self.logName = logName
          self.caseHandling = b"insensitive"
          self.isSuper = False
          self.verbose = verbose
          self.obfuscate = obfuscate
          self.obfuscateDict = {}
          self.files = []
          self.integed = []
          self.depots = {}
          self.streams = []
          self.changes = {}
          self.depotRE = re.compile(b'\/\/(.*?)/.*')
          self.filelog1RE = re.compile(b'\.\.\. #(\d+) change (\d+)')
          self.filelog2RE = re.compile(b'\.\.\. \.\.\. .* (\/\/.*?)#')
          self.importRE = re.compile(b'... Paths\d+ import.* (.*)')
          try:
            self.log = open(logName, "w")
            # for Python 2 compatibility purpose
            self.output = getattr(self.log, 'buffer', self.log)
          except IOError:
            sys.exit("Error: can\'t create " + logName)
          try:
            self.output.write(b"['files', '")
            for i in range(len(self.depotFiles)):
                depotFile = self.depotFiles[i]
                if self.obfuscate:
                   depotFile = self.obfuscateFilepath(depotFile)
                self.output.write(depotFile)
                if i+1 < len(self.depotFiles):
                   self.output.write(b"', '")
            self.output.write(b"']\n")                    
          except IOError:
            sys.exit("Error: can\'t write in " + logName)
          match = re.match(b'.*\/(\d+.\d)\/.*',self.runCmd(b'p4 -V', False)[-1])
          if match and float(match.group(1)) < minP4Version:
            sys.exit("Error: minimum supported P4 version is " + str(minP4Version))
             
      def obfuscateSave(self, filename):
          with gzip.open(filename + ".gz", 'wb') as gzfile:
               for depotFile in self.files:
                   listStr = []
                   for str in depotFile.split('/'):
                       if not str == "":
                          listStr.append(self.obfuscateDict[str])
                       else:
                          listStr.append("")
                   gzfile.write(depotFile + " <=> " + '/'.join(listStr) + "\n")

      def obfuscateFilepath(self, filepath):
          strList = []
          for str in filepath.split('/'):
              obfuscatedStr = ""
              if not str == "":
                 if str in self.obfuscateDict:
                    obfuscatedStr = self.obfuscateDict[str]
                 else:
                    obfuscatedStr = ''.join(((random.choice(string.letters) if char.isalpha() else char) for char in str))
                    self.obfuscateDict[str] = obfuscatedStr
              strList.append(obfuscatedStr)
          return('/'.join(strList))

      def obfuscateOutput(self, line):
          match = re.match(b'(... (?:desc|Description) )(.*)', line)
          if match:
             line = match.group(1) + "obfuscated description\\n"
          else:
             match = re.match(b'(... (?:clientFile|clientRoot|clientCwd|serverRoot|Root|Value) )(.*)', line)
             if match:
                line = match.group(1) + "/obfuscated/path"
             else:
                match = re.match(b'(... (?:Depot|depotFile|lbrFile|StreamDepth|stream|Stream|Name|path\d+ .*?|toFile|fromFile|path|user.*|client.*|User|Client|Owner|Email|FullName|Host|View\d+|Host|Map) )(.*)', line)
                if match:
                   line = match.group(1) + self.obfuscateFilepath(match.group(2))
                else:
                   match = re.match(b'(... (?:parent|Parent|baseParent|parentBase) )(.*)', line)
                   if match and not match.group(2) == "none":
                      line = match.group(1) + self.obfuscateFilepath(match.group(2))
                   else:
                      match = re.match(b'(p4 .* (?:depot .*|stream .*|istat .*|integed) \")(.*)(\")', line)
                      if match:
                         line = match.group(1) + self.obfuscateFilepath(match.group(2)) + match.group(3)
                      else:
                         match = re.match(b'(p4 .* fstat .* \")(.*)(@.*\")', line)
                         if match:
                            line = match.group(1) + self.obfuscateFilepath(match.group(2)) + match.group(3)
          return(line)

      def runCmd(self, cmd, toLog = True):
          # for Python 2 compatibility purpose
          stderr = getattr(sys.stderr, 'buffer', sys.stderr)
          tagged = False
          if re.search(b'-ztag', cmd):
             tagged = True
          filtered = False
          if re.search(b'-ztag -F', cmd):
            filtered = True
          result = []
          if toLog:
             result.append(cmd)
          if sys.platform == 'win32':
             process = subprocess.Popen(cmd.decode(), shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
          else:
             process = subprocess.Popen(cmd, shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE)
          (output, err) = process.communicate()
          exitCode = process.wait()          
          if self.verbose or err:
             stderr.write(cmd + b"\n")
          if self.verbose:
             stderr.write(output)
          if err:
             stderr.write(b"Error: " + err)
          if b"Perforce password (P4PASSWD) invalid or unset." in err:
             sys.exit()
          if exitCode == 0 and output:
             if filtered or not tagged:
                result = result + list(output.splitlines())
             else:           
                out = deque(output.splitlines())
                line = out.popleft()
                block = line
                while out:
                      nextLine = out.popleft()
                      match = re.search(b'^\.\.\. ', nextLine)
                      while out and not match:
                            block = block + b"\\n" + nextLine
                            nextLine = out.popleft()
                            match = re.search(b'^\.\.\. ', nextLine)
                      result.append(block)
                      line = nextLine
                      block = line
             if toLog == True:
                if result:
                   self.output.write(b"[")
                   for i,item in enumerate(result):
                       if self.obfuscate:
                          item = self.obfuscateOutput(item)
                       self.output.write(b"'")
                       self.output.write(item.replace(b"\\",b"\\\\").replace(b"\\\\n",b"\\n").replace(b"'",b"\\'"))
                       self.output.write(b"'")
                       if i < len(result)-1:
                          self.output.write(b", ")
                   self.output.write(b"]\n")
          return(result)

      def getConfigData(self):
          for line in self.runCmd(b'p4 -ztag info'):
              match = re.search(b'\.\.\. caseHandling (.*)', line.strip())
              if match:
                 self.caseHandling = match.group(1)
                 break
          self.runCmd(b'p4 set')
          self.runCmd(b'p4 -ztag user -o')
          self.runCmd(b'p4 -ztag client -o')
          self.runCmd(b'p4 -ztag protects')
          if re.search(b'... permMax super', self.runCmd(b"p4 -ztag protects -m")[-1]):
             self.isSuper = True
             self.runCmd(b'p4 -ztag configure show')
          self.runCmd(b'p4 counters -e change')
          self.runCmd(b'p4 counters -e traits')

      def addParentStream(self, stream):
          stream = stream.replace(b"(",b"\(").replace(b")",b"\)").replace(b"&",b"\&")
          for parent in self.runCmd(b'p4 -F %Parent% streams -m1 -F Stream="' + stream + b'"', False):
              if not parent == b"none" and parent not in self.streams:
                 self.streams.append(parent)
                 self.addParentStream(parent)

      def addVirtualStream(self, stream):
          stream = stream.replace(b"(",b"\(").replace(b")",b"\)").replace(b"&",b"\&")
          for virtual in self.runCmd(b'p4 -F %Stream% streams -F "Parent=' + stream + b' & Type=virtual"', False):
              if virtual not in self.streams:
                 self.streams.append(virtual)
                 self.addVirtualStream(virtual)

      def getFilesHistory(self, depotFile):
          if (self.caseHandling == b"insensitive" and depotFile.lower() not in map(lambda name: name.lower(), self.files)) or \
             (self.caseHandling == b"sensitive" and depotFile not in self.files):
             filelog = self.runCmd(b'p4 filelog -1 -l -t "' + depotFile + b'"', False)
             if len(filelog) > 2:
                self.files.append(depotFile)
                self.parseDepotFile(depotFile)
                for line in filelog:
                    match = self.filelog1RE.match(line.strip())
                    if match:
                       if match.group(2) not in self.changes:
                          self.changes[match.group(2)] = []
                       self.changes[match.group(2)].append(depotFile + b"#" + match.group(1))
                    else:
                       match = self.filelog2RE.match(line.strip())
                       if match:
                          self.getFilesHistory(match.group(1))

      def parseDepotFile(self, depotFile):
          match = self.depotRE.match(depotFile)
          if match:
             depot = match.group(1)
             if depot not in self.depots:
                streamDepth = None
                result = self.runCmd(b'p4 -ztag -F "%Type% %StreamDepth%" depot -o "' + depot + b'"', False)
                if result:
                   if result[0][0:6] == b"stream":
                      if len(result[0]) > 7:
                         streamDepth = result[0][7:]
                      else:
                         # for release with stream depot and no StreamDepth field
                         streamDepth = b"//" + depot + b"/1"
                   self.depots[depot] = streamDepth
             else:
               streamDepth = self.depots[depot]
             if streamDepth:
                stream = b""
                nbSlash = 0
                for i, char in enumerate(depotFile):
                    if (not python3 and char == b'/') or (python3 and char == 47):
                       nbSlash += 1
                    if nbSlash > streamDepth.count(b'/'):
                       stream = depotFile[:i]
                       break
                stream = stream.replace(b"(",b"\(").replace(b")",b"\)").replace(b"&",b"\&")
                if stream not in self.streams and self.runCmd(b'p4 streams -F "Stream=' + stream + b'"', False):
                   self.streams.append(stream)
                   self.addVirtualStream(stream)
                   self.addParentStream(stream)

      def getDepotsData(self):
          for depot in self.depots:
              self.runCmd(b'p4 -ztag depot -o "' + depot + b'"')

      def getStreamsData(self):
          for stream in self.streams:
              spec = self.runCmd(b'p4 -ztag stream -o "' + stream + b'"')
              if "... Type virtual" not in spec:
                 self.runCmd(b'p4 -ztag istat -s "' + stream + b'"')
              for line in spec:
                  match = self.importRE.match(line)
                  if match:
                     self.parseDepotFile(match.group(1))

      def getFilesData(self):
          flags = b"-Ol -Oi" 
          if self.isSuper:
             flags = flags + b" -Oc"
          for change in self.changes:
              for depotFile in self.changes[change]:
                  if python3:
                     filename = bytearray(depotFile)[0:depotFile.find(b"#")]
                  else:
                     filename = depotFile[0:depotFile.find(b"#")]
                  self.runCmd(b'p4 -ztag fstat ' + flags + b' "' + filename + b'@=' + change + b'"')

      def getIntegedData(self):
          for change in self.changes:
              for depotFile in self.changes[change]:
                  if python3:
                     filename = bytearray(depotFile)[0:depotFile.find(b"#")]
                  else:
                     filename = depotFile[0:depotFile.find(b"#")]
                  for depotFile in self.runCmd(b'p4 -ztag -F%depotFile% files "' + filename + b'@=' + change + b'"', False):
                      if depotFile not in self.integed:
                         self.runCmd(b'p4 -ztag integed "' + depotFile + b'"')
                         self.integed.append(depotFile)
                       
      def getChangesData(self):
          for change in self.changes:
              self.runCmd(b'p4 -ztag changes -l @=' + change)

      def getData(self):
          self.getConfigData()
          for depotFile in self.depotFiles:
              for df in self.runCmd(b'p4 -ztag -F%depotFile% files "' + depotFile + b'"', False):
                  self.getFilesHistory(df)
          if len(self.files) == 0:
             sys.exit("Error: No file found, check " + self.logName)
          self.changes = OrderedDict(sorted(self.changes.items(), key=lambda t: t[0]))
          self.getStreamsData()
          self.getDepotsData()
          self.getFilesData()
          self.getIntegedData()
          self.getChangesData()
          self.log.close()
          if (self.obfuscate):
             self.obfuscateSave("IntegHistoryObfuscated.txt")
          try:
            with open(self.logName, 'rb') as uncompressed:
                 with gzip.open(self.logName + ".gz", 'wb') as compressed:
                      shutil.copyfileobj(uncompressed, compressed)
            if os.path.isfile(self.logName):
               os.remove(self.logName)
            return self.logName + ".gz"
          except:
            return self.logName
      
def setP4Env(p4port, p4user, p4client):
    output = subprocess.check_output(["p4", "set"], universal_newlines=True)
    for line in output.splitlines():
        match = re.search("P4PORT=(.*?) .*", line)
        if match:
           if not p4port:
              p4port = match.group(1)
        else:
            match = re.search("P4USER=(.*?) .*", line)
            if match:
               if not p4user:
                  p4user = match.group(1)
            else:
               match = re.search("P4CLIENT=(.*?) .*", line)
               if match:
                  if not p4client:
                     p4client = match.group(1)
    os.environ["P4CONFIG"] = ""
    if p4port:
       os.environ["P4PORT"] = p4port
    if p4user:
       os.environ["P4USER"] = p4user
    if p4client:
       os.environ["P4CLIENT"] = p4client

def main():
    parser = argparse.ArgumentParser(prog='IntegHistory', usage='%(prog)s [-v] [-o] [-P p4_path] [-p p4port] [-u p4user] [-c p4client] depotFiles(s)[revRange]')
    if python3:
       parser.add_argument("depotFiles", nargs="*", help="list of files in depot syntax separated with spaces", type=os.fsencode)
    else:
       parser.add_argument("depotFiles", nargs="*", help="list of files in depot syntax separated with spaces")
    parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")
    parser.add_argument("-o", "--obfuscate", help="obfuscate command output", action="store_true")
    parser.add_argument("-P", "--path", dest="path", help="P4 path value")
    parser.add_argument("-p", "--p4port", dest="p4port", help="server value")
    parser.add_argument("-u", "--p4user", dest="p4user", help="user value")
    parser.add_argument("-c", "--p4client", dest="p4client", help="client value")
    args = parser.parse_args()
    if not args.depotFiles:
       sys.exit(parser.print_help())
    for depotFile in args.depotFiles:
        if not re.search(b'\/\/.*', depotFile):
           sys.exit(parser.print_help())
    if args.p4port or args.p4user or args.p4client:
       setP4Env(args.p4port, args.p4user, args.p4client)
    if args.path:
       os.environ["PATH"] = args.path + ":" + os.environ["PATH"]
    logName = os.path.abspath(os.getcwd() + "/IntegHistory.log")
    history = IntegHistory(args.depotFiles, logName, args.verbose, args.obfuscate)
    print("Info: " + history.getData() + " created successfully")
  
if __name__ == '__main__':
    main()