p4.py #2

# Perforce source for convert extension.
#
# Copyright 2009, Frank Kingswood <frank@kingswood-consulting.co.uk>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from mercurial import util
from mercurial.i18n import _

from common import commit, converter_source, checktool, NoRepo
import marshal
import re

def loaditer(f):
    "Yield the dictionary objects generated by p4"
    try:
        while True:
            d = marshal.load(f)
            if not d:
                break
            yield d
    except EOFError:
        pass

def decodefilename(filename):
    """Perforce escapes special characters @, #, *, or %
    with %40, %23, %2A, or %25 respectively

    >>> decodefilename('portable-net45%252Bnetcore45%252Bwp8%252BMonoAndroid')
    'portable-net45%2Bnetcore45%2Bwp8%2BMonoAndroid'
    >>> decodefilename('//Depot/Directory/%2525/%2523/%23%40.%2A')
    '//Depot/Directory/%25/%23/#@.*'
    """
    replacements = [('%2A', '*'), ('%23', '#'), ('%40', '@'), ('%25', '%')]
    for k, v in replacements:
        filename = filename.replace(k, v)
    return filename

class p4_source(converter_source):
    def __init__(self, ui, path, revs=None):
        # avoid import cycle
        import convcmd

        super(p4_source, self).__init__(ui, path, revs=revs)

        if "/" in path and not path.startswith('//'):
            raise NoRepo(_('%s does not look like a P4 repository') % path)

        checktool('p4', abort=False)

        self.p4changes = {}
        self.heads = {}
        self.changeset = {}
        self.files = {}
        self.copies = {}
        self.tags = {}
        self.lastbranch = {}
        self.parent = {}
        self.encoding = self.ui.config('convert', 'p4.encoding',
                                       default=convcmd.orig_encoding)
        self.depotname = {}           # mapping from local name to depot name
        self.localname = {} # mapping from depot name to local name
        self.re_type = re.compile(
            "([a-z]+)?(text|binary|symlink|apple|resource|unicode|utf\d+)"
            "(\+\w+)?$")
        self.re_keywords = re.compile(
            r"\$(Id|Header|Date|DateTime|Change|File|Revision|Author)"
            r":[^$\n]*\$")
        self.re_keywords_old = re.compile("\$(Id|Header):[^$\n]*\$")

        if revs and len(revs) > 1:
            raise util.Abort(_("p4 source does not support specifying "
                               "multiple revisions"))
        self._parse(ui, path)

    def _parse_view(self, path):
        "Read changes affecting the path"
        cmd = 'p4 -G changes -s submitted %s' % util.shellquote(path)
        stdout = util.popen(cmd, mode='rb')
        for d in loaditer(stdout):
            c = d.get("change", None)
            if c:
                self.p4changes[c] = True

    def _parse(self, ui, path):
        "Prepare list of P4 filenames and revisions to import"
        ui.status(_('reading p4 views\n'))

        # read client spec or view
        if "/" in path:
            self._parse_view(path)
            if path.startswith("//") and path.endswith("/..."):
                views = {path[:-3]:""}
            else:
                views = {"//": ""}
        else:
            cmd = 'p4 -G client -o %s' % util.shellquote(path)
            clientspec = marshal.load(util.popen(cmd, mode='rb'))

            views = {}
            for client in clientspec:
                if client.startswith("View"):
                    sview, cview = clientspec[client].split()
                    self._parse_view(sview)
                    if sview.endswith("...") and cview.endswith("..."):
                        sview = sview[:-3]
                        cview = cview[:-3]
                    cview = cview[2:]
                    cview = cview[cview.find("/") + 1:]
                    views[sview] = cview

        # list of changes that affect our source files
        self.p4changes = self.p4changes.keys()
        self.p4changes.sort(key=int)

        # list with depot pathnames, longest first
        vieworder = views.keys()
        vieworder.sort(key=len, reverse=True)

        # handle revision limiting
        startrev = self.ui.config('convert', 'p4.startrev', default=0)
        self.p4changes = [x for x in self.p4changes
                          if ((not startrev or int(x) >= int(startrev)) and
                              (not self.revs or int(x) <= int(self.revs[0])))]

        # now read the full changelists to get the list of file revisions
        ui.status(_('collecting p4 changelists\n'))
        lastid = None
        for change in self.p4changes:
            cmd = "p4 -G describe -s %s" % change
            stdout = util.popen(cmd, mode='rb')
            d = marshal.load(stdout)
            desc = self.recode(d.get("desc", ""))
            shortdesc = desc.split("\n", 1)[0]
            t = '%s %s' % (d["change"], repr(shortdesc)[1:-1])
            ui.status(util.ellipsis(t, 80) + '\n')

            if lastid:
                parents = [lastid]
            else:
                parents = []

            date = (int(d["time"]), 0)     # timezone not set
            c = commit(author=self.recode(d["user"]),
                       date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'),
                       parents=parents, desc=desc, branch=None,
                       extra={"p4": change})

            files = []
            copies = {}
            copiedfiles = []
            i = 0
            while ("depotFile%d" % i) in d and ("rev%d" % i) in d:
                oldname = d["depotFile%d" % i]
                filename = None
                for v in vieworder:
                    if oldname.lower().startswith(v.lower()):
                        filename = decodefilename(views[v] + oldname[len(v):])
                        break
                if filename:
                    files.append((filename, d["rev%d" % i]))
                    self.depotname[filename] = oldname
                    if (d.get("action%d" % i) == "move/add"):
                        copiedfiles.append(filename)
                    self.localname[oldname] = filename
                i += 1

            # Collect information about copied files
            for filename in copiedfiles:
                oldname = self.depotname[filename]

                flcmd = 'p4 -G filelog %s' \
                      % util.shellquote(oldname)
                flstdout = util.popen(flcmd, mode='rb')

                copiedfilename = None
                for d in loaditer(flstdout):
                    copiedoldname = None

                    i = 0
                    while ("change%d" % i) in d:
                        if (d["change%d" % i] == change and
                            d["action%d" % i] == "move/add"):
                            j = 0
                            while ("file%d,%d" % (i, j)) in d:
                                if d["how%d,%d" % (i, j)] == "moved from":
                                    copiedoldname = d["file%d,%d" % (i, j)]
                                    break
                                j += 1
                        i += 1

                    if copiedoldname and copiedoldname in self.localname:
                        copiedfilename = self.localname[copiedoldname]
                        break

                if copiedfilename:
                    copies[filename] = copiedfilename
                else:
                    ui.warn(_("cannot find source for copied file: %s@%s\n")
                            % (filename, change))

            self.changeset[change] = c
            self.files[change] = files
            self.copies[change] = copies
            lastid = change

        if lastid:
            self.heads = [lastid]

    def getheads(self):
        return self.heads

    def getfile(self, name, rev):
        cmd = 'p4 -G print %s' \
            % util.shellquote("%s#%s" % (self.depotname[name], rev))

        lasterror = None
        while True:
            stdout = util.popen(cmd, mode='rb')

            mode = None
            contents = []
            keywords = None

            for d in loaditer(stdout):
                code = d["code"]
                data = d.get("data")

                if code == "error":
                    # if this is the first time error happened
                    # re-attempt getting the file
                    if not lasterror:
                        lasterror = IOError(d["generic"], data)
                        # this will exit inner-most for-loop
                        break
                    else:
                        raise lasterror

                elif code == "stat":
                    action = d.get("action")
                    if action in ["purge", "delete", "move/delete"]:
                        return None, None
                    p4type = self.re_type.match(d["type"])
                    if p4type:
                        mode = ""
                        flags = ((p4type.group(1) or "")
                               + (p4type.group(3) or ""))
                        if "x" in flags:
                            mode = "x"
                        if p4type.group(2) == "symlink":
                            mode = "l"
                        if "ko" in flags:
                            keywords = self.re_keywords_old
                        elif "k" in flags:
                            keywords = self.re_keywords

                elif code == "text" or code == "binary":
                    contents.append(data)

                lasterror = None

            if not lasterror:
                break

        if mode is None:
            return None, None

        contents = ''.join(contents)

        if keywords:
            contents = keywords.sub("$\\1$", contents)
        if mode == "l" and contents.endswith("\n"):
            contents = contents[:-1]

        return contents, mode

    def getchanges(self, rev, full):
        if full:
            raise util.Abort(_("convert from p4 do not support --full"))
        return self.files[rev], self.copies[rev], set()

    def getcommit(self, rev):
        return self.changeset[rev]

    def gettags(self):
        return self.tags

    def getchangedfiles(self, rev, i):
        return sorted([x[0] for x in self.files[rev]])


class p4_sink(converter_sink):
    def __init__(self, ui, path):
        converter_sink.__init__(self, ui, path)
        self.p4 = P4.P4()
        
        self.repo = hg.repository(ui, path.replace('-hg', ''))
        
        #ui.pushbuffer()
        #commands.log(self.ui, self.repo, date=None, rev=None, user=None)
        #print(ui.popbuffer())

    def getheads(self):
        print( "Called getheads()")
        return []

    def revmapfile(self):
        print( "Called revmapfile()")
        return "revmapfile"

    def authorfile(self):
        print( "Called authorfile()")
        return "authorfile"

    def join(self, branch, file):
        return os.path.join(self.root, branch, file)
    
    def fixFilename( self, name ):
        result = name.replace('%', '%25')
        result = result.replace('@', '%40')
        result = result.replace('#', '%23')
        result = result.replace('*', '%2A')
        return result
    
    def fixBranchname( self, name ):
        result = name.replace('%', '_PERCENT_')
        result = result.replace('@', '_AT_')
        result = result.replace('#', '_HASH_')
        result = result.replace('*', '_STAR_')
        result = result.replace(' ', '_')
        return result
    
    def putfile( self, file, data, mode ):
        # find the directories first
        targetPath,basename = os.path.split(file)
        if not os.path.isdir( targetPath ):
            os.makedirs( targetPath )

        with open(file, 'w') as f:
            f.write(data)
        # ignore mode for now
        
    def findMergedFiles(self, search, sortkey):
        self.ui.pushbuffer()
        commands.diff(self.ui, self.repo, rev=[str(search), str(sortkey)], change=None, stat=True, reverse=None)
        result = self.ui.popbuffer()
        print(result)
        resultLines = result.split('\n')[:-2] # cut off the last two lines containing summary and a blank line
        files = [ line.split('|')[0].strip() for line in resultLines ]
        return files
    
    def putcommit(self, files, copies, parents, commit, source, revmap):
        print( "Called putcommit() with")
        print( "*** files", [x[0] for x in files] )
        print( "*** copies", copies )
        print( "*** parents", parents )
        print( "*** commit.author", commit.author )
        print( "*** commit.date", commit.date )
        print( "*** commit.desc", commit.desc )
        print( "*** commit.parents", commit.parents )
        print( "*** commit.branch", commit.branch )
        # print( "*** commit.rev", commit.rev )
        print( "*** commit.extra", commit.extra )
        print( "*** commit.sortkey", commit.sortkey )
        
        edited = []
        added = []
        deleted = []
        copied = []
        copySource = []
        renamed = []
        copyData = {}
        
        change = ''    

        revs = set()
        
        date = commit.date[:-6] # cut the timezone off, Python2.X cannot deal with it yet
        timestamp = datetime.strptime(date, '%a %b %d %H:%M:%S %Y').strftime('%Y/%m/%d %H:%M:%S')
        
        if commit.author in self.users:
            self.p4.user = commit.author
            self.p4.password = self.users[commit.author]
        else:
            self.p4.user = self.superuser
            self.p4.password = self.superticket
            r = self.p4.run_login('-p', commit.author)
            self.p4.user = commit.author # users are mapped via authorfile to legit Perforce users
            self.p4.password = r[0]
            
            self.users[self.p4.user] = self.p4.password

        # now create branches if necessary

        cbranch = self.fixBranchname(commit.branch)
        branch = self.fixBranchname(self.next_branch)
        pbranches = [ (x[0], self.fixBranchname(x[1])) for x in self.next_pbranches ]
        
        if not pbranches:
            pass        
        elif len(pbranches) == 1:
            change, parent = pbranches[0]
            if parent != branch:
                print("creating branch %s from %s @ %s" % (branch, parent, change))
                with self.p4.at_exception_level(1):
                    if change and change != '0':
                        self.p4.run_integrate('%s/...@%s' % (parent, change), '%s/...' % branch)
                    else:
                        self.p4.run_integrate('%s/...' % parent, '%s/...' % branch)
                    
                    print( "after integrate; warnings : ", self.p4.warnings )
                    # might need a resolve here, just in case the target exists
                    self.p4.run_resolve('-at')
                
                change = self.p4.fetch_change()
                if 'Files' in change:
                    change._description = 'Created branch %s from %s @ %s' % (branch, parent, change)
                    submitted = self.p4.run_submit(change)
                    # remember the change if this is just a branch with no edited changes.
                    # Otherwise the next submit will override this number 
                    for s in reversed( submitted ):
                        if 'submittedChange' in s:
                            change = s['submittedChange'] # the last entry in the array contains the change number
                            break
                else:
                    print("Nothing to integrate. Maybe a rebase?")
            else:
                # in some cases Mercurial does not tell us all the files that have changed
                # if the change originates from a push or pull between repos
                # find the missing files here
                 
                searchRev = commit.parents[0]
                mergedFiles = self.findMergedFiles(searchRev, commit.sortkey)

                officiallyEditedFiles = [ x[0] for x in files ]
                for m in mergedFiles:
                    if m not in officiallyEditedFiles:
                        files.append( (m, str(commit.sortkey)) )
                
                print files

        elif len(pbranches) == 2:
            targetChange, tgt = pbranches[0]
            sourceChange, src = pbranches[1]
            
            if tgt == branch and src == branch:
                # merge between two repositories in the same branch
                
                mergedFiles = []
                for search in commit.parents:
                    mergedFiles += self.findMergedFiles(search, commit.sortkey)
                
                # add all the merged files to the list of files 
                officiallyEditedFiles = [ x[0] for x in files ]
                for m in mergedFiles:
                    if m not in officiallyEditedFiles:
                        files.append( (m, str(commit.sortkey)) )
                
                print files
                
            elif tgt == branch:
                print("merging to %s@%s with %s@%s" % (tgt, targetChange, src, sourceChange) )
                if sourceChange:
                    with self.p4.at_exception_level(1):
                        for f,v in files:
                            copied = self.p4.run_copy('%s/%s@%s' % (src, f, sourceChange), '%s/%s' % (tgt, f) )
                else:
                    with self.p4.at_exception_level(1):
                        for f,v in files:
                            copied = self.p4.run_copy('%s/%s' % (src, f), '%s/%s' % (tgt,f) )
            elif src == branch:
                print("merging from %s@%s with %s@%s" % (tgt, targetChange, src, sourceChange))
                if targetChange:
                    with self.p4.at_exception_level(1):
                        for f,v in files:
                            self.p4.run_copy('%s/%s@%s' % (tgt, f, targetChange), '%s/%s' % (src,f) )
                else:
                    with self.p4.at_exception_level(1):
                        for f,v in files:
                            self.p4.run_copy('%s/%s' % (tgt,f), '%s/%s' % (src,f) )
            else:
                print("Serious trouble in setbranch: parent branches do not match target branch")
        else:
           print( "Serious problem in setbranches: more than 2 parent branches ???")

        executables = []
        
        for f, v in files:
            target = self.join(cbranch, f)
            fixedTarget = self.fixFilename(target)
            revs.add(v)

            try:
                data, mode = source.getfile(f, v)
                if 'x' in mode:
                    executables.append(fixedTarget)
                    
            except IOError:
                deleted.append(fixedTarget)
            else:
                print("* %s mode: %s" % (f, mode))
                if f in copies:
                    print("*** file %s is a copy from %s" % (f, copies[f]))
                    s = self.fixFilename( self.join(cbranch, copies[f]) )
                    copied.append( (fixedTarget, s) )
                    copySource.append( (s, fixedTarget) )
                    copyData[fixedTarget] = (data, mode)
                else:
                    with self.p4.at_exception_level(1):
                        fileinfo = self.p4.run_fstat(fixedTarget) # warning if file does not exist
                        print( fileinfo, self.p4.warnings, self.p4.errors )
                        if (len(self.p4.warnings) > 0) \
                                or ('headAction' in fileinfo[0] and 'delete' in fileinfo[0]['headAction'] ) \
                                or ('action' in fileinfo[0] and 'branch' in fileinfo[0]['action'] ):
                            added.append(target)
                            print("*a %s" % target)
                        else:
                            self.p4.run_sync(fixedTarget) # ignore warning that file already synced
                            edited.append(fixedTarget)
                            print("*e %s" % target)
                self.putfile(target, data, mode)
        
        # time to do some processing
        
        print( "Total added ", len(added))
        print( "Total edited ", len(edited))
        
        if len(added) > 0 : 
            self.p4.run_add('-f', added)
        if len(edited) > 0 : 
            self.p4.run_edit(edited)
        
        # if a file was the source of a copy and deleted, it was a rename
        
        trueCopies = []
        for i in copySource:
            if i[0] in deleted:
                renamed.append( i )
                deleted.remove(i[0])
            else:
                trueCopies.append( i )                
        
        # now process renames, copies and deletes
        
        if len(deleted) > 0 : 
            with self.p4.at_exception_level(1):
                self.p4.run_delete('-v', deleted)
                print("warnings after delete: ", self.p4.warnings)
        
        with self.p4.at_exception_level(1):
            for i in renamed:
                print("moving %s to %s" % (i[0], i[1]))
                self.p4.run_sync('-f', i[0], i[1]) # in case we are renaming into an existing target
                fileinfo = self.p4.run_fstat(fixedTarget) # warning if file does not exist
                if len(self.p4.warnings) > 0:
                    # target does not exist yet ==> move it
                    self.p4.run_edit(i[0])
                    self.p4.run_move('-k', i[0], i[1]) # file content is already in workspace
                    print("warnings after move: ", self.p4.warnings)
                    
                else:
                    # downgrade the move to an integrate and delete, it might have been a 
                    # forced rename in Mercurial
                    self.p4.run_integrate(i[0], i[1])
                    self.p4.run_resolve('-at', i[1])
                    self.p4.run_edit(i[1])
                    self.p4.run_delete(i[0])
                    
                (data, mode) = copyData[i[1]]
                self.putfile(i[1], data, mode) # need to overwrite the file, in case it is a dirty edit

        for i in trueCopies:
            print(">>> Trying to copy %s to %s" % (i[0], i[1]))
            with self.p4.at_exception_level(0):
                self.p4.run_copy(i[0], i[1])
                if self.p4.errors or self.p4.warnings:
                    print("Copy errors ", self.p4.errors, " warnings ", self.p4.warnings)
            with self.p4.at_exception_level(1):
                self.p4.run_integrate(i[0], i[1])
                self.p4.run_resolve('-at')
                self.p4.run_sync(i[1]) # if copy and integrate say the file is already integrated without syncing it
                if self.p4.warnings:
                    print(">>>Sync in trueCopies warning: ", self.p4.warnings)
                self.p4.run_edit(i[1])
                if self.p4.warnings:
                    print(">>>Edit in trueCopies warning, downgrading to add: ", self.p4.warnings)
                    self.p4.run_add(i[1])
                    if self.p4.warnings:
                        print(">>>Add in trueCopies warning: ", self.p4.warnings)
            (data, mode) = copyData[i[1]]
            self.putfile(i[1], data, mode) # need to overwrite the file, in case it is a dirty edit
        
        # if we have any files of type +x, change the type here
        
        if len(executables) > 0:
            for x in self.p4.run_opened():
                print('open files: ', x)
            print('changing file type to +x for %s' % str(executables))
            self.p4.run_reopen('-t+x', executables)
        
        # time to submit the lot
        
        l = len( self.p4.run_opened() )
        if l > 0: # the first entry in OpenOffice_DEV300 is empty
            print("*** Open files : ", l)
            
            description = 'Comment: ' + commit.desc
            description += '\nDate: ' + commit.date
            description += '\nBranch: ' + cbranch
            description += '\nUser: ' + commit.author
            description += '\nRev:'
            for i in revs: # not sure if there can be more than one. Better be safe
                description += ' ' + i
            description += '\nParents: ' + str(commit.parents)
            description += '\nSortKey: ' + str(commit.sortkey)
            
            ch = self.p4.fetch_change()
            ch._description = description
            with self.p4.at_exception_level(0):
                submitted = self.p4.run_submit(ch, '-frevertunchanged')
                print( submitted )

                if self.p4.errors:
                    print( "*** putcommit(): nothing to submit. Errors : ", self.p4.errors, " Warnings: ", self.p4.warnings )
                    self.p4.delete_change(submitted[0]['change'])
                    if self.p4.errors or self.p4.warnings:
                        print( "delete change had problems. Errors =:", self.p4.errors, " Warnings : ", self.p4.warnings)
                    return '0'
                else:
                    for s in reversed( submitted ):
                        if 'submittedChange' in s:
                            change = s['submittedChange'] # the last entry in the array contains the change number
                            break
            
            # fixing the timestamp to resemble original timestamp from Mercurial
            # need to be super user to do that
            if change:
                self.p4.user = self.superuser
                self.p4.password = self.superticket
    
                ch = self.p4.fetch_change(change)
                ch._date = timestamp
                self.p4.save_change(ch, '-f')
            
        print( "*** End of putcommit(%s) ***" % change)
        return change

    def puttags(self, tags):
        print( "Called puttags() with")
        print( "***", tags )
        
        with self.p4.at_exception_level(1):
            for name, change in tags.items():
                self.p4.run_tag('-l'+name, '//%s/...@%s' % (self.p4.client, change))
                if self.p4.warnings:
                    print( '>>>', self.p4.warnings)
        
        print( "*** End of puttags() ***")
        return None, None

    def setbranch(self, branch, pbranches):
        print( "Called setbranch() with")
        print( "***", branch )
        print( "***", pbranches )

        self.next_branch = branch
        self.next_pbranches = pbranches

        print( "*** End of setbranch() ***")
    
    def setfilemapmode(self, active):
        print( "Called setfilemapmode() with")
        print( "***", active )
        print( "*** End of setfilemapmode() ***")
    
    def before(self):
        print( "Called before")
        self.p4.port = self.ui.config('convert', 'p4.port', default=self.p4.port)
        self.p4.client = self.ui.config('convert', 'p4.client', default=self.p4.client)
        self.p4.user = self.ui.config('convert', 'p4.user', default=self.p4.user)
        password = self.ui.config('convert', 'p4.password', default=self.p4.password)
        self.superuser = self.p4.user
        self.p4.connect()
        
        print( self.p4 )
        print( self.p4.ticket_file )
        
        if password != self.p4.password:
            print( 'logging on ...')
            self.p4.password = password
            self.p4.run_login()
        
        self.superticket = self.p4.password
        self.users = {}
        
        client = self.p4.fetch_client()
        self.root = client._root
        self.p4.cwd = self.root
                
    def after(self):
        print( "Called after")
        self.p4.disconnect()