# Perforce source for convert extension. # # Copyright 2009, Frank Kingswood # # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. from mercurial import util from mercurial.i18n import _ from common import commit, converter_source, checktool, NoRepo import marshal import re def loaditer(f): "Yield the dictionary objects generated by p4" try: while True: d = marshal.load(f) if not d: break yield d except EOFError: pass def decodefilename(filename): """Perforce escapes special characters @, #, *, or % with %40, %23, %2A, or %25 respectively >>> decodefilename('portable-net45%252Bnetcore45%252Bwp8%252BMonoAndroid') 'portable-net45%2Bnetcore45%2Bwp8%2BMonoAndroid' >>> decodefilename('//Depot/Directory/%2525/%2523/%23%40.%2A') '//Depot/Directory/%25/%23/#@.*' """ replacements = [('%2A', '*'), ('%23', '#'), ('%40', '@'), ('%25', '%')] for k, v in replacements: filename = filename.replace(k, v) return filename class p4_source(converter_source): def __init__(self, ui, path, revs=None): # avoid import cycle import convcmd super(p4_source, self).__init__(ui, path, revs=revs) if "/" in path and not path.startswith('//'): raise NoRepo(_('%s does not look like a P4 repository') % path) checktool('p4', abort=False) self.p4changes = {} self.heads = {} self.changeset = {} self.files = {} self.copies = {} self.tags = {} self.lastbranch = {} self.parent = {} self.encoding = self.ui.config('convert', 'p4.encoding', default=convcmd.orig_encoding) self.depotname = {} # mapping from local name to depot name self.localname = {} # mapping from depot name to local name self.re_type = re.compile( "([a-z]+)?(text|binary|symlink|apple|resource|unicode|utf\d+)" "(\+\w+)?$") self.re_keywords = re.compile( r"\$(Id|Header|Date|DateTime|Change|File|Revision|Author)" r":[^$\n]*\$") self.re_keywords_old = re.compile("\$(Id|Header):[^$\n]*\$") if revs and len(revs) > 1: raise util.Abort(_("p4 source does not support specifying " "multiple revisions")) self._parse(ui, path) def _parse_view(self, path): "Read changes affecting the path" cmd = 'p4 -G changes -s submitted %s' % util.shellquote(path) stdout = util.popen(cmd, mode='rb') for d in loaditer(stdout): c = d.get("change", None) if c: self.p4changes[c] = True def _parse(self, ui, path): "Prepare list of P4 filenames and revisions to import" ui.status(_('reading p4 views\n')) # read client spec or view if "/" in path: self._parse_view(path) if path.startswith("//") and path.endswith("/..."): views = {path[:-3]:""} else: views = {"//": ""} else: cmd = 'p4 -G client -o %s' % util.shellquote(path) clientspec = marshal.load(util.popen(cmd, mode='rb')) views = {} for client in clientspec: if client.startswith("View"): sview, cview = clientspec[client].split() self._parse_view(sview) if sview.endswith("...") and cview.endswith("..."): sview = sview[:-3] cview = cview[:-3] cview = cview[2:] cview = cview[cview.find("/") + 1:] views[sview] = cview # list of changes that affect our source files self.p4changes = self.p4changes.keys() self.p4changes.sort(key=int) # list with depot pathnames, longest first vieworder = views.keys() vieworder.sort(key=len, reverse=True) # handle revision limiting startrev = self.ui.config('convert', 'p4.startrev', default=0) self.p4changes = [x for x in self.p4changes if ((not startrev or int(x) >= int(startrev)) and (not self.revs or int(x) <= int(self.revs[0])))] # now read the full changelists to get the list of file revisions ui.status(_('collecting p4 changelists\n')) lastid = None for change in self.p4changes: cmd = "p4 -G describe -s %s" % change stdout = util.popen(cmd, mode='rb') d = marshal.load(stdout) desc = self.recode(d.get("desc", "")) shortdesc = desc.split("\n", 1)[0] t = '%s %s' % (d["change"], repr(shortdesc)[1:-1]) ui.status(util.ellipsis(t, 80) + '\n') if lastid: parents = [lastid] else: parents = [] date = (int(d["time"]), 0) # timezone not set c = commit(author=self.recode(d["user"]), date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'), parents=parents, desc=desc, branch=None, extra={"p4": change}) files = [] copies = {} copiedfiles = [] i = 0 while ("depotFile%d" % i) in d and ("rev%d" % i) in d: oldname = d["depotFile%d" % i] filename = None for v in vieworder: if oldname.lower().startswith(v.lower()): filename = decodefilename(views[v] + oldname[len(v):]) break if filename: files.append((filename, d["rev%d" % i])) self.depotname[filename] = oldname if (d.get("action%d" % i) == "move/add"): copiedfiles.append(filename) self.localname[oldname] = filename i += 1 # Collect information about copied files for filename in copiedfiles: oldname = self.depotname[filename] flcmd = 'p4 -G filelog %s' \ % util.shellquote(oldname) flstdout = util.popen(flcmd, mode='rb') copiedfilename = None for d in loaditer(flstdout): copiedoldname = None i = 0 while ("change%d" % i) in d: if (d["change%d" % i] == change and d["action%d" % i] == "move/add"): j = 0 while ("file%d,%d" % (i, j)) in d: if d["how%d,%d" % (i, j)] == "moved from": copiedoldname = d["file%d,%d" % (i, j)] break j += 1 i += 1 if copiedoldname and copiedoldname in self.localname: copiedfilename = self.localname[copiedoldname] break if copiedfilename: copies[filename] = copiedfilename else: ui.warn(_("cannot find source for copied file: %s@%s\n") % (filename, change)) self.changeset[change] = c self.files[change] = files self.copies[change] = copies lastid = change if lastid: self.heads = [lastid] def getheads(self): return self.heads def getfile(self, name, rev): cmd = 'p4 -G print %s' \ % util.shellquote("%s#%s" % (self.depotname[name], rev)) lasterror = None while True: stdout = util.popen(cmd, mode='rb') mode = None contents = [] keywords = None for d in loaditer(stdout): code = d["code"] data = d.get("data") if code == "error": # if this is the first time error happened # re-attempt getting the file if not lasterror: lasterror = IOError(d["generic"], data) # this will exit inner-most for-loop break else: raise lasterror elif code == "stat": action = d.get("action") if action in ["purge", "delete", "move/delete"]: return None, None p4type = self.re_type.match(d["type"]) if p4type: mode = "" flags = ((p4type.group(1) or "") + (p4type.group(3) or "")) if "x" in flags: mode = "x" if p4type.group(2) == "symlink": mode = "l" if "ko" in flags: keywords = self.re_keywords_old elif "k" in flags: keywords = self.re_keywords elif code == "text" or code == "binary": contents.append(data) lasterror = None if not lasterror: break if mode is None: return None, None contents = ''.join(contents) if keywords: contents = keywords.sub("$\\1$", contents) if mode == "l" and contents.endswith("\n"): contents = contents[:-1] return contents, mode def getchanges(self, rev, full): if full: raise util.Abort(_("convert from p4 do not support --full")) return self.files[rev], self.copies[rev], set() def getcommit(self, rev): return self.changeset[rev] def gettags(self): return self.tags def getchangedfiles(self, rev, i): return sorted([x[0] for x in self.files[rev]]) class p4_sink(converter_sink): def __init__(self, ui, path): converter_sink.__init__(self, ui, path) self.p4 = P4.P4() self.repo = hg.repository(ui, path.replace('-hg', '')) #ui.pushbuffer() #commands.log(self.ui, self.repo, date=None, rev=None, user=None) #print(ui.popbuffer()) def getheads(self): print( "Called getheads()") return [] def revmapfile(self): print( "Called revmapfile()") return "revmapfile" def authorfile(self): print( "Called authorfile()") return "authorfile" def join(self, branch, file): return os.path.join(self.root, branch, file) def fixFilename( self, name ): result = name.replace('%', '%25') result = result.replace('@', '%40') result = result.replace('#', '%23') result = result.replace('*', '%2A') return result def fixBranchname( self, name ): result = name.replace('%', '_PERCENT_') result = result.replace('@', '_AT_') result = result.replace('#', '_HASH_') result = result.replace('*', '_STAR_') result = result.replace(' ', '_') return result def putfile( self, file, data, mode ): # find the directories first targetPath,basename = os.path.split(file) if not os.path.isdir( targetPath ): os.makedirs( targetPath ) with open(file, 'w') as f: f.write(data) # ignore mode for now def findMergedFiles(self, search, sortkey): self.ui.pushbuffer() commands.diff(self.ui, self.repo, rev=[str(search), str(sortkey)], change=None, stat=True, reverse=None) result = self.ui.popbuffer() print(result) resultLines = result.split('\n')[:-2] # cut off the last two lines containing summary and a blank line files = [ line.split('|')[0].strip() for line in resultLines ] return files def putcommit(self, files, copies, parents, commit, source, revmap): print( "Called putcommit() with") print( "*** files", [x[0] for x in files] ) print( "*** copies", copies ) print( "*** parents", parents ) print( "*** commit.author", commit.author ) print( "*** commit.date", commit.date ) print( "*** commit.desc", commit.desc ) print( "*** commit.parents", commit.parents ) print( "*** commit.branch", commit.branch ) # print( "*** commit.rev", commit.rev ) print( "*** commit.extra", commit.extra ) print( "*** commit.sortkey", commit.sortkey ) edited = [] added = [] deleted = [] copied = [] copySource = [] renamed = [] copyData = {} change = '' revs = set() date = commit.date[:-6] # cut the timezone off, Python2.X cannot deal with it yet timestamp = datetime.strptime(date, '%a %b %d %H:%M:%S %Y').strftime('%Y/%m/%d %H:%M:%S') if commit.author in self.users: self.p4.user = commit.author self.p4.password = self.users[commit.author] else: self.p4.user = self.superuser self.p4.password = self.superticket r = self.p4.run_login('-p', commit.author) self.p4.user = commit.author # users are mapped via authorfile to legit Perforce users self.p4.password = r[0] self.users[self.p4.user] = self.p4.password # now create branches if necessary cbranch = self.fixBranchname(commit.branch) branch = self.fixBranchname(self.next_branch) pbranches = [ (x[0], self.fixBranchname(x[1])) for x in self.next_pbranches ] if not pbranches: pass elif len(pbranches) == 1: change, parent = pbranches[0] if parent != branch: print("creating branch %s from %s @ %s" % (branch, parent, change)) with self.p4.at_exception_level(1): if change and change != '0': self.p4.run_integrate('%s/...@%s' % (parent, change), '%s/...' % branch) else: self.p4.run_integrate('%s/...' % parent, '%s/...' % branch) print( "after integrate; warnings : ", self.p4.warnings ) # might need a resolve here, just in case the target exists self.p4.run_resolve('-at') change = self.p4.fetch_change() if 'Files' in change: change._description = 'Created branch %s from %s @ %s' % (branch, parent, change) submitted = self.p4.run_submit(change) # remember the change if this is just a branch with no edited changes. # Otherwise the next submit will override this number for s in reversed( submitted ): if 'submittedChange' in s: change = s['submittedChange'] # the last entry in the array contains the change number break else: print("Nothing to integrate. Maybe a rebase?") else: # in some cases Mercurial does not tell us all the files that have changed # if the change originates from a push or pull between repos # find the missing files here searchRev = commit.parents[0] mergedFiles = self.findMergedFiles(searchRev, commit.sortkey) officiallyEditedFiles = [ x[0] for x in files ] for m in mergedFiles: if m not in officiallyEditedFiles: files.append( (m, str(commit.sortkey)) ) print files elif len(pbranches) == 2: targetChange, tgt = pbranches[0] sourceChange, src = pbranches[1] if tgt == branch and src == branch: # merge between two repositories in the same branch mergedFiles = [] for search in commit.parents: mergedFiles += self.findMergedFiles(search, commit.sortkey) # add all the merged files to the list of files officiallyEditedFiles = [ x[0] for x in files ] for m in mergedFiles: if m not in officiallyEditedFiles: files.append( (m, str(commit.sortkey)) ) print files elif tgt == branch: print("merging to %s@%s with %s@%s" % (tgt, targetChange, src, sourceChange) ) if sourceChange: with self.p4.at_exception_level(1): for f,v in files: copied = self.p4.run_copy('%s/%s@%s' % (src, f, sourceChange), '%s/%s' % (tgt, f) ) else: with self.p4.at_exception_level(1): for f,v in files: copied = self.p4.run_copy('%s/%s' % (src, f), '%s/%s' % (tgt,f) ) elif src == branch: print("merging from %s@%s with %s@%s" % (tgt, targetChange, src, sourceChange)) if targetChange: with self.p4.at_exception_level(1): for f,v in files: self.p4.run_copy('%s/%s@%s' % (tgt, f, targetChange), '%s/%s' % (src,f) ) else: with self.p4.at_exception_level(1): for f,v in files: self.p4.run_copy('%s/%s' % (tgt,f), '%s/%s' % (src,f) ) else: print("Serious trouble in setbranch: parent branches do not match target branch") else: print( "Serious problem in setbranches: more than 2 parent branches ???") executables = [] for f, v in files: target = self.join(cbranch, f) fixedTarget = self.fixFilename(target) revs.add(v) try: data, mode = source.getfile(f, v) if 'x' in mode: executables.append(fixedTarget) except IOError: deleted.append(fixedTarget) else: print("* %s mode: %s" % (f, mode)) if f in copies: print("*** file %s is a copy from %s" % (f, copies[f])) s = self.fixFilename( self.join(cbranch, copies[f]) ) copied.append( (fixedTarget, s) ) copySource.append( (s, fixedTarget) ) copyData[fixedTarget] = (data, mode) else: with self.p4.at_exception_level(1): fileinfo = self.p4.run_fstat(fixedTarget) # warning if file does not exist print( fileinfo, self.p4.warnings, self.p4.errors ) if (len(self.p4.warnings) > 0) \ or ('headAction' in fileinfo[0] and 'delete' in fileinfo[0]['headAction'] ) \ or ('action' in fileinfo[0] and 'branch' in fileinfo[0]['action'] ): added.append(target) print("*a %s" % target) else: self.p4.run_sync(fixedTarget) # ignore warning that file already synced edited.append(fixedTarget) print("*e %s" % target) self.putfile(target, data, mode) # time to do some processing print( "Total added ", len(added)) print( "Total edited ", len(edited)) if len(added) > 0 : self.p4.run_add('-f', added) if len(edited) > 0 : self.p4.run_edit(edited) # if a file was the source of a copy and deleted, it was a rename trueCopies = [] for i in copySource: if i[0] in deleted: renamed.append( i ) deleted.remove(i[0]) else: trueCopies.append( i ) # now process renames, copies and deletes if len(deleted) > 0 : with self.p4.at_exception_level(1): self.p4.run_delete('-v', deleted) print("warnings after delete: ", self.p4.warnings) with self.p4.at_exception_level(1): for i in renamed: print("moving %s to %s" % (i[0], i[1])) self.p4.run_sync('-f', i[0], i[1]) # in case we are renaming into an existing target fileinfo = self.p4.run_fstat(fixedTarget) # warning if file does not exist if len(self.p4.warnings) > 0: # target does not exist yet ==> move it self.p4.run_edit(i[0]) self.p4.run_move('-k', i[0], i[1]) # file content is already in workspace print("warnings after move: ", self.p4.warnings) else: # downgrade the move to an integrate and delete, it might have been a # forced rename in Mercurial self.p4.run_integrate(i[0], i[1]) self.p4.run_resolve('-at', i[1]) self.p4.run_edit(i[1]) self.p4.run_delete(i[0]) (data, mode) = copyData[i[1]] self.putfile(i[1], data, mode) # need to overwrite the file, in case it is a dirty edit for i in trueCopies: print(">>> Trying to copy %s to %s" % (i[0], i[1])) with self.p4.at_exception_level(0): self.p4.run_copy(i[0], i[1]) if self.p4.errors or self.p4.warnings: print("Copy errors ", self.p4.errors, " warnings ", self.p4.warnings) with self.p4.at_exception_level(1): self.p4.run_integrate(i[0], i[1]) self.p4.run_resolve('-at') self.p4.run_sync(i[1]) # if copy and integrate say the file is already integrated without syncing it if self.p4.warnings: print(">>>Sync in trueCopies warning: ", self.p4.warnings) self.p4.run_edit(i[1]) if self.p4.warnings: print(">>>Edit in trueCopies warning, downgrading to add: ", self.p4.warnings) self.p4.run_add(i[1]) if self.p4.warnings: print(">>>Add in trueCopies warning: ", self.p4.warnings) (data, mode) = copyData[i[1]] self.putfile(i[1], data, mode) # need to overwrite the file, in case it is a dirty edit # if we have any files of type +x, change the type here if len(executables) > 0: for x in self.p4.run_opened(): print('open files: ', x) print('changing file type to +x for %s' % str(executables)) self.p4.run_reopen('-t+x', executables) # time to submit the lot l = len( self.p4.run_opened() ) if l > 0: # the first entry in OpenOffice_DEV300 is empty print("*** Open files : ", l) description = 'Comment: ' + commit.desc description += '\nDate: ' + commit.date description += '\nBranch: ' + cbranch description += '\nUser: ' + commit.author description += '\nRev:' for i in revs: # not sure if there can be more than one. Better be safe description += ' ' + i description += '\nParents: ' + str(commit.parents) description += '\nSortKey: ' + str(commit.sortkey) ch = self.p4.fetch_change() ch._description = description with self.p4.at_exception_level(0): submitted = self.p4.run_submit(ch, '-frevertunchanged') print( submitted ) if self.p4.errors: print( "*** putcommit(): nothing to submit. Errors : ", self.p4.errors, " Warnings: ", self.p4.warnings ) self.p4.delete_change(submitted[0]['change']) if self.p4.errors or self.p4.warnings: print( "delete change had problems. Errors =:", self.p4.errors, " Warnings : ", self.p4.warnings) return '0' else: for s in reversed( submitted ): if 'submittedChange' in s: change = s['submittedChange'] # the last entry in the array contains the change number break # fixing the timestamp to resemble original timestamp from Mercurial # need to be super user to do that if change: self.p4.user = self.superuser self.p4.password = self.superticket ch = self.p4.fetch_change(change) ch._date = timestamp self.p4.save_change(ch, '-f') print( "*** End of putcommit(%s) ***" % change) return change def puttags(self, tags): print( "Called puttags() with") print( "***", tags ) with self.p4.at_exception_level(1): for name, change in tags.items(): self.p4.run_tag('-l'+name, '//%s/...@%s' % (self.p4.client, change)) if self.p4.warnings: print( '>>>', self.p4.warnings) print( "*** End of puttags() ***") return None, None def setbranch(self, branch, pbranches): print( "Called setbranch() with") print( "***", branch ) print( "***", pbranches ) self.next_branch = branch self.next_pbranches = pbranches print( "*** End of setbranch() ***") def setfilemapmode(self, active): print( "Called setfilemapmode() with") print( "***", active ) print( "*** End of setfilemapmode() ***") def before(self): print( "Called before") self.p4.port = self.ui.config('convert', 'p4.port', default=self.p4.port) self.p4.client = self.ui.config('convert', 'p4.client', default=self.p4.client) self.p4.user = self.ui.config('convert', 'p4.user', default=self.p4.user) password = self.ui.config('convert', 'p4.password', default=self.p4.password) self.superuser = self.p4.user self.p4.connect() print( self.p4 ) print( self.p4.ticket_file ) if password != self.p4.password: print( 'logging on ...') self.p4.password = password self.p4.run_login() self.superticket = self.p4.password self.users = {} client = self.p4.fetch_client() self.root = client._root self.p4.cwd = self.root def after(self): print( "Called after") self.p4.disconnect()