//private/robey/tools/bzr2p4/bzr2p4.py#1 - add change 140865 (text) #!/usr/bin/env python2.4 import logging import marshal import os import re import subprocess import sys import textwrap import time import StringIO if os.environ.get('BZRPATH', None) is not None: sys.path.append(os.environ['BZRPATH']) try: import bzrlib except ImportError: sys.stderr.write('\n') sys.stderr.write('Can\'t find bzrlib on the python path.\n') sys.stderr.write('Try putting it in either PYTHONPATH or BZRPATH.\n') sys.stderr.write('\n') sys.exit(1) import bzrlib.branch import bzrlib.diff import bzrlib.revision import bzrlib.textfile P4_EXE = 'p4' PATCH_EXE = 'patch' bzr_branch = '/Users/robey/danger/private/robey/imp' last_revid = None log = logging.getLogger('bzr2p4') def log_to(stream, level): h = logging.StreamHandler(stream) log.setLevel(level) h.setLevel(level) h.setFormatter(logging.Formatter('%(levelname)-.3s [%(asctime)s.%(msecs)03d] %(name)s: %(message)s', '%Y%m%d-%H:%M:%S')) log.addHandler(h) def patch(filename, diff): args = [ PATCH_EXE, '-s', '-t', filename ] try: log.debug('%r', args) process = subprocess.Popen(args, bufsize=-1, stdin=subprocess.PIPE, stdout=None, stderr=None, close_fds=True) process.stdin.write(diff) process.stdin.close() ret = process.wait() if ret != 0: raise Exception('patch returned %d' % (ret,)) except Exception, e: log.error('Exception executing (patch) %s: %s', ' '.join(args), e) def p4(*in_args, **kw): """ execute the string cmd as a perforce command, and return the output as an array of hashes. no interpretation of the results is done. any exceptions on popen are reraised. """ ret = [] if kw.get('raw', False): args = [ P4_EXE ] raw = True else: args = [ P4_EXE, '-G' ] raw = False args.extend(list(in_args)) if 'stdin' in kw: stdin_data = kw['stdin'] stdin = subprocess.PIPE else: stdin = None try: log.debug('%r', args) if stdin is not None: log.debug('stdin = %r', stdin_data) process = subprocess.Popen(args, bufsize=-1, stdin=stdin, stdout=subprocess.PIPE, stderr=None, close_fds=True) if stdin is not None: process.stdin.write(stdin_data) process.stdin.close() if raw: ret.append(process.stdout.read()) else: while True: try: ret.append(marshal.load(process.stdout)) except EOFError, e: break process.stdout.close() process.wait() except Exception, e: log.error('Exception executing (p4) %s: %s', ' '.join(args), e) return ret def p4_where(path): out = p4('where', path) return out[0]['data'].split(' ')[0] def make_p4_changelist(path, rev, comment=None): description = rev.message if not isinstance(description, list): description = description.split('\n') if (len(description) > 1) and (description[-1] == ''): description = description[:-1] if len(description) == 1: # robey likes to type really long -m commit lines; wrap them description = textwrap.wrap(description[0], 70) # add special notes: datetime = time.strftime('%d %b %Y %H:%M:%S', time.localtime(rev.timestamp)) description.append('') description.append('# bazaar revision import') description.append('# commit by: %s' % (rev.committer,)) description.append('# date/time: %s' % (datetime,)) description.append('# inventory: %s' % (rev.inventory_sha1,)) if comment is not None: description.append('# ' + comment) query = { 'Change': 'new', 'Description': '\n'.join(description) + '\n' } out = p4('change', '-i', stdin=marshal.dumps(query, 0)) m = re.match(r'Change (\d+) created', out[0]['data']) if m is None: log.debug('Bad response: %r', out) raise Exception('Can\'t parse changelist #.') return int(m.group(1)) def get_revision_path(branch, old_rev_id, new_rev_id): """ get the list of revision ids to traverse from one revision to another. """ base_ancestry = set(branch.repository.get_ancestry(old_rev_id)) return [r for r in branch.repository.get_ancestry(new_rev_id) if r not in base_ancestry] def tree_lines(tree, file_id): if not file_id in tree: return [] tree_file = bzrlib.textfile.text_file(tree.get_file(file_id)) return tree_file.readlines() def get_patch(file_id, old_tree, new_tree, old_path, new_path): """ @raise errors.BinaryFile: if it's not a text file """ old_lines = tree_lines(old_tree, file_id) new_lines = tree_lines(new_tree, file_id) buffer = StringIO.StringIO() bzrlib.diff.internal_diff(old_path, old_lines, new_path, new_lines, buffer) return buffer.getvalue() class DeltaWorker (object): def __init__(self, branch, rev_id): self.branch = branch self.rev_id = rev_id self.target = '.' self.get_delta() def get_delta(self): """ for a specific revision on a branch, get the Delta object describing the changes in that revision, and save copies of the prior and current revision tree. """ self.rev = self.branch.repository.get_revision(self.rev_id) self.rev_tree = self.branch.repository.revision_tree(self.rev_id) if self.rev.parent_ids: base_id = self.rev.parent_ids[-1] else: base_id = bzrlib.revision.NULL_REVISION self.base_tree = self.branch.repository.revision_tree(base_id) self.delta = self.rev_tree.changes_from(self.base_tree, want_unchanged=True, include_root=True) def set_target(self, path): self.target = path def process(self): # perforce can't handle renaming a file AND modifying it at the same # time. so if we have any of those, we need to break the commit into # two stages. self.two_stage = len([r for r in self.delta.renamed if r[4] == True]) > 0 if self.two_stage: self.process_renames_only() self.process_non_renames() else: self.process_all() def process_renames_only(self): self.changelist = make_p4_changelist(self.target, self.rev, comment='(phase 1/2)') for r in self.delta.renames: self.handle_rename(*r, **dict(stage=1)) def process_all(self): self.changelist = make_p4_changelist(self.target, self.rev) for a in self.delta.added: self.handle_add(*a) for r in self.delta.removed: self.handle_remove(*r) for r in self.delta.renamed: self.handle_rename(*r) for m in self.delta.modified: self.handle_modify(*m) # FIXME: commit def handle_add(self, path, id, kind): log.debug('add %r kind %r', path, kind) filename = os.path.join(self.target, path) if kind == 'directory': if not os.path.isdir(filename): os.mkdir(filename) return if kind != 'file': raise Exception('Don\'t know how to add objects of type %r' % (kind,)) f = open(filename, 'w') f.write(''.join(tree_lines(self.rev_tree, id))) f.close() if self.rev_tree.is_executable(id): os.chmod(filename, 0664) p4('add', '-c', str(self.changelist), filename) def handle_remove(self, path, id, kind): log.debug('remove %r kind %r', path, kind) filename = os.path.join(self.target, path) if kind == 'directory': try: os.path.rmdir(filename) except OSError, e: log.debug('error rmdir %r: %s', filename, e) return if kind != 'file': raise Exception('Don\'t know how to remove objects of type %r' % (kind,)) p4('delete', '-c', str(self.changelist), filename) def handle_rename(self, old_path, new_path, id, kind, text_modified, meta_modified, stage=0): log.debug('rename %r => %r, kind %r', old_path, new_path, kind) old_filename = os.path.join(self.target, old_path) new_filename = os.path.join(self.target, new_path) if kind == 'directory': # ignore return if kind != 'file': raise Exception('Don\'t know how to rename objects of type %r' % (kind,)) if (stage == 0) or (stage == 1): # move the file p4('integrate', '-c', str(self.changelist), old_filename, new_filename) p4('delete', '-c', str(self.changelist), old_filename) if (stage == 0) or (stage == 2): self.handle_modify(new_path, id, kind, text_modifed, meta_modified, old_path=old_path) def handle_modify(self, path, id, kind, text_modified, meta_modified, old_path=None): log.debug('modify %r kind %r', path, kind) filename = os.path.join(self.target, path) if kind != 'file': raise Exception('Don\'t know how to modify objects of type %r' % (kind,)) if old_path is None: old_path = path p4('edit', '-c', str(self.changelist), filename) patch(filename, get_patch(id, self.base_tree, self.rev_tree, old_path, path)) log_to(sys.stdout, logging.DEBUG) b = bzrlib.branch.Branch.open(bzr_branch) if last_revid is None: last_revid = bzrlib.revision.NULL_REVISION base_revision = bzrlib.revision.common_ancestor(b.last_revision(), last_revid, b.repository) if base_revision == bzrlib.revision.NULL_REVISION: base_revision = None path = get_revision_path(b, base_revision, b.last_revision()) print len(path) worker = DeltaWorker(b, path[1]) worker.set_target('/Users/robey/danger/private/robey/tools/bzr2p4/test') print worker.process()
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#42 | 7411 | Matt McClure |
Fixes a regression I introduced in the last several changes. bzr: revno 232, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Thu 2009-09-24 18:05:58 -0400 |
||
#41 | 7410 | Matt McClure |
Migrates trailing revision ranges from Bazaar to Perforce. bzr: revno 231, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Thu 2009-09-24 17:27:33 -0400 |
||
#40 | 7409 | Matt McClure |
Start of a scratch fix for migrating a revision range that is not the beginning of a branch. I observed that it doesn't work in trying to use bzr2p4 to migrate a range from this branch to Perforce. bzr: revno 230, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Thu 2009-09-24 13:28:38 -0400 |
||
#39 | 7407 | Matt McClure |
Parameterizes bzr2p4 by a revision range. bzr: revno 228, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Thu 2009-09-24 12:15:49 -0400 |
||
#38 | 7399 | Matt McClure |
Fixes two tests that broke as a result of variable tree root IDs introduced in Bazaar. Gets the root ID from the tree instead of using a hardcoded string. |
||
#37 | 7351 | Matt McClure | Updates bzrp4 to pass tests with Bazaar revno 4615. | ||
#36 | 7149 | Matt McClure | Reduces memory usage so that p4-fast-export can handle a large number of files at a single changelist, and very large individual files. | ||
#35 | 7119 | Matt McClure | Improves the test coverage reporting by importing python modules rather than forking processes. | ||
#34 | 7094 | Matt McClure |
bzr2p4.py migrates changes in the order that 'bzr log' displays them. TODO: I need to update the tests accordingly. bzr: revno 94.1.1, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-12-28 12:09:05 -0500 |
||
#33 | 7092 | Matt McClure |
Converts all source files to UNIX line endings. bzr: revno 88.2.25, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sat 2008-12-27 14:36:39 -0500 |
||
#32 | 7084 | Matt McClure |
Changes the Bazaar comment addendums to prefix all lines with 'bzr: '. bzr: revno 88.2.17, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Fri 2008-12-19 13:34:21 -0500 |
||
#31 | 7081 | Matt McClure |
Appends Bazaar author, date, and revision number to the Perforce changelist. bzr: revno 88.2.14, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Mon 2008-12-15 21:33:24 -0500 |
||
#30 | 7079 | Matt McClure |
Renames methods. bzr: revno 88.2.12, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-12-14 15:51:06 -0500 |
||
#29 | 7078 | Matt McClure |
Adds a 'bzr' note to each change description. bzr: revno 88.2.11, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-12-14 15:25:54 -0500 |
||
#28 | 7075 | Matt McClure |
I'm in the middle of changing the tagline that bzr2p4 adds to the Perforce change description. bzr: revno 88.2.8, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Tue 2008-12-02 09:43:39 -0500 |
||
#27 | 7074 | Matt McClure |
Eliminates deprecated use of None where intent is NULL_REVISION. Updates NEWS and TODO. bzr: revno 88.2.7, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Mon 2008-12-01 21:44:51 -0500 |
||
#26 | 7072 | Matt McClure |
Removes a handful of print/debug statements I had added in revno 8. bzr: revno 88.2.5, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Mon 2008-12-01 21:31:10 -0500 |
||
#25 | 7070 | Matt McClure |
Removes unnecessary text-wrapping code. bzr: revno 88.2.3, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Mon 2008-12-01 21:10:59 -0500 |
||
#24 | 7069 | Matt McClure |
Puts back the "incomplete revision" identifiers that I mistakenly removed in the previous revision. bzr: revno 88.2.2, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Mon 2008-12-01 19:11:27 -0500 |
||
#23 | 7068 | Matt McClure |
Removes Bazaar metadata from Perforce commit descriptions. bzr: revno 88.2.1, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Mon 2008-12-01 18:26:45 -0500 |
||
#22 | 7006 | Matt McClure |
Merges mainline changes into feature branch. bzr: revno 69.2.3, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sat 2008-08-23 13:40:15 -0400 |
||
#21 | 7003 | Matt McClure |
Snapshot. bzr: revno 69.2.1, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-08-10 10:47:26 -0400 |
||
#20 | 6979 | Matt McClure |
Cleans up and refactors a bit. bzr: revno 71.1.6, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-08-17 00:28:26 -0400 |
||
#19 | 6977 | Matt McClure |
Passes dogfood test and the test that isolates the delete/rename case that previously failed. bzr: revno 71.1.4, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sat 2008-08-16 23:18:40 -0400 |
||
#18 | 6976 | Matt McClure |
Work-in-progress: steps toward making the test from revid:mlm@aya.yale.edu-20080813012433-5c2is22zjh3hxcg8 pass. bzr: revno 71.1.3, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sat 2008-08-16 19:35:09 -0400 |
||
#17 | 6975 | Matt McClure |
Extracts methods. bzr: revno 71.1.2, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sat 2008-08-16 18:19:54 -0400 |
||
#16 | 6974 | Matt McClure |
Adds a failing test that isolates a problem I first observed in the bzr2p4 dogfood test. bzr: revno 71.1.1, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Tue 2008-08-12 21:24:33 -0400 |
||
#15 | 6972 | Matt McClure |
Uses two-phase migration from Bazaar to Perforce when a Bazaar revision contains a rename and an add of the renamed file. bzr: revno 69.1.2, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-08-10 21:51:14 -0400 |
||
#14 | 6961 | Matt McClure |
Extracts a method. bzr: revno 61, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-08-03 14:58:54 -0400 |
||
#13 | 6960 | Matt McClure |
Migrates branches with merge revisions, so that test_dogfood passes. bzr: revno 60, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-08-03 14:46:27 -0400 |
||
#12 | 6953 | Matt McClure |
TestDogFood works from a Cygwin shell, assuming you're using Cygwin p4. bzr: revno 54, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sat 2008-08-02 11:24:59 -0400 |
||
#11 | 6920 | Matt McClure |
Writes the entire revision contents instead of using patch. The dog food test almost passes now. It fails because Perforce has made all files read-only in the client directory before the test tear down attempts to remove the directory tree. bzr: revno 21, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-06-29 21:05:41 -0400 |
||
#10 | 6919 | Matt McClure |
I'm in the middle of implementing a "dog food" test that syncs bzr2p4 to Perforce. It doesn't work yet. Patch fails to apply some hunks. I think the trouble is CRLF line endings. I tried giving the --binary option to patch, and that made it fail on an earlier revision containing a Perforce keyword expansion. I'm considering abandoning the use of patch even before getting the dog food test to pass. I could bzr checkout or bzr cat to the staging directory instead of relying on patch to apply the diff. bzr: revno 20, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Sun 2008-06-29 19:57:41 -0400 |
||
#9 | 6917 | Matt McClure |
Documents the environment I need to make bzr2p4.py run. Uses default python instead of hardcoding to 2.4. bzr: revno 18, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Fri 2008-06-27 17:57:23 -0400 |
||
#8 | 6907 | Matt McClure |
Works from Cygwin shell using a shell script that wraps p4. bzr: revno 8, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Thu 2008-05-01 10:53:37 -0400 |
||
#7 | 6906 | Matt McClure |
Uses Graph.find_unique_lca instead of deprecated Revision.common_ancestor. Locks branch for read. bzr: revno 7, part 1 of 1 bzr: author Matt McClure <mlm@aya.yale.edu> bzr: committed Mon 2008-04-21 11:42:23 -0400 |
||
#6 | 6905 | Matt McClure |
add the gpl. bzr: revno 6, part 1 of 1 bzr: author Robey Pointer <robey@lag.net> bzr: committed Sat 2006-12-30 15:26:54 -0800 |
||
#5 | 6904 | Matt McClure |
remove silly p4 lines and add a check so an error is displayed if no command-line params are given and there is no saved config. bzr: revno 5, part 1 of 1 bzr: author Robey Pointer <robey@lag.net> bzr: committed Sat 2006-12-30 15:23:38 -0800 |
||
#4 | 6903 | Matt McClure |
final small tweaks necessary to get my own 97-revision branch imported. bzr: revno 4, part 1 of 1 bzr: author Robey Pointer <robey@lag.net> bzr: committed Sat 2006-12-30 15:17:31 -0800 |
||
#3 | 6902 | Matt McClure |
one more try... from 2006-11-22. bzr: revno 3, part 1 of 1 bzr: author Robey Pointer <robey@lag.net> bzr: committed Sat 2006-12-30 15:16:37 -0800 |
||
#2 | 6901 | Matt McClure |
2nd draft from 2006-11-22. bzr: revno 2, part 1 of 1 bzr: author Robey Pointer <robey@lag.net> bzr: committed Sat 2006-12-30 15:15:48 -0800 |
||
#1 | 6900 | Matt McClure |
original draft from 2006-11-02. bzr: revno 1, part 1 of 1 bzr: author Robey Pointer <robey@lag.net> bzr: committed Sat 2006-12-30 15:14:48 -0800 |