# Perforce Defect Tracking Integration Project
#
#
# P4I.PY -- PYTHON INTERFACE TO PERFORCE
#
# Gareth Rees, Ravenbrook Limited, 2000-09-25
# Robert Cowham, Vaccaperna Systems Ltd, 2005-08-08
#
#
# 1. INTRODUCTION
#
# This module defines the 'p4' class, which provides an interface to
# Perforce.
#
# It has been changed to use P4Python rather than p4 command line with -G option
#
# The intended readership of this document is project developers.
#
# This document is not confidential.
#
#
# 1.1. Using the p4 class
#
# To use this class, create an instance, passing appropriate parameters
# if necessary (if parameters are missing, the interface doesn't supply
# values for them, so Perforce will pick up its normal defaults from
# environment variables).
#
# import p4i
# p4i = p4i.p4(port = 'perforce:1666', user = 'root')
#
# The 'run' method takes a Perforce command and returns a list of
# dictionaries; for example:
#
# >>> for c in p4i.run('changes -m 2'):
# ... print c['change'], c['desc']
# ...
# 10021 Explaining how to use the autom
# 10020 Archiving new mail
#
# To pass information to Perforce, supply a dictionary as the second
# argument, for example:
#
# >>> job = p4i.run('job -o job000001')[0]
# >>> job['Title'] = string.replace(job['Title'], 'p4dti', 'P4DTI')
# >>> p4i.run('job -i', job)
# [{'code': 'info', 'data': 'Job job000001 saved.', 'level': 0}]
#
# Note the [0] at the end of line 1 of the above example: the run()
# method always returns a list, even of 1 element. This point is easy
# to forget.
import catalog
import marshal
import os
import re
import string
import tempfile
import types
import portable
import p4dti_exceptions
from p4 import P4
error = 'Perforce error'
# 2. THE P4 CLASS
class p4:
client = None
client_executable = None
logger = None
password = None
port = None
user = None
config_file = None
# 2.1. Create an instance
#
# We supply a default value for the client_executable parameter, but
# for no others; Perforce will use its own default values if these
# are not supplied. If logger is None then no messages will be
# logged.
#
# We check that the server and client are recent enough to support
# various options required for the operation of the P4DTI. See
# the method check_changelevels.
def __init__(self, client = None, client_executable = 'p4',
logger = None, password = None, port = None,
user = None, config_file = None):
self.client = client
self.client_executable = client_executable
self.logger = logger
self.password = password
self.port = port
self.user = user
self.config_file = config_file
self.p4p = P4()
if self.port:
self.p4p.port = self.port
if self.user:
self.p4p.user = self.user
if self.client:
self.p4p.client = self.client
if self.password:
self.p4p.password = self.password
self.p4p.parse_forms()
self.p4p.exception_level = 1
self._connect()
# If config_file is specified then create and fill
# config file.
# In future we might check whether the config file
# already exists.
if self.config_file:
f = open(self.config_file, 'w')
portable.protect_file(self.config_file)
f.write("P4PASSWD="+self.password+"\n")
f.close()
# Set P4CONFIG environment variable so that the p4 command
# picks up the config file. Relies on putenv() being
# implemented, which it is on any POSIX system, and Windows.
os.environ["P4CONFIG"]=self.config_file
# discover and check the client and server changelevels.
self.check_changelevels()
# 2.2. Write a message to the log
#
# But only if a logger was supplied.
def log(self, id, args = ()):
if self.logger:
msg = catalog.msg(id, args)
self.logger.log(msg)
# Connect to server
def _connect(self):
if self.p4p.connected:
try:
self.p4p.disconnect()
except:
pass
try:
self.p4p.connect()
except:
raise p4dti_exceptions.P4ServerDown(catalog.msg(738))
# 2.3. Run a Perforce command
#
# run(arguments, input): Run the Perforce client with the given
# command-line arguments, passing the dictionary 'input' to the
# client's standard input.
#
# The arguments should be a Perforce command and its arguments, like
# "jobs -o //foo/...". Options should generally include -i or -o to
# avoid forms being put up interactively.
#
# Return a list of dictionaries containing the output of the
# Perforce command. (Each dictionary contains one Perforce entity,
# so "job -o" will return a list of one element, but "jobs -o" will
# return a list of many elements.)
def run(self, arguments, input = None):
assert isinstance(arguments, types.StringType)
assert input is None or isinstance(input, types.DictType) or isinstance(input, types.StringType)
if self.p4p.dropped():
self._connect()
# Pass the input dictionary (if any) to Perforce.
if input:
# "Perforce input: '%s'."
self.log(700, input)
self.p4p.input = input
command = string.split(arguments, ' ')
# "Perforce command: '%s'."
self.log(701, command)
try:
results = self.p4p.run(command)
except:
errors = string.join(self.p4p.errors, '\n')
# "Perforce command: '%s'."
self.log(734, command)
# "Perforce status: '%s'."
self.log(735, errors)
# "The Perforce client exited with errors %s."
raise error, catalog.msg(736, errors)
# "Perforce results: '%s'."
self.log(703, results)
if self.p4p.dropped():
# "The connection to the Perforce server has gone down."
raise p4dti_exceptions.P4ServerDown(catalog.msg(737))
return results
# 2.4. Does the Perforce server support a feature?
#
# supports(feature) returns 1 if the Perforce server has the
# feature, 0 if it does not. You can interrogate the following
# features:
#
# fix_update Does Perforce update 'always' fields in a job when it
# is changed using the 'fix' command?
# p4dti Is the Perforce version supported by the P4DTI?
def supports(self, feature):
if feature == 'p4dti':
return self.server_changelevel >= 18974
elif feature == 'fix_update':
return self.server_changelevel >= 29455
else:
return 0
# 2.5. Check the Perforce server changelevels.
#
# We check that the Perforce server is recent enough
# to support various operations required by the P4DTI, and store
# the client and server changelevels in the p4 object for other
# subsequent checks (for example, those made by the 'supports'
# function above).
#
# We don't need to check client level since P4Python is only valid
# with clients of 2004.2 and later and is thus fine.
#
# We check that the Perforce server named by the port parameter is
# recent enough that it supports p4 -G jobspec -i.
#
# We get the server changelevel by running "p4 info" and parsing
# the output (because the output format of "p4 -G info" is
# different in Perforce 2003.2beta from previous Perforce
# releases, and may change again in future). It should contain a
# line which looks like "Server version: P4D/FREEBSD4/2002.2/40318
# (2003/01/17)" In this example, the changelevel is 40318. If no
# line looks like this, then raise an error anyway (this makes the
# module fragile if Perforce change the format of the output of
# "p4 info".
#
# Note that for "p4 info" we do not need the user, the client, or
# the password.
def check_changelevels(self):
# now server changelevel.
self.server_changelevel = 0
p4p = P4()
if self.port:
p4p.port = self.port
try:
p4p.connect()
results = p4p.run("info")
except:
errors = string.join(p4p.errors, '\n')
# "The Perforce client exited with errors %s."
raise error, catalog.msg(736, errors)
p4p.disconnect()
for result in results:
match = re.search('Server version: '
'[^/]+/[^/]+/[^/]+/([0-9]+)', result)
if match:
self.server_changelevel = int(match.group(1))
if not self.supports('p4dti'):
# "The Perforce server changelevel %d is not supported by
# the P4DTI. See the P4DTI release notes for Perforce
# server versions supported by the P4DTI."
raise error, catalog.msg(834, self.server_changelevel)
# 3. HANDLING JOBSPECS
#
# Jobspecs passed to or from Perforce ("p4 jobspec -i"
# or "p4 jobspec -o") look like this:
#
# { 'Comments': '# Form comments...',
# 'Fields': ['101 Job word 32 required',
# '102 State select 32 required']
# 'Values': ['', '_new/assigned/closed/verified/deferred']
# 'Presets': ['', '_new']
# ...
# }
#
# Jobspec structures in the rest of the P4DTI look like this
# [GDR 2000-10-16, 8.4]:
#
# ('# A Perforce Job Specification.\n'
# ...,
# [(101, 'Job', 'word', 32, 'required', None, None, None, None),
# (102, 'Status', 'select', 10, 'required', 'open', 'open/suspended/closed/duplicate', None, None),
# ...])
#
# The elements in each tuple being:
#
# 0: number;
# 1: name;
# 2: "datatype" (word/text/line/select/date);
# 3: length (note: relates to GUI display only);
# 4: "persistence" (optional/default/required/once/always);
# 5: default, or None;
# 6: possible values for select fields, as /-delimited string, or None;
# 7: string describing the field (for the jobspec comment), or None;
# 8: a translator object (not used in this module) or None).
#
# The comment is not parsed on reading the jobspec, but is
# constructed (from the per-field comments) when writing it.
# 3.1. Jobspec Utilities
#
# compare_field_by_number: this is a function for passing to
# sort() which allows us to sort jobspec field descriptions based
# on the field number.
def compare_field_by_number(self, x, y):
if x[0] < y[0]:
return -1
elif x[0] > y[0]:
return 1
else:
# "Jobspec fields '%s' and '%s' have the same
# number %d."
raise error, catalog.msg(710, (x[1], y[1], x[0]))
# jobspec_attribute_names[i] is the name of attribute i in a
# jobspec representation tuple. Used for generating messages
# about jobspecs.
jobspec_attribute_names = [
'code',
'name',
'datatype',
'length',
'fieldtype',
'preset',
'values',
'comment',
'translator', # not really needed
]
# jobspec_map builds a map from a jobspec, mapping one of the
# tuple elements (e.g. number, name) to the whole tuple.
def jobspec_map(self, jobspec, index):
map = {}
comment, fields = jobspec
for field in fields:
map[field[index]] = field
return map
# 3.2. Install a new jobspec
def install_jobspec(self, description):
comment, fields = description
assert isinstance(fields, types.ListType)
# "Installing jobspec from comment '%s' and fields %s."
self.log(712, (comment, fields))
for field in fields:
assert isinstance(field, types.TupleType)
assert len(field) >= 8
def make_comment(field):
if field[7] == None:
return ""
else:
return "# %s: %s\n" % (field[1], field[7])
# we will need the jobspec as a dictionary in order to
# give it to Perforce.
jobspec_dict = self.run("jobspec -o")[0]
fields.sort(self.compare_field_by_number)
jobspec_dict['Fields'] = []
for field in fields:
jobspec_dict['Fields'].append("%s %s %s %s %s"
% field[0:5])
jobspec_dict['Values'] = []
for field in fields:
if field[6] != None:
jobspec_dict['Values'].append("%s %s" % (field[1],
field[6]))
jobspec_dict['Presets'] = []
for field in fields:
if field[5] != None:
jobspec_dict['Presets'].append("%s %s" % (field[1],
field[5]))
jobspec_dict['Comments'] = (comment +
string.join(map(make_comment,
fields),
""))
self.run('jobspec -i', jobspec_dict)
# 3.3. Get the jobspec.
#
# Get the jobspec and convert it into P4DTI representation.
#
# Does very little checking on the output of 'jobspec -o'.
# Ought to validate it much more thoroughly than this.
def get_jobspec(self):
jobspec_dict = self.run('jobspec -o')[0]
fields = []
fields_dict = {}
comment = ""
for v in jobspec_dict['Fields']:
words = string.split(v)
name = words[1]
if not fields_dict.has_key(name):
fields_dict[name] = {}
fields_dict[name]['code'] = int(words[0])
fields_dict[name]['datatype'] = words[2]
fields_dict[name]['length'] = int(words[3])
fields_dict[name]['disposition'] = words[4]
for v in jobspec_dict['Presets']:
space = string.find(v,' ')
name = v[0:space]
preset = v[space+1:]
if not fields_dict.has_key(name):
fields_dict[name] = {}
fields_dict[name]['preset'] = preset
for v in jobspec_dict['Values']:
space = string.find(v,' ')
name = v[0:space]
values = v[space+1:]
if not fields_dict.has_key(name):
fields_dict[name] = {}
fields_dict[name]['values'] = values
comment = jobspec_dict['Comments']
for k,v in fields_dict.items():
fields.append((v['code'],
k,
v['datatype'],
v['length'],
v['disposition'],
v.get('preset', None),
v.get('values', None),
None,
None))
fields.sort(self.compare_field_by_number)
# "Decoded jobspec as comment '%s' and fields %s."
self.log(711, (comment, fields))
return comment, fields
# 3.4. Extending the current jobspec.
#
# extend_jobspec adds the given fields to the current jobspec if
# not already present.
def extend_jobspec(self, description, force = 0):
current_jobspec = self.get_jobspec()
comment, field_list = current_jobspec
_, new_fields = description
new_fields.sort(self.compare_field_by_number)
current_fields = self.jobspec_map(current_jobspec, 1)
new_field_names = map(lambda x: x[1], new_fields)
field_numbers = map(lambda x: x[0], field_list)
# counters for finding a free field number.
free_number_p4dti = 194
free_number = 106
for field_spec in new_fields:
field = field_spec[1]
if current_fields.has_key(field):
current_spec = current_fields[field]
if (current_spec[2] != field_spec[2] or
current_spec[3] != field_spec[3] or
current_spec[4] != field_spec[4] or
current_spec[5] != field_spec[5] or
current_spec[6] != field_spec[6]):
if force:
# "Forcing replacement of field '%s' in jobspec."
self.log(727, field)
current_fields[field] = ((current_spec[0],) +
field_spec[1:7] +
(None,None,))
else:
# "Retaining field '%s' in jobspec despite change."
self.log(728, field)
else:
# "No change to field '%s' in jobspec."
self.log(733, field)
else:
if field_spec[0] in field_numbers:
# Field numbering clashes; find a free field number.
if field[0:6] == 'P4DTI-':
while free_number_p4dti in field_numbers:
free_number_p4dti = free_number_p4dti - 1
number = free_number_p4dti
else:
while free_number in field_numbers:
free_number = free_number + 1
number = free_number
if free_number >= free_number_p4dti:
# "Too many fields in jobspec."
raise error, catalog.msg(730)
field_spec = (number, ) + field_spec[1:]
# "Adding field '%s' to jobspec."
self.log(729, field)
current_fields[field] = field_spec
field_numbers.append(field_spec[0])
# Also report jobspec names fields not touched.
for field in current_fields.keys():
if field not in new_field_names:
# "Retaining unknown field '%s' in jobspec."
self.log(732, field)
self.install_jobspec((comment, current_fields.values()))
# 3.5. Jobspec validation.
#
# jobspec_has_p4dti_fields: Does the jobspec include all the P4DTI
# fields, with the right types etc. The set of things we actually
# require is fairly limited. For instance, we don't insist on
# having particular field numbers.
#
# Note that the P4DTI-filespecs field is not required for correct
# operation of the P4DTI.
p4dti_fields = {
'P4DTI-rid': {2: 'word',
4: 'required',
5: 'None',
},
'P4DTI-issue-id': {2: 'word',
4: 'required',
5: 'None',
},
'P4DTI-user': {2: 'word',
4: 'always',
5: '$user',
},
'P4DTI-filespecs': {},
}
def jobspec_has_p4dti_fields(self, jobspec, warn = 1):
map = self.jobspec_map(jobspec, 1)
correct = 1
for k,v in self.p4dti_fields.items():
if map.has_key(k):
for i, value in v.items():
if map[k][i] != value:
if warn:
# "Jobspec P4DTI field '%s' has incorrect
# attribute '%s': '%s' (should be '%s')."
self.log(714, (k, self.jobspec_attribute_names[i],
map[k][i], value))
correct = 0
elif v:
if warn:
# "Jobspec does not have required P4DTI field '%s'."
self.log(713, k)
correct = 0
return correct
# validate_jobspec: look at a jobspec and find out whether we can
# run P4DTI with it.
def validate_jobspec(self, jobspec):
if not self.jobspec_has_p4dti_fields(jobspec):
# "Jobspec does not support P4DTI."
raise error, catalog.msg(715)
# increasing order of restriction on Perforce job fields, based on
# datatype:
restriction_order = {
'text': 1,
'line': 2,
'word': 3,
'select': 4,
'date': 5,
}
# check_jobspec: does the current jobspec include the fields we want?
# Warn on any problem areas, error if they will be fatal.
def check_jobspec(self, description):
satisfactory = 1
_, wanted_fields = description
actual_jobspec = self.get_jobspec()
self.validate_jobspec(actual_jobspec)
actual_fields = self.jobspec_map(actual_jobspec, 1)
wanted_fields = self.jobspec_map(description, 1)
# remove P4DTI fields, which are checked by validate_jobspec()
for field in self.p4dti_fields.keys():
if actual_fields.has_key(field):
del actual_fields[field]
if wanted_fields.has_key(field):
del wanted_fields[field]
shared_fields = []
# check that all wanted fields are present.
for field in wanted_fields.keys():
if actual_fields.has_key(field):
shared_fields.append(field)
else: # field is absent.
# "Jobspec does not have field '%s'."
self.log(716, field)
satisfactory = 0
for field in shared_fields:
# field is present
actual_spec = actual_fields[field]
wanted_spec = wanted_fields[field]
del actual_fields[field]
# check datatype
actual_type = actual_spec[2]
wanted_type = wanted_spec[2]
if actual_type == wanted_type:
# matching datatypes
if actual_type == 'select':
# select fields should have matching values.
actual_values = string.split(actual_spec[6], '/')
wanted_values = string.split(wanted_spec[6], '/')
shared_values = []
for value in wanted_values:
if value in actual_values:
shared_values.append(value)
for value in shared_values:
actual_values.remove(value)
wanted_values.remove(value)
if wanted_values:
if len(wanted_values) > 1:
# "The jobspec does not allow values '%s'
# in field '%s', so these values cannot be
# replicated from the defect tracker."
self.log(718, (string.join(wanted_values, '/'), field))
else:
# "The jobspec does not allow value '%s'
# in field '%s', so this value cannot be
# replicated from the defect tracker."
self.log(719, (wanted_values[0], field))
if actual_values:
if len(actual_values) > 1:
# "Field '%s' in the jobspec allows values
# '%s', which cannot be replicated to the
# defect tracker."
self.log(720, (field, string.join(actual_values, '/')))
else:
# "Field '%s' in the jobspec allows value
# '%s', which cannot be replicated to the
# defect tracker."
self.log(721, (field, actual_values[0]))
elif ((wanted_type == 'date' and (actual_type == 'word' or
actual_type == 'select')) or
(actual_type == 'date' and (wanted_type == 'word' or
wanted_type == 'select'))):
# "Field '%s' in the jobspec should be a '%s' field,
# not '%s'. This field cannot be replicated to or
# from the defect tracker."
self.log(724, (field, wanted_type, actual_type))
satisfactory = 0
else:
wanted_order = self.restriction_order[wanted_type]
actual_order = self.restriction_order.get(actual_type, None)
if actual_order is None:
# "Jobspec field '%s' has unknown datatype '%s'
# which may cause problems when replicating this
# field."
self.log(731, (field, actual_type))
elif wanted_order > actual_order:
# "Jobspec field '%s' has a less restrictive
# datatype ('%s' not '%s') which may cause
# problems replicating this field to the defect
# tracker."
self.log(723, (field, actual_type, wanted_type))
else:
# "Jobspec field '%s' has a more restrictive
# datatype ('%s' not '%s') which may cause
# problems replicating this field from the defect
# tracker."
self.log(722, (field, actual_type, wanted_type))
# check persistence
if actual_spec[4] != wanted_spec[4]:
# "Field '%s' in the jobspec should have persistence
# '%s', not '%s'. There may be problems replicating
# this field to or from the defect tracker."
self.log(725, (field, wanted_spec[4], actual_spec[4]))
if actual_fields:
for field in actual_fields.keys():
# "Perforce job field '%s' will not be replicated to the
# defect tracker."
self.log(726, field)
# Possibly should also check that some of the
# Perforce-required fields are present. See the lengthy
# comment below (under "jobspec_has_p4_fields").
if not satisfactory:
# "Current jobspec cannot be used for replication."
raise error, catalog.msg(717)
# Notes for writing a function "jobspec_has_p4_fields": Does the
# jobspec have the fields which are required by Perforce?
#
# In the default Perforce jobspec. the first five fields look like
# this:
#
# 101 Job word 32 required
# 102 Status select 10 required
# 103 User word 32 required
# 104 Date date 20 always
# 105 Description text 0 required
#
# Perforce documentation emphasizes that the names and types of
# the first five fields should not be changed. But in fact, there
# isn't much actually required for correct operation of Perforce:
#
# Field 101:
# - the job name, used in various commands and automatically generated
# by Perforce server if a job is created with value 'new' in this
# field.
# - required;
# - a word;
#
# Field 102:
# - the job status, used in various commands;
# - required;
# - a select;
# - if the Values don't include 'closed' then things will break
# (because 'p4 fix' will set it to 'closed' anyway).
#
# Field 103:
# - the job user.
# - Output by "p4 jobs" if it is a "word".
#
# Field 104:
# - the date.
# - Output by "p4 jobs" if it is a "date".
#
# Field 105:
# - the job description, output by various commands;
# - required;
# - text or line.
# A. REFERENCES
#
# [GDR 2000-10-16] "Perforce Defect Tracking Integration Integrator's
# Guide"; Gareth Rees; Ravenbrook Limited; 2000-10-16;
# .
#
#
# B. DOCUMENT HISTORY
#
# 2000-09-25 GDR Created. Moved Perforce interface from replicator.py.
#
# 2005-07-13 RHGC Modified to use P4Python for Perforce interface.
#
#
# C. COPYRIGHT AND LICENSE
#
# This file is copyright (c) 2001 Perforce Software, Inc. All rights
# reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.
#
#
# $Id: //info.ravenbrook.com/project/p4dti/version/2.1/code/replicator/p4.py#2 $