# Perforce Defect Tracking Integration Project
# <http://www.ravenbrook.com/project/p4dti/>
#
# BUILD.PY -- BUILD THE P4DTI AND THE INTEGRATION KIT
#
# Gareth Rees, Ravenbrook Limited, 2001-07-10
#
#
# 1. INTRODUCTION
#
# This Python script builds a release of the P4DTI or the Integration
# Kit.
#
# See [GDR 2001-07-13] for design and instructions in how to use it.
import sys
sys.path.append('../code/replicator')
import getopt
import getpass
import os
import p4
import re
import socket
import string
import tempfile
import types
from relocate_xhtml import relocater
class builder:
# 2. STUFF
# 2.1. Configuration parameters
# Base filespec for the project (no trailing /).
project_filespec = "//info.ravenbrook.com/project/p4dti"
# The set of names of build targets, in the form of a map from
# target name to target description.
target_desc = { 'bugzilla': 'Perforce/Bugzilla integration',
'kit': 'integration kit',
'manuals': 'online manuals',
}
# The path to the WinZip command-line executable (Windows only).
wzzip_path = '"c:\\program files\\winzip\\wzzip.exe"'
# 2.2. Build variables
build_dir = None # Directory containing build sources.
build_tool_version = None # Version of this tool ("1.2").
changelevel = None # Changelevel of build sources.
client_name = None # Temporary client used to sync build sources.
p4i = None # Perforce interface (default client).
p4i_build = None # Perforce interface (temporary client).
release = None # The release ("1.2.3").
release_filespec = None # Filespec for release (no trailing /).
relocater = None # URL relocater object (see relocate_xhtml.py).
targets = None # List of build targets.
temp_dir = None # Temporary directory to build release in.
version = None # Version from which release is built ("1.2").
version_filespec = None # Filespec for build sources.
# 2.3. Miscellaneous utilities
# error(msg) prints an error message and exits.
def error(self, msg):
if isinstance(msg, types.ListType):
sys.stderr.writelines(msg)
else:
sys.stderr.write(msg + "\n")
sys.stderr.flush()
self.query("Continue?")
# usage() prints a usage message and exits. An optional error
# message (if supplied) is prepended.
def usage(self, error_msg = None):
msg = []
if error_msg:
msg.append(error_msg + "\n")
msg.append("Usage: %s -r RELEASE -t TARGET1 -t TARGET2 ...\n"
% sys.argv[0])
msg.append("Where TARGET is one of:\n")
targets = self.target_desc.keys()
targets.sort()
for t in targets:
msg.append(" %s -- %s\n" % (t, self.target_desc[t]))
self.error(msg)
# query(msg) asks the yes/no question given by the argument and
# reads a response from standard input. If the answer is not yes,
# it exits.
def query(self, msg):
sys.stdout.write(msg + " ")
sys.stdout.flush()
if sys.stdin.readline()[0] not in 'yY':
sys.exit(1)
# progress(msg) prints a progress message to standard output.
def progress(self, msg):
sys.stdout.write(msg + "\n")
sys.stdout.flush()
# sh(cmd) prints and then runs a shell command.
def sh(self, cmd):
sys.stdout.write("Running " + cmd + "\n")
sys.stdout.flush()
os.system(cmd)
# filespec_to_path(filespec) returns the filesystem path
# corresponding to the given filespec, in the default client.
def filespec_to_path(self, filespec):
# 'p4 where' gives you three names for a file: server filespec,
# client filespec and file system path. We want the third of
# these.
data = self.p4i.run('where %s' % filespec)[0]['data']
return string.split(data, ' ')[2]
# 3. INITIALISE THE BUILDER
def __init__(self):
client = p4.p4().run('client -o')[0]['Client']
self.p4i = p4.p4(client = client)
self.targets = []
self.read_command_line()
self.check_parameters()
# 3.1. Read the command-line arguments
def read_command_line(self):
opts, paths = getopt.getopt(sys.argv[1:], 'r:t:c:',
['release=',
'target=',
'changelevel='])
changelevel = None
release = None
targets = []
version = None
for o, a in opts:
if o in ('-r', '--release'):
release = a
elif o in ('-t', '--target'):
targets.append(a)
elif o in ('-c', '--changelevel'):
changelevel = a
if paths or release == None or targets == []:
self.usage()
# Check changelevel argument.
if changelevel == None:
changelevel = self.p4i.run('counter change')[0]['data']
self.progress("Chose changelevel %s" % changelevel)
elif re.match('^[0-9]+$', changelevel) == None:
self.error("No such changelevel: %s" % changelevel)
# Check release argument.
match = re.match("([0-9]+\\.[0-9]+)\\.[0-9]+", release)
if match:
version = match.group(1)
else:
self.usage("No such release: '%s'." % release)
# Check target arguments.
for t in targets:
if not self.target_desc.has_key(t):
self.usage("No such build target: %s." % t)
# Record arguments.
self.changelevel = changelevel
self.client_name = ('release-%s-%s'
% (release,
string.split(socket.gethostname(),
'.')[0]))
self.release = release
self.release_filespec = ("%s/release/%s"
% (self.project_filespec, release))
self.targets = targets
self.temp_dir = tempfile.mktemp()
self.build_dir = os.path.join(self.temp_dir, 'build')
self.version = version
self.version_filespec = ("%s/version/%s"
% (self.project_filespec, version))
match = re.search('version/([0-9]+\\.[0-9]+)/tool',
'$Id: //info.ravenbrook.com/project/p4dti/version/2.0/tool/build.py#3 $')
if match == None:
self.error("Couldn't establish build tool version.")
else:
self.build_tool_version = match.group(1)
self.relocater = relocater(self.build_dir,
'/project/p4dti/version/%s' % self.version,
self.build_dir)
# 3.2. Check parameters
def check_parameters(self):
# 3.2.1. Does the release already exist?
try:
self.p4i.run("files %s/..." % self.release_filespec)
except p4.error:
# No, it doesn't exist.
pass
else:
self.error("Release %s already exists." % self.release)
# Check that build tool version matches release version.
if self.version != self.build_tool_version:
self.error("Build tool version %s doesn't match release "
"version %s."
% (self.build_tool_version, self.version))
# Are there any files open in the sources?
if self.p4i.run('opened %s/...' % self.version_filespec):
self.error("You have files open in the version %s sources."
% self.version)
# Check that user has brought readme, release notes and RPM spec
# up to date for the release.
for f in ('readme.txt', 'release-notes.txt',
'packaging/linux/p4dti.spec'):
data = self.p4i.run('print %s/%s'
% (self.version_filespec, f))
found = 0
for chunk in data:
if chunk.has_key('data'):
if string.find(chunk['data'], self.release) >= 0:
found = 1
break
if not found:
self.error("%s not up to date for release %s"
% (f, self.release))
# 4. COMMON BUILD STEPS
def start_build(self):
self.progress("Building release %s of the P4DTI."
% self.release)
self.progress("From %s/...@%s"
% (self.version_filespec, self.changelevel))
# 4.1. Make sure release directory exists
release_dir = self.filespec_to_path(self.release_filespec)
if not os.path.isdir(release_dir):
os.makedirs(release_dir)
# 4.2. Make the build directory
os.makedirs(self.build_dir)
self.progress("Building in %s." % self.build_dir)
# 4.3. Make a temporary client
#
# This is used to sync the sources. Note that the client has
# the 'allwrite' option: this is so that we can update documents
# in place when we convert the URLs.
clientspec = {
'Options': 'allwrite',
'Client': self.client_name,
'Root': self.build_dir,
'View0': ('%s/... //%s/...'
% (self.version_filespec, self.client_name)),
'View1': ('-%s/manual/....awk //%s/manual/....awk'
% (self.version_filespec, self.client_name)),
'Description': ('Temporary client for building release %s '
'of the P4DTI.' % self.release),
}
self.progress("Creating client %s." % self.client_name)
self.p4i.run('client -i', clientspec)
self.p4i_temp = p4.p4(client = self.client_name)
# 4.4. Sync the build sources
#
# Sync the sources to the temporary workspace, then use p4 flush
# to tell the server to forget that the files are synced.
def sync_sources(self):
self.progress("Populating temporary workspace.")
self.p4i_temp.run('sync -f %s/...@%s' % (self.version_filespec,
self.changelevel))
self.p4i_temp.run('flush %s/...#none' % self.version_filespec)
# 4.5. Make manuals relocatable
#
# By rewriting URLs in the manuals. The manuals argument is a list
# of manual directories, e.g., ['ag', 'ug'].
def relocate_manuals(self, manuals):
dist = map(lambda p, d=self.build_dir: '%s/manual/%s' % (d, p),
manuals)
self.relocater.relocate_distribution(dist)
# 5. BUILD THE TARGETS
def build(self):
cwd = os.getcwd()
self.start_build()
for t in self.targets:
getattr(self, 'build_' + t)()
# Don't chop the branch we're standing on!
os.chdir(cwd)
self.sh('rm -rf %s' % self.temp_dir)
self.p4i.run('client -d %s' % self.client_name)
# 5.1. Build the integration kit
def build_kit(self):
self.progress("\nCreating the integration kit.")
self.sync_sources()
# Convert URLs.
self.relocater.relocate_distribution(self.build_dir)
# To get the tarball to unpack to a directory with the right
# name, we copy all the sources to a directory with the right
# name (kit_dir), and run the tar command in the parent of
# that directory (kit_parent_dir) using the -C option to tar.
# The reason why we don't just use temp_dir as the kit_parent_dir
# is so that the ZIP comes out right; see below.
kit_parent_dir = os.path.join(self.temp_dir, 'parent')
kit_dir = os.path.join(kit_parent_dir,
'p4dti-kit-%s' % self.release)
os.makedirs(kit_dir)
os.chdir(self.build_dir)
self.sh('cp -pr . %s' % kit_dir)
if os.name == 'posix':
# Create a tarball of the version sources.
tarball_filespec = ('%s/p4dti-kit-%s.tar.gz'
% (self.release_filespec, self.release))
tarball_path = self.filespec_to_path(tarball_filespec)
self.sh('tar -C %s -c -f - p4dti-kit-%s | gzip -c > %s'
% (kit_parent_dir, self.release, tarball_path))
self.p4i.run('add %s' % tarball_filespec)
elif os.name == 'nt':
# Create a ZIP archive of the version sources. Use -rp to get
# path names into the archive. Unfortunately this option
# discounts any path component named on the command line, so to
# get the kit directory into the paths stored in the archive, we
# start zipping from the parent directory. (We can't just use
# temp_dir for the parent directory, because then we'd get the
# build sources and other cruft in temp_dir).
zip_filespec = ('%s/p4dti-kit-%s.zip'
% (self.release_filespec, self.release))
zip_path = self.filespec_to_path(zip_filespec)
self.sh('%s -rp %s %s'
% (self.wzzip_path, zip_path, kit_parent_dir))
self.p4i.run('add %s' % zip_filespec)
else:
self.error("Can't build the kit on %s." % os.name)
# 5.2. Build the TeamTrack integration.
# This section is obsolete. P4DTI for TeamTrack is now
# maintained and supported by TeamShare Inc. NB 2003-05-14.
# 5.3. Build the Bugzilla integration
def build_bugzilla(self):
self.progress("\nCreating the Bugzilla integration.")
self.sync_sources()
manuals = ['ag', 'ug', 'aag']
self.relocate_manuals(manuals)
# Create a directory containing the materials we need.
if os.name == 'nt':
bz_parent_dir = os.path.join(self.temp_dir, 'parent')
bz_dir = os.path.join(bz_parent_dir, 'P4DTI-%s'
% self.release)
else:
bz_dir = os.path.join(self.temp_dir, 'p4dti-bugzilla-%s'
% self.release)
os.makedirs(bz_dir)
if os.name == 'nt':
# Ask user to build the Eventlog DLL.
self.progress("** Start Microsoft Visual C++ 6.0.")
self.progress("** Chose File -> Open Workspace.")
path = os.path.join(self.build_dir, 'code', 'p4dti.dsw')
self.progress("** Select %s." % path)
for f, g in (('eventlog', 'eventlog.dll'),):
path = os.path.join(self.build_dir, 'code', 'replicator', g)
if not os.path.isfile(path):
self.progress("** Choose Build > Set Active "
"Configuration > %s Win32 Release." % f)
self.progress("** Choose Build > Rebuild All.")
self.query("** When %s is built, enter 'yes':" % f)
if not os.path.isfile(path):
self.query("Are you sure you built %s?" % f)
self.progress("** Now quit Microsoft Visual C++ 6.0.")
# Copy materials.
os.chdir(self.build_dir)
materials = ['readme.txt', 'release-notes.txt', 'license.txt',
'code/replicator/*.py']
if os.name == "nt":
materials.append('code/replicator/eventlog.dll')
else:
materials.append('packaging/linux/startup-script')
for f in materials:
self.sh('cp %s %s' % (f, bz_dir))
for f in manuals:
src = os.path.join('manual', f)
dest = os.path.join(bz_dir, f)
os.chdir(self.build_dir)
self.sh('cp -pr %s %s' % (src, dest))
# Edit config.py so that it specifies dt_name = "Bugzilla"
# rather than anything else. See job000360.
config_path = os.path.join(bz_dir, "config.py")
config_stream = open(config_path, "r+")
contents = config_stream.readlines()
i = 0
deletions = []
for l in contents:
if re.match("^(# *)?dt_name *=", l):
deletions.append(i)
i = i + 1
deletions.reverse()
for d in deletions:
del contents[d]
contents[d: d] = ['dt_name = "Bugzilla"\n']
config_stream.seek(0)
config_stream.writelines(contents)
config_stream.truncate()
config_stream.close()
# Make the Bugzilla patches.
bugzilla_versions = [
('2.14.4', 'import/2002-09-30/bugzilla-2.14.4/bugzilla-2.14.4'),
('2.14.5', 'import/2003-01-02/bugzilla-2.14.5/bugzilla-2.14.5'),
]
if os.name != "nt":
bugzilla_versions.extend([
('2.16.1', 'import/2002-09-30/bugzilla-2.16.1/bugzilla-2.16.1'),
('2.16.2', 'import/2003-01-02/bugzilla-2.16.2/bugzilla-2.16.2'),
('2.16.3', 'import/2003-04-25/bugzilla-2.16.3/bugzilla-2.16.3'),
])
for v, import_dir in bugzilla_versions:
orig_filespec = ('%s/%s'
% (self.project_filespec, import_dir))
orig_dir = self.filespec_to_path(orig_filespec)
self.p4i.run('sync -f %s/...@%s' % (orig_filespec,
self.changelevel))
patched_dir = os.path.join(self.build_dir, 'code',
'bugzilla-%s' % v)
os.chdir(patched_dir)
patch_path = os.path.join(bz_dir, 'bugzilla-%s-patch' % v)
self.sh('diff -r -u -N %s . > %s' % (orig_dir, patch_path))
if os.name == 'nt':
# Create a ZIP archive containing the release.
zip_filespec = ('%s/p4dti-bugzilla-%s.zip'
% (self.release_filespec, self.release))
zip_path = self.filespec_to_path(zip_filespec)
self.sh('%s -pr %s %s'
% (self.wzzip_path, zip_path, bz_parent_dir))
self.p4i.run('add %s' % zip_path)
# Ask user to make a self-extracting executable.
exe_filespec = ('%s/p4dti-bugzilla-%s.exe'
% (self.release_filespec, self.release))
exe_path = self.filespec_to_path(exe_filespec)
self.progress("** Run WinZip on %s." % zip_path)
self.progress("** Make sure that this copy of WinZip is licensed.")
self.progress("** Choose Actions -> Make .Exe File.")
self.progress("** Specify C:\Program Files for the default "
"unpack directory.")
self.query("** Then enter yes:")
if not os.path.isfile(exe_path):
self.query("Are you sure you built %s?" % exe_path)
self.p4i.run('add %s' % exe_path)
else: # Linux
# Make a tarball
tarball_filespec = ('%s/p4dti-bugzilla-%s.tar.gz'
% (self.release_filespec, self.release))
tarball_path = self.filespec_to_path(tarball_filespec)
self.sh('tar -C %s -c -f - p4dti-bugzilla-%s | gzip -c > %s'
% (self.temp_dir, self.release, tarball_path))
self.p4i.run('add %s' % tarball_filespec)
# Make the RPM.
rpm_filespec = ('%s/p4dti-%s-1.i386.rpm'
% (self.release_filespec, self.release))
rpm_path = self.filespec_to_path(rpm_filespec)
self.sh('echo "%_topdir $HOME/rpm" > $HOME/.rpmmacros')
self.sh('mkdir -p $HOME/rpm/{BUILD,SRPMS,RPMS/i386,SOURCES}')
self.sh('cp %s $HOME/rpm/SOURCES' % tarball_path)
self.sh('rpm --define "packager %s@ravenbrook.com" '
'-ba %s/packaging/linux/p4dti.spec'
% (getpass.getuser(), self.build_dir))
self.sh('cp $HOME/rpm/RPMS/i386/p4dti-%s-1.i386.rpm %s'
% (self.release, rpm_path))
self.p4i.run('add %s' % rpm_filespec)
# Remove the RPM directories.
self.sh('rm -rf $HOME/rpm')
# 5.4. Build the online copies of the manuals.
def build_manuals(self):
self.progress("\nCreating the online manuals.")
self.sync_sources()
for f in ('readme.txt', 'release-notes.txt'):
self.p4i.run('integ %s/%s %s/%s'
% (self.version_filespec, f,
self.release_filespec, f))
manuals = ['ag', 'ug', 'ig', 'aag']
self.relocate_manuals(manuals)
release_dir = self.filespec_to_path(self.release_filespec)
for f in manuals:
src_dir = os.path.join(self.build_dir, 'manual', f)
dest_dir = os.path.join(release_dir, f)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
for g in os.listdir(src_dir):
# Skip AppleWorks files (.awk).
if os.path.splitext(g)[1] == '.awk':
continue
src_path = os.path.join(src_dir, g)
dest_path = os.path.join(release_dir, f, g)
self.sh('cp %s %s' % (src_path, dest_path))
self.p4i.run('add %s' % dest_path)
if __name__ == '__main__':
builder().build()
# A. REFERENCES
#
# [GDR 2001-07-13] "Build automation design"; Gareth Rees; Ravenbrook
# Limited; 2001-07-13;
# <http://www.ravenbrook.com/project/p4dti/version/2.0/design/build/>.
#
# [GDR 2001-09-12] "Using the Windows event log"; Gareth Rees;
# Ravenbrook Limited; 2001-09-12;
# <http://www.ravenbrook.com/project/p4dti/version/2.0/design/win32-eventlog/>.
#
#
# B. DOCUMENT HISTORY
#
# 2001-07-10 GDR Created.
#
# 2001-07-14 GDR Fixed defects in Bugzilla and manual builders. Improved
# the cleanup at the end.
#
# 2001-07-18 GDR Changed install directory for TeamTrack integration to
# P4DTI-RELEASE, as given in readme. Include p4dti.reg in TeamTrack
# integration.
#
# 2001-09-03 NB added Bugzilla 2.14.
#
# 2001-09-12 GDR Build the event message file for the TeamTrack
# integration.
#
# 2001-09-26 GDR Edit config.py when building Bugzilla integration (see
# job000360).
#
# 2001-09-19 GDR Truncate config.py when editing it.
#
# 2001-11-14 GDR Added Advanced Administrator's Guide (AAG).
#
# 2001-11-27 GDR Support building of multiple targets on Unix platforms.
#
# 2002-01-31 NB Added Bugzilla 2.14.1.
#
# 2002-04-12 NB Removed "mc" step from TeamTrack build, as this is now
# handled by the eventlog Visual Studio project.
#
# 2002-06-26 RB Made the kit build on either NT or Posix systems, so that
# the release-build procedure can get kits with the right line endings.
#
# 2002-09-26 NB Added Bugzilla 2.14.2, 2.14.3, and 2.16.
#
# 2002-10-03 NB Add Bugzilla 2.14.4 and 2.16.1.
#
# 2002-11-20 NB 'p4 print' marshalled data has changed.
#
# 2003-05-15 NB Remove teamtrack integration build steps (and move
# eventlog build code to Bugzilla build function).
#
#
# C. COPYRIGHT AND LICENCE
#
# This file is copyright (c) 2001 Perforce Software, Inc. All
# rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.
#
#
# $Id: //info.ravenbrook.com/project/p4dti/version/2.0/tool/build.py#3 $