# perforce_sample_data.py -- generate random activity in Perforce.
# Gareth Rees, Ravenbrook Limited, 2000-09-25
# $Id: //info.ravenbrook.com/project/p4dti/version/2.0/test/perforce_sample_data.py#1 $
#
# This file is copyright (c) 2001 Perforce Software, Inc. All rights
# reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDERS AND CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.
import sys
sys.path.append('../code/replicator')
import logger
import os
import p4
import random
import re
import socket
import string
import time
# Configuration. tester is the person testing the software and domain is their
# e-mail domain. rid is the replicator identifier for the replicator that's
# going to replicate the sample repository. p4_user is the userid for the
# replicator. p4_client is the client that the replicator uses.
# p4_client_root is the root directory of the sample repository.
tester = 'gdr'
domain = 'ravenbrook.com'
rid = 'replicator0'
p4_user = 'P4DTI-%s' % rid
p4_client = 'p4dti-%s' % socket.gethostname()
p4_client_root = 'd:\\p4dti-test'
p4_server = 'sandpiper.ravenbrook.com:1666'
logger = logger.file_logger()
# p4_run(...). This is just a wrapper for p4.run that provides defaults for
# the input, user and client arguments. See the p4 module for details.
def p4_run(arguments, input=None, user=p4_user, client=p4_client):
return p4.p4( user = user,
client = client,
port = p4_server,
logger = logger).run(arguments, input=input)
# The structure of the sample repository, represented as a list whose
# first element is a directory name, and the remaining elements are the
# entries in that directory (which may be other directories).
repository = [ '//depot/',
[ 'project/',
[ 'compiler/',
[ 'doc/', 'goals', 'requirements', 'plan', 'design' ],
[ 'src/',
'assembler.c', 'assembler.h', 'compiler.h', 'lexer.c',
'lexer.h', 'main.c', 'parser.c', 'parser.h', 'register.c',
'register.h' ],
[ 'build/', 'cc' ] ],
[ 'editor/',
[ 'doc/', 'goals', 'requirements', 'plan', 'design' ],
[ 'src/', 'input.c', 'input.h', 'output.c', 'output.h',
'buffer.c', 'buffer.h', 'commands.c', 'commands.h',
'main.c' ],
[ 'build/', 'vi' ] ] ] ]
# Transform the repository structure so that all filespecs include the full
# path.
def expand_filespecs(parent, self):
self[0] = parent + self[0]
for i in range(len(self) - 1):
if (type(self[i+1]) == type([])):
expand_filespecs(self[0], self[i+1])
else:
self[i+1] = self[0] + self[i+1]
expand_filespecs('', repository)
# choose_k(choices, k). Choose k of the choices randomly and return them as a
# list.
def choose_k(choices, k):
n = len(choices)
i = 0
results = []
while i < n and k > 0:
if (random.uniform(0,1) < float(k)/float(n-i)):
results.append(choices[i])
k = k - 1
i = i + 1
return results
# random_filespecs(min_level). Returns a random list of filespecs from the
# repository. The distribution is skewed so that generally sensible results
# are returned.
def random_filespecs(min_level = 0):
# The deepest directory in the repository (0 is the depot).
max_level = 3
# Distribution of lengths of paths in returned filespecs (less 1).
levels = [0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3]
# Distribution of number of filespecs to return.
n_filespecs = [0,0,0,0,1,1,1,1,2,2,3,4,5]
# How many filespecs to return?
k = random.choice(n_filespecs)
# Pick a level at which to choose a directory.
if (k > 2):
level = max_level
else:
level = random.choice(levels)
if level < min_level:
level = min_level
# Pick a directory from which to choose the filespecs.
dir = repository
for i in range(level):
dir = random.choice(dir[1:])
# Now choose k out of n filespecs from the selected directory.
filespecs = choose_k(dir[1:], k)
if level < max_level:
filespecs = map(lambda (f): f[0] + '...', filespecs)
return filespecs
# add_file(filespec). If the file doesn't exist, create it and add it to the
# Perforce repository (but don't submit the addition). Return the number of
# files added to Perforce.
def add_file(filespec):
# Make the file exist in the client.
name = string.split(p4_run('where ' + filespec)[0]['data'])[2]
if not os.access(name, 0555):
stream = open(name, 'w')
stream.write('Created.\n')
stream.close()
# Now make the file exist on the server.
try:
p4_run('files ' + filespec)
except:
p4_run('add ' + name)
print 'Added', name
return 1
return 0
# add_directory(dir). Add files and directories to Perforce. The dir argument
# should be a repository structure (see above). If the directory specified by
# the first argument doesn't exist, create it. Then recurse to add the
# subdirectories or files. Don't submit the additions. Return the number of
# files added to Perforce.
def add_directory(dir):
if dir[0] == '//depot/':
name = p4_client_root
else:
# The [:-1] is necessary because p4 where doesn't understand directory
# names ending in slashes.
name = string.split(p4_run('where ' + dir[0][:-1])[0]['data'])[2]
if not os.access(name, 0777):
os.mkdir(name)
print 'Created directory', name
n_added = 0
for i in dir[1:]:
if type(i) == type(''):
n_added = n_added + add_file(i)
else:
n_added = n_added + add_directory(i)
return n_added
# init_repository(). Adds all the files in the 'repository' structure to the
# sample Perforce repository. Submit the additions.
def init_repository():
if add_directory(repository):
change = p4_run('change -o')[0]
change['Jobs'] = ''
change['Description'] = 'Files automatically added to sample repository'
p4_run('submit -i', change)
# users. A list of structures describing sample users. The 'name'
# key is their user name. The 'client' key is the Perforce client they
# use to edit files in the sample repository.
users = [
{ 'name': 'gdr', 'client': 'gdr-grouse' },
{ 'name': 'rb', 'client': 'rb-skylark' },
{ 'name': 'nb', 'client': 'nb-thrush' },
{ 'name': 'lmb', 'client': 'lmb-blackbird' },
]
# make_user(user). Make a Perforce user and set their e-mail address (unless
# it already exists).
def make_user(user):
u = p4_run('user -o', user=user)[0]
email = '%s+%s@%s' % (tester, user, domain)
if u['Email'] != email:
u['Email'] = email
p4_run('user -i', u, user=user)
print 'Added user', user
# make_client(user, clientname). Make a Perforce client (unless it already
# exists) belonging to the given user and with the given name. Its root
# directory is given by p4_client_root, plus the user name.
def make_client(user, clientname):
# Get the list of clients.
clients = p4_run('clients')
exists = 0
for c in clients:
if c['client'] == clientname:
exists = 1
client = p4_run('client -o', user=user, client=clientname)[0]
if not exists:
client['Root'] = '%s\\%s' % (p4_client_root, user)
client['View0'] = '//depot/... //%s/...' % clientname
p4_run('client -i', client, user=user, client=clientname)
print 'Added client', clientname, 'for user', user
if not os.access(client['Root'], 0777):
os.mkdir(client['Root'])
print 'Created directory', client['Root']
# init_clients(). Create Perforce clients for the replicator (p4_user,
# p4_client) and the sample users specified in the users data structure (if the
# clients don't already exist).
def init_clients():
make_user(p4_user)
make_client(p4_user, p4_client)
for u in users:
make_user(u['name'])
make_client(u['name'], u['client'])
# Status values for Perforce jobs in the sample repository.
statuses = [ '_new', 'assigned', 'closed', 'verified' ]
# make_change(jobs). Make a change in the sample repository by picking some
# files, editing them, submitting the edits, and making fix records associating
# the change with some jobs. The jobs argument should be a list of jobs for
# which fix records can be added. The Status field in the fixed jobs is
# changed to reflect the new status (but the other fields are not changed and
# may be out of date).
def make_change(all_jobs):
# User who will make the change.
user = random.choice(users)
# Sync their workspace. Note that sync returns an error if everything is
# up to date.
try:
p4_run('sync', user=user['name'], client=user['client'])
except p4.error:
pass
# Files affected by the change.
filespecs = []
while not filespecs:
filespecs = random_filespecs(min_level = 3)
# Jobs fixed by the change.
k = random.choice([0,1,1,1,1,1,1,2,2,3])
jobs = choose_k(all_jobs, k)
# Edit the files.
for f in filespecs:
p4_run('edit ' + f, user=user['name'], client=user['client'])
where = p4_run('where ' + f, user=user['name'], client=user['client'])
filename = string.split(where[0]['data'])[2]
assert(os.access(filename, 0777))
stream = open(filename, 'a')
now = time.asctime(time.localtime(time.time()))
stream.write('Edited at %s by %s\n' % (now, user['name']))
stream.close()
print filename, 'edited at', now, 'by', user['name']
# Submit the change (don't do the fixes at this stage since the Perforce
# job interface doesn't allow you to specify the status).
change = p4_run('change -o', user=user['name'], client=user['client'])[0]
change['Jobs'] = ''
change['Description'] = ('Automatically generated change comment.\n'
'Edited files: ' + str(filespecs))
results = p4_run('submit -i', change, user=user['name'],
client=user['client'])
# Work out the change number from the message returned by Perforce, which
# should be the last element in the results array.
match = re.compile('Change ([0-9]+) submitted\\.$').match(results[-1]['data'])
assert(match)
change_number = int(match.group(1))
# Now make the fix records.
for job in jobs:
# What's the current status of the job?
status = statuses.index(job['State'])
# Change the status keyword by moving it forward in the statuses array
# with probability 90%.
if (status == 0 or (status < len(statuses) - 1
and random.uniform(0,1) < 0.9)):
status = status + 1
else:
status = status - 1
# Fix the job.
p4_run('fix -s %s -c %d %s'
% (statuses[status], change_number, job['Job']),
user=user['name'], client=user['client'])
# Record the changed status so that we don't have to query Perforce
# again.
job['State'] = statuses[status]
def run(n_changes = 10):
init_clients()
init_repository()
# Get the set of jobs.
jobs = p4_run('jobs')
# Make some changes that fix those jobs.
for _ in range(n_changes):
make_change(jobs)
if __name__ == '__main__':
run()
# B. Document History
#
# 2000-09-25 GDR Created.
#
# 2000-09-26 GDR Creates clients and workspaces for the sample users, edits
# files, and fixes jobs.
#
# 2000-10-03 GDR Only report "Adding client" when actually doing so.
#
# 2000-10-10 GDR Changed Perforce job status fields so that they correspond
# with the issue states in the TeamTrack sample database.
#
# 2001-02-26 GDR Make users with appropriate e-mail addresses.
#
# 2001-02-27 GDR Don't use os.access() to test if a file exists on the server;
# use 'p4 files' instead.
#
# 2001-03-02 RB Transferred copyright to Perforce under their license.
#
# 2001-03-15 GDR Use revised p4 interface.