test_cbd.sh #1

#.!/bin/bash
#------------------------------------------------------------------------------
# Copyright (c) 2008-2013 Perforce Software, Inc.  Provided for use as defined
# in the Perforce Consulting Services Agreement.
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG="/p4/1/logs/test/cbd/test_cbd.$(date +'%Y%m%d-%H%M%S').log"
export VERBOSITY=${VERBOSITY:-3}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=BogusValueNonExistentFile

[[ -r "$P4U_ENV" ]] || {
   echo -e "\nError: Cannot load environment from: $P4U_ENV\n\n"
   exit 1
}

. $P4U_ENV
. $P4U_LIB/libcore.sh
. $P4U_LIB/libp4u.sh

declare Version=1.1.1
declare LogDir=
declare RunHost=waukperforce02
declare SampleDepotURL=ftp://ftp.perforce.com/perforce/tools/sampledepot.tar.gz
declare SampleDepotHome=/p4/1/tmp/sd
declare SampleFileGZFile=sampledepot.tar.gz
declare SampleDepotP4ROOT=$SampleDepotHome/PerforceSample
declare SampleDepotP4PORT=7473
declare SampleDepotBrokerPort=8223
declare TestTag=test_cbd
declare TestWS=bruno_jam.$TestTag
declare TestWSRoot=$SampleDepotHome/ws/$TestWS
declare Platform=linux26x86_64
declare p4Version=r14.1
declare p4URL=ftp://ftp.perforce.com/perforce/$p4Version/bin.$Platform/p4
declare p4dURL=ftp://ftp.perforce.com/perforce/$p4Version/bin.$Platform/p4d
declare p4brokerURL=ftp://ftp.perforce.com/perforce/$p4Version/bin.$Platform/p4broker
declare p4Exe=$SampleDepotHome/p4
declare p4dExe=$SampleDepotHome/p4d
declare p4brokerExe=$SampleDepotHome/p4broker
declare p4="$p4Exe -p $SampleDepotBrokerPort -u bruno"

export P4ROOT=$SampleDepotP4ROOT
export P4USER=bruno
export P4PORT=$SampleDepotP4PORT
export P4JOURNAL=$SampleDepotP4ROOT/journal
export P4LOG=$SampleDepotP4ROOT/log
export P4NAME=master
export P4AUDIT=audit
export P4DEBUG=server=4
export P4TICKETS=$PWD/.p4tickets

declare -i SilentMode=0

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit $OverallReturnStatus
}

#------------------------------------------------------------------------------
# Function: get_pids()
#
# Safely get just the appropraite processes.  Requires fully qualified path
# to a copy of p4d/p4broker used for this instance.
#
# Examples:
#   p4dPids=get_pids($p4dExe)
#   p4brokerPids=get_pids($p4brokerExe)
#------------------------------------------------------------------------------
function get_pids()
{
   [[ -z "$1" ]] && bail "get_pids(): No executable specified."
   exe=$1

   pids=$(/bin/ps -f -C $(basename $exe)|grep $exe|awk '{print $2}')
   echo $pids
}

#------------------------------------------------------------------------------
# Function: shutdown_servers()
#------------------------------------------------------------------------------
function shutdown_servers()
{
   vvmsg "CALL shutdown_servers($*)"

   p4dPids=$(get_pids "$SampleDepotHome/p4d")
   if [[ -n "$p4dPids" ]]; then
      runCmd "kill -9 $p4dPids" "Killing P4D pids [$p4dPids]."
   else
      msg "Verified: No P4D processes for the Sample Depot are running."
   fi

   p4brokerPids=$(get_pids "$SampleDepotHome/p4broker")
   if [[ -n "$p4brokerPids" ]]; then
      runCmd "kill -9 $p4brokerPids" "Killing P4Broker pids [$p4brokerPids]."
   else
      msg "Verified: No P4Broker processes for the Sample Depot are running."
   fi

   msg "Waiting a second to ensure pids stop."
   sleep 1
}

#------------------------------------------------------------------------------
# Function: get_sample_depot()
#------------------------------------------------------------------------------
function get_sample_depot()
{
   vvmsg "CALL get_sample_depot($*)"
   msg "Getting Sample Depot."

   if [[ -d $SampleDepotHome ]]; then
      runCmd "/bin/rm -rf $SampleDepotHome" "Blasting existing Sample Depot home dir [$SampleDepotHome]." ||\
         bail "Failed to blast existing Sample Depot home dir [$SampleDepotHome]."
   fi

   runCmd "/bin/mkdir -p $SampleDepotHome" "Creating empty new Sample Depot home dir [$SampleDepotHome]" ||\
      bail "Failed to create new Sample Depot root [$SampleDepotHome]."

   cd $SampleDepotHome || bail "Failed to cd to Sample Depot home dir [$SampleDepotHome]."

   msg "Operating in [$PWD]."

   runCmd "/usr/bin/wget -q $SampleDepotURL" \
      "Pulling Sample Depot from [$SampleDepotURL]." ||\
      bail "Failed to pull Sample Depot."

   msg "Sample Depot pulled successfully."

   msg "Now, pulling Perforce executables for $p4Version on $Platform."

   for exeUrl in $p4URL $p4dURL $p4brokerURL; do
      exe=${exeUrl##*/}
      runCmd "/usr/bin/wget -q $exeUrl" \
         "Pulling exe from [$exeUrl]." ||\
      bail "Failed to pull exe from [$exeUrl]."
      runCmd "chmod +x $exe"
      runCmd "./$exe -V" "Obtained [$exe] version:" 0 1
   done

   msg "Perforce executables pulled."
}

#------------------------------------------------------------------------------
# Function: init_sample_depot()
#
# This picks up where get_sample_depot leaves off.
# The $SampleDepotHome folder will exist and contain the sample_depot.tar.gz
# file, as downloaded from the Perforce FTP site.
#------------------------------------------------------------------------------
function init_sample_depot()
{
   vvmsg "CALL init_sample_depot($*)"

   tmpZipFile=tmp.tar.gz
   tmpTarFile=tmp.tar
   brokerCfg=p4broker.cbd.cfg

   cd $SampleDepotHome || bail "Failed to cd to Sample Depot Home dir [$SampleDepotHome]."
   msg "Operating in [$PWD]."

   [[ -r "$SampleFileGZFile" ]] || bail "Cannot access Sample Depot GZip file [$SampleFileGZFile]."

   runCmd "/bin/rm -f $tmpZipFile $tmpTarFile" || bail "Unexpected Error!"
   runCmd "/bin/cp -p $SampleFileGZFile $tmpZipFile" || bail "Unexpected Error!"
   runCmd "/bin/gunzip $tmpZipFile" || bail "Unexpected Error!"
   runCmd "/bin/tar -xpf $tmpTarFile" || bail "Failed to expand tar file [$tmpTarFile]."

   cd PerforceSample ||\
      bail "Exploded tarfile is missing expected 'PerforceSample' dir."

   [[ $PWD != $SampleDepotP4ROOT ]] && bail "Unexpected Error: Bad value for \$PWD."

   msg "Operating in [$PWD]."

   echo "P4PORT=$SampleDepotBrokerPort
P4USER=bruno
P4CLIENT=NoWorkspaceDefined
P4TICKETS=$P4TICKETS
P4IGNORE=.p4ignore" > .p4config

   export P4CONFIG=$PWD/.p4config

   msg "Generated this .p4config file:\n$(cat .p4config)\n"

   runCmd "/bin/rm -rf server.locks"
   runCmd "/bin/rm -f db.* rdb.* state"
   runCmd "$p4dExe -r $P4ROOT -jr checkpoint" \
      "Loading Sample Depot checkpoint." ||\
      bail "Failed to load Sample Depot checkpoint."
   runCmd "$p4dExe -r $P4ROOT -xu" \
      "Upgrading Sample Depot databases." ||\
      bail "Failed to upgrade Sample Depot checkpoint."

   runCmd "$p4dExe -r $P4ROOT -p $P4PORT -d -In $P4NAME -J $P4JOURNAL -L $P4LOG -A $P4AUDIT -v $P4DEBUG < /dev/null" \
      "Starting Sample Depot P4D on port $P4PORT."

   msg "Writing p4broker [$PWD/$brokerCfg]."

   echo "target      = localhost:$SampleDepotP4PORT;
listen      = $SampleDepotBrokerPort;
directory   = $SampleDepotP4ROOT;
logfile     = $SampleDepotP4ROOT/p4broker.log;
debug-level = server=1;
admin-name  = \"Tom Tyler\";
admin-phone = 617-513-2414;
admin-email = \"ttyler@perforce.com\";
compress = false;
redirection  = selective;
command: ^(sync|update|flush)\$
{
   action = filter;
   execute = \"/p4/common/bin/cbd/scripts/wssync.sh\";
}
" > $brokerCfg

   runCmd "$p4brokerExe -d -c $brokerCfg < /dev/null" \
      "Starting Sample Depot P4Broker $SampleDepotBrokerPort."

   p4brokerPids=$(get_pids "$p4brokerExe")

   if [[ -n "$p4brokerPids" ]]; then
      p4brokerPidFile=$PWD/.p4broker.pid
      echo $p4brokerPids > $p4brokerPidFile
      msg "P4Broker process pids: $(cat $p4brokerPidFile)."
   else
      bail "Could not determine pid for p4broker."
   fi 
}

#------------------------------------------------------------------------------
# Function: init_cbd
#
# Intialize the CBD stuff.  This needs to be restartable/idempotent.
#------------------------------------------------------------------------------
function init_cbd()
{
   vvmsg "CALL init_cbd($*)"
   ws=$1
   wsRoot=$2

   msg "Loading CBD Stream and Workspace Template Update triggers."

   echo -e "Triggers:\n\tSSTemplateUpdate change-commit //...cbdsst \"/p4/common/bin/cbd/triggers/SSTemplateUpdate.py -v DEBUG %changelist%\"\n\tWSTemplateUpdate change-commit //...cbdwst \"/p4/common/bin/cbd/triggers/WSTemplateUpdate.py -v DEBUG %changelist%\"" | $p4 triggers -i

   msg "Triggers table is now:"
   $p4 triggers -o | egrep -v '^#'

   msg "Generating workspace for bruno."

   sstFile=$wsRoot/jam.cbdsst

   runCmd "$p4 client -d $ws"

   msg "Making bruno a super user."

   if [[ "$NO_OP" != 1 ]]; then
      msg "Running: $p4 protect -o \| $p4 protect -i"
      $p4 protect -o | $p4 protect -i
   else
      msg "NO_OP: Would run: $p4 protect -o \| $p4 protect -i"
   fi

   echo -e "Client:\t$ws\n
Owner:\tbruno\n
Description:\n\tJam stream workspace.\n
Root:\t$wsRoot\n
Options:\tnoallwrite noclobber nocompress unlocked modtime rmdir\n
SubmitOptions:\tleaveunchanged\n
LineEnd:\tlocal\n
Stream: //jam/rel2.1\n" | $p4 client -f -i

   if [[ -d "$wsRoot" ]]; then
      runCmd "/bin/rm -rf $wsRoot" ||\
         bail "Failed to cleanup test workspace root dir [$wsRoot]."
   fi

   runCmd "/bin/mkdir -p $wsRoot"

   msg "Cleanup from earlier runs."
   runCmd "$p4 -c $ws obliterate -y $sstFile"
   runCmd "/bin/rm -f $sstFile"

   echo "P4PORT=$SampleDepotBrokerPort
P4USER=bruno
P4CLIENT=$ws
P4TICKETS=$P4TICKETS
P4IGNORE=.p4ignore" > $wsRoot/.p4config

   msg "Generated P4CONFIG file $wsRoot/.p4config file:\n$(cat $wsRoot/.p4config)\n"

   msg "Generating sample Stream Spec Template file."

   echo -e "Description:\n\tStream spec for __EDITME_STREAM__.\n\nPaths:\n\tshare ...\n\timport pb/... //pb/1.5.1-p/...@8883\n\timport gwt/... //gwt-streams/release1.5/...@12047\n\n" > $sstFile

   runCmd "$p4 -c $ws add $sstFile" "Adding Stream Spec Template file."
   runCmd "$p4 -c $ws submit -r -d Added. $sstFile"

   sleep 1
   runCmd "cat /p4/1/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="

   echo -e "Description:\n\tStream spec for __EDITME_STREAM__.\n\nPaths:\n\tshare ...\n\timport pb/... //pb/1.5.1-p/...@8906\n\timport gwt/... //gwt-streams/release1.5/...@12048\n\n" > $sstFile

   runCmd "$p4 -c $ws submit -r -d Updated. $sstFile"
   sleep 1
   runCmd "cat /p4/1/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="

   echo -e "Description:\n\tStream spec for __EDITME_STREAM__.\n\nPaths:\n\tshare ...\n\timport pb/... //pb/1.5.1-p/...@8906\n\timport gwt/... //gwt-streams/release1.5/...@12050\n\n" > $sstFile

   runCmd "$p4 -c $ws submit -d Re-Updated. $sstFile"
   sleep 1
   runCmd "cat /p4/1/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="

   echo -e "Client: bruno.LT-8BQ0VM1.Jam\n
Owner: bruno\n
Host: LT-8BQ0VM1\n
Description:\n\tCreated by bruno.\n
Root: C:\\p4\\Jam
Options: noallwrite noclobber nocompress unlocked modtime rmdir\n
SubmitOptions: leaveunchanged\n
LineEnd: local\n
Stream: //jam/rel2.1\n\n" | $p4 client -f -i

   msg "Adjusting flow of change, 2.1 -> 2.2 -> 2.3 -> main -> dev*"

   echo -e "Stream: //jam/rel2.1\n
Owner: bruno\n
Name:  rel2.1\n
Parent: //jam/rel2.2\n
Type:  release\n
Description:\n\tJam 2.1 release\n
Options: allsubmit unlocked toparent nofromparent\n
Paths:\n\tshare ...\n" | $p4 stream -f -i

   echo -e "Stream: //jam/rel2.2\n
Owner: bruno\n
Name: rel2.2\n
Parent: //jam/rel2.3\n
Type: release\n
Description:\n\tJam 2.2 release\n
Options: allsubmit unlocked toparent nofromparent\n
Paths:\n\tshare ...\n" | $p4 stream -f -i

echo -e "Stream: //jam/rel2.3\n
Owner: bruno\n
Name: rel2.3\n
Parent: //jam/main\n
Type: release\n
Description:\n\tJam 2.3 release stream\n
Options: allsubmit unlocked toparent nofromparent\n
Paths:\n\tshare ...\n" | $p4 stream -f -i

echo -e "Stream: //jam/dev2.3\n
Owner: bruno\n
Name: dev2.3\n
Parent: //jam/main\n
Type: development\n
Description:\n\tJam 2.3 development stream\n
Options: allsubmit unlocked toparent fromparent\n
Paths:\n\tshare ...\n" | $p4 stream -f -i

}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT [-G|-I|-C] [-L <log>] [-si] [-v<n>] [-n] [-D]

or

$THISSCRIPT [-h|-man]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION: CBD Tester

If your sync command looks like this:  This happens:                                 Comments:
-------------------------------------  -------------------------------------------   --------------------------------------------------------

p4 sync                                Rewritten to use version specifier for path.  In all cases, flags ('-n', '-k', etc. are passed thru.)
p4 sync //<L1>/...                     Rewritten to use version specifier for path.
p4 sync //<L1>/<L2>/...                Rewritten to use version specifier for path.
p4 sync //<L1>/<L2>/<L3+>/...          Rewritten to use version specifier for path.
p4 sync //< L1>/...#head               Rewritten to use version specifier for path.  P4V passes #head.
p4 sync //< L1>/<L2>/...head           Rewritten to use version specifier for path.
p4 sync //<L1>/<L2>/<L3+>/...head      Passed thru using #head literal
p4 sync //<L1>/...@MyVSpec             Passed thru using @MyVSpec                    Warning displayed.
p4 sync //<L1>/<L2>/...@MyVSpec        Passed thru sync using @MyVSpec               Warning displayed.
p4 sync //<L1>/<L2>/<L3+>/...@MyVSpec  Passed thru sync using @MyVSpec               Warning displayed.
p4 sync ../some/relative/path/...      Passed thru verbatim.                         Need to warn about how this could be abused.


OPTIONS:
 -G	Start from scratch, by getting a pristine copy of the Sample Depot
	from the Perforce web site, then installing the CBD scripts.

	Using -G implies -I and -C.

	The existing Sample Depot and related Perforce server instance is
	blasted.

 -I	Start almost from scratch, using an existing copy of the downloaded
	Sample Depot.  This picks up where '-G' leaves off.

	Using -I implies -C.

 -C	Initialize CBD on the Sample depot.  This picks up where '-I' leaves
	off.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	$(dirname ${P4U_LOG}).

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

-si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -n	No-Op.  Prints commands instead of running them.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

FILES:

EXAMPLES:

SEE ALSO:
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i GetSampleDepot=0
declare -i InitSampleDepot=0
declare -i InitCBD=0
declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-G) GetSampleDepot=1; InitSampleDepot=1; InitCBD=1;;
      (-I) InitSampleDepot=1; InitCBD=1;;
      (-C) InitCBD=1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i OverallReturnStatus=0

if [[ "${P4U_LOG}" != off ]]; then
   LogDir=${P4U_LOG%/*}
   if [[ ! -d $LogDir ]]; then
      /bin/mkdir -p $LogDir || bail "Couldn't create log dir [$LogDir]."
   fi

   touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee ${P4U_LOG})
      exec 2>&1
   else
      exec >${P4U_LOG}
      exec 2>&1
   fi

   initlog
fi

if [[ ${HOSTNAME%%\.*} != $RunHost ]]; then
   bail "Run this only on $RunHost."
   exit 1
else
   echo "Verified: Running on $RunHost."
fi

# Prestart Prep
if [[ $GetSampleDepot -eq 1 || $InitSampleDepot -eq 1 ]]; then
   shutdown_servers
fi

[[ $GetSampleDepot -eq 1 ]] && get_sample_depot
[[ $InitSampleDepot -eq 1 ]] && init_sample_depot
[[ $InitCBD -eq 1 ]] && init_cbd "$TestWS" "$TestWSRoot"

p4="$p4Exe -p $SampleDepotBrokerPort -u bruno -c $TestWS"

msg "Loading test data."
declare -i i=1
declare -A syncCmd
declare -A notes

syncCmd[$i]="$p4 sync"
notes[$i]="Expect rewrite with all import paths using keys."
i=$((i+1))

syncCmd[$i]="$p4 sync @12106"
notes[$i]="Expect rewrite with all import paths using file, *.sst file rev 1."
i=$((i+1))

syncCmd[$i]="$p4 sync @12107"
notes[$i]="Expect rewrite with all import paths using file, *.sst file rev 2."
i=$((i+1))

syncCmd[$i]="$p4 sync @12108"
notes[$i]="Expect rewrite with all import paths using file, *.sst file rev 3."
i=$((i+1))

syncCmd[$i]="$p4 sync //..."
notes[$i]="Expect rewrite with all import paths using keys."
i=$((i+1))

syncCmd[$i]="$p4 sync //jam/..."
notes[$i]="Expect rewrite with all import paths using keys."
i=$((i+1))

syncCmd[$i]="$p4 sync //jam/...@12106"
notes[$i]="Expect rewrite with all import paths using file, *.sst file rev 1."
i=$((i+1))

syncCmd[$i]="$p4 sync //pb/...@12106"
notes[$i]="Expect rewrite of import paths using file specified revision, ignoring keys."
i=$((i+1))

syncCmd[$i]="$p4 sync //jam/rel2.1/..."
notes[$i]="Expect rewrite with all import paths using keys."
i=$((i+1))

syncCmd[$i]="$p4 sync //jam/rel2.1/...@12106"
notes[$i]="Expect rewrite with all import paths using file, *.sst file rev 1."
i=$((i+1))

syncCmd[$i]="$p4 sync //jam/rel2.1/src/..."
notes[$i]="3+Levels, No rewrite expected.  Pass thru verbatim."
i=$((i+1))

syncCmd[$i]="$p4 sync //jam/rel2.1/src/...@12106"
notes[$i]="3+Levels, No rewrite expected.  Pass thru verbatim."
i=$((i+1))

syncCmd[$i]="$p4 sync -n -k //jam/rel2.1/src/...@12106"
notes[$i]="Just check that all flags are passed along, and don't otherwise affect processing."
i=$((i+1))

syncCmd[$i]="$p4 flush //jam/rel2.1/src/...@12106"
notes[$i]="Flush should be handled same as sync."
i=$((i+1))

syncCmd[$i]="$p4 flush //jam/rel2.1/src/...@12106"
notes[$i]="Flush should be handled same as sync."
i=$((i+1))

msg "Starting test runs.  Running $i tests."

for j in `seq 1 $((i-1))`; do
   runCmd "${syncCmd[$j]}" "\n=====Testing command: ${syncCmd[$j]} ======"
   cat /p4/1/logs/cbd.log
   echo -e "Expected Result Comments:\n${notes[$j]}\n"
done

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\nAll processing completed successfully.\n"
else
   msg "${H}\nProcessing completed, but with errors.  Scan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took about $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus
syncCmd[$i]="$p4 sync -n -k //jam/rel2.1/src/...@12106"
notes[$i]="3+Levels, No rewrite expected.  Pass thru verbatim."
i=$((i+1))

msg "Starting test runs.  Running $i tests."

for j in `seq 1 $((i-1))`; do
   runCmd "${syncCmd[$j]}" "\n=====Testing command: ${syncCmd[$j]} ======"
   cat /p4/1/logs/cbd.log
done

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\nAll processing completed successfully.\n"
else
   msg "${H}\nProcessing completed, but with errors.  Scan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took about $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus