test_cbd.sh #7

#!/bin/bash
#------------------------------------------------------------------------------
set -u

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG="/tmp/test_cbd.$(date +'%Y%m%d-%H%M%S').log"
export VERBOSITY=${VERBOSITY:-3}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=BogusValueNonExistentFile

[[ -r "$P4U_ENV" ]] || {
   echo -e "\nError: Cannot load environment from: $P4U_ENV.  Aborting.\n\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   source $bash_lib
done

declare Version=1.4.0
declare -i SilentMode=0
declare -i OverallReturnStatus=0

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Override of the default bail() function. This sets the global
# OverallReturnStatus.
function bail () {
   echo -e "\nFATAL: ${1:-Unknown Error}\n"
   OverallReturnStatus=${2:-1}
   exit ${2:-1};
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit $OverallReturnStatus
}

#------------------------------------------------------------------------------
# Function: get_pids()
#
# Safely get just the appropraite processes.  Requires fully qualified path
# to a copy of p4d/p4broker used for this instance.
#
# Examples:
#   p4dPids=get_pids($p4dExe)
#   p4brokerPids=get_pids($p4brokerExe)
#------------------------------------------------------------------------------
function get_pids()
{
   [[ -z "$1" ]] && bail "get_pids(): No executable specified."
   exe=$1

   if [[ $ThisOS == Darwin ]]; then
      pids=$(/bin/ps -ef | grep "$exe" | grep "$(id -u)" | grep -v grep | awk '{print $2}')
   else
      pids=$(/bin/ps -f -C $(basename $exe) | grep $exe|awk '{print $2}')
   fi

   echo $pids
}

#------------------------------------------------------------------------------
# Function: shutdown_servers()
#------------------------------------------------------------------------------
function shutdown_servers()
{
   vvmsg "CALL shutdown_servers($*)"

   p4dPids=$(get_pids "$SampleDepotHome/p4d")
   if [[ -n "$p4dPids" ]]; then
      runCmd "kill -9 $p4dPids" "Killing P4D pids [$p4dPids]."
   else
      msg "Verified: No P4D processes for the Sample Depot are running."
   fi

   p4brokerPids=$(get_pids "$SampleDepotHome/p4broker")
   if [[ -n "$p4brokerPids" ]]; then
      runCmd "kill -9 $p4brokerPids" "Killing P4Broker pids [$p4brokerPids]."
   else
      msg "Verified: No P4Broker processes for the Sample Depot are running."
   fi

   msg "Waiting a second to ensure pids stop."
   sleep 1
}

#------------------------------------------------------------------------------
# Function: get_sample_depot()
#------------------------------------------------------------------------------
function get_sample_depot()
{
   vvmsg "CALL get_sample_depot($*)"
   msg "Getting Sample Depot."

   if [[ -d $SampleDepotHome ]]; then
      runCmd "/bin/rm -rf $SampleDepotHome" "Blasting existing Sample Depot home dir [$SampleDepotHome]." ||\
         bail "Failed to blast existing Sample Depot home dir [$SampleDepotHome]."
   fi

   runCmd "/bin/mkdir -p $SampleDepotHome" "Creating empty new Sample Depot home dir [$SampleDepotHome]" ||\
      bail "Failed to create new Sample Depot root [$SampleDepotHome]."

   cd $SampleDepotHome || bail "Failed to cd to Sample Depot home dir [$SampleDepotHome]."

   msg "Operating in [$PWD]."

   runCmd "wget -q $SampleDepotURL" \
      "Pulling Sample Depot from [$SampleDepotURL]." ||\
      bail "Failed to pull Sample Depot."

   msg "Sample Depot pulled successfully."

   msg "Now, pulling Perforce executables for $p4Version on $Platform."

   for exeUrl in $p4URL $p4dURL $p4brokerURL; do
      exe=${exeUrl##*/}
      runCmd "wget -q $exeUrl" \
         "Pulling exe from [$exeUrl]." ||\
      bail "Failed to pull exe from [$exeUrl]."
      runCmd "chmod +x $exe" "Adjusting exe perms (chmod +x)."
      runCmd "./$exe -V" "Obtained [$exe] version:" 0 1
   done

   msg "Perforce executables pulled."
}

#------------------------------------------------------------------------------
# Function: init_sample_depot()
#
# This picks up where get_sample_depot leaves off.
# The $SampleDepotHome folder will exist and contain the sample_depot.tar.gz
# file, as downloaded from the Perforce FTP site.
#------------------------------------------------------------------------------
function init_sample_depot()
{
   vvmsg "CALL init_sample_depot($*)"

   if [[ ! -d "$SampleDepotHome" ]]; then
      msg "Info: Sample Depot Home dir [$SampleDepotHome] does not exist.  Operating as if '-G' was specified."
      shutdown_servers
      get_sample_depot
   fi

   cd $SampleDepotHome || bail "Failed to cd to Sample Depot Home dir [$SampleDepotHome]."
   msg "Operating in [$PWD]."

   [[ -r "$SampleFileGZFile" ]] || bail "Cannot access Sample Depot GZip file [$SampleFileGZFile]."

   runCmd "tar -xzpf $SampleFileGZFile" || bail "Failed to expand tar file [$tmpTarFile]."

   cd PerforceSample ||\
      bail "Exploded tarfile is missing expected 'PerforceSample' dir."

   [[ $PWD != $SampleDepotP4ROOT ]] && bail "Unexpected Error: Bad value for \$PWD."

   msg "Operating in [$PWD]."

   echo "P4PORT=$SampleDepotBrokerPort
P4USER=bruno
P4CLIENT=NoWorkspaceDefined
P4TICKETS=$P4TICKETS
P4IGNORE=.p4ignore" > .p4config

   export P4CONFIG=$PWD/.p4config

   msg "Generated this .p4config file:\n$(cat .p4config)\n"

   runCmd "/bin/rm -rf server.locks"
   runCmd "/bin/rm -f db.* rdb.* state log journal audit"
   runCmd "$p4dExe -r $P4ROOT -jr checkpoint" \
      "Loading Sample Depot checkpoint." ||\
      bail "Failed to load Sample Depot checkpoint."
   runCmd "$p4dExe -r $P4ROOT -xu" \
      "Upgrading Sample Depot databases." ||\
      bail "Failed to upgrade Sample Depot checkpoint."

   runCmd "$p4dExe -r $P4ROOT -p $P4PORT -d -J $P4JOURNAL -L $P4LOG -v $P4DEBUG < /dev/null" \
      "Starting Sample Depot P4D on port $P4PORT."

   msg "Writing p4broker config file [$BrokerCfg]."

   echo "target      = localhost:$SampleDepotP4PORT;
listen      = $SampleDepotBrokerPort;
directory   = $SampleDepotP4ROOT;
logfile     = $SampleDepotP4ROOT/p4broker.log;
debug-level = server=1;
admin-name  = \"Tom Tyler\";
admin-phone = 617-513-2414;
admin-email = \"ttyler@perforce.com\";
compress = false;
redirection  = selective;

command: ^(sync|update|flush)\$
{
   action = filter;
   execute = \"$CBD_HOME/scripts/wssync.sh\";
}

command: ^(sbi)\$
{
   action = filter;
   execute = \"$CBD_HOME/scripts/sbi.pl\";
}


" > $BrokerCfg

   runCmd "$p4brokerExe -d -c $BrokerCfg < /dev/null" \
      "Starting Sample Depot P4Broker $SampleDepotBrokerPort."

   p4brokerPids=$(get_pids "$p4brokerExe")

   if [[ -n "$p4brokerPids" ]]; then
      p4brokerPidFile=$PWD/.p4broker.pid
      echo $p4brokerPids > $p4brokerPidFile
      msg "P4Broker process pids: $(cat $p4brokerPidFile)."
   else
      bail "Could not determine pid for p4broker."
   fi 
}

#------------------------------------------------------------------------------
# Function: init_cbd
#
# Intialize the CBD stuff.  This needs to be restartable/idempotent.
#------------------------------------------------------------------------------
function init_cbd()
{
   vvmsg "CALL init_cbd($*)"
   ws=$1
   wsRoot=$2
   tmpFile=/tmp/tmp.test_cbd.$$.$RANDOM

   if [[ ! -f "$BrokerCfg" ]]; then
      msg "Info: Broker Config file [$BrokerCfg] does not exist.  Operating as if '-I' was specified."
      shutdown_servers
      init_sample_depot
   fi

   if [[ -d "$CBD_HOME/logs" ]]; then
      runCmd "/bin/rm -rf $CBD_HOME/logs" "Cleaning up logs from earlier runs."
   fi

   if [[ ! -d "$CBD_HOME/logs" ]]; then
      runCmd "/bin/mkdir -p $CBD_HOME/logs" "Creating empty log dir." ||\
         bail "Failed to create CBD Logs dir [$CBD_HOME/logs]."
   fi

   msg "Loading CBD Stream and Workspace Template Update triggers."

   echo -e "Triggers:\n\tSSTemplateUpdate change-content //...cbdsst \"$CBD_HOME/triggers/SSTemplateUpdate.py -v DEBUG -L $CBD_HOME/logs/SSTemplateUpdate.log %changelist%\"\n\tSSTemplateUpdate change-content //...cbdwst \"$CBD_HOME/triggers/WSTemplateUpdate.py -v DEBUG -L $CBD_HOME/logs/SSTemplateUpdate.log %changelist%\"" > $tmpFile
   $p4 -s triggers -i < $tmpFile || cat $tmpFile

   msg "Triggers table is now:"
   $p4 triggers -o | egrep -v '^#'

   msg "Generating workspace for bruno."

   sstFile=$wsRoot/jam.cbdsst

   runCmd "$p4 client -d $ws"

   msg "Making bruno a super user."

   if [[ "$NO_OP" != 1 ]]; then
      msg "Running: $p4 protect -o \| $p4 -s protect -i"
      $p4 protect -o | $p4 -s protect -i
   else
      msg "NO_OP: Would run: $p4 protect -o \| $p4 -s protect -i"
   fi

   echo -e "Client:\t$ws\n
Owner:\tbruno\n
Description:\n\tJam stream workspace.\n
Root:\t$wsRoot\n
Options:\tnoallwrite noclobber nocompress unlocked modtime rmdir\n
SubmitOptions:\tleaveunchanged\n
LineEnd:\tlocal\n
Stream: //jam/rel2.1\n" > $tmpFile
   $p4 -s client -f -i < $tmpFile || cat $tmpFile

   if [[ -d "$wsRoot" ]]; then
      runCmd "/bin/rm -rf $wsRoot" ||\
         bail "Failed to cleanup test workspace root dir [$wsRoot]."
   fi

   runCmd "/bin/mkdir -p $wsRoot"

   msg "Cleanup from earlier runs."
   runCmd "$p4 -c $ws obliterate -y $sstFile"
   runCmd "/bin/rm -f $sstFile"

   echo "P4PORT=$SampleDepotBrokerPort
P4USER=bruno
P4CLIENT=$ws
P4TICKETS=$P4TICKETS
P4IGNORE=.p4ignore" > $wsRoot/.p4config

   msg "Generated P4CONFIG file $wsRoot/.p4config file:\n$(cat $wsRoot/.p4config)\n"

   echo -e "Client: bruno.LT-8BQ0VM1.Jam\n
Owner: bruno\n
Host: LT-8BQ0VM1\n
Description:\n\tCreated by bruno.\n
Root: C:\\p4\\Jam
Options: noallwrite noclobber nocompress unlocked modtime rmdir\n
SubmitOptions: leaveunchanged\n
LineEnd: local\n
Stream: //jam/rel2.1\n\n" > $tmpFile

   $p4 -s client -f -i < $tmpFile || cat $tmpFile

   msg "Adjusting flow of change, 2.1 -> 2.2 -> 2.3 -> main -> dev*"

   echo -e "Stream: //jam/rel2.1\n
Owner: bruno\n
Name:  rel2.1\n
Parent: //jam/rel2.2\n
Type:  release\n
Description:\n\tJam 2.1 release\n
Options: allsubmit unlocked toparent nofromparent\n
Paths:\n\tshare ...\n" > $tmpFile

   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Stream: //jam/rel2.2\n
Owner: bruno\n
Name: rel2.2\n
Parent: //jam/rel2.3\n
Type: release\n
Description:\n\tJam 2.2 release\n
Options: allsubmit unlocked toparent nofromparent\n
Paths:\n\tshare ...\n" > $tmpFile

   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Stream: //jam/rel2.3\n
Owner: bruno\n
Name: rel2.3\n
Parent: //jam/main\n
Type: release\n
Description:\n\tJam 2.3 release stream\n
Options: allsubmit unlocked toparent nofromparent\n
Paths:\n\tshare ...\n" > $tmpFile

   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Stream: //jam/dev2.3\n
Owner: bruno\n
Name: dev2.3\n
Parent: //jam/main\n
Type: development\n
Description:\n\tJam 2.3 development stream\n
Options: allsubmit unlocked toparent fromparent\n
Paths:\n\tshare ...\n" > $tmpFile

   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Stream: //jam/task-4491\n
Owner: bruno\n
Name: task-4491\n
Parent: //jam/dev2.3\n
Type: task\n
Description:\n\tTask stream for task 4491.\n
Options: allsubmit unlocked toparent fromparent\n
Paths:\n\tshare ...\n" > $tmpFile
   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Client: jsmith.TomsLaptop.depot\n
Owner: jsmith\n
Host: LT-8BQ0VM1\n
Description:\n\tCreated by jsmith.\n
Root: C:\\p4\\CBD\n
Options: noallwrite noclobber nocompress unlocked modtime rmdir\n
SubmitOptions:  leaveunchanged\n
LineEnd:        local\n
Stream: //jam/task-4491\n\n" > $tmpFile

   $p4 -s client -f -i < $tmpFile || cat $tmpFile

   msg "Generating sample Stream Spec Template file."

   echo -e "Description:\n\tStream spec for __EDITME_STREAM__.\n\nPaths:\n\tshare ...\n\timport pb/... //pb/1.5.1-p/...@8883\n\timport gwt/... //gwt-streams/release1.5/...@12047\n\n" > $sstFile

   runCmd "$p4 -c $ws add $sstFile" "Adding Stream Spec Template file."
   runCmd "$p4 -c $ws submit -r -d Added. $sstFile"
   runCmd "$p4 -c $ws tag -l Jam-2.0.1.01 //jam/rel2.1/..." "Applying Jam-2.0.1.01 label to //jam/rel2.1/...." 1 0 0

   sleep 1
   runCmd "cat $CBD_HOME/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $CBD_HOME/logs/SSTemplateUpdate.log"

   echo -e "Description:\n\tStream spec for __EDITME_STREAM__.\n\nPaths:\n\tshare ...\n\timport pb/... //pb/1.5.1-p/...@8906\n\timport gwt/... //gwt-streams/release1.5/...@12048\n\n" > $sstFile

   runCmd "$p4 -c $ws submit -r -d Updated. $sstFile"
   runCmd "$p4 -c $ws tag -l Jam-2.0.1.02 //jam/rel2.1/..." "Applying Jam-2.0.1.02 label to //jam/rel2.1/...." 1 0 0
   sleep 1
   runCmd "cat $CBD_HOME/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $CBD_HOME/logs/SSTemplateUpdate.log"

   echo -e "Description:\n\tStream spec for __EDITME_STREAM__.\n\nPaths:\n\tshare ...\n\timport pb/... //pb/1.5.1-p/...@8906\n\timport gwt/... //gwt-streams/release1.5/...@12050\n\n" > $sstFile

   runCmd "$p4 -c $ws submit -d Re-Updated. $sstFile"
   runCmd "$p4 -c $ws tag -l Jam-2.0.1.03 //jam/rel2.1/..." "Applying Jam-2.0.1.03 label to //jam/rel2.1/...." 1 0 0
   sleep 1
   runCmd "cat $CBD_HOME/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $CBD_HOME/logs/SSTemplateUpdate.log"

   runCmd "$p4 -c $ws populate //jam/rel2.1/jam.cbdsst //jam/rel2.2/jam.cbdsst" \
      "Populating CBD stream spec template file in jam-rel2.2."
   runCmd "cat $CBD_HOME/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="

   runCmd "$p4 -c $ws populate //jam/rel2.2/jam.cbdsst //jam/rel2.3/jam.cbdsst" \
      "Populating CBD stream spec template file in jam-rel2.3."
   runCmd "cat $CBD_HOME/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="

   runCmd "$p4 -c $ws populate //jam/rel2.3/jam.cbdsst //jam/main/jam.cbdsst" \
      "Populating CBD stream spec template file in main."
   runCmd "cat $CBD_HOME/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="

   runCmd "$p4 -c $ws populate //jam/main/jam.cbdsst //jam/dev2.3/jam.cbdsst" \
      "Populating CBD stream spec template file in dev2.3."
   runCmd "cat $CBD_HOME/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="

   runCmd "$p4 -c $ws populate //jam/dev2.3/... //jam/task-4491/..." \
      "Populating task stream jam-task4491."
   runCmd "cat $CBD_HOME/logs/SSTemplateUpdate.log" "== Stream Spec Trigger Log =="

   msg "Setting a password for user earl."
   echo -e "Abcdefg123\nAbcdefg123" > $tmpFile
   $p4 -u bruno passwd earl < $tmpFile

   msg "Creating locaked earl_jams workspace."

   echo -e "Client: earl_jams\n\nOwner:\tearl\n\nDescription:\n\tCreated by earl.\n\nRoot: c:\\p4\\CT\n\nOptions: noallwrite noclobber nocompress locked modtime rmdir\n\nSubmitOptions: leaveunchanged\n\nLineEnd: local\n\nStream: //jam/rel2.1\n\n" > $tmpFile

   $p4 -s client -f -i < $tmpFile || cat $tmpFile

   runCmd "/bin/rm -f $tmpFile" "Removing temp file [$tmpFile]."
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT [-G|-I|-C] [-d <cfg_dir>] [-f] [-L <log>] [-si] [-v<n>] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script runs test for the Component Based Develoment (CBD)
	system, supplemental automation that enhances the CBD capabilities
	of a Perforce Helix server.

OPTIONS:
 -d <cfg_dir>
	Specify the configuration directory where configuration and test
	data are expected to exist.  By default, scripts are expected to live
	in a directory named '/shared' if that directory exists, or else in
	the current directory at the time this script was called.

	The use of the /shared folder is consistent with the 'helix-01' test
	virtual machine setup for automated testing.

 -f	Use '-f' to bypass any invalid test entries in the command line test
	data.

 -G	Starts test preparation from scratch, by:
	* Getting a pristine copy of the Sample Depot tarfile from the
	Perforce FTP server,
	* Downloading Helix executables p4/p4d/p4broker

	Using -G implies -I and -C.

	The existing Sample Depot and related Perforce server instance are
	blasted with -G.

	The '-G' is implied if the configured Sample Depot Home dir does not
	exist.  It can be specified to force a clean start with the latest
	exeutables.

 -I	Continues test preparation almost from scratch, using an existing copy
	of the downloaded Sample Depot.  This picks up where '-G' leaves off.

	With -G or -I, Any exisitng p4d or p4broker processes from earlier test
	runs are killed.
	
	Next, it:
	* Configures the broker for CBD operation.
	* Blasts and resets the P4ROOT for the Sample Depot checkpoint.
	* Starts 'p4d' and 'p4broker' services.

	Using -I implies -C.

 -C	Initialize CBD on the Sample depot.  This picks up where '-I' leaves
	off.

	It:
	* Installs the CBD triggers.
	* Creates several streams for testing.
	* Creates and submits *.cbdsst files, causing the CBD trigger to
	fire and generate 'keys' data.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	$(dirname ${P4U_LOG}).

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

 -si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

FILES:
	The test.<hostname>.cfg files are CBD Test Configuration Files.  They define
	a set of host-specific values that define how CBD is tested, where
	test assets are aquired from, which versions of p4/p4d are used, etc.

	The cli_tests.txt file defines the command line tests to be executed,
	including expected exit codes and output for each.

	Each bit.*.txt (broker input test) file define one test simulated with
	a handcrafted broker input file.  The bit.cfg file defines the list of
	bit.*.txt files to be executed.

	The auto_test_cbd.sh wrapper script call this script with options
	optimzed for use in continuous integration build tests.  For example,
	the normally helpful self-logging features of this script are disasbled
	for CI builds, since the build system does it's own snazzy and webby
	log capture.

	ti.sh is a manual test support script that parses and verifies the
	syntax of the the cli_tests.txt test input data file.

EXAMPLES:
	For typical usage, no arguments are needed:

	cd /p4/common/bin/cbd/test
	./test_cbd.sh
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i RequireValidTests=1
declare -i GetSampleDepot=0
declare -i InitSampleDepot=0
declare -i InitCBD=0
declare -i shiftArgs=0
declare CfgDir=Undefined
declare CfgFile=

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-d) CfgDir=$2; shiftArgs=1;;
      (-f) RequireValidTests=0;;
      (-G) GetSampleDepot=1; InitSampleDepot=1; InitCBD=1;;
      (-I) InitSampleDepot=1; InitCBD=1;;
      (-C) InitCBD=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

if [[ "${P4U_LOG}" != off ]]; then
   LogDir="$(dirname $P4U_LOG)"
   if [[ ! -d $LogDir ]]; then
      /bin/mkdir -p "$LogDir" || bail "Couldn't create log dir [$LogDir]."
   fi

   touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee ${P4U_LOG})
      exec 2>&1
   else
      exec >${P4U_LOG}
      exec 2>&1
   fi

   initlog
fi

if [[ $CfgDir == Undefined ]]; then
   if [[ -d /shared ]]; then
      CfgDir=/shared
   else
      CfgDir=$PWD
   fi
fi

CfgFile=$CfgDir/test_cbd.${HOSTNAME%%\.*}.cfg

if [[ -r "$CfgFile" ]]; then
   source "$CfgFile" || bail "Failed to load config file [$CfgFile]. Aborting."
else
   bail "Missing config file [$CfgFile]. Aborting."
fi

# Sanity check on values loaded from the config file.
[[ -z "$SampleDepotHome" ]] && bail "Environment loaded from  is missing variable definitions. Aborting."
[[ -z "$CBD_HOME" ]] && bail "Environment loaded from  is missing variable definitions. Aborting."

declare p4Exe=$SampleDepotHome/p4
declare p4dExe=$SampleDepotHome/p4d
declare p4brokerExe=$SampleDepotHome/p4broker
declare BrokerCfg=$SampleDepotHome/PerforceSample/p4broker.cbd.cfg
declare p4="$p4Exe -p $SampleDepotBrokerPort -u bruno"
declare p4c="$p4 -c $TestWS"

declare ThisOS=$(uname -s)

export P4ROOT=$SampleDepotP4ROOT
export P4USER=bruno
export P4PORT=$SampleDepotP4PORT
export P4JOURNAL=$SampleDepotP4ROOT/journal
export P4LOG=$SampleDepotP4ROOT/log
export P4DEBUG=server=4
export P4TICKETS=$PWD/.p4tickets
export P4ENVIRO=$PWD/.p4enviro
export P4TRUST=$PWD/.p4trust
unset P4NAME
unset P4AUDIT

if [[ ${HOSTNAME%%\.*} != $RunHost ]]; then
   bail "Run this only on $RunHost."
   exit 1
else
   echo "Verified: Running on $RunHost."
fi

declare P4TestCmd=
declare BITFile=
declare ExpectedExit=
declare -i ExpectedExitCode
declare -i ActualExitCode
declare ExpectedString=
declare -i ExpectedStringInOutput
declare Comments=
declare CLITestDataOK=1
declare CLITestDataFile=$CfgDir/cli_tests.txt
declare BITestCfgFile=$CfgDir/bit.cfg
declare OutputFile=/tmp/cbd.test_output.$$.$RANDOM
declare -i AllTestsPassed
declare -i TestPassed
declare -i Line=0
declare -i TestCount=0
declare -i TestPassedCount=0
declare -i TestFailedCount=0
GARBAGE+=" $OutputFile"

msg "Loading and verifying command line test data from $CLITestDataFile."
[[ -r "$CLITestDataFile" ]] || bail "Missing command line test data file [$CLITestDataFile]."

# See the $CLITestDataFile file for details on the expected file format.
# Short version: We're expecting one-line entries like this:
# <P4Cmd>|<ExitCode>|<Output>|<Comments>
# The '<P4Cmd>' command *must* start with 'p4 '
while read entry; do
   Line=$((Line+1))
   [[ -z "$(echo $entry)" ]] && continue
   [[ $entry == "#"* ]] && continue
   P4TestCmd=${entry%%|*}
   ExpectedExit=${entry#*|}
   ExpectedExit=${ExpectedExit%%|*}
   ExpectedString=${entry%|*}
   ExpectedString=${ExpectedString##*|}
   Comments=${entry##*|}

   if [[ $P4TestCmd == "p4 "* ]]; then
      P4TestCmd=${P4TestCmd/p4 /$p4c }
   elif [[ $P4TestCmd == "p4:"* ]]; then
      # Parse entries like: "p4:user:client command args ..."
      User=${P4TestCmd#p4:}
      User=${User%%:*}
      Workspace=${P4TestCmd#p4:}
      Workspace=${Workspace#*:}
      Workspace=${Workspace%% *}
      P4TestCmd=${P4TestCmd/p4:$User:$Workspace /$p4 -u $User -c $Workspace }
   else
      echo -e "Warning: Entry on line $Line of $CLITestDataFile has a bogus 'p4' command.  Must start with 'p4 '.  Skipping this test."
      CLITestDataOK=0
      continue
   fi

   if [[ $ExpectedExit == U ]]; then
      ExpectedExit=Undefined
   else
      if [[ $ExpectedExit =~ [0-9]+ ]]; then
         ExpectedExitCode=$ExpectedExit
      else
         echo -e "Warning: Entry on line $Line of $CLITestDataFile has a bogus exit code.  Must be numeric or 'U', value is $ExpectedExit. Skipping this test."
         DataOK=0
         continue
      fi
   fi

   echo "C:[$P4TestCmd] E:[$ExpectedExit] S:[$ExpectedString] Comments: $Comments"
done < $CLITestDataFile

if [[ $CLITestDataOK -eq 1 ]]; then
   echo "Verified: All test entries are OK."
else
   if [[ $RequireValidTests -eq 1 ]]; then
      bail "The command line test data file [$CLITestDataFile] contained invalid entries. Use '-f' to bypassing bad tests and continue. Aborting."
   else
      echo -e "\nWarning: Some tests were skipped due to invalid entries in $CLITestDataFile. Continuing due to '-f'.\n"
   fi
fi

msg "$H\nPre-start test preparations."

if [[ $GetSampleDepot -eq 1 || $InitSampleDepot -eq 1 ]]; then
   shutdown_servers
fi

[[ $GetSampleDepot -eq 1 ]] && get_sample_depot
[[ $InitSampleDepot -eq 1 ]] && init_sample_depot
[[ $InitCBD -eq 1 ]] && init_cbd "$TestWS" "$TestWSRoot"

msg "$H\nExecuting Command Line Tests."

# It might seem inelegant to simply read-the config file in again, as opposed to storing and re-using the results
# from the data verification run above.  But with 'bash', storing results tends to have bad but subtle side effects,
# like having quote characters disappear.

AllTestsPassed=1

while read entry; do
   [[ -z "$(echo $entry)" ]] && continue
   [[ $entry == "#"* ]] && continue

   TestPassed=1
   P4TestCmd=${entry%%|*}
   ExpectedExit=${entry#*|}
   ExpectedExit=${ExpectedExit%%|*}
   ExpectedString=${entry%|*}
   ExpectedString=${ExpectedString##*|}

   # If the ExpectedString from the data file is prefixed with 'LOG:',
   # set ExpectedStringInOutput=0, indicating the cbd.log file shoudl be
   # searched instead of the command outpout.
   if [[ "$ExpectedString" == "LOG:"* ]]; then
      ExpectedString=${ExpectedString#LOG:}
      ExpectedStringInOutput=0
   else
      ExpectedStringInOutput=1
   fi

   Comments=${entry##*|}

   if [[ $P4TestCmd == "p4 "* ]]; then
      P4TestCmd=${P4TestCmd/p4 /$p4c }
   elif [[ $P4TestCmd == "p4:"* ]]; then
      # Parse entries like: "p4:user:client command args ..."
      User=${P4TestCmd#p4:}
      User=${User%%:*}
      Workspace=${P4TestCmd#p4:}
      Workspace=${Workspace#*:}
      Workspace=${Workspace%% *}
      P4TestCmd=${P4TestCmd/p4:$User:$Workspace /$p4 -u $User -c $Workspace }
   else
      continue
   fi


   if [[ $ExpectedExit == U ]]; then
      ExpectedExit=Undefined
   else
      if [[ $ExpectedExit =~ [0-9]+ ]]; then
         ExpectedExitCode=$ExpectedExit
      else
         continue
      fi
   fi

   TestCount=$((TestCount+1))
   msg "$H\nTest $TestCount\nTesting Command: $P4TestCmd\nExpected Exit: $ExpectedExit"
   if [[ $ExpectedStringInOutput -eq 1 ]]; then
      msg "Expected string in output: $ExpectedString\nComments: $Comments\n"
   else
      msg "Expected string in cbd.log: $ExpectedString\nComments: $Comments\n"
   fi
   $P4TestCmd > $OutputFile 2>&1
   ActualExitCode=$?

   echo -e "\n== Command Output =="
   cat $OutputFile

   echo TEST_EXIT_CODE: Actual $ActualExitCode, Expected $ExpectedExit

   if [[ $ExpectedExit == U ]]; then
      echo "Ignoring TEST_EXIT_CODE due to 'U' value."
   else
      [[ $ExpectedExitCode -ne $ActualExitCode ]] && TestPassed=0
   fi

   if [[ -f "$CBD_HOME/logs/cbd.log" ]]; then
      echo -e "\n== Contents of cbd.log =="
      cat $CBD_HOME/logs/cbd.log
   fi

   if [[ -n "$ExpectedString" ]]; then
      # Check for the expected string in the command output or the cbd.log file.
      if [[ $ExpectedStringInOutput -eq 1 ]]; then
         grep "$ExpectedString" $OutputFile > /dev/null 2>&1
         if [[ $? -eq 0 ]]; then
            echo -e "\nExpected string [$ExpectedString] found in command outpout."
         else
            echo -e "\nExpected string [$ExpectedString] NOT found in command outpout."
            TestPassed=0
         fi
      else
         if [[ -f "$CBD_HOME/logs/cbd.log" ]]; then
            grep "$ExpectedString" "$CBD_HOME/logs/cbd.log" > /dev/null 2>&1
            if [[ $? -eq 0 ]]; then
               echo -e "\nExpected string [$ExpectedString] found in cbd.log."
            else
               echo -e "\nExpected string [$ExpectedString] NOT found in cbd.log."
               TestPassed=0
            fi
         else
            echo "String [$ExpectedString] expected in cbd.log, but cbd.log is not there."
            TestPassed=0
         fi
      fi
   else
      echo "No expected string defined, skipping check for this test."
   fi

   /bin/rm -f "$CBD_HOME/logs/cbd.log" "$OutputFile"

   if [[ $TestPassed -eq 1 ]]; then
      echo -e "\nTEST $TestCount PASSED\n"
      TestPassedCount=$((TestPassedCount+1))
   else
      echo -e "\nTEST $TestCount FAILED\n"
      TestFailedCount=$((TestFailedCount+1))
      AllTestsPassed=0
   fi

done < $CLITestDataFile

msg "$H\nExecuting Broker Input Simulation Tests."

[[ -r "$BITestCfgFile" ]] || bail "Missing broker input test data file [$BITestCfgFile]."

while read entry; do
   [[ -z "$(echo $entry)" ]] && continue
   [[ $entry == "#"* ]] && continue

   TestPassed=1

   BITFile=${entry%%|*}

   ExpectedExit=${entry#*|}
   ExpectedExit=${ExpectedExit%%|*}
   ExpectedString=${entry%|*}
   ExpectedString=${ExpectedString##*|}

   # If the ExpectedString from the data file is prefixed with 'LOG:',
   # set ExpectedStringInOutput=0, indicating the cbd.log file shoudl be
   # searched instead of the command outpout.
   if [[ "$ExpectedString" == "LOG:"* ]]; then
      ExpectedString=${ExpectedString#LOG:}
      ExpectedStringInOutput=0
   else
      ExpectedStringInOutput=1
   fi

   Comments=${entry##*|}

   TestCount=$((TestCount+1))
   msg "$H\nTest $TestCount\nTesting Broker Input File: $BITFile\nExpected Exit: $ExpectedExit"

   if [[ $ExpectedStringInOutput -eq 1 ]]; then
      msg "Expected string in output: $ExpectedString\nComments: $Comments\n"
   else
      msg "Expected string in cbd.log: $ExpectedString\nComments: $Comments\n"
   fi

   if [[ -r "$CfgDir/$BITFile" ]]; then
      msg "Contents of $BITFile:"
      cat $CfgDir/$BITFile

      $CBD_HOME/scripts/wssync.sh < $CfgDir/$BITFile > $OutputFile 2>&1
      ActualExitCode=$?

      echo -e "\n== Command Output =="
      cat $OutputFile

      echo TEST_EXIT_CODE: Actual $ActualExitCode, Expected $ExpectedExit

      if [[ $ExpectedExit == U ]]; then
         echo "Ignoring TEST_EXIT_CODE due to 'U' value."
      else
         [[ $ExpectedExitCode -ne $ActualExitCode ]] && TestPassed=0
      fi

      if [[ -f "$CBD_HOME/logs/cbd.log" ]]; then
         echo -e "\n== Contents of cbd.log =="
         cat $CBD_HOME/logs/cbd.log
      fi

      if [[ -n "$ExpectedString" ]]; then
         # Check for the expected string in the command output or the cbd.log file.
         if [[ $ExpectedStringInOutput -eq 1 ]]; then
            grep "$ExpectedString" $OutputFile > /dev/null 2>&1
            if [[ $? -eq 0 ]]; then
               echo -e "\nExpected string [$ExpectedString] found in command outpout."
            else
               echo -e "\nExpected string [$ExpectedString] NOT found in command outpout."
               TestPassed=0
            fi
         else
            if [[ -f "$CBD_HOME/logs/cbd.log" ]]; then
               grep "$ExpectedString" "$CBD_HOME/logs/cbd.log" > /dev/null 2>&1
               if [[ $? -eq 0 ]]; then
                  echo -e "\nExpected string [$ExpectedString] found in cbd.log."
               else
                  echo -e "\nExpected string [$ExpectedString] NOT found in cbd.log."
                  TestPassed=0
               fi
            else
               echo "String [$ExpectedString] expected in cbd.log, but cbd.log is not there."
               TestPassed=0
            fi
         fi
      else
         echo "No expected string defined, skipping check for this test."
      fi

      /bin/rm -f "$CBD_HOME/logs/cbd.log" "$OutputFile"

   else
      errmsg "Missing broker input test file [$CfgDir/$BITFile]."
      TestPassed=0
   fi

   if [[ $TestPassed -eq 1 ]]; then
      echo -e "\nTEST $TestCount PASSED\n"
      TestPassedCount=$((TestPassedCount+1))
   else
      echo -e "\nTEST $TestCount FAILED\n"
      TestFailedCount=$((TestFailedCount+1))
      AllTestsPassed=0
   fi

done < $BITestCfgFile

[[ $AllTestsPassed -eq 0 ]] && OverallReturnStatus=1

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\nAll $TestCount tests PASSED.\n"
else
   msg "${H}\nProcessing completed, but with errors.\nTests Executed (Passed/Failed): $TestCount ($TestPassedCount/$TestFailedCount)\n\nScan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took about $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus