test_cbd.sh #13

#!/bin/bash
#------------------------------------------------------------------------------
set -u

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG="/tmp/test_cbd.$(date +'%Y%m%d-%H%M%S').log"
export VERBOSITY=${VERBOSITY:-3}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=BogusValueNonExistentFile

[[ -r "$P4U_ENV" ]] || {
   echo -e "\nError: Cannot load environment from: $P4U_ENV.  Aborting.\n\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   source $bash_lib
done

declare Version=1.6.9
declare -i SilentMode=0
declare -i OverallReturnStatus=0

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Override of the default bail() function. This sets the global
# OverallReturnStatus.
function bail () {
   echo -e "\nFATAL: ${1:-Unknown Error}\n"
   OverallReturnStatus=${2:-1}
   exit ${2:-1};
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit $OverallReturnStatus
}

#------------------------------------------------------------------------------
# Function: get_pids()
#
# Safely get just the appropraite processes.  Requires fully qualified path
# to a copy of p4d/p4broker used for this instance.
#
# Examples:
#   p4dPids=get_pids($p4dExe)
#   p4brokerPids=get_pids($p4brokerExe)
#------------------------------------------------------------------------------
function get_pids()
{
   [[ -z "$1" ]] && bail "get_pids(): No executable specified."
   exe=$1

   if [[ $ThisOS == Darwin ]]; then
      pids=$(/bin/ps -ef | grep "$exe" | grep "$(id -u)" | grep -v grep | awk '{print $2}')
   else
      pids=$(/bin/ps -f -C $(basename $exe) | grep $exe|awk '{print $2}')
   fi

   echo $pids
}

#------------------------------------------------------------------------------
# Function: shutdown_servers()
#------------------------------------------------------------------------------
function shutdown_servers()
{
   vvmsg "CALL shutdown_servers($*)"

   # If we are using the SDP, the sdp reset script, called elsewhwere, will
   # take care of shutting down p4d and p4broker processes started SDP-style.
   [[ $UseSDP -eq 1 ]] && return 0

   p4dPids=$(get_pids "$SampleDepotHome/p4d")
   p4brokerPids=$(get_pids "$SampleDepotHome/p4broker")

   if [[ -n "$p4dPids" ]]; then
      runCmd "kill -9 $p4dPids" "Killing P4D pids [$p4dPids]."
   else
      msg "Verified: No P4D processes for the Sample Depot are running."
   fi

   if [[ -n "$p4brokerPids" ]]; then
      runCmd "kill -9 $p4brokerPids" "Killing P4Broker pids [$p4brokerPids]."
   else
      msg "Verified: No P4Broker processes for the Sample Depot are running."
   fi

   msg "Waiting a second to ensure pids stop."
   sleep 1

   return 0
}

#------------------------------------------------------------------------------
# Function: get_sample_depot()
#------------------------------------------------------------------------------
function get_sample_depot()
{
   vvmsg "CALL get_sample_depot($*)"
   msg "Getting Sample Depot."
   declare -i gotFile=0

   if [[ -d $SampleDepotHome ]]; then
      runCmd "/bin/rm -rf $SampleDepotHome" "Blasting existing Sample Depot home dir [$SampleDepotHome]." ||\
         bail "Failed to blast existing Sample Depot home dir [$SampleDepotHome]."
   fi

   runCmd "/bin/mkdir -p $SampleDepotHome" "Creating empty new Sample Depot home dir [$SampleDepotHome]" ||\
      bail "Failed to create new Sample Depot root [$SampleDepotHome]."

   cd $SampleDepotHome || bail "Failed to cd to Sample Depot home dir [$SampleDepotHome]."

   msg "Operating in [$PWD]."

   if [[ $UseSDP -eq 1 && -f /depotdata/$SampleFileGZFile ]]; then
      runCmd "ln -s /depotdata/$SampleFileGZFile" \
         "Symlinking existing Sample Depot tarfile using one downloaded by SDP Helix Installer." ||\
         "Failed to create symlink in $PWD to /depotdata/$SampleFileGZFile."
      gotFile=1
   fi

   if [[ $gotFile -eq 0 ]]; then
      runCmd "wget -q $SampleDepotURL" \
         "Pulling Sample Depot from [$SampleDepotURL]." ||\
         bail "Failed to pull Sample Depot."
   fi

   msg "Sample Depot pulled successfully."

   msg "Now, pulling Perforce executables for $p4Version on $Platform."

   for exeUrl in $p4URL $p4dURL $p4brokerURL; do
      exe=${exeUrl##*/}
      if [[ $UseSDP -eq 0 ]]; then
         runCmd "wget -q $exeUrl" \
            "Pulling exe from [$exeUrl]." ||\
         bail "Failed to pull exe from [$exeUrl]."
         runCmd "chmod +x $exe" "Adjusting exe perms (chmod +x)."
         runCmd "./$exe -V" "Obtained [$exe] version:" 0 1
      fi
   done

   msg "Perforce executables pulled."
}

#------------------------------------------------------------------------------
# Function: init_sample_depot()
#
# This picks up where get_sample_depot leaves off.
# The $SampleDepotHome folder will exist and contain the sample_depot.tar.gz
# file, as downloaded from the Perforce FTP site.
#------------------------------------------------------------------------------
function init_sample_depot()
{
   vvmsg "CALL init_sample_depot($*)"

   if [[ $UseSDP -eq 1 ]]; then
      resetScriptURL=https://swarm.workshop.perforce.com/downloads/guest/tom_tyler/helix-installer/main/src/reset_sdp.sh
      resetScript=/depotdata/reset/reset_sdp.sh
      resetScriptArgs="-B -no_ssl"
      resetLog=/tmp/reset_sdp.$$.log

      if [[ ! -x $resetScript ]]; then
         msg "SDP Reset script [$resetScript] does not exist.  Attempting to acquire it."
         if [[ ! -d /depotdata/reset ]]; then
            runCmd "mkdir -p /depotdata/reset" \
               "Creating dir: /depotdata/reset" ||\
               bail "Failed to create dir: /depotdata/reset"
         fi

         cd $(dirname $resetScript) || bail "Could not cd to $(dirname $resetScript)."
         runCmd "wget -q $resetScriptURL" \
            "Getting Helix Installer SDP Reset script from this URL: $resetScriptURL" ||\
            bail "Unable to acquire Helix Installer SDP Reset script."

         runCmd "chmod +x $resetScript" "chmod +x" ||\
            bail "Failed to set perms on Helix Installer SDP reset script."
      fi

      cd $(dirname $resetScript) || bail "Could not cd to $(dirname $resetScript)."
      echo -e "Calling Helix Installer SDP Reset script $resetScript, with args $resetScriptArgs.\nLogging to: $resetLog"
      sudo $resetScript $resetScriptArgs > $resetLog 2>&1 ||\
         bail "SDP Reset failed. Aborting!  Check the log: $resetLog"

      echo -e "# Added by CBD test automation.\nexport CBD_HOME=$CBD_HOME\n" >> /p4/common/bin/p4_vars

   else
      if [[ ! -d "$SampleDepotHome" ]]; then
         msg "Info: Sample Depot Home dir [$SampleDepotHome] does not exist.  Operating as if '-G' was specified."
         shutdown_servers
         get_sample_depot
      fi

      cd $SampleDepotHome || bail "Failed to cd to Sample Depot Home dir [$SampleDepotHome]."
      msg "Operating in [$PWD]."
      [[ -r "$SampleFileGZFile" ]] || bail "Cannot access Sample Depot GZip file [$SampleFileGZFile]."

      runCmd "tar -xzpf $SampleFileGZFile" || bail "Failed to expand tar file [$tmpTarFile]."

      cd PerforceSample ||\
      bail "Expanded tarfile is missing expected 'PerforceSample' dir."

      [[ $PWD != $SampleDepotP4ROOT ]] && bail "Unexpected Error: Bad value for \$PWD."
   fi

   cd $SampleDepotHome
   msg "Operating in $PWD."

   echo "P4PORT=$SampleDepotBrokerPort
P4USER=bruno
P4CLIENT=NoWorkspaceDefined
P4TICKETS=$P4TICKETS
P4IGNORE=.p4ignore" > .p4config

   export P4CONFIG=$PWD/.p4config

   msg "Generated this .p4config file in $PWD:\n$(cat .p4config)\n"

   if [[ $UseSDP -eq 1 ]]; then
      msg "Restarting SDP Instance 1 with sample depot $P4PORT."
      /p4/1/bin/p4d_1_init restart
      /p4/1/bin/p4broker_1_init restart
      sleep 1
   else
      cd $P4ROOT || bail "Failed to cd to $P4ROOT."

      msg "Operating in [$PWD]."

      runCmd "/bin/rm -rf server.locks"
      runCmd "/bin/rm -f db.* rdb.* state log journal audit"
      runCmd "$p4dExe -r $P4ROOT -jr $SampleDepotHome/PerforceSample/checkpoint" \
         "Loading Sample Depot checkpoint." ||\
         bail "Failed to load Sample Depot checkpoint."
      runCmd "$p4dExe -r $P4ROOT -xu" \
         "Upgrading Sample Depot databases." ||\
         bail "Failed to upgrade Sample Depot checkpoint."
      runCmd "$p4dExe -r $P4ROOT -p $P4PORT -d -J $P4JOURNAL -L $P4LOG -v $P4DEBUG < /dev/null" \
         "Starting Sample Depot P4D on port $P4PORT."
   fi

   msg "Writing p4broker config file [$BrokerCfg]."

   echo "target      = $SampleDepotP4PORT;
listen      = $SampleDepotBrokerPort;
directory   = $SampleDepotP4ROOT;
logfile     = $SampleDepotP4ROOT/p4broker.log;
debug-level = server=1;
admin-name  = \"John Doe\";
admin-phone = 555-555-1212;
admin-email = \"jdoe@p4demo.com\";
compress = false;
redirection  = selective;

command: ^(sync|update|flush)\$
{
   action = filter;
   execute = \"$CBD_HOME/scripts/wssync.sh\";
}

command: ^(sbi)\$
{
   action = filter;
   execute = \"$CBD_HOME/scripts/sbi.pl\";
}


" > $BrokerCfg

   if [[ $UseSDP -eq 1 ]]; then
      msg "Using SDP broker."
   else
      runCmd "$p4brokerExe -d -c $BrokerCfg < /dev/null" \
         "Starting Sample Depot P4Broker $SampleDepotBrokerPort."
      p4brokerPids=$(get_pids "$p4brokerExe")

      if [[ -n "$p4brokerPids" ]]; then
         p4brokerPidFile=$PWD/.p4broker.pid
         echo $p4brokerPids > $p4brokerPidFile
         msg "P4Broker process pids: $(cat $p4brokerPidFile)."
      else
         bail "Could not determine pid for p4broker."
      fi 
   fi
}

#------------------------------------------------------------------------------
# Function: init_cbd
#
# Intialize the CBD stuff.  This needs to be restartable/idempotent.
#------------------------------------------------------------------------------
function init_cbd()
{
   vvmsg "CALL init_cbd($*)"
   declare ws=$1
   declare wsRoot=$2
   declare tmpFile=/tmp/tmp.test_cbd.init_cbd.$$.$RANDOM

   if [[ ! -f "$BrokerCfg" ]]; then
      msg "Info: Broker Config file [$BrokerCfg] does not exist.  Operating as if '-I' was specified."
      shutdown_servers
      init_sample_depot
   fi

   if [[ -d "$CBD_HOME/logs" ]]; then
      runCmd "/bin/rm -rf $CBD_HOME/logs" "Cleaning up logs from earlier runs."
   fi

   if [[ ! -d "$CBD_HOME/logs" ]]; then
      runCmd "/bin/mkdir -p $CBD_HOME/logs" "Creating empty log dir." ||\
         bail "Failed to create CBD Logs dir [$CBD_HOME/logs]."
   fi

   msg "Loading CBD Stream and Workspace Template Update triggers."

   echo -e "Triggers:\n\tSSTemplateUpdate change-content //...cbdsst \"$CBD_HOME/triggers/SSTemplateUpdate.py -v DEBUG -L $SSTULog %changelist%\"\n\tSSTemplateUpdateC change-commit //...cbdsst \"$CBD_HOME/triggers/SSTemplateUpdate.py -v DEBUG -L $SSTULog %changelist%\"" > $tmpFile
   $p4 -s triggers -i < $tmpFile || cat $tmpFile

   msg "Triggers table is now:"
   $p4 triggers -o | egrep -v '^#'

   msg "Generating workspace for bruno."

   sstFile=$wsRoot/jam.cbdsst

   runCmd "$p4 client -d $ws"

   msg "Making bruno a super user."

   if [[ "$NO_OP" != 1 ]]; then
      msg "Running: $p4 protect -o \| $p4 -s protect -i"
      $p4 protect -o | $p4 -s protect -i
   else
      msg "NO_OP: Would run: $p4 protect -o \| $p4 -s protect -i"
   fi

   echo -e "Client:\t$ws\n
Owner:\tbruno\n
Description:\n\tJam stream workspace.\n
Root:\t$wsRoot\n
Options:\tnoallwrite noclobber nocompress unlocked modtime rmdir\n
SubmitOptions:\tleaveunchanged\n
LineEnd:\tlocal\n
Stream: //jam/rel2.1\n" > $tmpFile
   $p4 -s client -f -i < $tmpFile || cat $tmpFile

   if [[ -d "$wsRoot" ]]; then
      runCmd "/bin/rm -rf $wsRoot" ||\
         bail "Failed to cleanup test workspace root dir [$wsRoot]."
   fi

   runCmd "/bin/mkdir -p $wsRoot"

   msg "Cleanup from earlier runs."
   runCmd "/bin/rm -f $sstFile"

   echo "P4PORT=$SampleDepotBrokerPort
P4USER=bruno
P4CLIENT=$ws
P4TICKETS=$P4TICKETS
P4IGNORE=.p4ignore" > $wsRoot/.p4config

   msg "Generated P4CONFIG file $wsRoot/.p4config file:\n$(cat $wsRoot/.p4config)\n"

   echo -e "Client: bruno.LT-8BQ0VM1.Jam\n
Owner: bruno\n
Host: LT-8BQ0VM1\n
Description:\n\tCreated by bruno.\n
Root: C:\\p4\\Jam
Options: noallwrite noclobber nocompress unlocked modtime rmdir\n
SubmitOptions: leaveunchanged\n
LineEnd: local\n
Stream: //jam/rel2.1\n\n" > $tmpFile

   $p4 -s client -f -i < $tmpFile || cat $tmpFile

   msg "Adjusting flow of change, 2.1 -> 2.2 -> 2.3 -> main -> dev*"

   echo -e "Stream: //jam/rel2.1\n
Owner: bruno\n
Name:  rel2.1\n
Parent: //jam/rel2.2\n
Type:  release\n
Description:\n\tJam 2.1 release\n
Options: allsubmit unlocked toparent nofromparent\n
Paths:\n\tshare ...\n" > $tmpFile

   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Stream: //jam/rel2.2\n
Owner: bruno\n
Name: rel2.2\n
Parent: //jam/rel2.3\n
Type: release\n
Description:\n\tJam 2.2 release\n
Options: allsubmit unlocked toparent nofromparent\n
Paths:\n\tshare ...\n" > $tmpFile

   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Stream: //jam/rel2.3\n
Owner: bruno\n
Name: rel2.3\n
Parent: //jam/main\n
Type: release\n
Description:\n\tJam 2.3 release stream\n
Options: allsubmit unlocked toparent nofromparent\n
Paths:\n\tshare ...\n" > $tmpFile

   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Stream: //jam/dev2.3\n
Owner: bruno\n
Name: dev2.3\n
Parent: //jam/main\n
Type: development\n
Description:\n\tJam 2.3 development stream\n
Options: allsubmit unlocked toparent fromparent\n
Paths:\n\tshare ...\n" > $tmpFile

   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Stream: //jam/task-4491\n
Owner: bruno\n
Name: task-4491\n
Parent: //jam/dev2.3\n
Type: task\n
Description:\n\tTask stream for task 4491.\n
Options: allsubmit unlocked toparent fromparent\n
Paths:\n\tshare ...\n" > $tmpFile
   $p4 -s stream -f -i < $tmpFile || cat $tmpFile

   echo -e "Client: jsmith.TomsLaptop.depot\n
Owner: jsmith\n
Host: LT-8BQ0VM1\n
Description:\n\tCreated by jsmith.\n
Root: C:\\p4\\CBD\n
Options: noallwrite noclobber nocompress unlocked modtime rmdir\n
SubmitOptions:  leaveunchanged\n
LineEnd:        local\n
Stream: //jam/task-4491\n\n" > $tmpFile

   $p4 -s client -f -i < $tmpFile || cat $tmpFile

   msg "Generating sample Stream Spec Template file."

   echo -e "Stream: __EDITME_STREAM__\n\nDescription:\n\tStream spec for __EDITME_STREAM__.\n\nPaths:\n\tshare ...\n\timport pb/... //pb/1.5.1-p/...@8883\n\timport gwt/... //gwt-streams/release1.5/...@12047\n\n" > $sstFile

   runCmd "/bin/rm -f $SSTULog"
   runCmd "$p4 -c $ws add $sstFile" "Adding Stream Spec Template file."
   runCmd "$p4 -c $ws submit -r -d Added. $sstFile" "Submitting $sstFile." ||\
      bail "Failed to submit $sstFile."
   runCmd "$p4 -c $ws tag -l Jam-2.0.1.01 //jam/rel2.1/..." "Applying Jam-2.0.1.01 label to //jam/rel2.1/...." 1 0 0

   sleep 1
   runCmd "cat $SSTULog" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $SSTULog"

   echo -e "Stream: __EDITME_STREAM__\n\nDescription:\n\tStream spec for __EDITME_STREAM__.\n\nPaths:\n\tshare ...\n\timport pb/... //pb/1.5.1-p/...@8906\n\timport gwt/... //gwt-streams/release1.5/...@12048\n\n" > $sstFile

   runCmd "$p4 -c $ws submit -r -d Updated. $sstFile" " Submitting $sstFile" ||\
      bail "Failed to submit $sstFile."
   runCmd "$p4 -c $ws tag -l Jam-2.0.1.02 //jam/rel2.1/..." "Applying Jam-2.0.1.02 label to //jam/rel2.1/...." 1 0 0
   sleep 1
   runCmd "cat $SSTULog" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $SSTULog"

   echo -e "Stream: __EDITME_STREAM__\n\nDescription:\n\tStream spec for __EDITME_STREAM__.\n\nPaths:\n\tshare ...\n\timport pb/... //pb/1.5.1-p/...@8906\n\timport gwt/... //gwt-streams/release1.5/...@12050\n\n" > $sstFile

   runCmd "$p4 -c $ws submit -d Re-Updated. $sstFile" "Submitting $sstFile." ||\
      bail "Failed to submit $sstFile."
   runCmd "$p4 -c $ws tag -l Jam-2.0.1.03 //jam/rel2.1/..." "Applying Jam-2.0.1.03 label to //jam/rel2.1/...." 1 0 0
   sleep 1
   runCmd "cat $SSTULog" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $SSTULog"

   runCmd "$p4 -c $ws populate //jam/rel2.1/jam.cbdsst //jam/rel2.2/jam.cbdsst" \
      "Populating CBD stream spec template file in jam-rel2.2."
   runCmd "cat $SSTULog" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $SSTULog"

   runCmd "$p4 -c $ws populate //jam/rel2.2/jam.cbdsst //jam/rel2.3/jam.cbdsst" \
      "Populating CBD stream spec template file in jam-rel2.3."
   runCmd "cat $SSTULog" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $SSTULog"

   runCmd "$p4 -c $ws populate //jam/rel2.3/jam.cbdsst //jam/main/jam.cbdsst" \
      "Populating CBD stream spec template file in main."
   runCmd "cat $SSTULog" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $SSTULog"

   runCmd "$p4 -c $ws populate //jam/main/jam.cbdsst //jam/dev2.3/jam.cbdsst" \
      "Populating CBD stream spec template file in dev2.3."
   runCmd "cat $SSTULog" "== Stream Spec Trigger Log =="
   runCmd "/bin/rm -f $SSTULog"

   runCmd "$p4 -c $ws populate //jam/dev2.3/... //jam/task-4491/..." \
      "Populating task stream jam-task4491."
   runCmd "cat $SSTULog" "== Stream Spec Trigger Log =="

   msg "Setting a password for user earl."
   echo -e "Abcdefg123\nAbcdefg123" > $tmpFile
   $p4 -u bruno passwd earl < $tmpFile

   msg "Creating locaked earl_jams workspace."

   echo -e "Client: earl_jams\n\nOwner:\tearl\n\nDescription:\n\tCreated by earl.\n\nRoot: C:\\p4\\CT\n\nOptions: noallwrite noclobber nocompress locked modtime rmdir\n\nSubmitOptions: leaveunchanged\n\nLineEnd: local\n\nStream: //jam/rel2.1\n\n" > $tmpFile

   $p4 -s client -f -i < $tmpFile || cat $tmpFile

   runCmd "/bin/rm -f $tmpFile" "Removing temp file [$tmpFile]." 0 0 0
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT [-G|-I|-C] [-S] [-d <cfg_dir>] [-a] [-f] [-L <log>] [-si] [-v<n>] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script runs test for the Component Based Develoment (CBD)
	system, supplemental automation that enhances the CBD capabilities
	of a Perforce Helix server.

OPTIONS:
 -S	Specify the '-S' flag to test the SDP executables and Sample Depot
	tar downloaded by the Helix Installer.  Using '-S' cause the
	'reset_sdp' script from the Helix Installer project to be used to
	bootstrap the SDP and install the Sample Depot.  If '-S' is not
	used, the script uses its own internal logic for acquring the sample
	Depot and laoding it into a Helix Server.

 -a	Use '-a' to indicate automated continuous integration testing.  In
	this mode, the p4d and p4broker services started for testing are
	shutown upcon completion.  (For manual testing, it is more convenient
	to leave them running).

 -d <cfg_dir>
	Specify the configuration directory where configuration and test
	data are expected to exist.  By default, scripts are expected to live
	in a directory named '/shared' if that directory exists, or else in
	a 'shared' directory under the current directory at the time this script
	was called, or else in a 'shared' directory under the directory where
	this exists (if called with a fully qualified path).  It looks real
	hard to try to find the config dir.

	The use of the /shared folder is consistent with the 'helix-01' test
	virtual machine setup for automated testing.

 -f	Use '-f' to bypass any invalid test entries in the command line test
	data.

 -G	Starts test preparation from scratch, by:
	* Getting a pristine copy of the Sample Depot tarfile from the
	Perforce FTP server,
	* Downloading Helix executables p4/p4d/p4broker

	Using -G implies -I and -C.

	The existing Sample Depot and related Perforce server instance are
	blasted with -G.

	The '-G' is implied if the configured Sample Depot Home dir does not
	exist.  It can be specified to force a clean start with the latest
	exeutables.

 -I	Continues test preparation almost from scratch, using an existing copy
	of the downloaded Sample Depot.  This picks up where '-G' leaves off.

	With -G or -I, Any exisitng p4d or p4broker processes from earlier test
	runs are killed.
	
	Next, it:
	* Configures the broker for CBD operation.
	* Blasts and resets the P4ROOT for the Sample Depot checkpoint.
	* Starts 'p4d' and 'p4broker' services.

	Using -I implies -C.

 -C	Initialize CBD on the Sample depot.  This picks up where '-I' leaves
	off.

	It:
	* Installs the CBD triggers.
	* Creates several streams for testing.
	* Creates and submits *.cbdsst files, causing the CBD trigger to
	fire and generate 'keys' data.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	$(dirname ${P4U_LOG}).

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

 -si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

FILES:
	The test.<hostname>.cfg files are CBD Test Configuration Files.  They define
	a set of host-specific values that define how CBD is tested, where
	test assets are aquired from, which versions of p4/p4d are used, etc.

	The cli_tests.cfg file defines the command line tests to be executed,
	including expected exit codes and output for each.

	Each bit.*.txt (broker input test) file define one test simulated with
	a handcrafted broker input file.  The bit.cfg file defines the list of
	bit.*.txt files to be executed.

	The auto_test_cbd.sh wrapper script call this script with options
	optimzed for use in continuous integration build tests.  For example,
	the normally helpful self-logging features of this script are disasbled
	for CI builds, since the build system does it's own snazzy and webby
	log capture.

	ti.sh is a manual test support script that parses and verifies the
	syntax of the the cli_tests.cfg test input data file.

EXAMPLES:
	For typical usage, no arguments are needed:

	cd /p4/common/bin/cbd/test
	./test_cbd.sh
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i CIAutomation=0
declare -i UseSDP=0
declare -i RequireValidTests=1
declare -i GetSampleDepot=0
declare -i InitSampleDepot=0
declare -i InitCBD=0
declare -i shiftArgs=0
declare ThisOS=$(uname -s)
declare ThisHost=$(hostname -s)
declare CfgDir=Undefined
declare CfgFile=

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-d) CfgDir=$2; shiftArgs=1;;
      (-S) UseSDP=1;;
      (-a) CIAutomation=1;;
      (-f) RequireValidTests=0;;
      (-G) GetSampleDepot=1; InitSampleDepot=1; InitCBD=1;;
      (-I) InitSampleDepot=1; InitCBD=1;;
      (-C) InitCBD=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

if [[ "${P4U_LOG}" != off ]]; then
   LogDir="$(dirname $P4U_LOG)"
   if [[ ! -d $LogDir ]]; then
      /bin/mkdir -p "$LogDir" || bail "Couldn't create log dir [$LogDir]."
   fi

   touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee ${P4U_LOG})
      exec 2>&1
   else
      exec >${P4U_LOG}
      exec 2>&1
   fi

   initlog
fi

if [[ $CfgDir == Undefined ]]; then
   if [[ -d /shared ]]; then
      CfgDir=/shared
   elif [[ "$CMDLINE" == /* ]]; then
      CfgDir=$(dirname $0)/shared
   else
      CfgDir=$PWD/shared
   fi
fi

CfgFile=$CfgDir/test_cbd.$ThisHost.cfg

if [[ -r "$CfgFile" ]]; then
   source "$CfgFile" || bail "Failed to load config file [$CfgFile]. Aborting."
else
   bail "Missing config file [$CfgFile]. Aborting."
fi

# Sanity check on values loaded from the config file.
[[ -z "$SampleDepotHome" ]] && bail "Environment loaded from  is missing variable definitions. Aborting."
[[ -z "$CBD_HOME" ]] && bail "Environment loaded from  is missing variable definitions. Aborting."

if [[ $UseSDP -eq 1 ]]; then
   declare p4Exe=/p4/1/bin/p4_1
   declare p4dExe=/p4/1/bin/p4d_1
   declare p4brokerExe=/p4/1/bin/p4broker_1
   declare BrokerCfg=/p4/common/config/p4_1.broker.cfg
else
   declare p4Exe=$SampleDepotHome/p4
   declare p4dExe=$SampleDepotHome/p4d
   declare p4brokerExe=$SampleDepotHome/p4broker
   declare BrokerCfg=$SampleDepotHome/PerforceSample/p4broker.cbd.cfg
fi

declare p4="$p4Exe -p $SampleDepotBrokerPort -u bruno"
declare p4c="$p4 -c $TestWS"

if [[ $UseSDP -eq 1 ]]; then
   CBDLog=/p4/1/logs/cbd.log
   SSTULog=/p4/1/logs/SSTemplateUpdate.log
   SSUTLog=/p4/1/logs/SSUpdateTemplate.log
else
   CBDLog="$CBD_HOME/logs/cbd.log"
   SSTULog="$CBD_HOME/logs/SSTemplateUpdate.log"
   SSUTLog="$CBD_HOME/logs/SSUpdateTemplate.log"
fi

export P4ROOT=$SampleDepotP4ROOT
export P4USER=bruno
export P4PORT=$SampleDepotP4PORT
export P4JOURNAL=$SampleDepotP4ROOT/journal
export P4LOG=$SampleDepotP4ROOT/log
export P4DEBUG=server=4
export P4TICKETS=$PWD/.p4tickets
export P4ENVIRO=$PWD/.p4enviro
export P4TRUST=$PWD/.p4trust
unset P4NAME
unset P4AUDIT

if [[ $ThisHost != $RunHost ]]; then
   bail "Not configured to run on host $ThisHost.  Run only on $RunHost, configured in $CfgFile."
   exit 1
else
   echo "Verified: Running on $RunHost."
fi

declare P4TestCmd=
declare BITFile=
declare BITFile2=/tmp/tmp.test_cbd.bit.$$.RANDOM
declare ExpectedExit=
declare -i ExpectedExitCode
declare -i ActualExitCode
declare ExpectedString=
declare -i ExpectedStringInOutput
declare Comments=
declare CLITestDataOK=1
declare CLITestDataFile=$CfgDir/cli_tests.cfg
declare BITestCfgFile=$CfgDir/bit.cfg
declare ScriptedTestCfgFile=$CfgDir/scripted_tests.cfg
declare OutputFile=/tmp/cbd.test_output.$$.$RANDOM
declare -i AllTestsPassed
declare -i TestPassed
declare -i Line=0
declare -i TestCount=0
declare -i TestPassedCount=0
declare -i TestFailedCount=0
GARBAGE+=" $OutputFile"

msg "Loading and verifying command line test data from $CLITestDataFile."
[[ -r "$CLITestDataFile" ]] || bail "Missing command line test data file [$CLITestDataFile]."

# See the $CLITestDataFile file for details on the expected file format.
# Short version: We're expecting one-line entries like this:
# <P4Cmd>|<ExitCode>|<Output>|<Comments>
# The '<P4Cmd>' command *must* start with 'p4 '
while read entry; do
   Line=$((Line+1))
   [[ -z "$(echo $entry)" ]] && continue
   [[ $entry == "#"* ]] && continue
   P4TestCmd=${entry%%|*}
   ExpectedExit=${entry#*|}
   ExpectedExit=${ExpectedExit%%|*}
   ExpectedString=${entry%|*}
   ExpectedString=${ExpectedString##*|}
   Comments=${entry##*|}

   if [[ $P4TestCmd == "p4 "* ]]; then
      P4TestCmd=${P4TestCmd/p4 /$p4c }
   elif [[ $P4TestCmd == "p4:"* ]]; then
      # Parse entries like: "p4:user:client command args ..."
      User=${P4TestCmd#p4:}
      User=${User%%:*}
      Workspace=${P4TestCmd#p4:}
      Workspace=${Workspace#*:}
      Workspace=${Workspace%% *}
      P4TestCmd=${P4TestCmd/p4:$User:$Workspace /$p4 -u $User -c $Workspace }
   else
      echo -e "Warning: Entry on line $Line of $CLITestDataFile has a bogus 'p4' command.  Must start with 'p4 '.  Skipping this test."
      CLITestDataOK=0
      continue
   fi

   if [[ $ExpectedExit == U ]]; then
      ExpectedExit=Undefined
   else
      if [[ $ExpectedExit =~ [0-9]+ ]]; then
         ExpectedExitCode=$ExpectedExit
      else
         echo -e "Warning: Entry on line $Line of $CLITestDataFile has a bogus exit code.  Must be numeric or 'U', value is $ExpectedExit. Skipping this test."
         DataOK=0
         continue
      fi
   fi

   echo "C:[$P4TestCmd] E:[$ExpectedExit] S:[$ExpectedString] Comments: $Comments"
done < $CLITestDataFile

if [[ $CLITestDataOK -eq 1 ]]; then
   echo "Verified: All test entries are OK."
else
   if [[ $RequireValidTests -eq 1 ]]; then
      bail "The command line test data file [$CLITestDataFile] contained invalid entries. Use '-f' to bypassing bad tests and continue. Aborting."
   else
      echo -e "\nWarning: Some tests were skipped due to invalid entries in $CLITestDataFile. Continuing due to '-f'.\n"
   fi
fi

msg "$H\nPre-start test preparations."

if [[ $GetSampleDepot -eq 1 || $InitSampleDepot -eq 1 ]]; then
   shutdown_servers
fi

[[ $GetSampleDepot -eq 1 ]] && get_sample_depot
[[ $InitSampleDepot -eq 1 ]] && init_sample_depot

# Using the SDP will override several of the settings
# just set above.
if [[ $UseSDP -eq 1 ]]; then
   source /p4/common/bin/p4_vars 1 ||\
      bail "Failed to load SDP environment file p4_vars."
   msg "Loaded SDP Environment file.  Logging in."
   $p4 -u perforce login < /p4/common/bin/adminpass
   $p4 -u perforce -p $P4BROKERPORT login < /p4/common/bin/adminpass
   runCmd "$p4 -u perforce login -s"
   runCmd "$p4 -u perforce -p $P4BROKERPORT login -s"

   echo "Setting default P4PORT to P4BROKERPORT for CBD testing."
   export P4PORT=$P4BROKERPORT
fi

[[ $InitCBD -eq 1 ]] && init_cbd "$TestWS" "$TestWSRoot"

msg "$H\nExecuting Command Line Tests."

# It might seem inelegant to simply read-the config file in again, as opposed to storing and re-using the results
# from the data verification run above.  But with 'bash', storing results tends to have bad but subtle side effects,
# like having quote characters disappear.

AllTestsPassed=1

while read entry; do
   [[ -z "$(echo $entry)" ]] && continue
   [[ $entry == "#"* ]] && continue

   TestPassed=1
   P4TestCmd=${entry%%|*}
   ExpectedExit=${entry#*|}
   ExpectedExit=${ExpectedExit%%|*}
   ExpectedString=${entry%|*}
   ExpectedString=${ExpectedString##*|}

   # If the ExpectedString from the data file is prefixed with 'CBDLOG:',
   # set ExpectedStringInOutput=0, indicating the cbd.log file shoudl be
   # searched instead of the command outpout.
   if [[ "$ExpectedString" == "CBDLOG:"* ]]; then
      ExpectedString=${ExpectedString#CBDLOG:}
      ExpectedStringInOutput=0
   else
      ExpectedStringInOutput=1
   fi

   Comments=${entry##*|}

   if [[ $P4TestCmd == "p4 "* ]]; then
      P4TestCmd=${P4TestCmd/p4 /$p4c }
   elif [[ $P4TestCmd == "p4:"* ]]; then
      # Parse entries like: "p4:user:client command args ..."
      User=${P4TestCmd#p4:}
      User=${User%%:*}
      Workspace=${P4TestCmd#p4:}
      Workspace=${Workspace#*:}
      Workspace=${Workspace%% *}
      P4TestCmd=${P4TestCmd/p4:$User:$Workspace /$p4 -u $User -c $Workspace }
   else
      continue
   fi

   if [[ $ExpectedExit == U ]]; then
      ExpectedExit=Undefined
   else
      if [[ $ExpectedExit =~ [0-9]+ ]]; then
         ExpectedExitCode=$ExpectedExit
      else
         continue
      fi
   fi

   TestCount=$((TestCount+1))
   msg "$H\nTest $TestCount\nTesting Command: $P4TestCmd\nExpected Exit: $ExpectedExit"
   if [[ $ExpectedStringInOutput -eq 1 ]]; then
      msg "Expected string in output: $ExpectedString\nComments: $Comments\n"
   else
      msg "Expected string in cbd.log: $ExpectedString\nComments: $Comments\n"
   fi
   $P4TestCmd > $OutputFile 2>&1
   ActualExitCode=$?

   msg "\n== Command Output =="
   cat $OutputFile

   echo TEST_EXIT_CODE: Actual $ActualExitCode, Expected $ExpectedExit

   if [[ $ExpectedExit == U ]]; then
      echo "Ignoring TEST_EXIT_CODE due to 'U' value."
   else
      [[ $ExpectedExitCode -ne $ActualExitCode ]] && TestPassed=0
   fi

   if [[ -f $CBDLog ]]; then
      echo -e "\n== Contents of cbd.log =="
      cat $CBDLog
   fi

   if [[ -n "$ExpectedString" ]]; then
      # Check for the expected string in the command output or the cbd.log file.
      if [[ $ExpectedStringInOutput -eq 1 ]]; then
         grep "$ExpectedString" $OutputFile > /dev/null 2>&1
         if [[ $? -eq 0 ]]; then
            echo -e "\nExpected string [$ExpectedString] found in command outpout."
         else
            echo -e "\nExpected string [$ExpectedString] NOT found in command outpout."
            TestPassed=0
         fi
      else
         if [[ -f $CBDLog ]]; then
            grep "$ExpectedString" $CBDLog > /dev/null 2>&1
            if [[ $? -eq 0 ]]; then
               echo -e "\nExpected string [$ExpectedString] found in cbd.log."
            else
               echo -e "\nExpected string [$ExpectedString] NOT found in cbd.log."
               TestPassed=0
            fi
         else
            echo "String [$ExpectedString] expected in cbd.log, but cbd.log is not there."
            TestPassed=0
         fi
      fi
   else
      echo "No expected string defined, skipping check for this test."
   fi

   runCmd "/bin/rm -f $CBDLog $OutputFile" "Removing cbd.log and output file between tests." 0 0 0

   if [[ $TestPassed -eq 1 ]]; then
      echo -e "\nTEST $TestCount PASSED\n"
      TestPassedCount=$((TestPassedCount+1))
   else
      echo -e "\nTEST $TestCount FAILED\n"
      TestFailedCount=$((TestFailedCount+1))
      AllTestsPassed=0
   fi

done < $CLITestDataFile

msg "$H\nExecuting Broker Input Simulation Tests."

[[ -r "$BITestCfgFile" ]] || bail "Missing broker input test data file [$BITestCfgFile]."

while read entry; do
   [[ -z "$(echo $entry)" ]] && continue
   [[ $entry == "#"* ]] && continue

   TestPassed=1

   BITFile=${entry%%|*}

   ExpectedExit=${entry#*|}
   ExpectedExit=${ExpectedExit%%|*}
   ExpectedString=${entry%|*}
   ExpectedString=${ExpectedString##*|}

   # If the ExpectedString from the data file is prefixed with 'CBDLOG:',
   # set ExpectedStringInOutput=0, indicating the cbd.log file shoudl be
   # searched instead of the command outpout.
   if [[ "$ExpectedString" == "CBDLOG:"* ]]; then
      ExpectedString=${ExpectedString#CBDLOG:}
      ExpectedStringInOutput=0
   else
      ExpectedStringInOutput=1
   fi

   if [[ $ExpectedExit != U ]]; then
      ExpectedExitCode=$ExpectedExit
   fi

   Comments=${entry##*|}

   TestCount=$((TestCount+1))
   msg "$H\nTest $TestCount\nTesting Broker Input File: $BITFile\nExpected Exit: $ExpectedExit"

   if [[ $ExpectedStringInOutput -eq 1 ]]; then
      msg "Expected string in output: $ExpectedString\nComments: $Comments\n"
   else
      msg "Expected string in cbd.log: $ExpectedString\nComments: $Comments\n"
   fi

   if [[ -r "$CfgDir/$BITFile" ]]; then
      msg "Contents of original $BITFile:"
      cat $CfgDir/$BITFile

      sed "s:__EDITME_P4PORT__:$SampleDepotP4PORT:g" $CfgDir/$BITFile |\
         sed "s:__EDITME_P4BROKERPORT__:$SampleDepotBrokerPort:g" |\
         sed "s:__EDITME_P4CLIENT__:$TestWS:g" |\
         sed "s:__EDITME_WSROOT__:$TestWSRoot:g" > $BITFile2

      msg "Broker input test file tweaks:"
      diff $CfgDir/$BITFile $BITFile2

      $CBD_HOME/scripts/wssync.sh < $BITFile2 > $OutputFile 2>&1
      ActualExitCode=$?

      msg "\n== Command Output =="
      cat $OutputFile

      echo TEST_EXIT_CODE: Actual $ActualExitCode, Expected $ExpectedExit

      if [[ $ExpectedExit == U ]]; then
         echo "Ignoring TEST_EXIT_CODE due to 'U' value."
      else
         [[ $ExpectedExitCode -ne $ActualExitCode ]] && TestPassed=0
      fi

      if [[ -f "$CBDLog" ]]; then
         echo -e "\n== Contents of cbd.log =="
         cat $CBDLog
      fi

      if [[ -n "$ExpectedString" ]]; then
         # Check for the expected string in the command output or the cbd.log file.
         if [[ $ExpectedStringInOutput -eq 1 ]]; then
            grep "$ExpectedString" $OutputFile > /dev/null 2>&1
            if [[ $? -eq 0 ]]; then
               echo -e "\nExpected string [$ExpectedString] found in command outpout."
            else
               echo -e "\nExpected string [$ExpectedString] NOT found in command outpout."
               TestPassed=0
            fi
         else
            if [[ -f $CBDLog ]]; then
               grep "$ExpectedString" "$CBDLog" > /dev/null 2>&1
               if [[ $? -eq 0 ]]; then
                  echo -e "\nExpected string [$ExpectedString] found in cbd.log."
               else
                  echo -e "\nExpected string [$ExpectedString] NOT found in cbd.log."
                  TestPassed=0
               fi
            else
               echo "String [$ExpectedString] expected in cbd.log, but cbd.log is not there."
               TestPassed=0
            fi
         fi
      else
         echo "No expected string defined, skipping check for this test."
      fi

      runCmd "/bin/rm -f $CBDLog $OutputFile" "Removing cbd.log and output file between test." 0 0 0

   else
      errmsg "Missing broker input test file [$CfgDir/$BITFile]."
      TestPassed=0
   fi

   if [[ $TestPassed -eq 1 ]]; then
      echo -e "\nTEST $TestCount PASSED\n"
      TestPassedCount=$((TestPassedCount+1))
   else
      echo -e "\nTEST $TestCount FAILED\n"
      TestFailedCount=$((TestFailedCount+1))
      AllTestsPassed=0
   fi

done < $BITestCfgFile

msg "${H}\nExecuting scripted tests."

[[ -r "$ScriptedTestCfgFile" ]] || bail "Missing scripted test config file [$ScriptedTestCfgFile]."

runCmd "/bin/rm -f $CBDLog $SSUTLog $SSTULog $OutputFile" \
   "Removing various *.log and output file between tests." 0 0 0

Line=0
while read entry; do
   Line=$((Line+1))
   [[ -z "$(echo $entry)" ]] && continue
   [[ $entry == "#"* ]] && continue
   TestCount=$((TestCount+1))
   TestPassed=1

   TestScript=$CfgDir/${entry%%|*}

   ExpectedExit=${entry#*|}
   ExpectedExit=${ExpectedExit%%|*}
   ExpectedString=${entry%|*}
   ExpectedString=${ExpectedString##*|}
   LogToCheck=

   # If the ExpectedString from the data file is prefixed with 'CBDLOG:',
   # set ExpectedStringInOutput=0, indicating the cbd.log file shoudl be
   # searched instead of the command outpout.
   if [[ "$ExpectedString" == "CBDLOG:"* ]]; then
      ExpectedString=${ExpectedString#CBDLOG:}
      ExpectedStringInOutput=0
      LogToCheck=$CBDLog
   elif [[ "$ExpectedString" == "SSTULOG:"* ]]; then
      ExpectedString=${ExpectedString#SSTULOG:}
      ExpectedStringInOutput=0
      LogToCheck=$SSTULog
   elif [[ "$ExpectedString" == "SSUTLOG:"* ]]; then
      ExpectedString=${ExpectedString#SSUTLOG:}
      ExpectedStringInOutput=0
      LogToCheck=$SSUTLog
   else
      ExpectedStringInOutput=1
   fi

   if [[ $ExpectedExit == U ]]; then
      ExpectedExit=Undefined
   else
      ExpectedExitCode=$ExpectedExit
   fi

   Comments=${entry##*|}

   msg "$H\nTest $TestCount\nTest Script: $TestScript\nExpected Exit: $ExpectedExit"

   if [[ ! -x $TestScript ]]; then
      errmsg "Missing or Non-Executable Test Script file [$TestScript]."
      TestPassed=0
   fi

   echo -e "S:[$TestScript] E:[$ExpectedExit] S:[$ExpectedString]\nComments: $Comments"

   TestScriptCmd="$TestScript $TestCount $p4Exe $P4PORT bruno $TestWS"
   $TestScriptCmd > $OutputFile 2>&1
   ActualExitCode=$?

   msg "\n== Test Script Output =="
   cat $OutputFile

   echo TEST_EXIT_CODE: Actual $ActualExitCode, Expected $ExpectedExit

   if [[ $ExpectedExit == U ]]; then
      if [[ $ActualExitCode -ne 2 ]]; then
         echo "Ignoring TEST_EXIT_CODE due to 'U' value."
      else
         echo "Test exit code 2 indicates test did not run correctly.  Failing test."
         TestPasswd=0
      fi
   else
      [[ $ExpectedExitCode -ne $ActualExitCode ]] && TestPassed=0
   fi

   if [[ -f $CBDLog ]]; then
      echo -e "\n== Contents of cbd.log for Test $TestCount =="
      cat $CBDLog
   fi

   if [[ -f $SSTULog ]]; then
      echo -e "\n== Contents of SSTemplateUpdate.log for Test $TestCount =="
      cat $SSTULog
   fi

   if [[ -f $SSUTLog ]]; then
      echo -e "\n== Contents of SSUpdateTemplate.log for Test $TestCount =="
      cat $SSUTLog
   fi

   if [[ -n "$ExpectedString" ]]; then
      # Check for the expected string in the command output or the cbd.log file.
      if [[ $ExpectedStringInOutput -eq 1 ]]; then
         grep "$ExpectedString" $OutputFile > /dev/null 2>&1
         if [[ $? -eq 0 ]]; then
            echo -e "\nExpected string [$ExpectedString] found in command outpout."
         else
            echo -e "\nExpected string [$ExpectedString] NOT found in command outpout."
            TestPassed=0
         fi
      else
         if [[ -f $CBDLog && $LogToCheck == $CBDLog ]]; then
            grep "$ExpectedString" $CBDLog > /dev/null 2>&1
            if [[ $? -eq 0 ]]; then
               echo -e "\nExpected string [$ExpectedString] found in cbd.log."
            else
               echo -e "\nExpected string [$ExpectedString] NOT found in cbd.log."
               TestPassed=0
            fi
         elif [[ -f $SSTULog && $LogToCheck == $SSTULog ]]; then
            grep "$ExpectedString" $SSTULog > /dev/null 2>&1
            if [[ $? -eq 0 ]]; then
               echo -e "\nExpected string [$ExpectedString] found in SSTemplateUpdate.log."
            else
               echo -e "\nExpected string [$ExpectedString] NOT found in SSTemplateUpdate.log."
               TestPassed=0
            fi
         elif [[ -f $SSUTLog && $LogToCheck == $SSUTLog ]]; then
            grep "$ExpectedString" $SSUTLog > /dev/null 2>&1
            if [[ $? -eq 0 ]]; then
               echo -e "\nExpected string [$ExpectedString] found in SSUpdateTemplate.log."
            else
               echo -e "\nExpected string [$ExpectedString] NOT found in SSUpdateTemplate.log."
               TestPassed=0
            fi
         else
            echo "String [$ExpectedString] expected in $LogToCheck, but that log is missing."
            TestPassed=0
         fi
      fi
   else
      echo "No expected string defined, skipping check for this test."
   fi

   runCmd "/bin/rm -f $CBDLog $SSUTLog $SSTULog $OutputFile" \
      "Removing various *.log and output file between tests." 0 0 0

   if [[ $TestPassed -eq 1 ]]; then
      echo -e "\nTEST $TestCount PASSED\n"
      TestPassedCount=$((TestPassedCount+1))
   else
      echo -e "\nTEST $TestCount FAILED\n"
      TestFailedCount=$((TestFailedCount+1))
      AllTestsPassed=0
   fi

done < $ScriptedTestCfgFile

if [[ $CIAutomation -eq 1 ]]; then
   msg "Testing down servers after test completion due to '-a' flag."
   shutdown_servers
fi

[[ $AllTestsPassed -eq 0 ]] && OverallReturnStatus=1

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\nAll $TestCount tests PASSED.\n"
else
   msg "${H}\nProcessing completed, but with errors.\nTests Executed (Passed/Failed): $TestCount ($TestPassedCount/$TestFailedCount)\n\nScan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took about $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus