run_scripted_tests.sh #1

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
set -u

#==============================================================================
# Declarations and Environment

declare ThisScript="${0##*/}"
declare Version=2.7.2
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare CmdLine="${0} $*"
declare -i AbortOnTestFailure=0
declare -i Debug=${SDP_DEBUG:-0}
declare -i ErrorCount=0
declare -i SilentMode=0
declare -i TestCount=0
declare -i TestID=0
declare -i TestPassCount=0
declare -i TestFailCount=0
declare -i TestSkipCount=0
declare -i NoOp=0
declare -i LoadSDPEnv=1
declare -i ScriptedTestDataOK=0
declare -i AllScriptedTestDataOK=1
declare -i i=0
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare H3="||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||"
declare H4="++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
declare Log=
declare LogLink="/tmp/${ThisScript%.sh}.log"

# Values loaded from general config file.
declare RunHost=
declare TestTag=

declare -i DisplayFullLogs=0
declare UserTestGroups=
declare -i InUserTestGroup=0
declare TestGroup=
declare TestScript=
declare ExpectedExit=
declare ExpectedString=
declare TestLog=
declare TestLogHost=
declare TmpLocalLog=
declare -i RequireValidTests=1
declare -i ExpectedExitCode
declare -i ActualExitCode
declare -i ExpectedStringInOutput=0
declare -i DoBSWPreflight=1
declare Comments=
declare CfgDir="/p4/sdp/test/bsw"
declare CfgFile=
declare -a TestGroupList
declare -a TestScriptList
declare -a ExpectedExitList
declare -a TestLogList
declare -a ExpectedStringList=
declare -a TestLogHostList
declare -a CommentsList
declare ScriptedTestDataFile=
declare OutputFile="/tmp/sdp.test_output.$$.$RANDOM"
declare -i TestPassed
declare -i Line=0
declare -i GarbageCount=0
declare -a Garbage

# For Preflight checks of the Lab Environment
declare TmpFile=
declare CommitJournalCounter=

Garbage[GarbageCount]="$OutputFile"
GarbageCount+=1

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "$ErrorCount"; }

function pass () { TestCount+=1; TestPassCount+=1; msg "PASS Test $TestCount"; }
function fail () {
   TestCount+=1
   TestFailCount+=1
   msg "FAIL Test $TestCount"
   if [[ "$AbortOnTestFailure" -eq 1 ]]; then
      msg "Aborting after first test failure due to '-e'."
      exit 1
   fi
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate {
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   local g=

   dbg "$ThisScript: EXITCODE: $ErrorCount"

   if [[ "$Debug" -eq 0 ]]; then
      for g in "${Garbage[@]}"; do
         echo rm -rf "$g"
      done
   else
      msg "Due to '-d' (debug) mode, skipping removal of these:"
      for g in "${Garbage[@]}"; do
         msg "$g"
      done
   fi

   # Stop logging.
   [[ "$Log" == off ]] || msg "\\nLog is: $Log\\n${H1}\\n"

   # With the trap removed, exit.
   exit "$((ErrorCount+TestFailCount+TestSkipCount))"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}

   msg "USAGE for $ThisScript v$Version:

$ThisScript [-cd <cfg_dir>] [-sp] [-no_env] [-FL] [-g <test_group_1>[,<test_group_2>,...]] [-e] [-f] [-L <log>] [-si] [-d|-D]

or

$ThisScript [-h|-man]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script runs tests for the Server Deployment Package (SDP).

	Tests are defined in the file: $ScriptedTestDataFile

OPTIONS:
 -cd	Specify '-cd <cfg_dir>' to specify an alternate location for the
	configuration directory. This is expected to include a the
	${ScriptedTestDataFile##*/} file and the test.<hostname>.cfg file.
	See FILES for more information.

	The default is: $CfgDir

	This option implies '-sp'.

 -sp	Specify '-sp' to skip preflight checks, which are run by default
 	to ensure the servers in the Battle School Workshop Lab
	Environment, used for certain tests, are only and working
	properly.

 -no_env
 	Specify '-no_env' to avoid loading the SDP Shell Environment (p4_vars).

 -FL	Specify '-FL' to display full logs files.  The logged output
	indicates whether the 'grep' expression detected expected output
	in logs or command ouptut.  With this option, the entire
	contents of each log scanned is also displayed.

	This will greatly increase the size of the generated log file for
	this script.  It is intended to be useful for interactive
	debugging.
	
 -g <test_group_1>[,<test_group_2>,...]
	Specify '-g <test_group>' to specify a comma-delimited list of
	test groups to run.  If not specified, the default is to execute
	all test groups.

 -e	Use '-e' to abort immediately after the first test failure.

	By default, this script attempts to execute all tests in the
	test suite (in hopes of illuminating the most issues).  The
	'-e' option changes the behavior to stop execution after
	a test failure, so as to preserve the state of the system.
	This may be useful for interactive debugging of scripted tests.

 -f	Use '-f' to bypass any invalid test entries in the scripted test
	data file and process remaining valid tets. By default, no tests
	are executed if any are invalid.

LOGGING OPTIONS:
 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	/tmp/${ThisScript%.sh}.<timestamp>.log

 -si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.

LOGGING:
	This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  It is not necessary (nor harmful)
	to use redirection operators like '> log' or '2>&1' or 'tee'.

	This script uses a fixed log symlink /tmp/${ThisScript%.sh}.log
	which is updated to point to the newly created log for each run, so that
	this symlink reliably points to the most recent execution of the script
	(unless '-L off' was used).

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

DEBUGGING OPTIONS:

 -d	Enable debug messages, and prevents cleanup of some interim files that
	may be useful for debugging (with details noted in the log).
      
 -D     Use bash 'set -x' extreme debugging verbosity.  Implies '-d'.

FILES:
	The test.<hostname>.cfg files are SDP Test Configuration Files.  They
	define a set of host-specific values that define how SDP is tested,
	where test assets are aquired from, which versions of p4/p4d are used,
	etc.

	The ${ScriptedTestDataFile##*/} file defines the scripted tests to be
	executed, including expected exit codes and strings to grep for in
	logs/output for each.

	The /p4/sdp/test directory will not appear on a customer-deployed
	SDP in the real world. The /p4/sdp directory will exist, but the 'test'
	directory and 'bsw' subdirectory are populated only when using the
	//sdp/dev_insitu stream in a Battle School Lab Environment.

EXAMPLES:
	Example 1: Typical Usage

	For typical usage, no arguments are needed. Run this in a Battle School
	Lab Environment as perforce@bos-helix-01, after first having run 'lab qa'
	as student@bsw-lab-ui to prepare the environment.

	\$ cd /p4/sdp/test/bsw
	\$ ./$ThisScript

	Example 2: Run specific test group(s).

	\$ cd /p4/sdp/test/bsw
	\$ ./$ThisScript -g LoadCheckpoint

	See test group names as defined in scripted_tests.cfg to help select
	values for '-g'.
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-cd) CfgDir="$2"; DoBSWPreflight=0; shiftArgs=1;;
      (-no_env) LoadSDPEnv=0;;
      (-sp) DoBSWPreflight=0;;
      (-FL) DisplayFullLogs=1;;
      (-g) UserTestGroups="$2"; shiftArgs=1;;
      (-e) AbortOnTestFailure=1;;
      (-f) RequireValidTests=0;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-L) Log="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) NoOp=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown option ($1).";;
      (*) usage -h "Unknown parameter ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ -n "$Log" ]] || Log="/tmp/${ThisScript%.sh}.$(date +'%Y-%m-%d-%H%M%S').log"

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

if [[ "$Log" != off ]]; then
   touch "$Log" || bail "Couldn't touch log file [$Log]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   # Setup /tmp/test_sdp.log symlink so it points to the current log.
   rm -f "$LogLink"
   ln -s "$Log" "$LogLink"

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@$ThisHost on $(date) as pid $$:\\nInitial Command Line:\\n$CmdLine\\n"

ScriptedTestDataFile="$CfgDir/scripted_tests.cfg"
CfgFile="$CfgDir/test_sdp.$ThisHost.cfg"

[[ -d "$CfgDir" ]] || bail "Missing confg dir [$CfgDir]."

if [[ -r "$CfgFile" ]]; then
   # shellcheck disable=SC1090
   source "$CfgFile" || bail "Failed to load config file [$CfgFile]."
else
   bail "Missing config file [$CfgFile]."
fi

if [[ "$DoBSWPreflight" -eq 1 ]]; then
   msg "Pre-start check: Verify Battle School Lab Environment is running and replicating OK."

   TmpFile=$(mktemp)
   p4master_run 1 p4 -ztag -F %ServerID%:%AppliedJournal% servers -J > "$TmpFile" ||\
      bail "Could not check 'p4 servers -J'. Aborting."

   CommitServerID=$(grep -E "(commit|master).1" "$TmpFile" | cut -d ':' -f1)
   CommitJournalCounter=$(grep -E "(commit|master).1" "$TmpFile" | cut -d ':' -f2)

   for grepString in ${CommitServerID}:${CommitJournalCounter} p4d_edge_syd:${CommitJournalCounter} p4d_fs_nyc:${CommitJournalCounter} p4d_ha_bos:${CommitJournalCounter}; do
      if ! grep -q "$grepString" "$TmpFile"; then
         errmsg "Unexpected: ServerID ${grepString%:*} is not on journal $CommitJournalCounter."
      fi
      if grep -q "${grepString}:0" "$TmpFile"; then
         errmsg "Unexpected: ServerID ${grepString%:*} is not replicating beyond journal $CommitJournalCounter yet."
      fi
   done

   if [[ "$ErrorCount" -eq 0 ]]; then
      msg "Verified: BSW Lab servers and replicating beyond sequence 0 for journal $CommitJournalCounter."
   else
      bail "Aboring before running scripted tests due to issues with Lab Environment."
   fi
else
   dbg "Skipping BSW Prelfight checks due to '-sp' or '-cd'."
fi

# Sanity check on values loaded from the config file.
[[ -z "$TestTag" ]] && \
   bail "Environment loaded from $CfgFile is missing variable definition for TestTag."

if [[ $ThisHost == "$RunHost" ]]; then
   msg "Verified: Running on $RunHost."
else
   bail "Not configured to run on host $ThisHost.  Run only on $RunHost, as configured in $CfgFile."
fi

msg "$H1\\nPre-start test preparations."

if [[ "$LoadSDPEnv" -eq 1 && -r /p4/common/bin/p4_vars ]]; then
   msg "Loading SDP Environment file."

   # shellcheck disable=SC1091
   source /p4/common/bin/p4_vars 1 ||\
      bail "Failed to load SDP environment file p4_vars."
else
   if [[ "$LoadSDPEnv" -eq 0 ]]; then
      dbg "Skipping load of SDP Environment due to '-no_env'."
   else
      dbg "Skipping load of SDP Environment due to missing /p4/common/bin/p4_vars."
   fi
fi

msg "Loading and verifying scripted test data from: $ScriptedTestDataFile"
[[ -r "$ScriptedTestDataFile" ]] || bail "Missing command line test data file [$ScriptedTestDataFile]."

# See the file scripted_tests.cfg for details on expected format of entries
# in that file we are about to parse. Short version is that we're expecting
# lines of this form:
#
# <TestGroup>|<TestScript>|<ExitCode>|[<Host>:]<TestLog>|<ExpectedStringRegex>|<Comments>

AllScriptedTestDataOK=1

i=0; Line=0; while read -r entry; do
   Line=$((Line+1))

   # Ignore blank lines and comments (but still increment line counter).
   # shellcheck disable=SC2116
   [[ -z "$(echo "$entry")" ]] && continue
   [[ $entry == "#"* ]] && continue

   ScriptedTestDataOK=1
   TestGroup=${entry%%|*}

   # Shift to next '|' char.
   entry=${entry#*|}

   # TestScript is the first field.
   if [[ "$entry" =~ ^no_script ]]; then
      TestScript="no_script"
      ExpectedExit="U"
   else
      TestScript="$CfgDir/${entry%%|*}"

      # ExpectedExit is the next field.
      ExpectedExit="${entry#*|}"
      ExpectedExit="${ExpectedExit%%|*}"
   fi

   # If user specified a comma-delimted list of test groups with '-g',
   # run only tests in specified groups.
   if [[ -n "$UserTestGroups" ]]; then
      InUserTestGroup=0
      for g in ${UserTestGroups//,/ }; do
         if [[ "${TestGroup^^}" == "${g^^}" ]]; then
            InUserTestGroup=1
         fi
      done
      if [[ "$InUserTestGroup" -eq 1 ]]; then
         dbg "Test Group [$TestGroup] is in user-specified test group list [$UserTestGroups]. Processing it."
      else
         dbg "Test Group [$TestGroup] is NOT in user-specified test group list [$UserTestGroups]. Skipping it."
         # Bypass remainder of the outer for loop if group isn't in users-specified test group list.
         continue
      fi
   fi

   # TestLog is the next field.
   TestLog="${entry#*|}"
   TestLog="${TestLog#*|}"
   TestLog="${TestLog%%|*}"

   # TestLog might look like syd-helix-04:/p4/1/logs/log, indicating to grep the log on
   # host syd-helix-04.
   if [[ "$TestLog" == *:* ]]; then
      TestLogHost="${TestLog%%:*}"
      TestLog="${TestLog##*:}"
   else
      TestLogHost=
   fi

   # ExpectedString is the next field.
   ExpectedString="${entry#*|}"
   ExpectedString="${ExpectedString#*|}"
   ExpectedString="${ExpectedString#*|}"
   ExpectedString="${ExpectedString%%|*}"

   # Comments is right-most field; strip everything to the left.
   Comments="${entry##*|}"

   if [[ -n "$TestLogHost" ]]; then
      dbg "TG:[$TestGroup]: S:[$TestScript] E:[$ExpectedExit] L:[${TestLogHost}:$TestLog] S:[$ExpectedString]\\nComments: $Comments"
   else
      dbg "TG:[$TestGroup]: S:[$TestScript] E:[$ExpectedExit] L:[$TestLog] S:[$ExpectedString]\\nComments: $Comments"
   fi

   # Do sanity tests on the loaded scripted test configuration data.
   if [[ "$ExpectedExit" == U || "$ExpectedExit" == N || "$ExpectedExit" =~ ^[0-9]+$ ]]; then
      dbg "ExpectedExit value [$ExpectedExit] is valid."
   else
      errmsg "Invalid format of expected exit code [$ExpectedExit] on line $Line of $ScriptedTestDataFile; should be numeric or 'U'."
      ScriptedTestDataOK=0
   fi

   if [[ ! "$TestLog" == /* ]]; then
      if [[ "$TestLog" != output ]]; then
         errmsg "Invalid format of TestLog [$TestLog] on line $Line of $ScriptedTestDataFile. An absolute path or the special value 'output' is expected."
         ScriptedTestDataOK=0
      fi
   fi

   if [[ "$TestScript" != no_script ]]; then
      if [[ ! -x "$TestScript" ]]; then
         errmsg "Missing or Non-Executable Test Script file [$TestScript]."
         ScriptedTestDataOK=0
      fi
   fi

   if [[ "$TestLog" == output && "$TestScript" == no_script ]]; then
      errmsg "Invalid use of both 'no_script' and 'output' on line $Line of $ScriptedTestDataFile. If no script is run, there can be no output!"
      ScriptedTestDataOK=0
   fi

   if [[ "$ScriptedTestDataOK" -eq 1 ]]; then
      TestGroupList[i]="$TestGroup"
      TestScriptList[i]="$TestScript"
      ExpectedExitList[i]="$ExpectedExit"
      TestLogList[i]="$TestLog"
      TestLogHostList[i]="$TestLogHost"
      ExpectedStringList[i]="$ExpectedString"
      CommentsList[i]="$Comments"
      i+=1
   else
      TestSkipCount+=1
      AllScriptedTestDataOK=0
   fi
done < "$ScriptedTestDataFile"

if [[ "$AllScriptedTestDataOK" -eq 1 ]]; then
   msg "Verified: All test entries are OK."
else
   if [[ "$RequireValidTests" -eq 1 ]]; then
      bail "The scripted test data file [$ScriptedTestDataFile] contained invalid entries. Use '-f' to bypass bad tests and continue using the good ones."
   else
      msg "\\n$TestSkipCount tests will be skipped due to invalid entries in $ScriptedTestDataFile. Continuing due to '-f'.\\n"
   fi
fi

msg "${H3}\\nExecuting Scripted Tests."

[[ -r "$ScriptedTestDataFile" ]] || bail "Missing scripted test config file [$ScriptedTestDataFile]."

for ((i=0; i<${#TestScriptList[@]}; i++)); do
   TestID=$((i+1))
   TestScript="${TestScriptList[i]}"
   TestGroup="${TestGroupList[i]}"
   ExpectedExit="${ExpectedExitList[i]}"
   ExpectedString="${ExpectedStringList[i]}"
   TestLog="${TestLogList[i]}"
   TestLogHost="${TestLogHostList[i]}"
   Comments="${CommentsList[i]}"
   TestPassed=1

   if [[ $ExpectedExit == U ]]; then
      ExpectedExit=Undefined
   elif [[ $ExpectedExit == N ]]; then
      ExpectedExit=NonZero
   else
      ExpectedExitCode="$ExpectedExit"
   fi

   if [[ "$TestLog" == "output" ]]; then
      ExpectedStringInOutput=1
   else
      ExpectedStringInOutput=0
   fi

   msg "${H2}\\nTest $TestID - Group $TestGroup - $Comments"
   if [[ -n "$TestLogHost" ]]; then
      msg "TG:[$TestGroup] S:[$TestScript] E:[$ExpectedExit] L:[${TestLogHost}:$TestLog] S:[$ExpectedString]"
   else
      msg "TG:[$TestGroup] S:[$TestScript] E:[$ExpectedExit] L:[$TestLog] S:[$ExpectedString]"
   fi

   if [[ "$TestScript" != no_script ]]; then
      if [[ "$NoOp" -eq 0 ]]; then
         "$TestScript" > "$OutputFile" 2>&1
         ActualExitCode=$?
      else
         msg "NO_OP: Would run: $TestScript > $OutputFile 2>&1"
         echo > "$OutputFile"
         ActualExitCode=0
      fi

      msg "\\n== Test Script Output =="
      cat "$OutputFile"

      msg "TEST_EXIT_CODE: Actual $ActualExitCode, Expected $ExpectedExit"

      if [[ "$ExpectedExit" == Undefined ]]; then
         if [[ "$ActualExitCode" -ne 2 ]]; then
            msg "Ignoring TEST_EXIT_CODE due to 'U' value."
         else
            msg "Test exit code 2 indicates test did not run correctly.  Failing test."
            TestPassed=0
         fi
      elif [[ "$ExpectedExit" == NonZero ]]; then
         [[ "$ActualExitCode" -gt 0 ]] || TestPassed=0
      else
         [[ "$ExpectedExitCode" -ne "$ActualExitCode" ]] && TestPassed=0
      fi
   fi

   if [[ -n "$ExpectedString" ]]; then
      # Check for the expected string in the command output or the script log file.
      if [[ "$ExpectedStringInOutput" -eq 1 ]]; then
         if grep -q -E "$ExpectedString" "$OutputFile"; then
            msg "\\nExpected string [$ExpectedString] found in command output."
         else
            msg "\\nExpected string [$ExpectedString] NOT found in command output."
            TestPassed=0
         fi
      else
         if [[ -n "$TestLogHost" ]]; then
            TmpLocalLog=$(mktemp)
            if rsync --copy-links "$TestLogHost:$TestLog" "$TmpLocalLog"; then
               if grep -E -q "$ExpectedString" "$TmpLocalLog"; then
                  if [[ "$DisplayFullLogs" -eq 1 ]]; then
                     msg "\\nExpected string [$ExpectedString] found in this log [${TestLogHost}:$TestLog]:\\n${H4}"
                     cat "$TmpLocalLog"
                     msg "${H4}"
                  else
                     msg "\\nExpected string [$ExpectedString] found in log [${TestLogHost}:$TestLog]."
                  fi
               else
                  if [[ "$DisplayFullLogs" -eq 1 ]]; then
                     msg "\\nExpected string [$ExpectedString] NOT found in this log [${TestLogHost}:$TestLog]:\\n${H4}"
                     cat "$TmpLocalLog"
                     msg "${H4}"
                  else
                     msg "\\nExpected string [$ExpectedString] NOT found in log [${TestLogHost}:$TestLog]."
                  fi
                  TestPassed=0
               fi
            else
               msg "String [$ExpectedString] expected in log [${TestLogHost}:$TestLog], but that log could not be copied locally to $ThisHost."
               TestPassed=0
            fi
            rm -f "$TmpLocalLog"
         else
            if [[ -r "$TestLog" ]]; then
               if grep -E -q "$ExpectedString" "$TestLog"; then
                  if [[ "$DisplayFullLogs" -eq 1 ]]; then
                     msg "\\nExpected string [$ExpectedString] found in this log [$TestLog]:\\n${H4}"
                     cat "$TestLog"
                     msg "${H4}"
                  else
                     msg "\\nExpected string [$ExpectedString] found in log [$TestLog]."
                  fi
               else
                  if [[ "$DisplayFullLogs" -eq 1 ]]; then
                     msg "\\nExpected string [$ExpectedString] NOT found in this log [$TestLog]:\\n${H4}"
                     cat "$TestLog"
                     msg "${H4}"
                  else
                     msg "\\nExpected string [$ExpectedString] NOT found in log [$TestLog]."
                  fi
                  TestPassed=0
               fi
            else
               msg "String [$ExpectedString] expected in log [$TestLog], but that log is missing."
               TestPassed=0
            fi
         fi
      fi
   else
      msg "No expected string defined, skipping check for expected string for this test."
   fi

   if [[ "$TestPassed" -eq 1 ]]; then
      pass
   else
      fail
   fi
done

if [[ "$TestPassCount" -ge 1 && "$TestFailCount" -eq 0 && "$ErrorCount" -eq 0 ]]; then
   msg "${H1}\\nScripted Test Run completed.  Summary:\\nALL $TestCount tests PASSED.\\n"
else
   msg "${H1}\\nScripted Test Run completed.  Summary:
   Total Tests:   $TestCount
   PASS Count:    $TestPassCount
   FAIL Count:    $TestFailCount"

   if [[ "$TestSkipCount" -ne 0 ]]; then
      msg "   SKIP Count:    $TestSkipCount"
   fi

   if [[ "$DisplayFullLogs" -eq 1 ]]; then
      msg "   Full Logs:     YES"
   else
      msg "   Full Logs:     NO"
   fi

   if [[ "$ErrorCount" -eq 0 ]]; then
      msg "\\nThere were no errors in test setup. Test results should be valid."
   else
      errmsg "There were $ErrorCount errors in test setup. Test results may not be valid."
   fi

   msg "\\nScan above output carefully."
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took about $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit "$((ErrorCount+TestFailCount+TestSkipCount))"