run_scripted_tests.sh #2

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
set -u

#==============================================================================
# Declarations and Environment

declare ThisScript="${0##*/}"
declare Version=2.0.0
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare CmdLine="${0} $*"
declare -i Debug=${SDP_DEBUG:-0}
declare -i ErrorCount=0
declare -i SilentMode=0
declare -i TestCount=0
declare -i TestPassCount=0
declare -i TestFailCount=0
declare -i TestSkipCount=0
declare -i NoOp=0
declare -i ScriptedTestDataOK=0
declare -i AllScriptedTestDataOK=1
declare -i i=0
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare H3="||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||"
declare Log=
declare LogLink="/tmp/${ThisScript%.sh}.log"

# Values loaded from general config file.
declare RunHost=
declare TestTag=

declare ExpectedExit=
declare TestLog=
declare TestLogHost=
declare -i RequireValidTests=1
declare -i ExpectedExitCode
declare -i ActualExitCode
declare ExpectedString=
declare -i ExpectedStringInOutput=0
declare Comments=
declare CfgDir=/p4/sdp/test/bsw
declare CfgFile=
declare -a TestScriptList
declare -a ExpectedExitList
declare -a TestLogList
declare -a ExpectedStringList=
declare -a TestLogHostList
declare -a CommentsList
declare ScriptedTestDataFile="$CfgDir/scripted_tests.cfg"
declare OutputFile=/tmp/sdp.test_output.$$.$RANDOM
declare -i TestPassed
declare -i Line=0
declare -i TestCount=0
declare -i TestPassCount=0
declare -i TestFailCount=0
GARBAGE+=" $OutputFile"

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "$ErrorCount"; }

function pass () { TestCount+=1; TestPassCount+=1; msg "PASS Test $TestCount: $*"; }
function fail () { TestCount+=1; TestFailCount+=1; msg "FAIL Test $TestCount: $*"; }

#------------------------------------------------------------------------------
# Function: terminate
function terminate {
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   dbg "$ThisScript: EXITCODE: $ErrorCount"

   # Stop logging.
   [[ "$Log" == off ]] || msg "\\nLog is: $Log\\n${H1}\\n"

   # With the trap removed, exit.
   exit "$((ErrorCount+TestFailCount+TestSkipCount))"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}

   msg "USAGE for $ThisScript v$Version:

$ThisScript [-d <cfg_dir>] [-f] [-L <log>] [-si] [-v<n>] [-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script runs tests for the Server Deployment Package (SDP).

OPTIONS:
 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -f	Use '-f' to bypass any invalid test entries in the command line test
	data.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	/tmp/${ThisScript%.sh}.<timestamp>.log

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  It is not necessary (nor harmful)
	to use redirection operators like '> log' or '2>&1' or 'tee'.

 -si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -D     Set extreme debugging verbosity, for debugging the test suite itself.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

FILES:
	The test.<hostname>.cfg files are SDP Test Configuration Files.  They
	define a set of host-specific values that define how SDP is tested,
	where test assets are aquired from, which versions of p4/p4d are used,
	etc.

	The scripted_test_data.cfg file defines the command line tests to be executed,
	including expected exit codes and output for each.

	Note that the /p4/sdp/test directory will not appear on a customer-deployed
	SDP in the real world. The /p4/sdp directory will exist, but the 'test'
	directory is populated only when using the //sdp/dev_insitu stream.

EXAMPLES:
	For typical usage, no arguments are needed. Run this in a Battle School
	Lab Environment as perforce@bos-helix-01, after first having run 'lab qa'
	as student@bsw-lab-ui to prepare the environment.

	\$ cd /p4/sdp/test/bsw
	\$ ./$ThisScript
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-d) CfgDir=$2; shiftArgs=1;;
      (-f) RequireValidTests=0;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-L) Log="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) NoOp=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ -n "$Log" ]] || Log="/tmp/${ThisScript%.sh}.$(date +'%Y-%m-%d-%H%M%S').log"

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

if [[ "$Log" != off ]]; then
   touch "$Log" || bail "Couldn't touch log file [$Log]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   # Setup /tmp/test_sdp.log symlink so it points to the current log.
   rm -f "$LogLink"
   ln -s "$Log" "$LogLink"

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@$ThisHost on $(date) as pid $$:\\nInitial Command Line:\\n$CmdLine\\n"

CfgFile="$CfgDir/test_sdp.$ThisHost.cfg"

if [[ -r "$CfgFile" ]]; then
   # shellcheck disable=SC1090
   source "$CfgFile" || bail "Failed to load config file [$CfgFile]."
else
   bail "Missing config file [$CfgFile]."
fi

# Sanity check on values loaded from the config file.
[[ -z "$TestTag" ]] && bail "Environment loaded from $CfgFile is missing variable definition for TestTag."

if [[ $ThisHost == "$RunHost" ]]; then
   msg "Verified: Running on $RunHost."
else
   bail "Not configured to run on host $ThisHost.  Run only on $RunHost, as configured in $CfgFile."
fi

msg "$H1\\nPre-start test preparations."

# shellcheck disable=SC1091
source /p4/common/bin/p4_vars 1 ||\
   bail "Failed to load SDP environment file p4_vars."
msg "Loaded SDP Environment file.  Logging in."

msg "Loading and verifying scripted test data from: $ScriptedTestDataFile"
[[ -r "$ScriptedTestDataFile" ]] || bail "Missing command line test data file [$ScriptedTestDataFile]."

# See the file scripted_tests.cfg for details on expected format of entries
# in that file we are about to parse. Short version is that we're expecting
# lines of this form:
#
# <TestScript>|<ExitCode>|[<Host>:]<TestLog>|<ExpectedStringRegex>|<Comments>

AllScriptedTestDataOK=1

Line=0; while read -r entry; do
   Line=$((Line+1))

   # Ignore blank lines and comments (but still increment line counter).
   # shellcheck disable=SC2116
   [[ -z "$(echo "$entry")" ]] && continue
   [[ $entry == "#"* ]] && continue

   ScriptedTestDataOK=1

   # TestScript is the first field.
   if [[ "$entry" =~ ^no_script ]]; then
      TestScript="no_script"
      ExpectedExit="U"
   else
      TestScript="$CfgDir/${entry%%|*}"

      # ExpectedExit is 2nd field.
      ExpectedExit="${entry#*|}"
      ExpectedExit="${ExpectedExit%%|*}"
   fi

   # TestLog is 3rd field.
   TestLog="${entry#*|}"
   TestLog="${TestLog#*|}"
   TestLog="${TestLog%%|*}"

   # TestLog might look like syd-helix-04:/p4/1/logs/log, indicating to grep the log on
   # host syd-helix-04.
   if [[ "$TestLog" == *:* ]]; then
      TestLogHost="${TestLog%%:*}"
      TestLog="${TestLog##*:}"
   else
      TestLogHost=
   fi

   # ExpectedString is 4th field.
   ExpectedString="${entry#*|}"
   ExpectedString="${ExpectedString#*|}"
   ExpectedString="${ExpectedString#*|}"
   ExpectedString="${ExpectedString%%|*}"

   # Comments is right-most field; strip everything to the left.
   Comments="${entry##*|}"

   if [[ -n "$TestLogHost" ]]; then
      msg "S:[$TestScript] E:[$ExpectedExit] L:[${TestLogHost}:$TestLog] S:[$ExpectedString]\\nComments: $Comments"
   else
      msg "S:[$TestScript] E:[$ExpectedExit] L:[$TestLog] S:[$ExpectedString]\\nComments: $Comments"
   fi

   # Do sanity tests on the loaded scripted test configuration data.
   if [[ "$ExpectedExit" == U || "$ExpectedExit" =~ ^[0-9]+$ ]]; then
      dbg "ExpectedExit value [$ExpectedExit] is valid."
   else
      errmsg "Invalid format of expected exit code [$ExpectedExit] on line $Line of $ScriptedTestDataFile; should be numeric or 'U'."
      ScriptedTestDataOK=0
   fi

   if [[ ! "$TestLog" == /* ]]; then
      errmsg "Invalid format of TestLog [$TestLog] on line $Line of $ScriptedTestDataFile; absolute path expected."
      ScriptedTestDataOK=0
   fi

   if [[ "$TestScript" != no_script ]]; then
      if [[ ! -x "$TestScript" ]]; then
         errmsg "Missing or Non-Executable Test Script file [$TestScript]."
         TestPassed=0
      fi
   fi

   if [[ "$ScriptedTestDataOK" -eq 1 ]]; then
      TestScriptList[TestCount]="$TestScript"
      ExpectedExitList[TestCount]="$ExpectedExit"
      TestLogList[TestCount]="$TestLog"
      TestLogHostList[TestCount]="$TestLogHost"
      ExpectedStringList[TestCount]="$ExpectedString"
      CommentsList[TestCount]="$Comments"
      TestCount+=1
   else
      TestSkipCount+=1
      AllScriptedTestDataOK=0
   fi

done < "$ScriptedTestDataFile"

if [[ "$AllScriptedTestDataOK" -eq 1 ]]; then
   msg "Verified: All test entries are OK."
else
   if [[ "$RequireValidTests" -eq 1 ]]; then
      bail "The scripted test data file [$ScriptedTestDataFile] contained invalid entries. Use '-f' to bypass bad tests and continue using the good ones."
   else
      msg "\\n$TestSkipCount tests will be skipped due to invalid entries in $ScriptedTestDataFile. Continuing due to '-f'.\\n"
   fi
fi

msg "${H3}\\nExecuting Scripted Tests."

[[ -r "$ScriptedTestDataFile" ]] || bail "Missing scripted test config file [$ScriptedTestDataFile]."

for ((i=0; i<TestCount; i++)); do
   TestPassed=1
   TestScript="${TestScriptList[i]}"
   ExpectedExit="${ExpectedExitList[i]}"
   ExpectedString="${ExpectedStringList[i]}"
   TestLog="${TestLogList[i]}"
   TestLogHost="${TestLogHostList[i]}"
   Comments="${CommentsList[i]}"

   if [[ "$ExpectedExit" == U ]]; then
      ExpectedExit="Undefined"
   else
      ExpectedExitCode="$ExpectedExit"
   fi

   if [[ "$TestLog" == "output" ]]; then
      ExpectedStringInOutput=1
   else
      ExpectedStringInOutput=0
   fi

   msg "$H2\\nTest $TestCount"

   if [[ -n "$TestLogHost" ]]; then
      msg "S:[$TestScript] E:[$ExpectedExit] L:[${TestLogHost}:$TestLog] S:[$ExpectedString]\\nComments: $Comments"
   else
      msg "S:[$TestScript] E:[$ExpectedExit] L:[$TestLog] S:[$ExpectedString]\\nComments: $Comments"
   fi

   TestScriptCmd="$TestScript $TestCount"

   if [[ "$TestScript" != no_script ]]; then
      if [[ "$NoOp" -eq 0 ]]; then
         $TestScriptCmd > "$OutputFile" 2>&1
         ActualExitCode=$?
      else
         msg "NO_OP: Would run: $TestScriptCmd > $OutputFile 2>&1"
         echo > "$OutputFile"
         ActualExitCode=0
      fi

      msg "\\n== Test Script Output =="
      cat "$OutputFile"

      msg "TEST_EXIT_CODE: Actual $ActualExitCode, Expected $ExpectedExit"

      if [[ "$ExpectedExit" == U ]]; then
         if [[ "$ActualExitCode" -ne 2 ]]; then
            msg "Ignoring TEST_EXIT_CODE due to 'U' value."
         else
            msg "Test exit code 2 indicates test did not run correctly.  Failing test."
            TestPassed=0
         fi
      else
         [[ "$ExpectedExitCode" -ne "$ActualExitCode" ]] && TestPassed=0
      fi
   fi

   if [[ -n "$ExpectedString" ]]; then
      # Check for the expected string in the command output or the cbd.log file.
      if [[ "$ExpectedStringInOutput" -eq 1 ]]; then
         if grep -q -E "$ExpectedString" "$OutputFile"; then
            msg "\\nExpected string [$ExpectedString] found in command output."
         else
            msg "\\nExpected string [$ExpectedString] NOT found in command output."
            TestPassed=0
         fi
      else
         if [[ -n "$TestLogHost" ]]; then
            if ssh -q "$TestLogHost" "[[ -r $TestLog ]]"; then
               if ssh -q "$TestLogHost" "grep -q -E \"$ExpectedString\" $TestLog"; then
                  msg "\\nExpected string [$ExpectedString] found in log [${TestLogHost}:$TestLog]."
               else
                  msg "\\nExpected string [$ExpectedString] NOT found in log [${TestLogHost}:$TestLog]."
               fi
            else
               msg "String [$ExpectedString] expected in log [${TestLogHost}:$TestLog], but that log is missing."
               TestPassed=0
            fi
         else
            if [[ -r "$TestLog" ]]; then
               if grep -E -q "$ExpectedString" "$TestLog"; then
                  msg "\\nExpected string [$ExpectedString] found in log [$TestLog]."
               else
                  msg "\\nExpected string [$ExpectedString] NOT found in log [$TestLog]."
                  TestPassed=0
               fi
            else
               msg "String [$ExpectedString] expected in log [$TestLog], but that log is missing."
               TestPassed=0
            fi
         fi
      fi
   else
      msg "No expected string defined, skipping check for expected string for this test."
   fi

   if [[ $TestPassed -eq 1 ]]; then
      msg "\\nPASS TEST $TestCount\\n"
      TestPassCount=$((TestPassCount+1))
   else
      msg "\\nFAIL TEST $TestCount\\n"
      TestFailCount=$((TestFailCount+1))
   fi
done

if [[ "$TestPassCount" -ge 1 && "$TestFailCount" -eq 0 && "$ErrorCount" -eq 0 ]]; then
   msg "${H1}\\nALL $TestCount tests PASSED.\\n"
else
   msg "${H1}\\nTest Run completed.  Summary:
   Total Tests:   $TestCount
   PASS Count:    $TestPassCount
   FAIL Count:    $TestFailCount"

   if [[ "$TestSkipCount" -ne 0 ]]; then
      msg "SKIP Count:    $TestSkipCount"
   fi

   if [[ "$ErrorCount" -eq 0 ]]; then
      msg "There were no errors in test setup. Test results should be valid."
   else
      errmsg "There were $ErrorCount errors in test setup. Test results may not be valid."
   fi

   msg "\\nScan above output carefully."
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took about $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit "$((ErrorCount+TestFailCount+TestSkipCount))"