test_hi.sh #3

#!/bin/bash
#------------------------------------------------------------------------------
set -u

#==============================================================================
# Declarations and Environment Setup
#------------------------------------------------------------------------------
declare ThisScript=${0##*/}
declare Version="1.5.2"
declare Log=
declare Verbosity=3
declare HxDepots=
declare ResetDir=
declare SDPBranch=${SDP_BRANCH:-Unset}
declare HelixInstallerTestDir=
declare H1="\\n=============================================================================="
declare H2="\\n------------------------------------------------------------------------------"
declare -i ErrorCount=0
declare -i TestID=0
declare -i SilentMode=0
declare -i OverallExitCode=0
declare -i Debug=0
declare -i TestCrontab=1
declare -i ForceCleanup=0
declare TestLog=

# The OS environment variable should be set in the Docker configuration.
export OS="${OS:-UnsetOS}"

#==============================================================================
# Local Functions
function msg () { echo -e "$*"; }
function vmsg () { [[ "$Verbosity" -ge 4 ]] && msg "$*"; }
function vvmsg () { [[ "$Verbosity" -ge 5 ]] && msg "$*"; }
function dbg () { [[ "$Debug" -eq 1 ]] && msg "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "BAILING: ${1:-Unknown Error}"; exit "${2:-255}"; }
function run () {
   dbg "CALL run (${1:-}, ${2:-})"
   local cmdAndArgs="${1:-echo}"
   local desc="${2:-}"

   [[ -n "$desc" ]] && msg "$desc"
   msg "Running: $cmdAndArgs"
   $cmdAndArgs
   return $?
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   dbg "CALL usage (${1:-}, ${2:-})"
   declare style="${1:--h}"
   declare errorMessage="${2:-Unset}"

   if [[ "$errorMessage" != "Unset" ]]; then
      errmsg "Bad Usage:\\n\\n$errorMessage\\n\\n"
   fi

   msg "USAGE for $ThisScript v$Version:

$ThisScript [-v<n>] [-d|-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This is the test suite for the Helix Installer.

OPTIONS:
 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

	Levels 1 and 2 are reserved for future use.

	See EXAMPLES below for more detail on -v3, -v4, and -v5.

 -L <log>
	Specify the path to a log file to enable logging.  Or, specify
	'-L off' to disable logging using a default log file name:
	/tmp/$ThisScript.v$Version.<datestamp>.log

	NOTE: This script is self-logging if logging is enabled. That is,
	output displayed on the screen is simultaneously captured in the log
	file.  Do not run this script with redirection operators like '> log'
	or '2>&1', and do not use 'tee.'

-si	Operate silently.  All output (stdout and stderr) is redirected to the
	log only; no output appears on the terminal.  This cannot be used with
	'-L off'.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version.

EXIT CODE:
	If all tests are executed and all pass, the exit code is 0.

	If all tests are executed and some fail, the exit code is 1.

	If '-h' or '-man' (usage messages) are used, the exit code is 1.

	If there is a usage error to this script or something goes
	horribly wrong with testing, the exit code is 255.

EXAMPLES:
	Example 1: Test Summary

	Normally, run with no arguments (equivalent to -v3):
	$ThisScript

	This verbosity level is mainly intended to get a quick summary of which
	tests pass or fail, but without detail as to why they failed.

	Example 2: A Bit More

	For a bit more pedantic output:
	$ThisScript -v4

	This verbosity level shows the test summary, plus a bit more detail
	about output we're expecting in each test.  At this level, command
	output is suppressed except in cases where expected output for a
	test is defined but not contained in the output.
	
	Example 3: Full Details

	For a lot more pedantic output:
	$ThisScript -v5

	This verbosity level shows the test summary, plus a bit more detail
	about output we're expecting in each test, plus the full output of
	all commands, regardless of whether they passed or failed the tests.
" | less
   fi

   if [[ "$errorMessage" != "Unset" ]]; then
      exit 255
   else
      exit 1
   fi
}

#------------------------------------------------------------------------------
# Function run_test_cmd (cmd, desc, expectedExitCode, expectedInOutput)
#
# Runs a test command and determines test PASS/FAIL status.
# Increments global $TestID, and sets $TestLog.
# Increments global $ErrorCount if the test fails.
#
# After running a test command, the $TestLog can be checked multiple times
# (e.g with grep commands) until the next test resets it. Test logs are
# /tmp/test.<TestID>.log, so it is possible to grep for something in all
# logs as well.
#
# $1 - command to execute; required
# $2 - description of test
# $3 - expected exit code; default 0
# $4 - snippet of text to 'grep -q -E' for in output to pass.
#      The '-E' allows regex pattern greps, e.g. for "(this|that)"
#
# Test can only pass if command executes and returns expected exit code.
# If 4th parameter $expectedInOutput is defined, in addition to having the
# expected exit code, output must also contain the expected string for test
# to pass.
#------------------------------------------------------------------------------
function run_test_cmd () {
   dbg "CALL run_test_cmd (${1:-T} ${2:-}, ${3:-}, ${4:-})"
   local testCmd="${1:-Unset}"
   local desc="${2:-}"
   local -i expectedExitCode="${3:-0}"
   local expectedInOutput="${4:-}"
   local -i exitCode=0

   [[ "$testCmd" == Unset ]] && bail "TEST ERROR: No test specified."

   TestID+=1
   TestLog="/tmp/test.$TestID.log"
   msg "${H2}\\nTEST $TestID: $desc\\nRunning:\\n$testCmd\\nExpected Exit Code: $expectedExitCode"
   [[ -n "$expectedInOutput" ]] && msg "Expected in Output: $expectedInOutput"
   msg "Test log: $TestLog"

   $testCmd > "$TestLog" 2>&1
   exitCode="$?"

   msg "${H2}\\nBEGIN TEST $TestID OUTPUT"
   cat "$TestLog"
   msg "END TEST $TestID OUTPUT\\n"

   if [[ "$exitCode" -ne "$expectedExitCode" ]]; then
      errmsg "Exit code was $exitCode, but expected to be $expectedExitCode."
      msg "FAILED TEST $TestID"
      return 1
   else
      vmsg "Exit code was $expectedExitCode, as expected."
   fi

   if [[ -n "$expectedInOutput" ]]; then
      # This 'grep' should never generate stderr, so we do NOT redirect stderr
      # here, just in case something goes insanely wrong with the testing
      # mechanism itself.  The exit code determines the success/failure of
      # the test.
      if grep -q -E "$expectedInOutput" "$TestLog"; then
         vmsg "Expected output detected: $expectedInOutput"
      else
         errmsg "MISSING expected output: $expectedInOutput"
         msg "FAILED TEST $TestID"
         return 1
      fi
   else
      vmsg "No expected output defined for test $TestID. Skipping output grep check."
   fi

   msg "PASSED TEST $TestID"

   return 0
}

#------------------------------------------------------------------------------
# Function run_su_test_cmd (cmd, user, desc, expectedExitCode, expectedInOutput)
#
# Runs a test command with su -u 'user' and determines test PASS/FAIL status.
# Increments global $TestID, and sets $TestLog.
# Increments global $ErrorCount if the test fails.
#
# After running a test command, the $TestLog can be checked multiple times
# (e.g with grep commands) until the next test resets it. Test logs are
# /tmp/test.<TestID>.log, so it is possible to grep for something in all
# logs as well.
#
# $1 - command to execute; required
# $2 - operating system user to run as
# $3 - description of test
# $4 - expected exit code; default 0
# $5 - snippet of text to 'grep -q -E' for in output to pass.
#      The '-E' allows regex pattern greps, e.g. for "(this|that)"
#
# Test can only pass if command executes and returns expected exit code.
# If 5th parameter $expectedInOutput is defined, in addition to having the
# expected exit code, output must also contain the expected string for test
# to pass.
#------------------------------------------------------------------------------
function run_test_su_cmd () {
   dbg "CALL run_test_su_cmd (${1:-}, ${2:-}, ${3:-}, ${4:-}, ${5:-})"
   local testCmd="${1:-Unset}"
   local user="${2:-Unset}"
   local desc="${3:-}"
   local -i expectedExitCode="${4:-0}"
   local expectedInOutput="${5:-}"
   local -i exitCode=0

   [[ "$testCmd" == Unset ]] && bail "TEST ERROR: No test specified."
   [[ "$user" == Unset ]] && bail "TEST ERROR: No user specified."

   TestID+=1
   TestLog="/tmp/test.$TestID.log"
   msg "${H2}\\nTEST $TestID: $desc\\nRunning as $user:\\n$testCmd\\nExpected Exit Code: $expectedExitCode"
   [[ -n "$expectedInOutput" ]] && msg "Expected in Output: $expectedInOutput"
   msg "Test log: $TestLog"

   su - "$user" -c "$testCmd" > "$TestLog" 2>&1
   exitCode="$?"

   msg "${H1}\\nBEGIN TEST $TestID OUTPUT"
   cat "$TestLog"
   msg "END TEST $TestID OUTPUT\\n"

   if [[ "$exitCode" -ne "$expectedExitCode" ]]; then
      errmsg "Exit code was $exitCode, but expected to be $expectedExitCode."
      msg "FAILED TEST $TestID"
      return 1
   else
      vmsg "Exit code was $expectedExitCode, as expected."
   fi

   if [[ -n "$expectedInOutput" ]]; then
      # This 'grep' should never generate stderr, so we do NOT redirect stderr
      # here, just in case something goes insanely wrong with the testing
      # mechanism itself.  The exit code determines the success/failure of
      # the test.
      if grep -q -E "$expectedInOutput" "$TestLog"; then
         vmsg "Expected output detected: $expectedInOutput"
      else
         errmsg "MISSING expected output: $expectedInOutput"
         msg "FAILED TEST $TestID"
         return 1
      fi
   else
      vmsg "No expected output defined for test $TestID. Skipping output grep check."
   fi

   msg "PASSED TEST $TestID"

   return 0
}

#------------------------------------------------------------------------------
# Function check_log_test (expectedInLog)
#
# Increments global $TestID, and reads $TestLog.
# Increments global $ErrorCount if the test fails.
#
# After running a test command with run_test_cmd() or run_test_su_cmd(), the
# the $TestLog can be checked multiple times with grep commands using
# check_log_test.
#
# $1 - snippet of text to 'grep -q -E' for in output to pass.
#      The '-E' allows regex pattern greps, e.g. for "(this|that)"
#------------------------------------------------------------------------------
function check_log_test () {
   dbg "CALL check_log_test (${1:-})"
   local expectedInLog="${1:-Unset}"

   [[ "$expectedInLog" == Unset ]] && bail "TEST ERROR: No grep text specified."
   TestID+=1

   if grep -q -E "$expectedInLog" "$TestLog"; then
      vmsg "Expected output detected: $expectedInLog"
   else
      errmsg "MISSING expected output: $expectedInLog"
      msg "FAILED TEST $TestID"
      return 1
   fi

   msg "PASSED TEST $TestID"

   return 0
}

#------------------------------------------------------------------------------
# Function terminate, run on termination.
#------------------------------------------------------------------------------
function terminate () {
   trap - EXIT SIGINT SIGTERM

   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log${H2}"
}

#==============================================================================
# Command Line Parsing
declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-f) ForceCleanup=1;;
      (-V) msg "$ThisScript v$Version"; exit 1;;
      (-v1) Verbosity=1;;
      (-v2) Verbosity=2;;
      (-v3) Verbosity=3;;
      (-v4) Verbosity=4;;
      (-v5) Verbosity=5;;
      (-L) Log="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-d) Debug=1;; # Debug
      (-D) Debug=1; set -x;; # Debug; use 'set -x' mode.
      (*) usage -h "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

[[ "${Log:-Unset}" == "off" && "$SilentMode" -eq 1 ]] &&\
   bail "Usage Error: The '-L off' and '-si' options are mutually exclusive."

[[ "${Log:-Unset}" == "Unset" ]] && \
   Log="/tmp/${ThisScript%.sh}.v$Version.$(date +'%Y%m%d-%H%M%S').log"

#==============================================================================
# Main Program
#------------------------------------------------------------------------------
trap terminate EXIT SIGINT SIGTERM

if [[ "$Log" != off ]]; then
   touch "$Log" || bail "Couldn't touch log file [$Log]."

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec > "${Log}"
      exec 2>&1
   fi

   msg "Logging to: $Log"
fi

msg "${H1}\\nStarted ${0##*/} v$Version as $(whoami) on $OS at $(date)."

msg "$H1\\nTest Prep - Part 1."

if [[ "$OS" == "centos6" ]]; then
   msg "Note: Skipping crontab testing on $OS due to test env setup issues."
   TestCrontab=0
fi

# The SDP_BRANCH variable is passed in via the Docker environment, and can be set
# locally on the VM for manual testing.
if [[ "$SDPBranch" != Unset ]]; then
   msg "\\nInfo: SDP $SDPBranch selected for testing."
else
   SDPBranch="main"
   msg "\\nWarning: No SDP branch selected for testing; defaulting to $SDPBranch branch."
fi

HxDepots="hxdepots"
ResetDir="/$HxDepots/reset"
HelixInstallerSrcDir="/hi/src"
HelixInstallerTestDir="/hi/test"

msg "Ensuring required utilities are available."

if ! command -v rsync > /dev/null; then
   msg "Missing rsync, trying to get it."

   if [[ -r /etc/redhat-release ]]; then
      run "yum install -y rsync" "Installing package rsync" ||\
         bail "Failed to get package rsync."
   elif [[ -r /etc/lsb-release ]]; then
      run "apt-get install -y rsync" "Installing package rsync." ||\
         bail "Failed to get package rsync."
   else
      bail "Required rsync utility is missing, not sure how to get it on this OS."
   fi
fi

if [[ "$TestCrontab" -eq 1 ]]; then
   if ! command -v crontab > /dev/null; then
      msg "Missing crontab, trying to get it."

      if [[ -r /etc/redhat-release ]]; then
         run "yum install -y cronie" "Installing package cronie" ||\
            bail "Failed to get package cronie."
      elif [[ -r /etc/lsb-release ]]; then
         run "apt-get install -y cron" "Installing package cron." ||\
            bail "Failed to get package cron."
      else
         bail "Required crontab utility is missing, not sure how to get it on this OS."
      fi
   fi
fi

if [[ "$ForceCleanup" -eq 1 && -d "/$HxDepots" ]]; then
   run "rm -rf /$HxDepots" "Force cleanup due to '-f'."|| bail "Failed to do: rm -rf /$HxDepots"
fi

run "mkdir /$HxDepots" || bail "Failed to mkdir /$HxDepots"
run "rsync -a $HelixInstallerSrcDir/ $ResetDir" || bail "Failed to rsync to: $ResetDir"
cd "$ResetDir" || "Could not cd to: $ResetDir"

msg "Operating in: $PWD"

run "cp -pf $HelixInstallerTestDir/settings.fgs.cfg $ResetDir/." || \
   bail "Failed to copy cfg file to: $ResetDir"

msg "$H1\\nStarting Version check test."

msg "Cleaning logs from earlier runs (if any)."
if ! rm -f /tmp/test.*.log; then
   bail "Failed to remove logs from earlier logs with: rm -f /tmp/test.*.log"
fi

run_test_cmd './reset_sdp.sh -h' 'Version' 1 'reset_sdp.sh v'
run_test_cmd './reset_sdp.sh -man' 'Manual Page' 1 'DESCRIPTION:'

msg "$H1\\nNormal Flow Tests - fgs.\\n"

if [[ "$TestCrontab" -eq 1 ]]; then
   run_test_cmd "./reset_sdp.sh -b $SDPBranch -c settings.fgs.cfg -no_ssl -no_sd -no_ppr -no_tweaks -no_systemd -v -M" \
      'Basic usage test for instance fgs' 0 'SUCCESS'
else
   run_test_cmd "./reset_sdp.sh -b $SDPBranch -c settings.fgs.cfg -fast -no_ssl -no_sd -no_ppr -no_cron -no_tweaks -no_systemd -v -M" \
      'Basic usage test for instance fgs' 0 'SUCCESS'

   run_test_cmd 'cat /etc/sudoers.d/cooldude' 'Limited sudoers check' 0 'Cmnd_Alias P4_SVC'
fi

run_test_cmd 'id -u cooldude' 'Checking that OS user cooldude got created.' 0

run_test_cmd '/p4/common/bin/p4master_run fgs p4 set -q P4USER' \
   'Confirming P4USER is superman.' 0 'superman'

if [[ "$TestCrontab" -eq 1 ]]; then
   run_test_su_cmd 'crontab -l' 'cooldude' \
      'Checking crontab of OSUSER cooldude.' 0 'P4AdminList'
fi

msg "$H1\\nTest Prep - Part 2."

msg "Operating in: $PWD"

run_test_cmd "./reset_sdp.sh -X -R -c settings.fgs.cfg" \
   'Test of Extreme Cleanup with -X and -R flags' 0 'Extreme Cleanup complete'

msg "$H1\\nTest Prep - Part 3."

HxDepots="bigdisk"
ResetDir="/$HxDepots/reset"

if [[ "$ForceCleanup" -eq 1 && -d "/$HxDepots" ]]; then
   run "rm -rf /$HxDepots" "Force cleanup due to '-f'."|| bail "Failed to do: rm -rf /$HxDepots"
fi

run "mkdir /$HxDepots" || bail "Failed to mkdir /$HxDepots"
run "rsync -a $HelixInstallerSrcDir/ $ResetDir" || bail "Failed to rsync to: $ResetDir"
cd "$ResetDir" || "Could not cd to: $ResetDir"

msg "Operating in: $PWD"

run "cp -pf $HelixInstallerTestDir/settings.fgs2.cfg $ResetDir/." || \
   bail "Failed to copy cfg file to: $ResetDir"

msg "$H1\\nNormal Flow Tests - fgs2.\\n"

run_test_cmd "./reset_sdp.sh -b $SDPBranch -c settings.fgs2.cfg -fast -no_ssl -no_sd -no_ppr -no_cron -no_tweaks -no_systemd -no_firewall -no_sudo -v" \
   'Basic usage test for instance fgs2' 0 'SUCCESS'

check_log_test "Renamed to clarify this should not be executed again."
check_log_test "Skipping sudo setup due to"
check_log_test "Skipping firewall setup due"

run_test_cmd 'id -u cooldude' 'Checking that OS user cooldude got created.' 0

run_test_cmd 'id -g -n cooldude' 'Checking that OS user cooldude is in group heroes.' 0 heroes

run_test_cmd '/p4/common/bin/p4master_run fgs2 p4 set -q P4USER' \
   'Confirming P4USER is superman.' 0 'superman'

# Quiet Test Prep: Re-activate the reset_sdp.sh script.
mv reset_sdp.sh.txt reset_sdp.sh
chmod +x reset_sdp.sh

run_test_cmd "./reset_sdp.sh -b $SDPBranch -c settings.fgs2.cfg -fast -no_ssl -no_sd -no_ppr -no_cron -no_tweaks -no_systemd -no_sudo -ls -v" \
   'Bad usage test with -no_sudo and -ls.' 1 'options are mutually exclusive'

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "$H1\\nSummary: Executed $TestID tests, ALL PASSED."
else
   msg "$H1\\nSummary: Executed $TestID tests, $ErrorCount FAILED, $((TestID-ErrorCount)) passed."
   OverallExitCode=1
fi

msg "\\nTest suite executed in $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\n"

exit "$OverallExitCode"