hits.sh #5

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

declare ThisScript="${0##*/}"
declare Version=1.4.0
declare -i ForceRebuild=0
declare -i PullLatestImage=0
declare -i BuildDockerImages=1
declare -i RunDockerTests=1
declare -i BuiltImageCount=0
declare -i TestedOSCount=0
declare -a AllTestedOperatingSystems
declare -a InteractiveCommands
declare -i i=0

declare Cmd=
declare HelixInstallerBranch=dev
declare SDPBranch=main
declare WS=
declare WSRoot=
declare CopiedBranchRoot=
declare DockerDir=
declare DockerBaseFile=
declare DockerHIFile=
declare TestedOSList=
declare -i ErrorCount=0
declare -i PassCount=0
declare -i FailCount=0
declare -i WarningCount=0
declare -i Debug=0
declare -i NoOp=0
declare -i SilentMode=0
declare H1="\\n=============================================================================="
declare H2="\\n------------------------------------------------------------------------------"
declare Log=
declare -i i=0

AllTestedOperatingSystems[$i]="rocky8"
i+=1;
AllTestedOperatingSystems[$i]="rocky9"
i+=1;
AllTestedOperatingSystems[$i]="centos7"
i+=1;
AllTestedOperatingSystems[$i]="ubuntu20"
i+=1;
AllTestedOperatingSystems[$i]="ubuntu22"
i+=1;

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function pass () { msg "PASS $*"; PassCount+=1; }
function fail () { msg "FAIL $*"; FailCount+=1; }
function dbg () { [[ "$Debug" -eq 1 ]] && msg "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Function run ($cmd, $desc, $honorNoOp)
function run () {
   local cmdAndArgs="${1:-echo}"
   local desc="${2:-}"
   local -i honorNoOp="${3:-1}"
   local -i exitCode=0

   [[ -n "$desc" ]] && msg "$desc"
   msg "Running: $cmdAndArgs"

   # If we're not honoring NoOp mode or it isn't specified, execute the cmd.
   if [[ "$NoOp" -eq 0 || "$honorNoOp" -eq 0 ]]; then
      $cmdAndArgs
      exitCode=$?
   else
      msg "NO_OP: Would run: $cmdAndArgs"
   fi

   return $exitCode
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   dbg "$ThisScript: EXITCODE: $((ErrorCount+FailCount))"

   # Stop logging.
   [[ "$Log" == off ]] || msg "Log is: $Log${H1}"

   # With the trap removed, exit.
   exit "$((ErrorCount+FailCount))"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ "$errorMessage" != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n"
   fi

   echo "USAGE for $ThisScript v$Version:

$ThisScript [-B|-b] [-f] [-p] [-o <os1>[,<os2>,...]] [-hb <helix_installer_branch>] [-sb <sdp_branch>] [-n] [-D] [-L <log>]

or

$ThisScript [-h|-man]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script manages Docker environment for the Helix Installer Test Suite.

	The test_hi.sh runs the actual tests within VMs managed by this script.

OPTIONS:
 -b	Build Docker images, and then stop before running tests. By default,
 	tests are executed after images are built.

 -f	Specify '-f' to force a rebuild of the Docker image by passing '--no-cache'
 	option to the 'podman build' command.

 -p	Specify '-p' to pull the latest Docker source image by passing '--pull'
 	option to the 'podman build' command.

 -B	Specify '-B' as a shorthand for '-b -f -p'.

 -o	Specify a comma-delimited list of operating systems to build.
 	Valid operating system values are:

	${AllTestedOperatingSystems[*]}

 -hb <helix_installer_branch>
 	Specify the Helix Installer branch to test.  The default is dev.

 -sb <sdp_branch>
 	Specify the SDP branch to test.  The default is main, which always
	represents the latest version officially released for general
	availability.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to

	${ThisScript%.sh}.<DateTimeStamp>.log

	NOTE: This script is self-logging.  That is, output displayed on the
	screen is simultaneously captured in the log file.  Do not run this
	script with redirection operators like '> log' or '2>&1', and do not
	use 'tee'.

 -si	Operate silently.  All output (stdout and stderr) is redirected to the
 	log only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
	This is useful when running from cron, as it prevents automatic email
	from being sent by cron directly, as cron does when a script called
	from cron generates any output.  This script is then responsible for
	email handling, if any is to be done.

 -n	No-Op.  Prints podman, rsync, and other commands instead of running
 	them.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	Example 1: Normal test run on all operating systems - no arguments.
	$ThisScript

	Example 2: Test image build only
	$ThisScript -B

	Example 3: Test on a single operating system:
	$ThisScript -o centos7

	Example 4: See podman and other commands without executing them.
	$ThisScript -n
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-b) RunDockerTests=0;;
      (-f) ForceRebuild=1;;
      (-p) PullLatestImage=1;;
      (-B) RunDockerTests=0; ForceRebuild=1; PullLatestImage=1;;
      (-o) TestedOSList="${2/,/ }"; shiftArgs=1;;
      (-hb) HelixInstallerBranch="$2"; shiftArgs=1;;
      (-sb) SDPBranch="$2"; shiftArgs=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-L) Log="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) NoOp=1;;
      (-d) Debug=1;; # Debug verbosity.
      (-D) Debug=1; set -x;; # Extreme Debug; use 'set -x' mode.
      (*) usage -h "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usage -h "Cannot use '-si' with '-L off'."

[[ "${Log:-Unset}" == "Unset" ]] && \
   Log="${ThisScript%.sh}.$(date +'%Y%m%d-%H%M%S').log"

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

if [[ "$Log" != off ]]; then
   touch "$Log" || bail "Couldn't touch log file: $Log"

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec > "$Log"
      exec 2>&1
   fi

   msg "\\n${H1}\\nLog is: $Log"
fi

msg "Started $ThisScript v$Version as $USER@${HOSTNAME%%.*} at $(date)."

WS="$(p4 set -q P4CLIENT)"
WS="${WS#*=}"
WSRoot="$(p4 -ztag -F %Root% client -o)"
BranchRoot="$WSRoot/$HelixInstallerBranch"
CopiedBranchRoot="$BranchRoot/hi"
TestDir="$CopiedBranchRoot/test"
DockerDir="$TestDir/docker"

# Silent preflight check for basic utilities.
for util in podman rsync chmod; do
   if ! command -v $util > /dev/null 2>&1; then
      errmsg "Preflight failure: Required utility not found in PATH: $util"
   fi
done

[[ "$ErrorCount" -eq 0 ]] || bail "Aborting due to failed preflight checks."

# By default, test all supported operating systems.
[[ -z "$TestedOSList" ]] && TestedOSList="${AllTestedOperatingSystems[*]}"

# Copy workspace files to a separate test area.
run "rsync -a --exclude=.p4passwd --exclude .p4config --exclude '*.log' --exclude --delete ${BranchRoot}/ $CopiedBranchRoot" \
   "Copying files to test from ${BranchRoot}." ||\
   errmsg "Copy with rsync failed."

run "chmod 777 $CopiedBranchRoot" \
   "Opening perms for copied folder to be mounted." ||\
   errmsg "Adjusting perms failed for copied dir: $CopiedBranchRoot"

msg "\\nDoing test infrastructure preflight checks."

# Do a quick preflight check to enure needed files exist.
if [[ "$BuildDockerImages" -eq 1 ]]; then
   for os in $TestedOSList; do
      DockerBaseFile="$DockerDir/Dockerfile.${os}.base"
      DockerHIFile="$DockerDir/Dockerfile.${os}.hi"
      [[ -f "$DockerBaseFile" ]] || \
         errmsg "Missing Docker base file: $DockerBaseFile"
      [[ -f "$DockerHIFile" ]] || \
         errmsg "Missing Docker Helix Installer file: $DockerHIFile"
   done
fi

[[ "$ErrorCount" -eq 0 ]] || bail "Aborting due to failed pre-flight checks."

[[ -d "$DockerDir" ]] || bail "The expected Docker dir does not exist: $DockerDir"

if [[ "$BuildDockerImages" -eq 1 ]]; then
   msg "${H1}\\nBuilding Docker Images for: $TestedOSList"
   for os in $TestedOSList; do
      msg "${H2}\\nBuilding image for $os."
      DockerBaseFile="$DockerDir/Dockerfile.${os}.base"
      DockerHIFile="$DockerDir/Dockerfile.${os}.hi"

      # Build the base OS image.
      Cmd="podman build"
      [[ "$ForceRebuild" -eq 1 ]] && Cmd+=" --no-cache"
      [[ "$PullLatestImage" -eq 1 ]] && Cmd+=" --pull"
      Cmd+=" --rm=true -t=perforce_hi/${os}-base -f $DockerBaseFile $DockerDir"
      run "$Cmd" "Building base image for $os." ||\
         warnmsg "Errors were encountered building base image for $os, which may or may not\\nimpact tests."

      # Modify the image, adding in the Helix Installer bits.
      Cmd="podman build --rm=true -t=perforce_hi/${os}-hi -f $DockerHIFile $DockerDir"
      run "$Cmd" "Adding Helix Installer bits to base image for $os." ||\
         warnmsg "Errors encountered adding HI bits to base image for $os, which may or may not\\nimpact tests."

      InteractiveCommands[$BuiltImageCount]="podman run --rm -v ${CopiedBranchRoot}:/hi -e OS=${os} -e SDP_BRANCH=$SDPBranch -it perforce_hi/${os}-hi /bin/bash"
      BuiltImageCount+=1
   done

   msg "${H2}\\nDocker image processing summary:\\n\\t$BuiltImageCount images built/updated."

   if [[ "$WarningCount" -eq 0 ]]; then
      msg "\\nImages built cleanly with no errors."
   else
      warnmsg "$WarningCount Problems were encountered getting the images ready. These may or may not\\nimpact tests."
   fi
fi

if [[ "$RunDockerTests" -eq 1 ]]; then
   msg "${H1}\\nRunning tests on these operating systems: $TestedOSList"
   for os in $TestedOSList; do
      TestedOSCount+=1

      # Run the test suite.
      if run "podman run --rm -v ${CopiedBranchRoot}:/hi -e OS=${os} -e SDP_BRANCH=$SDPBranch perforce_hi/${os}-hi" "${H2}\\nTesting on $os"; then
         pass "on $os"
      else
         fail "on $os"
      fi
   done
fi

msg "${H2}\\nInteractive commands to get into VMs:"
i=0; while [[ $i -lt ${#InteractiveCommands[*]} ]]; do
   msg "\\t${InteractiveCommands[$i]}"
   i+=1
done

msg "\\nAfter running the above command to get on a VM, run:\\n\\n\\t/hi/test/podman/docker_entry.sh\\n"

if [[ "$TestedOSCount" -gt 0 ]]; then
   msg "\\nTest Operating System status summary:"
   if [[ "$Log" != "off" ]]; then
      grep -E '^(PASS|FAIL) on ' "$Log"
   fi

   if [[ "$FailCount" -eq 0 ]]; then
      msg "\\nSUCCESS: Tests passed on all $TestedOSCount operating systems."
   else
      msg "\\nTest failures detected. Test Summary:\\n\\tTests ran on $TestedOSCount operating systems.\\n\\t$PassCount succeeded.\\n\\t$FailCount failed.\\n"
   fi
else
   msg "\\nNo tests were executed."
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "\\nThat took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

# See the terminate() function, which is really where this script exits.
exit 0