ccheck.sh #14

#!/bin/bash
set -u

#==============================================================================
# Declarations and Environment
declare ThisScript=${0##*/}
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare Version=1.6.0
declare Args="$*"
declare CmdLine="$0 $Args"

declare -i ExitCode=0
declare -i ErrorCount=0
declare -i NoOp=1
declare -i Debug=${DEBUG:-0}
declare -i ConfigMatches=0
declare -i SilentMode=0
declare -i DoSecurityCheck=0
declare -i DoSSLCheck=0
declare -i EnableSSLAdvised=0
declare -i ApplyEasyFixes=0
declare -i EasyFixCount=0
declare -a EasyFixConfigurables
declare -a EasyFixValues
declare -i ApplyAllFixes=0
declare -i PlannedFixCount=0
declare -a PlannedFixConfigurables
declare -a PlannedFixAdvice
declare -a PlannedFixAutomated  # Set to 1 if Advice can be applied automatically, 0 if manual.
declare -i i=0 
declare Profile=prod
declare CfgFile=/p4/common/config/configurables.cfg
declare Log=
declare OldLogTimestamp=
declare LogTimestamp=
declare LogLink=
declare TmpFile=
declare TmpFile2=
declare SDPRoot=${SDP_ROOT:-/p4}
declare SDPInstance=
declare SDPInstanceVars=
declare TimeoutDelay=5s
declare SuperUser=
declare Cmd=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare -i Verbosity=0
declare -i i=0

# Associative arrays indexed by configurable, capturing data loaded from
# /p4/common/config/configurables.cfg.
declare Configurable=
declare ConfigurableType=
declare -A CurrentValue
declare -A CurrentValueByDefault
declare -A ExpectedValue
declare -A CompareStyle
declare -A Optionality
declare -A ServerIDType
declare -A SetNotes
declare ThisServerIDType=

# Categories is indexed by configurable, and is loaded with data from
# 'p4 configure help'; not to be confused with 'p4 help configure'.
declare Category
declare -A Categories

# Summary Counters
declare -i ConfigCount=0
declare -i ConfigPassCount=0
declare -i ConfigFailCount=0
declare -i ConfigFailRequiredCount=0
declare -i ConfigFailRecommendedCount=0
declare -i ConfigFailOptionalCount=0

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function msgn () { echo -n -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || echo -e "DEBUG: $*" >&2; }
function dbg2 () { [[ "$Debug" -ge 2 ]] && echo -e "DEBUG: $*" >&2; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

# The cfgreqpass() function is called when required configuration settings are
# as expected, and cfgrecpass() is called when recommended settings are
# as expected. Both increment ConfigPassCount.
function cfgreqpass () { [[ "$Verbosity" -ne 0 ]] && msg "GREAT: $*"; ConfigPassCount+=1; }
function cfgrecpass () { [[ "$Verbosity" -ne 0 ]] && msg "GOOD: $*"; ConfigPassCount+=1; }

# The cfgreqfail() function is called when required configuration settings are
# not as expected, and cfgrecfail() is called when recommended settings are not
# as expected. Both increment ConfigFailCount and contribute to an unhappy exit
# code.
function cfgoptfail () { msg "INFO: $*"; ConfigFailOptionalCount+=1; }
function cfgreqfail () { msg "BAD: $*"; ConfigFailCount+=1; ConfigFailRequiredCount+=1; }
function cfgrecfail () { msg "WARN: $*"; ConfigFailCount+=1; ConfigFailRecommendedCount+=1; }

#------------------------------------------------------------------------------
# Function: at_least_size_compare ("$currentValue" "$expectedValue")
# Return true if size 1 is at least as big as size 2. Do the needful conversion
# of units (just numeric bytes, K, M, G, T, P, E, etc.) to compare correctly.
function at_least_size_compare () {
   local s1=${1:-}
   local s2=${2:-}
   local num1=
   local unit1=
   local num2=
   local unit2=
   local multiplier1=1
   local multiplier2=1
   local val1=
   local val2=

   # If inputs are purely numeric, append a 'B' (bytes) suffix.
   [[ "$s1" =~ ^[0-9]+$ ]] && s1="${s1}B"
   [[ "$s2" =~ ^[0-9]+$ ]] && s2="${s2}B"

   if [[ ! "$s1" =~ ^[0-9]*([BKMGTPE]){1}$ || ! "$s2" =~ ^[0-9]*([BKMGTPE]){1}$ ]]; then
      errmsg "Cannot compare size value [$s1] with value [$s2]; the values are invalid."
      return 1
   fi

   # Extract the numeric part and unit from each value.
   [[ $s1 =~ ^([0-9]+)([BKMGTPE]) ]] && num1=${BASH_REMATCH[1]} && unit1=${BASH_REMATCH[2]}
   [[ $s2 =~ ^([0-9]+)([BKMGTPE]) ]] && num2=${BASH_REMATCH[1]} && unit2=${BASH_REMATCH[2]}

   # Convert units to base 10 multipliers.
   case $unit1 in
      B) multiplier1=1;;
      K) multiplier1=1000;;
      M) multiplier1=1000000;;
      G) multiplier1=1000000000;;
      T) multiplier1=1000000000000;;
      P) multiplier1=1000000000000000;;
      E) multiplier1=1000000000000000000;;
   esac

   case $unit2 in
      B) multiplier2=1;;
      K) multiplier2=1000;;
      M) multiplier2=1000000;;
      G) multiplier2=1000000000;;
      T) multiplier2=1000000000000;;
      P) multiplier2=1000000000000000;;
      E) multiplier2=1000000000000000000;;
   esac

   val1=$((num1 * multiplier1))
   val2=$((num2 * multiplier2))

   [[ "$val1" -ge "$val2" ]] && return 0
   return 1
}

#------------------------------------------------------------------------------
# Function: run ($cmdAndArgs, $desc, $honorNoOp, $showOutput)
#
# Runs a command, with optional description, showing command line to execute
# and optionally also the output, and capturing and returning the exit code.
# Honors the global NoOp unless honorNoOp is 0.
#
# Input:
# $1 - Command and arguments to execute. Defaults to 'echo'.
# $2 - Optional message to display describing what the command is doing.
# $3 - Honor NoOp mode. If set to one, honor the NoOp variable -- so display
#      rather than execute commands.
# $4 - Numeric flag to show output; '1' indicates to show output, 0 to
#      suppress it.
#------------------------------------------------------------------------------
function run () {
   local cmdAndArgs="${1:-echo}"
   local desc="${2:-}"
   local honorNoOp="${3:-1}"
   local -i showOutput="${4:-1}"
   local -i cmdReturnCode=0
   local log

   [[ -n "$desc" ]] && msg "$desc"
   msg "Executing: $cmdAndArgs"
   if [[ "$honorNoOp" -eq 1 && "$NoOp" -eq 0 ]]; then
      log="$(mktemp "$P4TMP/run.${ThisScript%.sh}.XXXXXXXXXXX")"
      # shellcheck disable=SC2086
      eval $cmdAndArgs > "$log" 2>&1
      cmdReturnCode=$?
   else
      msg "NO_OP: Would run: $cmdAndArgs"
      cmdReturnCode=0

      # In NoOp mode, no command was executed, so there is
      # output to show.
      showOutput=0
   fi

   if [[ "$showOutput" -eq 1 ]]; then
      echo "EXIT_CODE: $cmdReturnCode" >> "$log"
      cat "$log"
   fi

   if [[ "$honorNoOp" -eq 0 && "$NoOp" -eq 0 ]]; then
      /bin/rm -f "$log"
   fi

   return "$cmdReturnCode"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   local style=${1:--h}
   local usageErrorMessage=${2:-}

   [[ -n "$usageErrorMessage" ]] && msg "\\n\\nUsage Error:\\n\\n$usageErrorMessage\\n\\n"

   msg "USAGE for $ThisScript v$Version:

$ThisScript [<SDPInstance>] [-p <Profile>] [-sec [-no_ssl]] [-c <CfgFile>] [-y] [-v] [-d|-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script compares configurables set on the current server with best
	practices defined a data file.  The default data file is provided as:

	/p4/common/config/configurables.cfg

	Optionally, if the '-fix' or '-FIX' options are used, this script can make
	changes to p4d, via 'p4 configure' commands, to bring it in line with best
	practices.  A preview of proposed changes is then displayed.  The '-y'
	option can then be used to proceed with the operation.

OPTIONS:
 -p <Profile>
 	Specify a profile defined in the config file, such as 'demp' or 'hcc'.  A profile
	defines a set of expected configurable values that can differ from the expected
	values in other profiles.  For example, for a demo environment, the filesys.P4ROOT.min
	might have an expected value of 128M, while the expected value in a prod (production)
	profile might be 5G, and the same value might be 30G for 'prodent', the profile for
	production at large enterprise scale.

	The 'always' profile defines settings that always apply whether '-p' is specified
	or not. The profile specified with '-p' applies in addition to the 'always'
	configuration, adding to and possibly overriding settings from the 'always'
	configuration.

	The default profile is 'prod', the production profile.

	Specify the special value '-p none' to use only the settings defined in the
	'always' profile.

 -sec	Specify '-sec' to do a focused security check.

	This option also adds one additional check to ensure that the P4PORT value
	used is SSL-enabled (unless '-no_ssl' is specified).

 -no_ssl
	Specify '-no_ssl' with '-sec' to bypass the check for an SSL-enabled P4PORT.

 -c <CfgFile>
	Specify an alternate config file to define best practice configurables. This
	is intended primarily for testing. It can also be useful to define a site-local
	definition of best practices to compare against.  To use this option, first
	copy the default file to create a local copy in the /p4/common/site/config,
	e.g.
	
	cp -p /p4/common/config /p4/common/site/config/configurables.cfg

	Then reference it with '-c /p4/common/site/config/configurables.cfg'.

	WARNING: If you maintain a site-local copy of configurables.cfg, you will
	need to keep it current after SDP upgrades by manually merging in changes
	from the latest SDP version delivered with each release. Thus, using this
	option is discouraged.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to a log file
	pointed to by a symlink:

	\$LOGS/${ThisScript%.sh}.log

	The symlink is for convenience. It refers to the log from the most recent
	run where '-L' was not used.

	Each time this script is run, a new timestampped log is started, and
	the symlink updated to reference the new/latest log during startup.  Log
	files have timestamps that go to the second (or millisecond if needed)
	to differentiate logs.

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file. Using redirection operators like
	'> log' or '2>&1' are unnecessary, as is using 'tee' (though using 'tee'
	or redirects is safe and harmless).

 -fix	Specify -fix to take corrective action to resolve differences between
	current settings and the recommended/required values.  Optional settings
	are not affected by '-fix', only those indicated as Recommended or
	Required.

	When -fix is specified, this script determines on a per configurable basis
	whether it is safe to proceed immediately with the advised change, or if
	should be deferred until potentially disruptive impacts are understood.
	This determination is made based on the configuration file, which provides
	links go guidance documentation for configurables that are best changed
	with awareness of potential impact.  Such changes are displayed with '-fix',
	but require use of '-FIX' to process.

	This option previews advised changes by default. Use with -y to make changes.

 -FIX	Specify -FIX to make all changes that have been automated, even those that
	'-fix' would refuse to process immediately.

	In some cases even with -FIX, there may be follow up work to do. Follow up
	tasks will be indicated with 'TO DO:' comments in the output.

	The '-FIX' option implies '-fix'.

	This option previews advised changes by default. Use with -y to make changes.

 -y	Live operation mode.  By default, any commands that affect data, such as
	setting configurables, are displayed, but not executed.  With the '-y' option,
	commands may be executed.

 -d	Display debug messages.

 -D     Set extreme debugging verbosity using bash 'set -x' mode. Implies -d.

 -si	Silent Mode.  No output is displayed to the terminal (except for usage errors
	on startup). Output is captured in the log.  The '-si' cannot be used with
	'-L off'.

HELP OPTIONS:
 -h	Display short help message.
 -man	Display man-style help message.
 -V	Display script name and version.

FILES:
	The standard configurables config file is:

	$CfgFile

GENERAL ADVICE and DISCLAIMER:
	This script is based on a data file that represents generalized best
	practices. This data file should be considered a source of information
	to be checked against other sources, such as the System Administration
	Guide and/or documentation on specific configurables found here:

	https://help.perforce.com/helix-core/server-apps/cmdref/current/Content/CmdRef/configurables.alphabetical.html

	Before acting on information provided by the script, and especially before
	using the '-fix' and '-FIX' options, be sure to review the output carefully.

	Contact Perforce Technical Support for guidance as needed.

EXAMPLES:
	Example 1: Check configurables with the default profile, and no logging:

	$ThisScript -L off

	Example 2: Check configurables with the 'prod' (Production) profile:

	$ThisScript -p prod

	Example 3: Check configurables with the 'demo' profile, doing a verbose comparison:

	$ThisScript -p demo -v

	Example 4: Security focused check

	$ThisScript -sec

	Example 5: Security focused check, with non-disruptive fixes:

	Start with a preview by omitting the '-y' option:

	$ThisScript -sec -fix

	If the output is correct, append the '-y' to the command to make changes:

	$ThisScript -sec -fix -y

	Example 6: Security focused check, including non-disruptive fixes:

	Start with a preview by omitting the '-y' option:

	$ThisScript -sec -FIX

	If the output is correct, append the '-y' to the command to make changes:

	$ThisScript -sec -FIX -y

FUTURE ENHANCEMENTS:
	* Add multi-version support for backward compatibility. This version assumes
	  P4D 2024.2+ (though it may be useful for older versions).
"
   fi

   exit 2
}

#------------------------------------------------------------------------------
# Function: terminate
# shellcheck disable=SC2317
function terminate ()
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   ExitCode=$((ErrorCount+ConfigFailRequiredCount))
   msg "ExitCode: $ExitCode"

   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log\\n${H1}"

   # With the trap removed, exit.
   exit "$ExitCode"
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-V|-version|--version) msg "$ThisScript version $Version"; exit 0;;
      (-i) SDPInstance="$2"; shiftArgs=1;;
      (-sec) DoSecurityCheck=1; DoSSLCheck=1;;
      (-no_ssl) DoSSLCheck=0;;
      (-fix) ApplyEasyFixes=1;;
      (-FIX) ApplyEasyFixes=1; ApplyAllFixes=1;;
      (-p) Profile="$2"; shiftArgs=1;;
      (-c) CfgFile="$2"; shiftArgs=1;;
      (-L) Log="$2"; shiftArgs=1;;
      (-y|--yes) NoOp=0;;
      (-si) SilentMode=1;;
      (-v) Verbosity=1;;
      (-d) Debug=1;;
      (-d2) Debug=2;;
      (-D) Debug=2; set -x;; # Use bash 'set -x' extreme debug mode.
      (-*) usage -h "Unknown option ($1).";;
      (*)
         if [[ -z "$SDPInstance" ]]; then
            SDPInstance="$1"
	 else
            usage -h "Extra parameter [$1] is unknown; SDP Instance is already set to [$SDPInstance]."
	 fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usage -h "The '-si' option cannot be used with '-Log off'."

[[ -z "$SDPInstance" ]] && SDPInstance="${SDP_INSTANCE:-}"
[[ -z "$SDPInstance" ]] && usage -h "The SDP instance parameter is required unless SDP_INSTANCE is set. To set SDP_INSTANCE, do:\\n\\tsource $SDPRoot/common/bin/p4_vars INSTANCE\\n\\nreplacing INSTANCE with your SDP instance name."

SDPInstanceVars="$SDPRoot/common/config/p4_${SDPInstance}.vars"
[[ -r "$SDPInstanceVars" ]] || usage -h "The SDP instance specified [$SDPInstance] is missing Instance Vars file: $SDPInstanceVars"

# shellcheck disable=SC1090 disable=SC1091
source "$SDPRoot/common/bin/p4_vars" "$SDPInstance" ||\
   bail "Could not do: source \"$SDPRoot/common/bin/p4_vars\" \"$SDPInstance\""
# shellcheck disable=SC1090 disable=SC1091
source "$SDPRoot/common/bin/log_functions.sh" ||\
   bail "Could not do: source \"$SDPRoot/common/bin/log_functions.sh\""

[[ -e "$CfgFile" ]] || usage -h "The config file [$CfgFile] is missing."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

# If the user specifies a log file file with '-L', write to the specified file.
# If no log was specified, create a default log file using a timestamp in the
# LOGS dir, and immediately update the symlink for the default log to point to
# it.
if [[ "$Log" != off ]]; then
   # If $Log is not yet defined, set it to a reasonable default.
   if [[ -z "$Log" ]]; then
      LogTimestamp=$(date +'%Y-%m-%d-%H%M%S')
      Log="$LOGS/${ThisScript%.sh}.${LogTimestamp}.log"
      # Make sure we have a unique log file. Prefer a human-readable timestamp
      # using hours/minutes/seconds. Append milliseconds if needed to ensure
      # achieve a unique filename.
      while [[ -e "$Log" ]]; do
         LogTimestamp=$(date +'%Y-%m-%d-%H%M%S.%3N')
         Log="$LOGS/${ThisScript%.sh}.${LogTimestamp}.$i.log"
         i+=1
      done
   fi
   # The LogLink symlink has no timestamp. It points to the most recent log file.
   LogLink="$LOGS/ccheck.log"

   if [[ -e "$LogLink" ]]; then
      if [[ -L "$LogLink" ]]; then
         rm -f "$LogLink"
      else
         # If the name that should be a symlink is not a symlink, move it aside before
         # creating the symlink.
         OldLogTimestamp=$(get_old_log_timestamp "$LogLink")
         mv -f "$LogLink" "${LogLink%.log}.${OldLogTimestamp}.log" ||\
            bail "Could not move old log file aside; tried: mv -f \"$LogLink\" \"${LogLink%.log}.${OldLogTimestamp}.log\""
      fi
   fi

   touch "$Log" || bail "Couldn't touch new log file [$Log]."

   # Use a subshell so the 'cd' doesn't persist.
   ( cd "$LOGS"; ln -s "${Log##*/}" "${LogLink##*/}"; ) ||\
       bail "Couldn't initialize log symlink; tried: ln -s \"$Log\" \"$LogLink\""

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Starting $ThisScript v$Version as $ThisUser@$ThisHost on $(date) with \\n$CmdLine"

msg "Comparing with Profile: [$Profile] loaded from: $CfgFile\\n${H2}"

# For environments that define P4SUPER, use that for P4USER rather than P4USER.
SuperUser=$(p4 set -q P4SUPER)
SuperUser=${SuperUser##*=}

if [[ -n "$SuperUser" ]]; then
   export P4USER="$SuperUser"
fi

TmpFile=$(mktemp)

if [[ "$P4PORT" =~ ^ssl[46]*: ]]; then
   Cmd="timeout $TimeoutDelay p4 trust -y"
   dbg "Running: $Cmd [P4PORT=$P4PORT]"
   $Cmd > "$TmpFile" 2>&1 ||\
      bail "Could not do: $Cmd\\n$(cat "$TmpFile")\\n"
fi

Cmd="timeout $TimeoutDelay p4 info -s"
dbg "Running: $Cmd"
if $Cmd > "$TmpFile" 2>&1; then
   dbg "Verified: Helix Core is accessible."
   if [[ "$(p4 protects -m)" == "super" ]]; then
      dbg "Verified: super user access available."
   else
      bail "Could not verify super access [P4USER=$P4USER P4PORT=$P4PORT P4TICKETS=$P4TICKETS]."
   fi
else
   bail "Could not access Helix Core server [P4PORT=$P4PORT] with: $Cmd\\n$(cat "$TmpFile")\\n"
fi

if [[ "$DoSecurityCheck" -eq 1 && "$DoSSLCheck" -eq 1 ]]; then
   ConfigCount+=1
   if [[ $P4PORT =~ ^ssl[46]*: ]]; then
      cfgreqpass "The P4PORT [$P4PORT] is SSL-enabled."
   else
      cfgreqfail "The P4PORT [$P4PORT] is NOT SSL-enabled."
      EnableSSLAdvised=1
   fi
fi

#------------------------------------------------------------------------------
# P4D's categories for each configurable (Security, Performance, etc.) are
# accessible via 'p4 configure help'.
dbg "Checking configurable categories."
p4 -ztag -F "%Configurable%|%Category%" configure help > "$TmpFile" ||\
   bail "Could not get 'p4 configure help' output."

[[ "$Debug" -ge 2 ]] && dbg2 "FILE:\\n$(cat "$TmpFile")\\n\\n"

while read -r Line; do
   Configurable="${Line%%|*}"
   Category="${Line##*|}"
   Categories[$Configurable]="$Category"
   dbg2 "Configurable [$Configurable] has categories [$Category]."
done < "$TmpFile"

# Extract current configurables values using 'p4d -cshow'.
Cmd="p4d -r $P4ROOT -cshow"
if $Cmd > "$TmpFile" 2>&1; then
   dbg "Captured configurables:\\n$(cat "$TmpFile")\\n"
else
   bail "Could not capture configurables with: $Cmd\\n$(cat "$TmpFile")\\n"
fi

dbg "Comparing P4MASTER_ID=$P4MASTER_ID with SERVERID=$SERVERID."

# Determine whether the current ServerID is that of a commit, replica, standby, or edge.
### Use is_*() functions from backup_functions.sh, but avoid loading backup_functions.sh
### directly for now, due to compatibility issues with new bash template this script is
### based on. EDITME - Find a better way later.
# shellcheck disable=SC1091
if [[ "$P4MASTER_ID" == "$SERVERID" ]]; then
   ThisServerIDType=commit
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_replica "$SERVERID")" == YES ]]; then
   ThisServerIDType=replica
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_standby "$SERVERID")" == YES ]]; then
   ThisServerIDType=standby
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_edge "$SERVERID")" == YES ]]; then
   ThisServerIDType=edge
else
   errmsg "Could not determine server type for ServerID [$SERVERID]."
fi

dbg "This ServerID [$SERVERID] is of type [$ThisServerIDType]."

dbg "Loading current configurable values."
p4 -ztag -F "%Type%|%ServerName%|%Name%|%Value%" configure show allservers > "$TmpFile"
while read -r Line; do
   ConfigurableType="$(echo "$Line" | cut -d '|' -f1)"
   [[ "$ConfigurableType" == "configure" ]] || continue
   ServerName="$(echo "$Line"|cut -d '|' -f2)"

   # We are only concerned with configurables affecting the current ServerID.
   # That includes configurables set explicitly for the current ServerID,
   # as well as values set as global defaults with the 'any' config.
   [[ "$ServerName" == any || "$ServerName" == "$SERVERID" ]] || continue

   Configurable="$(echo "$Line"|cut -d '|' -f3)"

   # If a configurable is set both as a global default and explicitly
   # for the current ServerID, set the value to be that of the explicitly
   # set value. That overrides the global default effectively in p4d,
   # and we mirror that logic here.  We can't control whether the 'any'
   # or the explicit values appears first in the output of 'p4 configure
   # show allservers' (it depends where the ServerID name appears in
   # an listing by alphabetical order). So, ignore the 'any' value if
   # there already is a value defined for the configurable.

   [[ "$ServerName" == "any" && -n "${CurrentValue[$Configurable]:-}" ]] && continue

   currentValue="$(echo "$Line"|cut -d '|' -f4)"
   [[ -n "$currentValue" ]] || currentValue="unset"

   dbg "CurrentValue[$Configurable]=$currentValue"
   CurrentValue[$Configurable]="$currentValue"
done < "$TmpFile"

# Extract settings for the selected profile and the default profile.
# Normalize the profile to lowercase for searching in the config file.
grep '^always|' "$CfgFile" > "$TmpFile"

if [[ "$Profile" != none ]]; then
   TmpFile2=$(mktemp)
   grep "^${Profile,,}|" "$CfgFile" >> "$TmpFile2"
   if [[ -s "$TmpFile2" ]]; then
      cat "$TmpFile2" >> "$TmpFile" ||\
         bail "Could not append settings for profile $Profile to settings for default profile. Aborting."
      rm -f "$TmpFile2"
   else
      bail "No settings were found for profile [$Profile] in config file [$CfgFile]. Typo in the profile name? Aborting."
   fi
fi

# shellcheck disable=2094
while read -r Line; do
   Configurable="$(echo "$Line" | cut -d '|' -f 2)"

   if [[ "$DoSecurityCheck" -eq 1 ]]; then
      if [[ "${Categories[$Configurable]:-}" =~ Security ]]; then
         dbg "Considering security-related configurable: $Configurable"
      else
         dbg "Ignoring configurable unrelated to security: $Configurable"
         continue
      fi
   fi

   expectedValue="$(echo "$Line" | cut -d '|' -f 3)"
   compareStyle="$(echo "$Line" | cut -d '|' -f 4)"
   optionality="$(echo "$Line" | cut -d '|' -f 5)"
   serverIDType="$(echo "$Line" | cut -d '|' -f 6)"
   setNotes="$(echo "$Line" | cut -d '|' -f 7)"

   # If the setNotes value is Standard, apply the usual documentation URL for
   # configurables.
   if [[ "$setNotes" == Standard ]]; then
      setNotes="https://help.perforce.com/helix-core/server-apps/cmdref/current/Content/CmdRef/configurables.alphabetical.html#$Configurable"
   fi

   # Do substitutions for expectedValue, e.g. replacing SDP Instance.
   expectedValue="${expectedValue/__SDP_INSTANCE__/$SDPInstance}"
   expectedValue="${expectedValue/__P4ROOT__/$P4ROOT}"
   expectedValue="${expectedValue/__P4SERVER__/$P4SERVER}"
   expectedValue="${expectedValue/__KEEPLOGS__/$KEEPLOGS}"
   expectedValue="${expectedValue/__LOGS__/$LOGS}"

   currentValue="${CurrentValue[$Configurable]:-}"

   # For values that are not set explicitly, ask p4d for the current value,
   # which will be the default value.  If that has no value, use the value
   # 'unset'.
   if [[ -z "$currentValue" ]]; then
      dbg "No value explicitly set for $Configurable; checking default."
      currentValue="$(p4 -ztag -F %Value% configure show "$Configurable")"
      [[ -n "$currentValue" ]] || currentValue="unset"
      CurrentValue[$Configurable]="$currentValue"
      CurrentValueByDefault[$Configurable]=1
   fi

   dbg "C=[$Configurable] EV=[$expectedValue] CV=[$currentValue] CS=[$compareStyle] O=[$optionality] ST=[$serverIDType] SN=[$setNotes]"
   ExpectedValue[$Configurable]="$expectedValue"
   CompareStyle[$Configurable]="$compareStyle"
   Optionality[$Configurable]="$optionality"
   ServerIDType[$Configurable]="$serverIDType"
   SetNotes[$Configurable]="$setNotes"
done < "$TmpFile"

rm -f "$TmpFile"

for Configurable in "${!ExpectedValue[@]}"; do
   ConfigCount+=1
   dbg "Configurable: [$Configurable]\\n  EValue=[${ExpectedValue[$Configurable]}]\\n  CValue=[${CurrentValue[$Configurable]:-}]\\n  CompareStyle=[${CompareStyle[$Configurable]}]\\n  Optionality=[${Optionality[$Configurable]}]\\n  SetNotes=[${SetNotes[$Configurable]}]\\n"

   # Determine if CompareStyle loaded from config file is known/handled. While
   # CamelCase is preferred in the config file, normalize to uppercase for comparison
   # to be forgiving case variations (case-insensitive).
   case "${CompareStyle[$Configurable]^^}" in
      (ATLEAST) compareStyle=AtLeast;;
      (NOMORETHAN) compareStyle=NoMoreThan;;
      (CONTAINS) compareStyle=Contains;;
      (EXACT) compareStyle=Exact;;
      (SET) compareStyle=Set;;
      (UNSET) compareStyle=Unset;;
      (*)
         errmsg "Unknown CompareStyle [${CompareStyle[$Configurable]}] for configurable [$Configurable]. Treating as Exact."
         compareStyle=Exact
      ;;
   esac

   # Match expected vs. actual using the compare style.
   ConfigMatches=0
   expectedValue="${ExpectedValue[$Configurable]}"
   currentValue="${CurrentValue[$Configurable]:-}"

   case "$compareStyle" in
      (AtLeast)
         # For a numeric compare, change 'unset' to '0'.
         [[ "$currentValue" == unset ]] && currentValue=0
         if [[ "$currentValue" =~ ^[0-9]+$ && "$expectedValue" =~ ^[0-9]+$ ]]; then
            [[ "$currentValue" -ge "$expectedValue" ]] && ConfigMatches=1
         else
            # If the numbers aren't simple integers, try a size compare with sizes like 20M,
            # 2G, etc.  This function will display an appropriate error message if the
            # values are not valid for comparison.
            at_least_size_compare "$currentValue" "$expectedValue" && ConfigMatches=1
         fi
      ;;
      (NoMoreThan)
         # For a numeric compare, change 'unset' to '0'.
         [[ "$currentValue" == unset ]] && currentValue=0
         if [[ "$currentValue" =~ ^[0-9]+$ && "$expectedValue" =~ ^[0-9]+$ ]]; then
            [[ "$currentValue" -le "$expectedValue" ]] && ConfigMatches=1
         else
            # If the numbers aren't simple integers, try a size compare with sizes like 20M,
            # 2G, etc.  This function will display an appropriate error message if the
            # values are not valid for comparison.
            # Use the 'at_least' comparison function as above, but swap the
            # arguments to get the desired NoMoreThan comparison.
            at_least_size_compare "$expectedValue" "$currentValue" && ConfigMatches=1
         fi
      ;;
      (Contains)
         [[ $currentValue =~ $expectedValue ]] && ConfigMatches=1
      ;;
      (Exact)
         [[ "$currentValue" == "$expectedValue" ]] && ConfigMatches=1

	 # Special case for 'security' counter. If we expect a '0', that matches 'unset'.
	 [[ "$Configurable" == "security" && "$expectedValue" == "0" && "$currentValue" == "unset" ]] && \
            ConfigMatches=1
      ;;
      (Set) # If any value is set at all, it is considered to match for 'Set' compare.
         [[ -n "$currentValue" && "$currentValue" != unset ]] && ConfigMatches=1
      ;;
      (Unset) # If no value is set at all, it is considered to match for 'Unset' compare.
         if [[ "$currentValue" == unset || -n "${CurrentValueByDefault[$Configurable]:-}" ]]; then
            ConfigMatches=1
         fi
      ;;
   esac

   if [[ "${Optionality[$Configurable]}" == "Required" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgreqpass "Value for configurable [$Configurable] is at least [$expectedValue], as is required.";;
            (NoMoreThan) cfgreqpass "Value for configurable [$Configurable] is no more than [$expectedValue], as is required.";;
            (Contains) cfgreqpass "Value for configurable [$Configurable] contains required text [$expectedValue].";;
            (Exact) cfgreqpass "Value for configurable [$Configurable] matches required value [$expectedValue] exactly.";;
            (Set) cfgreqpass "A value for configurable [$Configurable] is set, as required. Value is [$currentValue].";;
            (Unset) cfgreqpass "The configurable [$Configurable] is unset (i.e. not explicitly set), exactly as is required. Default value is [$currentValue].";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgreqfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is required."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="p4 configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (NoMoreThan)
               cfgreqfail "Value for configurable [$Configurable] is [$currentValue]; it is required to be no more than [$expectedValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="p4 -s configure set $Configurable=$expectedValue"
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (Contains)
               cfgreqfail "Value for configurable [$Configurable] does not contain required text [$expectedValue]."
               PlannedFixConfigurables[PlannedFixCount]="$Configurable"
               PlannedFixAdvice[PlannedFixCount]="Set the configurable '$Configurable' to a value that contains '$expectedValue'."
               PlannedFixAutomated[PlannedFixCount]=0
               PlannedFixCount+=1
            ;;
            (Exact)
               cfgreqfail "Value for configurable [$Configurable] does not exactly match required value [$expectedValue]; value is: [$currentValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="p4 configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (Set)
               cfgreqfail "A value for configurable [$Configurable] is not set; it is required to have a value."
               PlannedFixConfigurables[PlannedFixCount]="$Configurable"
               if [[ "$Configurable" == auth.id ]]; then
                  PlannedFixAdvice[PlannedFixCount]="p4 configure set $Configurable=\"p4_${P4MASTER_ID}\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               else
                  PlannedFixAdvice[PlannedFixCount]="The configurable '$Configurable' should be set so it has a value."
                  PlannedFixAutomated[PlannedFixCount]=0
                  PlannedFixCount+=1
               fi
            ;;
            (Unset)
               cfgreqfail "The configurable [$Configurable] is required to be unset (i.e. not explicitly set), but it is explicitly set to [$currentValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="UNSET"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="p4 -s configure unset $Configurable"
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
         esac
      fi
   elif [[ "${Optionality[$Configurable]}" == "Recommended" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgrecpass "Value for configurable [$Configurable] is [$currentValue], which is at least [$expectedValue], as is recommended.";;
            (NoMoreThan) cfgrecpass "Value for configurable [$Configurable] is [$currentValue], which is no more than [$expectedValue], as is recommended.";;
            (Contains) cfgrecpass "Value for configurable [$Configurable] contains recommended text [$expectedValue].";;
            (Exact) cfgrecpass "Value for configurable [$Configurable] matches recommended value [$expectedValue] exactly.";;
            (Set) cfgrecpass "A value for configurable [$Configurable] is set, as recommended.";;
            (Unset) cfgrecpass "The configurable [$Configurable] is unset (i.e. not explicitly set), exactly as is recommended. Default value is [$currentValue].";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgrecfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is recommended."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="p4 -s configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (NoMoreThan)
               cfgrecfail "Value for configurable [$Configurable] is [$currentValue]; it is recommended to be no more than [$expectedValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="p4 -s configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (Contains)
               cfgrecfail "Value for configurable [$Configurable] does not contain recommended text [$expectedValue]."
               PlannedFixConfigurables[PlannedFixCount]="$Configurable"
               PlannedFixAdvice[PlannedFixCount]="TO DO: Set a value for '$Configurable' that contains '$expectedValue'."
               PlannedFixAutomated[PlannedFixCount]=0
               PlannedFixCount+=1
            ;;
            (Exact)
               cfgrecfail "Value for configurable [$Configurable] does not exactly match recommended value [$expectedValue]; value is: [$currentValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="p4 -s configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (Set)
               cfgrecfail "A value for configurable [$Configurable] is not set; setting a value is recommended."
               PlannedFixConfigurables[PlannedFixCount]="$Configurable"
               PlannedFixAdvice[PlannedFixCount]="TO DO: Set a value for '$Configurable'."
               PlannedFixAutomated[PlannedFixCount]=0
               PlannedFixCount+=1
            ;;
            (Unset)
               cfgrecfail "The configurable [$Configurable] is recommended to be unset (i.e. not explicitly set), but it is explicitly set to [$currentValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="UNSET"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="p4 -s configure unset $Configurable"
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
         esac
      fi
   elif [[ "${Optionality[$Configurable]}" == "Optional" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgrecpass "Value for configurable [$Configurable] is [$currentValue], which is at least [$expectedValue], as is suggested.";;
            (NoMoreThan) cfgrecpass "Value for configurable [$Configurable] is [$currentValue], which is no more than [$expectedValue], as is suggested.";;
            (Contains) cfgrecpass "Value for configurable [$Configurable] contains suggested text [$expectedValue].";;
            (Exact) cfgrecpass "Value for configurable [$Configurable] matches suggested value [$expectedValue] exactly.";;
            (Set) cfgrecpass "A value for configurable [$Configurable] is set, as suggested.";;
            (Unset) cfgrecpass "The configurable [$Configurable] is unset (i.e. not explicitly set), exactly as is suggested. Default value is [$currentValue].";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgoptfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is suggested."
            ;;
            (NoMoreThan)
               cfgoptfail "Value for configurable [$Configurable] is [$currentValue]; which is suggested to be no more than [$expectedValue]."
            ;;
            (Contains)
               cfgoptfail "Value for configurable [$Configurable] does not contain suggested text [$expectedValue]."
            ;;
            (Exact)
               cfgoptfail "Value for configurable [$Configurable] does not exactly match suggested value [$expectedValue]; value is: [$currentValue]."
            ;;
            (Set)
               cfgoptfail "A value for configurable [$Configurable] is not set; setting a value is suggested."
            ;;
            (Unset)
               cfgoptfail "The configurable [$Configurable] is suggested to be unset (i.e. not explicitly set), but it is explicitly set to [$currentValue]."
            ;;
         esac
      fi
   else
      errmsg "Unknown optionality [${Optionality[$Configurable]}]; should be 'Required' or 'Recommended'."
   fi
done

msg "\\nSummary:
   Configs Checked:               $ConfigCount
   Configs GOOD/GREAT:            $ConfigPassCount
   Configs BAD/WARN (Total):      $ConfigFailCount
      Configs BAD (Required):       $ConfigFailRequiredCount
      Configs WARN (Recommended):   $ConfigFailRecommendedCount

   Configs INFO:                  $ConfigFailOptionalCount (not counted for grading)."
if [[ "$ErrorCount" -ne 0 ]]; then
   msg "\\n   ERRORS:                        $ErrorCount (The existence of any errors at all indicates an inability to fully check configurable values)."
fi

msgn "\\nResult: "
ExitCode=$((ErrorCount+ConfigFailRequiredCount))

if [[ "$ConfigCount" -ne 0 ]]; then
   if [[ "$ConfigFailCount" -eq 0 ]]; then
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "PASS with Grade A: All $ConfigCount checked configurables pass, and there are no errors."
      else
         msg "FAIL: There were $ErrorCount errors attempting to check configurables.  All $ConfigCount checked configurables pass."
      fi
   elif [[ "$ConfigFailRequiredCount" -eq 0 ]]; then
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "PASS with Grade B: Of $ConfigCount configurables checked, all required values pass, but $ConfigFailRecommendedCount recommended settings were not optimal."
      else
         msg "FAIL: There were $ErrorCount errors attempting to check configurables.  Of $ConfigCount configurables checked, all required values pass, but $ConfigFailRecommendedCount recommended settings were not optimal."
      fi
   else
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "FAIL: Of $ConfigCount configurables, $ConfigFailRequiredCount required settings were not optimal, and $ConfigFailRecommendedCount recommended settings were not optimal."
      else
         msg "FAIL: There were $ErrorCount errors attempting to check configurables. Of $ConfigCount configurables, $ConfigFailRequiredCount required settings were not optimal, and $ConfigFailRecommendedCount recommended settings were not optimal."
      fi
   fi
else
   errmsg "No configs were checked. Something went wrong."
fi

#------------------------------------------------------------------------------
# FOR FUTURE ENHANCEMENT
# These '-n' checks are meaningless; they are here to suppress specific
# ShellCheck errors about unused variables.  These variables are intended
# for future use in a version that supports modifying configurables.
[[ -n "${ServerIDType[server]:-}" ]] && true
[[ "$NoOp" -eq 0 ]] && true

if [[ "$ApplyEasyFixes" -eq 1 ]]; then
   msg "${H2}\\nApplying fixes that can be applied non-disruptively."

   if [[ "$EasyFixCount" -gt 0 ]]; then
      for ((i=0; i < ${#EasyFixConfigurables[@]}; i++)); do
         if [[ "${EasyFixValues[i]}" != UNSET ]]; then
            run "p4 -s configure set ${EasyFixConfigurables[i]}=${EasyFixValues[i]}" ||\
               errmsg "Error setting '${EasyFixConfigurables[i]}' to '${EasyFixValues[i]}'."
         else
            run "p4 -s configure unset ${EasyFixConfigurables[i]}" ||\
               errmsg "Error unsetting '${EasyFixConfigurables[i]}'."
         fi
      done
   else
      msg "There are no fixes that can be applied non-disruptively."
   fi

   if [[ "$ApplyAllFixes" -eq 1 ]]; then
      msg "${H2}\\nApplying fixes that should be planned to avoid potential disruption."

      if [[ "$EnableSSLAdvised" -eq 1 ]]; then
         PlannedFixConfigurables[PlannedFixCount]="P4PORT"
         PlannedFixAdvice[PlannedFixCount]="TO DO: The SSL feature should be enabled for all P4PORT values on servers and clients in the topology."
         PlannedFixAutomated[PlannedFixCount]=0
         PlannedFixCount+=1
      fi

      if [[ "$PlannedFixCount" -gt 0 ]]; then
         for ((i=0; i < ${#PlannedFixConfigurables[@]}; i++)); do
            if [[ "${PlannedFixAutomated[i]}" == 1 ]]; then
               run "${PlannedFixAdvice[i]}" ||\
                  errmsg "Error implementing best practice for ${PlannedFixConfigurables[i]}."
            else
               msg "${PlannedFixAdvice[i]}"
            fi
         done
      else
         msg "There are no fixes to apply that would be non-disruptive."
      fi
   else
      msg "${H2}\\nDisplaying fixes that should be planned to avoid potential disruption."

      if [[ "$PlannedFixCount" -gt 0 ]]; then
         for ((i=0; i < ${#PlannedFixConfigurables[@]}; i++)); do
            if [[ "${PlannedFixAutomated[i]}" == 1 ]]; then
               # Prepend 'TO DO:' to commands that could be automated with '-FIX'.
               msg "TO DO: ${PlannedFixAdvice[i]}"
            else
               # The non-automated messages are already prefixed with 'TO DO:'.
               msg "${PlannedFixAdvice[i]}"
            fi
         done
      else
         msg "There are no fixes to apply that would be non-disruptive."
      fi
   fi
fi

#------------------------------------------------------------------------------
# The exit code is a happy zero if all required checks passed AND there are no errors doing checks.
# See the terminate() function where this script really exits.
exit "$ExitCode"