ccheck.sh #5

#!/bin/bash
set -u

#==============================================================================
# Declarations and Environment
declare ThisScript=${0##*/}
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare Version=1.2.1
declare Args="$*"
declare CmdLine="$0 $Args"

declare -i ExitCode=0
declare -i ErrorCount=0
declare -i NoOp=1
declare -i Debug=${DEBUG:-0}
declare -i ConfigMatches=0
declare -i SilentMode=0
declare -i i=0 
declare Profile=prod
declare CfgFile=/p4/common/config/configurables.cfg
declare Log=
declare OldLogTimestamp=
declare LogTimestamp=
declare LogLink=
declare TmpFile=
declare TmpFile2=
declare SDPRoot=${SDP_ROOT:-/p4}
declare SDPInstance=
declare SDPInstanceVars=
declare TimeoutDelay=5s
declare SuperUser=
declare Cmd=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare -i Verbosity=0

# Associative arrays indexed by configurable, capturing data loaded from
# /p4/common/config/configurables.cfg.
declare Configurable=
declare ConfigurableType=
declare -A CurrentValue
declare -A ExpectedValue
declare -A CompareStyle
declare -A Optionality
declare -A ServerIDType
declare -A SetNotes
declare ThisServerIDType=

# Associative arrays indexed by configurable, for new values, and when to set
# them.
declare -A SuggestedValue
declare -A SetWhen

# Summary Counters
declare -i ConfigCount=0
declare -i ConfigPassCount=0
declare -i ConfigFailCount=0
declare -i ConfigFailRequiredCount=0
declare -i ConfigFailRecommendedCount=0

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function msgn () { echo -n -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || echo -e "DEBUG: $*" >&2; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

# The cfgpass() is called when a config value matches expectations.
function cfgpass () { [[ "$Verbosity" -ne 0 ]] && msg "GOOD: $*"; ConfigPassCount+=1; }

# The cfgreqfail() function is called when required configuration settings are
# not as expected, and cfgrecfail() is called when recommended settings are not
# as expected. Both increment ConfigFailCount and contribute to an unhappy exit
# code.
function cfgreqfail () { msg "BAD: $*"; ConfigFailCount+=1; ConfigFailRequiredCount+=1; }
function cfgrecfail () { msg "BAD: $*"; ConfigFailCount+=1; ConfigFailRecommendedCount+=1; }

#------------------------------------------------------------------------------
# Function: at_least_size_compare ("$currentValue" "$expectedValue")
# Return true if size 1 is at least as big as size 2. Do the needful conversion
# of units (just numeric bytes, K, M, G, T, P, E, etc.) to compare correctly.
function at_least_size_compare () {
   local s1=${1:-}
   local s2=${2:-}
   local num1=
   local unit1=
   local num2=
   local unit2=
   local multiplier1=1
   local multiplier2=1
   local val1=
   local val2=

   # If inputs are purely numeric, append a 'B' (bytes) suffix.
   [[ "$s1" =~ ^[0-9]+$ ]] && s1="${s1}B"
   [[ "$s2" =~ ^[0-9]+$ ]] && s2="${s2}B"

   if [[ ! "$s1" =~ ^[0-9]*([BKMGTPE]){1}$ || ! "$s2" =~ ^[0-9]*([BKMGTPE]){1}$ ]]; then
      errmsg "Cannot compare size value [$s1] with value [$s2]; the values are invalid."
      return 1
   fi

   # Extract the numeric part and unit from each value.
   [[ $s1 =~ ^([0-9]+)([BKMGTPE]) ]] && num1=${BASH_REMATCH[1]} && unit1=${BASH_REMATCH[2]}
   [[ $s2 =~ ^([0-9]+)([BKMGTPE]) ]] && num2=${BASH_REMATCH[1]} && unit2=${BASH_REMATCH[2]}

   # Convert units to base 10 multipliers.
   case $unit1 in
      B) multiplier1=1;;
      K) multiplier1=1000;;
      M) multiplier1=1000000;;
      G) multiplier1=1000000000;;
      T) multiplier1=1000000000000;;
      P) multiplier1=1000000000000000;;
      E) multiplier1=1000000000000000000;;
   esac

   case $unit2 in
      B) multiplier2=1;;
      K) multiplier2=1000;;
      M) multiplier2=1000000;;
      G) multiplier2=1000000000;;
      T) multiplier2=1000000000000;;
      P) multiplier2=1000000000000000;;
      E) multiplier2=1000000000000000000;;
   esac

   val1=$((num1 * multiplier1))
   val2=$((num2 * multiplier2))

   [[ "$val1" -ge "$val2" ]] && return 0
   return 1
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   local style=${1:--h}
   local usageErrorMessage=${2:-}

   [[ -n "$usageErrorMessage" ]] && msg "\\n\\nUsage Error:\\n\\n$usageErrorMessage\\n\\n"

   msg "USAGE for $ThisScript v$Version:

$ThisScript [<SDPInstance>] [-p <Profile>] [-c <CfgFile>] [-y] [-v] [-d|-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script compares configurables set on the current server with best
	practices defined a data file.

OPTIONS:
 -p <Profile>
 	Specify a profile defined in the config file, such as 'demp' or 'hcc'.  A profile
	defines a set of expected configurable values that can differ from the expected
	values in other profiles.  For example, for a demo environment, the filesys.P4ROOT.min
	might have an expected value of 128M, while the expected value in a prod (production)
	profile might be 5G, and the same value might be 30G for 'prodent', the profile for
	production at large enterprise scale.

	The 'always' profile defines settings that always apply whether '-p' is specified
	or not. The profile specified with '-p' applies in addition to the 'always'
	configuration, adding to and possibly overriding settings from the 'always'
	configuration.

	The defaut profile is 'prod', the production profile.

	Specify the special value '-p none' to use only the settings defined in the
	'always' profile.

 -c <CfgFile>
	Specify an alternate config file that defines best practice configurables. This
	is intended for testing.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to
	\$LOGS/${ThisScript%.sh}.log

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file. Using redirection operators like
	'> log' or '2>&1' are unnecessary, nor is using 'tee'.

 -y	Live operation mode.  By default, any commands that affect data, such as
	setting configurables, are displayed, but not executed.  With the '-y' option,
	commands may be executed.

	This option is included for future needs. This current version of
	$ThisScript does not execute any commands that affect data.

 -d	Display debug messages.

 -D     Set extreme debugging verbosity using bash 'set -x' mode. Implies -d.

 -si	Silient Mode.  No output is displayed to the terminal (except for usage errors
	on startup). Output is captured in the log.  The '-si' cannot be used with
	'-L off'.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

FILES:
	The standard configurables config file is:

	$CfgFile

EXAMPLES:
	Example 1: Check configurables with the default profile, and no logging:

	$ThisScript -L off

	Example 2: Check configurables with the 'prod' (Production) profile:

	$ThisScript -p prod

	Example 3: Check configurables with the 'demo' profile, doing a verbose comparison:

	$ThisScript -p demo -v

FUTURE ENHANCEMENTS:
	Presently, this $ThisScript v$Version only reports configurables. It does not
	support changing configurables.

	As the script is currently only capable of reporting, the '-y' option has no
	effect.

	Some possible future enhancements are:
	* Extend reporting to suggesting configuration changes.

	* Provide an option to make changes to configurables that are safe to change
	  immediately, and provide guidance on those configurables that are best set with
	  guidance and plannning.

	* Provide a way to specify custom exemptions for certain configurables.

	* Added multi-version support for backward compatibility. This version assumes
	  P4D 2023.1+ (though will be useful for older versions).
"
   fi

   exit 2
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate ()
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   ExitCode=$((ErrorCount+ConfigFailRequiredCount))
   msg "ExitCode: $ExitCode"

   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log\\n${H1}"

   # With the trap removed, exit.
   exit "$ExitCode"
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-i) SDPInstance="$2"; shiftArgs=1;;
      (-p) Profile="$2"; shiftArgs=1;;
      (-c) CfgFile="$2"; shiftArgs=1;;
      (-L) Log="$2"; shiftArgs=1;;
      (-y) NoOp=0;;
      (-si) SilentMode=1;;
      (-v) Verbosity=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Use bash 'set -x' extreme debug mode.
      (-*) usage -h "Unknown option ($1).";;
      (*)
         if [[ -z "$SDPInstance" ]]; then
            SDPInstance="$1"
	 else
            usage -h "Extra parameter [$1] is unknown; SDP Instance is already set to [$SDPInstance]."
	 fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usage -h "The '-si' option cannot be used with '-Log off'."

[[ -z "$SDPInstance" ]] && SDPInstance="${SDP_INSTANCE:-}"
[[ -z "$SDPInstance" ]] && usage -h "The SDP instance parameter is required unless SDP_INSTANCE is set. To set SDP_INSTANCE, do:\\n\\tsource $SDPRoot/common/bin/p4_vars INSTANCE\\n\\nreplacing INSTANCE with your SDP instance name."

SDPInstanceVars="$SDPRoot/common/config/p4_${SDPInstance}.vars"
[[ -r "$SDPInstanceVars" ]] || usage -h "The SDP instance specified [$SDPInstance] is missing Instance Vars file: $SDPInstanceVars"

# shellcheck disable=SC1090
source "$SDPRoot/common/bin/p4_vars" "$SDPInstance" ||\
   bail "Could not do: source \"$SDPRoot/common/bin/p4_vars\" \"$SDPInstance\""

# shellcheck disable=SC1090
source "$SDPRoot/common/bin/log_functions.sh" ||\
   bail "Could not do: source \"$SDPRoot/common/bin/log_functions.sh\""

[[ -e "$CfgFile" ]] || usage -h "The config file [$CfgFile] is missing."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

# If the user specifies a log file file with '-L', write to the specified file.
# If no log was specified, create a default log file using a timestmap in the
# LOGS dir, and immediately update the symlink for the default log to point to
# it.
if [[ "$Log" != off ]]; then
   # If $Log is not yet defined, set it to a reasonable default.
   if [[ -z "$Log" ]]; then
      LogTimestamp=$(date +'%Y-%m-%d-%H%M%S')
      Log="$LOGS/${ThisScript%.sh}.${LogTimestamp}.log"
      # Make sure we have a unique log file. Prefer a human-readoable timetamp
      # using hours/minutes/seconds. Append milliseconds if needed to ensure
      # achieve a unique filename.
      while [[ -e "$Log" ]]; do
         LogTimestamp=$(date +'%Y-%m-%d-%H%M%S.%3N')
         Log="$LOGS/${ThisScript%.sh}.${LogTimestamp}.$i.log"
         i+=1
      done
   fi
   # The LogLink symlink has no timestamp. It points to the most recent log file.
   LogLink="$LOGS/ccheck.log"

   if [[ -e "$LogLink" ]]; then
      if [[ -L "$LogLink" ]]; then
         rm -f "$LogLink"
      else
         # If the name that should be a symlink is not a symlink, move it aside before
         # creating the symlink.
         OldLogTimestamp=$(get_old_log_timestamp "$LogLink")
         mv -f "$LogLink" "${LogLink%.log}.${OldLogTimestamp}.log" ||\
            bail "Could not move old log file aside; tried: mv -f \"$LogLink\" \"${LogLink%.log}.${OldLogTimestamp}.log\""
      fi
   fi

   touch "$Log" || bail "Couldn't touch new log file [$Log]."

   # Use a subshell so the 'cd' doesn't persist.
   ( cd "$LOGS"; ln -s "${Log##*/}" "${LogLink##*/}"; ) ||\
       bail "Couldn't initialize log symlink; tried: ln -s \"$Log\" \"$LogLink\""

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Starting $ThisScript v$Version as $ThisUser@$ThisHost on $(date) with \\n$CmdLine"

msg "Comparing with Profile: [$Profile] loaded from: $CfgFile\\n${H2}"

# For environments that define P4SUPER, use that for P4USER rather than P4USER.
SuperUser=$(p4 set -q P4SUPER)
SuperUser=${SuperUser##*=}

if [[ -n "$SuperUser" ]]; then
   export P4USER="$SuperUser"
fi

TmpFile=$(mktemp)

if [[ "$P4PORT" =~ ^ssl[46]*: ]]; then
   Cmd="timeout $TimeoutDelay p4 trust -y"
   dbg "Running: $Cmd [P4PORT=$P4PORT]"
   $Cmd > "$TmpFile" 2>&1 ||\
      bail "Could not do: $Cmd\\n$(cat "$TmpFile")\\n"
fi

Cmd="timeout $TimeoutDelay p4 info -s"
dbg "Running: $Cmd"
if $Cmd > "$TmpFile" 2>&1; then
   dbg "Verified: Helix Core is accessible."
   if [[ "$(p4 protects -m)" == "super" ]]; then
      dbg "Verified: super user access available."
   else
      bail "Could not verify super access [P4USER=$P4USER P4PORT=$P4PORT P4TICKETS=$P4TICKETS]."
   fi
else
   bail "Could not access Helix Core server [P4PORT=$P4PORT] with: $Cmd\\n$(cat "$TmpFile")\\n"
fi

# Extract current configurables values using 'p4d -cshow'.
Cmd="p4d -r $P4ROOT -cshow"
if $Cmd > "$TmpFile" 2>&1; then
   dbg "Captured configurables:\\n$(cat "$TmpFile")\\n"
else
   bail "Could not capture configurables with: $Cmd\\n$(cat "$TmpFile")\\n"
fi

dbg "Comparing P4MASTER_ID=$P4MASTER_ID with SERVERID=$SERVERID."

# Determine whether the current ServerID is that of a commmit, replica, standby, or edge.
### Use is_*() functions from backup_functions.sh, but avoid loading backup_functions.sh
### directly for now, due to compatibility issues with new bash template this script is
### based on. EDITME - Find a better way later.
# shellcheck disable=SC1091
if [[ "$P4MASTER_ID" == "$SERVERID" ]]; then
   ThisServerIDType=commit
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_replica "$SERVERID")" == YES ]]; then
   ThisServerIDType=replica
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_standby "$SERVERID")" == YES ]]; then
   ThisServerIDType=standby
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_edge "$SERVERID")" == YES ]]; then
   ThisServerIDType=edge
else
   errmsg "Could not determine server type for ServerID [$SERVERID]."
fi

dbg "This ServerID [$SERVERID] is of type [$ThisServerIDType]."

dbg "Loading current configurable values."
p4 -ztag -F "%Type%|%ServerName%|%Name%|%Value%" configure show allservers > "$TmpFile"
while read -r Line; do
   ConfigurableType="$(echo "$Line" | cut -d '|' -f1)"
   [[ "$ConfigurableType" == "configure" ]] || continue
   ServerName="$(echo "$Line"|cut -d '|' -f2)"

   # We are only concerned with configurables affecting the current ServerID.
   # That includes configurables set explicitly for the current ServerID,
   # as well as values set as global defaults with the 'any' config.
   [[ "$ServerName" == any || "$ServerName" == "$SERVERID" ]] || continue

   Configurable="$(echo "$Line"|cut -d '|' -f3)"

   # If a configurable is set both as a global default and explicitly
   # for the current ServerID, set the value to be that of the explicitly
   # set value. That overrides the global default effectively in p4d,
   # and we mirror that logic here.  We can't control whether the 'any'
   # or the explicit values appears first in the output of 'p4 configure
   # show allservers' (it depends where the ServerID name appears in
   # an listing by alphabetical order). So, ignore the 'any' value if
   # there already is a value defined for the configurable.

   [[ "$ServerName" == "any" && -n "${CurrentValue[$Configurable]:-}" ]] && continue

   currentValue="$(echo "$Line"|cut -d '|' -f4)"
   [[ -n "$currentValue" ]] || currentValue="unset"

   dbg "CurrentValue[$Configurable]=$currentValue"
   CurrentValue[$Configurable]="$currentValue"
done < "$TmpFile"

# Extract settings for the selected profile and the default profile.
# Normalize the profile to lowercase for searching in the config file.
grep '^always|' "$CfgFile" > "$TmpFile"

if [[ "$Profile" != none ]]; then
   TmpFile2=$(mktemp)
   grep "^${Profile,,}|" "$CfgFile" >> "$TmpFile2"
   if [[ -s "$TmpFile2" ]]; then
      cat "$TmpFile2" >> "$TmpFile" ||\
         bail "Could not append settings for profile $Profile to settings for default profile. Aborting."
      rm -f "$TmpFile2"
   else
      bail "No settings were found for profile [$Profile] in config file [$CfgFile]. Typo in the profile name? Aborting."
   fi
fi

# shellcheck disable=2094
while read -r Line; do
   Configurable="$(echo "$Line" | cut -d '|' -f 2)"
   expectedValue="$(echo "$Line" | cut -d '|' -f 3)"
   compareStyle="$(echo "$Line" | cut -d '|' -f 4)"
   optionality="$(echo "$Line" | cut -d '|' -f 5)"
   serverIDType="$(echo "$Line" | cut -d '|' -f 6)"
   setNotes="$(echo "$Line" | cut -d '|' -f 7)"

   # If the setNotes value is Standard, apply the usual documenton URL for
   # configurables.
   if [[ "$setNotes" == Standard ]]; then
      setNotes="https://help.perforce.com/helix-core/server-apps/cmdref/current/Content/CmdRef/configurables.alphabetical.html#$Configurable"
   fi

   # Do substitutions for expectedValue, e.g. replacing SDP Instance.
   expectedValue="${expectedValue/__SDP_INSTANCE__/$SDPInstance}"
   expectedValue="${expectedValue/__P4ROOT__/$P4ROOT}"
   expectedValue="${expectedValue/__P4SERVER__/$P4SERVER}"
   expectedValue="${expectedValue/__KEEPLOGS__/$KEEPLOGS}"
   expectedValue="${expectedValue/__LOGS__/$LOGS}"

   currentValue="${CurrentValue[$Configurable]:-}"

   # For values that are not set explicitly, ask p4d for the current value,
   # which will be the default value.  If that has no value, use the value
   # 'unset'.
   if [[ -z "$currentValue" ]]; then
      dbg "No value explicitly set for $Configurable; checking default."
      currentValue="$(p4 -ztag -F %Value% configure show "$Configurable")"
      [[ -n "$currentValue" ]] || currentValue="unset"
      CurrentValue[$Configurable]="$currentValue"
   fi

   dbg "C=[$Configurable] EV=[$expectedValue] CV=[$currentValue] CS=[$compareStyle] O=[$optionality] ST=[$serverIDType] SN=[$setNotes]"
   ExpectedValue[$Configurable]="$expectedValue"
   CompareStyle[$Configurable]="$compareStyle"
   Optionality[$Configurable]="$optionality"
   ServerIDType[$Configurable]="$serverIDType"
   SetNotes[$Configurable]="$setNotes"
done < "$TmpFile"

rm -f "$TmpFile"

for Configurable in "${!ExpectedValue[@]}"; do
   ConfigCount+=1
   dbg "Configurable: [$Configurable]\\n  EValue=[${ExpectedValue[$Configurable]}]\\n  CValue=[${CurrentValue[$Configurable]:-}]\\n  CompareStyle=[${CompareStyle[$Configurable]}]\\n  Optionality=[${Optionality[$Configurable]}]\\n  SetNotes=[${SetNotes[$Configurable]}]\\n"

   # Determine if CompareStyle loaded from config file is known/handled. While
   # CamelCase is preferred in the config file, normalize to uppercase for comparison
   # to be forgiving case variations (case-insensitive).
   case "${CompareStyle[$Configurable]^^}" in
      (ATLEAST) compareStyle=AtLeast;;
      (NOMORETHAN) compareStyle=NoMoreThan;;
      (CONTAINS) compareStyle=Contains;;
      (EXACT) compareStyle=Exact;;
      (SET) compareStyle=Set;;
      (UNSET) compareStyle=Unset;;
      (*)
         errmsg "Unknown CompareStyle [${CompareStyle[$Configurable]}] for configurable [$Configurable]. Treating as Exact."
         compareStyle=Exact
      ;;
   esac

   # Match expected vs. actual using the compare style.
   ConfigMatches=0
   expectedValue="${ExpectedValue[$Configurable]}"
   currentValue="${CurrentValue[$Configurable]:-}"

   case "$compareStyle" in
      (AtLeast)
         # For a numeric compare, change 'unset' to '0'.
         [[ "$currentValue" == unset ]] && currentValue=0
         if [[ "$currentValue" =~ ^[0-9]+$ && "$expectedValue" =~ ^[0-9]+$ ]]; then
            [[ "$currentValue" -ge "$expectedValue" ]] && ConfigMatches=1
         else
            # If the numbers aren't simple integers, try a size compare with sizes like 20M,
            # 2G, etc.  This function will display an appropriate error message if the
            # values are not valid for comparsion.
            at_least_size_compare "$currentValue" "$expectedValue" && ConfigMatches=1
         fi
      ;;
      (NoMoreThan)
         # For a numeric compare, change 'unset' to '0'.
         [[ "$currentValue" == unset ]] && currentValue=0
         if [[ "$currentValue" =~ ^[0-9]+$ && "$expectedValue" =~ ^[0-9]+$ ]]; then
            [[ "$currentValue" -le "$expectedValue" ]] && ConfigMatches=1
         else
            # If the numbers aren't simple integers, try a size compare with sizes like 20M,
            # 2G, etc.  This function will display an appropriate error message if the
            # values are not valid for comparsion.
            # Use the 'at_least' comparison function as above, but swap the
            # arguments to get the desired NoMoreThan comparison.
            at_least_size_compare "$expectedValue" "$currentValue" && ConfigMatches=1
         fi
      ;;
      (Contains)
         [[ $currentValue =~ $expectedValue ]] && ConfigMatches=1
      ;;
      (Exact)
         [[ "$currentValue" == "$expectedValue" ]] && ConfigMatches=1

	 # Special case for 'security' counter. If we expect a '0', that matches 'unset'.
	 [[ "$Configurable" == "security" && "$expectedValue" == "0" && "$currentValue" == "unset" ]] && \
            ConfigMatches=1
      ;;
      (Set) # If any value is set at all, it is considered to match for 'Set' compare.
         [[ -n "$currentValue" && "$currentValue" != unset ]] && ConfigMatches=1
      ;;
      (Unset) # If no value is set at all, it is considered to match for 'Unset' compare.
         [[ "$currentValue" == unset ]] && ConfigMatches=1
      ;;
   esac

   if [[ "${Optionality[$Configurable]}" == "Required" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgpass "Value for configurable [$Configurable] is at least [$expectedValue], as is required.";;
            (NoMoreThan) cfgpass "Value for configurable [$Configurable] is no more than [$expectedValue], as is required.";;
            (Contains) cfgpass "Value for configurable [$Configurable] contains required text [$expectedValue].";;
            (Exact) cfgpass "Value for configurable [$Configurable] matches required value [$expectedValue] exactly.";;
            (Set) cfgpass "A value for configurable [$Configurable] is set, as required. Value is [$currentValue].";;
            (Unset) cfgpass "The configurable [$Configurable] is unset, as required.";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgreqfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is required."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (NoMoreThan)
               cfgreqfail "Value for configurable [$Configurable] is [$currentValue]; it is required to be no more than [$expectedValue]."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (Contains)
               cfgreqfail "Value for configurable [$Configurable] does not contain required text [$expectedValue]."
               SuggestedValue[$Configurable]="EDIT_ME_Choose_Wisely_SomethingContaining_$expectedValue"
               SetWhen[$Configurable]="Later"
            ;;
            (Exact)
               cfgreqfail "Value for configurable [$Configurable] does not exactly match required value [$expectedValue]; value is: [$currentValue]."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (Set)
               cfgreqfail "A value for configurable [$Configurable] is set, as required."
               SuggestedValue[$Configurable]="EDIT_ME_ChooseWisely"
               SetWhen[$Configurable]="Later"
            ;;
            (Unset)
               cfgreqfail "The configurable [$Configurable] is required to be unset, but is set to [$currentValue]."
               SuggestedValue[$Configurable]="Unset"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
         esac
      fi
   elif [[ "${Optionality[$Configurable]}" == "Recommended" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgpass "Value for configurable [$Configurable] is [$currentValue], which is at least [$expectedValue], as is recommended.";;
            (NoMoreThan) cfgpass "Value for configurable [$Configurable] is [$currentValue], which is no more than [$expectedValue], as is recommended.";;
            (Contains) cfgpass "Value for configurable [$Configurable] contains recommended text [$expectedValue].";;
            (Exact) cfgpass "Value for configurable [$Configurable] matches recommended value [$expectedValue] exactly.";;
            (Set) cfgpass "A value for configurable [$Configurable] is set, as recommended.";;
            (Unset) cfgpass "The configurable [$Configurable] is unset, as recommended.";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgrecfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is recommended."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (NoMoreThan)
               cfgrecfail "Value for configurable [$Configurable] is [$currentValue]; is is recommended to be no more than [$expectedValue]."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (Contains)
               cfgrecfail "Value for configurable [$Configurable] does not contain recommended text [$expectedValue]."
               SuggestedValue[$Configurable]="EDIT_ME_Choose_Wisely_SomethingContaining_$expectedValue"
               SetWhen[$Configurable]="Later"
            ;;
            (Exact)
               cfgrecfail "Value for configurable [$Configurable] does not exactly match recommended value [$expectedValue]; value is: [$currentValue]."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (Set)
               cfgrecfail "A value for configurable [$Configurable] is not set, as recommended."
               SuggestedValue[$Configurable]="EDIT_ME_ChooseWisely"
               SetWhen[$Configurable]="Later"
            ;;
            (Unset)
               cfgrecfail "The configurable [$Configurable] is recommended to be unset, but is set to [$currentValue]."
               SuggestedValue[$Configurable]="Unset"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
         esac
      fi
   else
      errmsg "Unknown optionality [${Optionality[$Configurable]}]; should be 'Required' or 'Recommended'."
   fi
done

msg "\\nSummary:
   Configs Checked:                $ConfigCount
   Configs GOOD:                   $ConfigPassCount
   Configs BAD (Total):            $ConfigFailCount
      Configs BAD (Required):      $ConfigFailRequiredCount
      Configs BAD (Recommended):   $ConfigFailRecommendedCount"
if [[ "$ErrorCount" -ne 0 ]]; then
   msg "   ERRORS:                         $ErrorCount"
fi

msgn "\\nResult: "
ExitCode=$((ErrorCount+ConfigFailRequiredCount))

if [[ "$ConfigCount" -ne 0 ]]; then
   if [[ "$ConfigFailCount" -eq 0 ]]; then
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "PASS with Grade A: All $ConfigCount checked configurables pass, and there are no errors."
      else
         msg "FAIL: There were $ErrorCount errors attempting to check configurables.  All $ConfigCount checked configurables pass."
      fi
   elif [[ "$ConfigFailRequiredCount" -eq 0 ]]; then
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "PASS with Grade B: Of $ConfigCount configurables checked, all required values pass, but $ConfigFailRecommendedCount recommended settings were not optimal."
      else
         msg "FAIL: There were $ErrorCount errors attempting to check configurables.  Of $ConfigCount configurables checked, all required values pass, but $ConfigFailRecommendedCount recommended settings were not optimal."
      fi
   else
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "FAIL: Of $ConfigCount configurables, $ConfigFailRequiredCount required settings were not optimal, and $ConfigFailRecommendedCount recommended settings were not optimal."
      else
         msg "FAIL: There were $ErrorCount errors attempting to check configurables. Of $ConfigCount configurables, $ConfigFailRequiredCount required settings were not optimal, and $ConfigFailRecommendedCount recommended settings were not optimal."
      fi
   fi
else
   errmsg "No configs were checked. Something went wrong."
fi

#------------------------------------------------------------------------------
# FOR FUTURE ENHANCEMENT
# These '-n' checks are meaningless; they are here to suppress specific
# ShellCheck errors about unused variables.  These variables are intended
# for future use in a version that supports modifying configurables.
[[ -n "${SetWhen[server]:-}" ]] && true
[[ -n "${SuggestedValue[server]:-}" ]] && true
[[ -n "${ServerIDType[server]:-}" ]] && true
[[ "$NoOp" -eq 0 ]] && true
#------------------------------------------------------------------------------

# The exit code is a happy zero if all required checks passed AND there are no errors doing checks.
# See the terminate() function where this script really exits.
exit "$ExitCode"