ccheck.sh #5

#!/bin/bash
set -u

#==============================================================================
# Declarations and Environment
declare ThisScript=${0##*/}
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare Version=1.0.5
declare Args="$*"
declare CmdLine="$0 $Args"

declare -i ExitCode=0
declare -i ErrorCount=0
declare -i NoOp=1
declare -i Debug=${DEBUG:-0}
declare -i ConfigMatches=0
declare -i SilentMode=0
declare -i i=0 
declare Profile=default
declare CfgFile=/p4/common/config/configurables.cfg
declare Log=
declare OldLogTimestamp=
declare LogTimestamp=
declare LogLink=
declare TmpFile=
declare TmpFile2=
declare SDPRoot=${SDP_ROOT:-/p4}
declare SDPInstance=
declare SDPInstanceVars=
declare TimeoutDelay=5s
declare SuperUser=
declare Cmd=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare -i Verbosity=0

# Associative arrays indexed by configurable, capturing data loaded from
# /p4/common/config/configurables.cfg.
declare Configurable=
declare ConfigurableType=
declare CurrentValue=
declare -A CurrentValue
declare -A ExpectedValue
declare -A CompareStyle
declare -A Optionality
declare -A ServerIDType
declare -A SetNotes
declare ThisServerIDType=

# Associative arrays indexed by configurable, for new values, and when to set
# them.
declare -A SuggestedValue
declare -A SetWhen

# Summary Counters
declare -i ConfigCount=0
declare -i ConfigPassCount=0
declare -i ConfigFailCount=0
declare -i ConfigFailRequiredCount=0
declare -i ConfigFailRecommendedCount=0

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function msgn () { echo -n -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || echo -e "DEBUG: $*" >&2; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

# The cfgpass() is called when a config value matches expectations.
function cfgpass () { [[ "$Verbosity" -ne 0 ]] && msg "GOOD: $*"; ConfigPassCount+=1; }

# The cfgreqfail() function is called when required configuration settings are
# not as expected, and cfgrecfail() is called when recommended settings are not
# as expected. Both increment ConfigFailCount and contribute to an unhappy exit
# code.
function cfgreqfail () { msg "BAD: $*"; ConfigFailCount+=1; ConfigFailRequiredCount+=1; }
function cfgrecfail () { msg "BAD: $*"; ConfigFailCount+=1; ConfigFailRecommendedCount+=1; }

#------------------------------------------------------------------------------
# Return true if size 1 is at least as big as size 2.
# Usage: at_least_size_compare ("$currentValue" "$expectedValue")
function at_least_size_compare () {
   local s1=${1:-}
   local s2=${2:-}
   local num1=
   local unit1=
   local num2=
   local unit2=
   local multiplier1=1
   local multiplier2=1
   local val1=
   local val2=

   # If inputs are purely numeric, append a 'B' (bytes) suffix.
   [[ "$s1" =~ ^[0-9]+$ ]] && s1="${s1}B"
   [[ "$s2" =~ ^[0-9]+$ ]] && s2="${s2}B"

   if [[ ! "$s1" =~ ^[0-9]*([BKMGTPE]){1}$ || ! "$s2" =~ ^[0-9]*([BKMGTPE]){1}$ ]]; then
      errmsg "Cannot compare size value [$s1] with value [$s2]; the values are invalid."
      return 1
   fi

   # Extract the numeric part and unit from each value.
   [[ $s1 =~ ^([0-9]+)([BKMGTPE]) ]] && num1=${BASH_REMATCH[1]} && unit1=${BASH_REMATCH[2]}
   [[ $s2 =~ ^([0-9]+)([BKMGTPE]) ]] && num2=${BASH_REMATCH[1]} && unit2=${BASH_REMATCH[2]}

   # Convert units to base 10 multipliers.
   case $unit1 in
      B) multiplier1=1;;
      K) multiplier1=1000;;
      M) multiplier1=1000000;;
      G) multiplier1=1000000000;;
      T) multiplier1=1000000000000;;
      P) multiplier1=1000000000000000;;
      E) multiplier1=1000000000000000000;;
   esac

   case $unit2 in
      B) multiplier2=1;;
      K) multiplier2=1000;;
      M) multiplier2=1000000;;
      G) multiplier2=1000000000;;
      T) multiplier2=1000000000000;;
      P) multiplier2=1000000000000000;;
      E) multiplier2=1000000000000000000;;
   esac

   val1=$((num1 * multiplier1))
   val2=$((num2 * multiplier2))

   [[ "$val1" -ge "$val2" ]] && return 0
   return 1
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   local style=${1:--h}
   local usageErrorMessage=${2:-}

   [[ -n "$usageErrorMessage" ]] && msg "\\n\\nUsage Error:\\n\\n$usageErrorMessage\\n\\n"

   msg "USAGE for $ThisScript v$Version:

$ThisScript [<SDPInstance>] [-p <Profile>] [-c <CfgFile>] [-y] [-v] [-d|-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script compares configurables set on the current server with best
	practices defined a data file.

OPTIONS:
 -p <Profile>
 	Specify a profile defined in the config file, such as 'demp' or 'prod'.  A profile
	defines a set of expected configurable values that can differ from the expected
	values in other profiles.  For example, for a demo environment, the filesys.P4ROOT.min
	might have an expected value of 128M, while the expected value in a prod (production)
	profile might be 5G.

	There is a default profile which always applies whether '-p' is specified or not.

	The profile specified with '-p' applies in addition to the default configuration.

 -c <CfgFile>
	Specify an alternate config file that defines best practice configurables. This
	is intended for testing.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to
	\$LOGS/${ThisScript%.sh}.log

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file. Using redirection operators like
	'> log' or '2>&1' are unnecessary, nor is using 'tee'.

 -y	Live operation mode.  By default, any commands that affect data, such as
	setting configurables, are displayed, but not executed.  With the '-y' option,
	commands may be executed.

 -d	Display debug messages.

 -D     Set extreme debugging verbosity using bash 'set -x' mode. Implies -d.

 -si	Silient Mode.  No output is displayed to the terminal (except for usage errors
	on startup). Output is captured in the log.  The '-si' cannot be used with
	'-L off'.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

FILES:
	The standard configurables config file is:

	$CfgFile

EXAMPLES:
	Example 1: Check configurables with the default profile, and no logging:

	$ThisScript -L off

	Example 2: Check configurables with the 'prod' (Production) profile:

	$ThisScript -p prod

	Example 3: Check configurables with the 'demo' profile, doing a verbose comparison:

	$ThisScript -p demo -v

FUTURE ENHANCEMENTS:
	Presently, this $ThisScript v$Version only reports configurables. It does not
	support changing configurables.

	As the script is currently only capable of reporting, the '-y' option has no
	effect.

	Some possible future enhancements are:
	* Extend reporting to suggesting configuration changes.

	* Provide an option to make changes to configurables that are safe to change
	  immediately, and provide guidance on those configurables that are best set with
	  guidance and plannning.

	* Provide a way to specify custom exemptions for certain configurables.

	* Added multi-version support for backward compatibility. This version assumes
	  P4D 2023.1+ (though will be useful for older versions).
"
   fi

   exit 2
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate ()
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   ExitCode=$((ErrorCount+ConfigFailRequiredCount))
   msg "ExitCode: $ExitCode"

   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log\\n${H1}"

   # With the trap removed, exit.
   exit "$ExitCode"
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-i) SDPInstance="$2"; shiftArgs=1;;
      (-p) Profile="$2"; shiftArgs=1;;
      (-c) CfgFile="$2"; shiftArgs=1;;
      (-L) Log="$2"; shiftArgs=1;;
      (-y) NoOp=0;;
      (-si) SilentMode=1;;
      (-v) Verbosity=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Use bash 'set -x' extreme debug mode.
      (-*) usage -h "Unknown option ($1).";;
      (*)
         if [[ -z "$SDPInstance" ]]; then
            SDPInstance="$1"
	 else
            usage -h "Extra parameter [$1] is unknown; SDP Instance is already set to [$SDPInstance]."
	 fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usage -h "The '-si' option cannot be used with '-Log off'."

[[ -z "$SDPInstance" ]] && SDPInstance="${SDP_INSTANCE:-}"
[[ -z "$SDPInstance" ]] && usage -h "The SDP instance parameter is required unless SDP_INSTANCE is set. To set SDP_INSTANCE, do:\\n\\tsource $SDPRoot/common/bin/p4_vars INSTANCE\\n\\nreplacing INSTANCE with your SDP instance name."

SDPInstanceVars="$SDPRoot/common/config/p4_${SDPInstance}.vars"
[[ -r "$SDPInstanceVars" ]] || usage -h "The SDP instance specified [$SDPInstance] is missing Instance Vars file: $SDPInstanceVars"

# shellcheck disable=SC1090
source "$SDPRoot/common/bin/p4_vars" "$SDPInstance" ||\
   bail "Could not do: source \"$SDPRoot/common/bin/p4_vars\" \"$SDPInstance\""

# shellcheck disable=SC1090
source "$SDPRoot/common/bin/log_functions.sh" ||\
   bail "Could not do: source \"$SDPRoot/common/bin/log_functions.sh\""

[[ -e "$CfgFile" ]] || usage -h "The config file [$CfgFile] is missing."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

# If the user specifies a log file file with '-L', write to the specified file.
# If no log was specified, create a default log file using a timestmap in the
# LOGS dir, and immediately update the symlink for the default log to point to
# it.
if [[ "$Log" != off ]]; then
   # If $Log is not yet defined, set it to a reasonable default.
   if [[ -z "$Log" ]]; then
      LogTimestamp=$(date +'%Y-%m-%d-%H%M%S')
      Log="$LOGS/${ThisScript%.sh}.${LogTimestamp}.log"
      # Make sure we have a unique log file. Prefer a human-readoable timetamp
      # using hours/minutes/seconds. Append milliseconds if needed to ensure
      # achieve a unique filename.
      while [[ -e "$Log" ]]; do
         LogTimestamp=$(date +'%Y-%m-%d-%H%M%S.%3N')
         Log="$LOGS/${ThisScript%.sh}.${LogTimestamp}.$i.log"
         i+=1
      done
   fi
   # The LogLink symlink has no timestamp. It points to the most recent log file.
   LogLink="$LOGS/ccheck.log"

   if [[ -e "$LogLink" ]]; then
      if [[ -L "$LogLink" ]]; then
         rm -f "$LogLink"
      else
         # If the name that should be a symlink is not a symlink, move it aside before
         # creating the symlink.
         OldLogTimestamp=$(get_old_log_timestamp "$LogLink")
         mv -f "$LogLink" "${LogLink%.log}.${OldLogTimestamp}.log" ||\
            bail "Could not move old log file aside; tried: mv -f \"$LogLink\" \"${LogLink%.log}.${OldLogTimestamp}.log\""
      fi
   fi

   touch "$Log" || bail "Couldn't touch new log file [$Log]."

   # Use a subshell so the 'cd' doesn't persist.
   ( cd "$LOGS"; ln -s "${Log##*/}" "${LogLink##*/}"; ) ||\
       bail "Couldn't initialize log symlink; tried: ln -s \"$Log\" \"$LogLink\""

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Starting $ThisScript v$Version as $ThisUser@$ThisHost on $(date) with \\n$CmdLine"

msg "Comparing with Profile: [$Profile] loaded from: $CfgFile\\n${H2}"

# For environments that define P4SUPER, use that for P4USER rather than P4USER.
SuperUser=$(p4 set -q P4SUPER)
SuperUser=${SuperUser##*=}

if [[ -n "$SuperUser" ]]; then
   export P4USER="$SuperUser"
fi

TmpFile=$(mktemp)

if [[ "$P4PORT" =~ ^ssl[46]*: ]]; then
   Cmd="timeout $TimeoutDelay p4 trust -y"
   dbg "Running: $Cmd [P4PORT=$P4PORT]"
   $Cmd > "$TmpFile" 2>&1 ||\
      bail "Could not do: $Cmd\\n$(cat "$TmpFile")\\n"
fi

Cmd="timeout $TimeoutDelay p4 info -s"
dbg "Running: $Cmd"
if $Cmd > "$TmpFile" 2>&1; then
   dbg "Verified: Helix Core is accessible."
   if [[ "$(p4 protects -m)" == "super" ]]; then
      dbg "Verified: super user access available."
   else
      bail "Could not verify super access [P4USER=$P4USER P4PORT=$P4PORT P4TICKETS=$P4TICKETS]."
   fi
else
   bail "Could not access Helix Core server [P4PORT=$P4PORT] with: $Cmd\\n$(cat "$TmpFile")\\n"
fi

# Extract current configurables values using 'p4d -cshow'.
Cmd="p4d -r $P4ROOT -cshow"
if $Cmd > "$TmpFile" 2>&1; then
   dbg "Captured configurables:\\n$(cat "$TmpFile")\\n"
else
   bail "Could not capture configurables with: $Cmd\\n$(cat "$TmpFile")\\n"
fi

dbg "Comparing P4MASTER_ID=$P4MASTER_ID with SERVERID=$SERVERID."

# Determine whether the current ServerID is that of a commmit, replica, standby, or edge.
### Use is_*() functions from backup_functions.sh, but avoid loading backup_functions.sh
### directly for now, due to compatibility issues with new bash template this script is
### based on. EDITME - Find a better way later.
# shellcheck disable=SC1091
if [[ "$P4MASTER_ID" == "$SERVERID" ]]; then
   ThisServerIDType=commit
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_replica "$SERVERID")" == YES ]]; then
   ThisServerIDType=replica
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_standby "$SERVERID")" == YES ]]; then
   ThisServerIDType=standby
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_edge "$SERVERID")" == YES ]]; then
   ThisServerIDType=edge
else
   errmsg "Could not determine server type for ServerID [$SERVERID]."
fi

dbg "This ServerID [$SERVERID] is of type [$ThisServerIDType]."

dbg "Loading current configurable values."
p4 -ztag -F "%Type%|%ServerName%|%Name%|%Value%" configure show allservers > "$TmpFile"
while read -r Line; do
   ConfigurableType="$(echo "$Line" | cut -d '|' -f1)"
   [[ "$ConfigurableType" == "configure" ]] || continue
   ServerName="$(echo "$Line"|cut -d '|' -f2)"
   [[ "$ServerName" == any || "$ServerName" == "$SERVERID" ]] || continue

   Configurable="$(echo "$Line"|cut -d '|' -f3)"
   currentValue="$(echo "$Line"|cut -d '|' -f4)"
   CurrentValue[$Configurable]="$currentValue"
   #dbg "CurrentValue[$Configurable]=$currentValue"
done < "$TmpFile"

# Extract settings for the selected profile and the default profile.
# Normalize the profile to lowercase for searching in the config file.
grep '^default|' "$CfgFile" > "$TmpFile"

if [[ "$Profile" != default ]]; then
   TmpFile2=$(mktemp)
   grep "^${Profile,,}|" "$CfgFile" >> "$TmpFile2"
   if [[ -s "$TmpFile2" ]]; then
      cat "$TmpFile2" >> "$TmpFile" ||\
         bail "Could not append settings for profile $Profile to settings for default profile. Aborting."
      rm -f "$TmpFile2"
   else
      bail "No settings were found for profile [$Profile] in config file [$CfgFile]. Typo in the profile name? Aborting."
   fi
fi

# shellcheck disable=2094
while read -r Line; do
   Configurable="$(echo "$Line" | cut -d '|' -f 2)"
   expectedValue="$(echo "$Line" | cut -d '|' -f 3)"
   compareStyle="$(echo "$Line" | cut -d '|' -f 4)"
   optionality="$(echo "$Line" | cut -d '|' -f 5)"
   serverIDType="$(echo "$Line" | cut -d '|' -f 6)"
   setNotes="$(echo "$Line" | cut -d '|' -f 7)"

   # If the setNotes value is Standard, apply the usual documenton URL for
   # configurables.
   if [[ "$setNotes" == Standard ]]; then
      setNotes="https://www.perforce.com/manuals/cmdref/Content/CmdRef/configurables.alphabetical.html#$Configurable"
   fi

   # Do substitutions for expectedValue, e.g. replacing SDP Instance.
   expectedValue="${expectedValue/__SDP_INSTANCE__/$SDPInstance}"
   expectedValue="${expectedValue/__P4ROOT__/$P4ROOT}"
   expectedValue="${expectedValue/__P4SERVER__/$P4SERVER}"
   expectedValue="${expectedValue/__KEEPLOGS__/$KEEPLOGS}"
   expectedValue="${expectedValue/__LOGS__/$LOGS}"

   if [[ -n "${CurrentValue[$Configurable]:-}" ]]; then
      currentValue="${CurrentValue[$Configurable]:-}"
   else
      currentValue="unset"
   fi

   dbg "C=[$Configurable] EV=[$expectedValue] CV=[${CurrentValue[$Configurable]:-}] CS=[$compareStyle] O=[$optionality] ST=[$serverIDType] SN=[$setNotes]"
   ExpectedValue[$Configurable]="$expectedValue"
   CompareStyle[$Configurable]="$compareStyle"
   Optionality[$Configurable]="$optionality"
   ServerIDType[$Configurable]="$serverIDType"
   SetNotes[$Configurable]="$setNotes"
done < "$TmpFile"

rm -f "$TmpFile"

for Configurable in "${!ExpectedValue[@]}"; do
   ConfigCount+=1
   dbg "Configurable: [$Configurable]\\n  EValue=[${ExpectedValue[$Configurable]}]\\n  CValue=[${CurrentValue[$Configurable]:-}]\\n  CompareStyle=[${CompareStyle[$Configurable]}]\\n  Optionality=[${Optionality[$Configurable]}]\\n  SetNotes=[${SetNotes[$Configurable]}]\\n"

   # Determine if CompareStyle loaded from config file is known/handled. While
   # CamelCase is preferred in the config file, normalize to uppercase for comparison
   # to be forgiving case variations (case-insensitive).
   case "${CompareStyle[$Configurable]^^}" in
      (ATLEAST) compareStyle=AtLeast;;
      (NOMORETHAN) compareStyle=NoMoreThan;;
      (CONTAINS) compareStyle=Contains;;
      (EXACT) compareStyle=Exact;;
      (SET) compareStyle=Set;;
      (UNSET) compareStyle=Unset;;
      (*)
         errmsg "Unknown CompareStyle [${CompareStyle[$Configurable]}] for configurable [$Configurable]. Treating as Exact."
         compareStyle=Exact
      ;;
   esac

   # Match expected vs. actual using the compare style.
   ConfigMatches=0
   expectedValue="${ExpectedValue[$Configurable]}"
   currentValue="${CurrentValue[$Configurable]:-}"
   [[ -n "$currentValue" ]] || currentValue="unset"

   case "$compareStyle" in
      (AtLeast)
         # For a numeric compare, change 'unset' to '0'.
         [[ "$currentValue" == unset ]] && currentValue=0
         if [[ "$currentValue" =~ ^[0-9]+$ && "$expectedValue" =~ ^[0-9]+$ ]]; then
            [[ "$currentValue" -ge "$expectedValue" ]] && ConfigMatches=1
         else
            # If the numbers aren't simple integers, try a size compare with sizes like 20M,
            # 2G, etc.  This function will display an appropriate error message if the
            # values are not valid for comparsion.
            at_least_size_compare "$currentValue" "$expectedValue" && ConfigMatches=1
         fi
      ;;
      (NoMoreThan)
         # For a numeric compare, change 'unset' to '0'.
         [[ "$currentValue" == unset ]] && currentValue=0
         if [[ "$currentValue" =~ ^[0-9]+$ && "$expectedValue" =~ ^[0-9]+$ ]]; then
            [[ "$currentValue" -le "$expectedValue" ]] && ConfigMatches=1
         else
            # If the numbers aren't simple integers, try a size compare with sizes like 20M,
            # 2G, etc.  This function will display an appropriate error message if the
            # values are not valid for comparsion.
            # Use the 'at_least' comparison function as above, but swap the
            # arguments to get the desired NoMoreThan comparison.
            at_least_size_compare "$expectedValue" "$currentValue" && ConfigMatches=1
         fi
      ;;
      (Contains)
         [[ $currentValue =~ $expectedValue ]] && ConfigMatches=1
      ;;
      (Exact)
         [[ "$currentValue" == "$expectedValue" ]] && ConfigMatches=1

	 # Special case for 'security' counter. If we expect a '0', that matches 'unset'.
	 [[ "$Configurable" == "security" && "$expectedValue" == "0" && "$currentValue" == "unset" ]] && \
            ConfigMatches=1
      ;;
      (Set) # If any value is set at all, it is considered to match for 'Set' compare.
         [[ -n "$currentValue" ]] && ConfigMatches=1
      ;;
      (Unset) # If no value is set at all, it is considered to match for 'Unset' compare.
         [[ "$currentValue" == unset ]] && ConfigMatches=1
      ;;
   esac

   if [[ "${Optionality[$Configurable]}" == "Required" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgpass "Value for configurable [$Configurable] is at least [$expectedValue], as is required.";;
            (NoMoreThan) cfgpass "Value for configurable [$Configurable] is no more than [$expectedValue], as is required.";;
            (Contains) cfgpass "Value for configurable [$Configurable] contains required text [$expectedValue].";;
            (Exact) cfgpass "Value for configurable [$Configurable] matches required value [$expectedValue] exactly.";;
            (Set) cfgpass "A value for configurable [$Configurable] is set, as required. Value is [$currentValue].";;
            (Unset) cfgpass "The configurable [$Configurable] is unset, as required.";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgreqfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is required."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (NoMoreThan)
               cfgreqfail "Value for configurable [$Configurable] is [$currentValue]; it is required to be no more than [$expectedValue]."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (Contains)
               cfgreqfail "Value for configurable [$Configurable] does not contain required text [$expectedValue]."
               SuggestedValue[$Configurable]="EDIT_ME_Choose_Wisely_SomethingContaining_$expectedValue"
               SetWhen[$Configurable]="Later"
            ;;
            (Exact)
               cfgreqfail "Value for configurable [$Configurable] does not exactly match required value [$expectedValue]; value is: [$currentValue]."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (Set)
               cfgreqfail "A value for configurable [$Configurable] is set, as required."
               SuggestedValue[$Configurable]="EDIT_ME_ChooseWisely"
               SetWhen[$Configurable]="Later"
            ;;
            (Unset)
               cfgreqfail "The configurable [$Configurable] is required to be unset, but is set to [$currentValue]."
               SuggestedValue[$Configurable]="Unset"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
         esac
      fi
   elif [[ "${Optionality[$Configurable]}" == "Recommended" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgpass "Value for configurable [$Configurable] is [$currentValue], which is at least [$expectedValue], as is recommended.";;
            (NoMoreThan) cfgpass "Value for configurable [$Configurable] is [$currentValue], which is no more than [$expectedValue], as is recommended.";;
            (Contains) cfgpass "Value for configurable [$Configurable] contains recommended text [$expectedValue].";;
            (Exact) cfgpass "Value for configurable [$Configurable] matches recommended value [$expectedValue] exactly.";;
            (Set) cfgpass "A value for configurable [$Configurable] is set, as recommended.";;
            (Unset) cfgpass "The configurable [$Configurable] is unset, as recommended.";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgrecfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is recommended."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (NoMoreThan)
               cfgrecfail "Value for configurable [$Configurable] is [$currentValue]; is is recommended to be no more than [$expectedValue]."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (Contains)
               cfgrecfail "Value for configurable [$Configurable] does not contain recommended text [$expectedValue]."
               SuggestedValue[$Configurable]="EDIT_ME_Choose_Wisely_SomethingContaining_$expectedValue"
               SetWhen[$Configurable]="Later"
            ;;
            (Exact)
               cfgrecfail "Value for configurable [$Configurable] does not exactly match recommended value [$expectedValue]; value is: [$currentValue]."
               SuggestedValue[$Configurable]="$expectedValue"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
            (Set)
               cfgrecfail "A value for configurable [$Configurable] is not set, as recommended."
               SuggestedValue[$Configurable]="EDIT_ME_ChooseWisely"
               SetWhen[$Configurable]="Later"
            ;;
            (Unset)
               cfgrecfail "The configurable [$Configurable] is recommended to be unset, but is set to [$currentValue]."
               SuggestedValue[$Configurable]="Unset"
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  SetWhen[$Configurable]="Now"
               else
                  SetWhen[$Configurable]="Later"
               fi
            ;;
         esac
      fi
   else
      errmsg "Unknown optionality [${Optionality[$Configurable]}]; should be 'Required' or 'Recommended'."
   fi
done

msg "\\nSummary:
   Configs Checked:                $ConfigCount
   Configs GOOD:                   $ConfigPassCount
   Configs BAD (Total):            $ConfigFailCount
      Configs BAD (Required):      $ConfigFailRequiredCount
      Configs BAD (Recommended):   $ConfigFailRecommendedCount"
if [[ "$ErrorCount" -ne 0 ]]; then
   msg "   ERRORS:                         $ErrorCount"
fi

msgn "\\nResult: "
ExitCode=$((ErrorCount+ConfigFailRequiredCount))

if [[ "$ConfigCount" -ne 0 ]]; then
   if [[ "$ConfigFailCount" -eq 0 ]]; then
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "PASS with Grade A: All $ConfigCount checked configurables pass, and there are no errors."
      else
         msg "FAIL: There were $ErrorCount errors attempting to check configurables.  All $ConfigCount checked configurables pass."
      fi
   elif [[ "$ConfigFailRequiredCount" -eq 0 ]]; then
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "PASS with Grade B: Of $ConfigCount configurables checked, all required values pass, but $ConfigFailRecommendedCount recommended settings were not optimal."
      else
         msg "FAIL: There were $ErrorCount errors attempting to check configurables.  Of $ConfigCount configurables checked, all required values pass, but $ConfigFailRecommendedCount recommended settings were not optimal."
      fi
   else
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "FAIL: Of $ConfigCount configurables, $ConfigFailRequiredCount required settings were not optimal, and $ConfigFailRecommendedCount recommended settings were not optimal."
      else
         msg "FAIL: There were $ErrorCount errors attempting to check configurables. Of $ConfigCount configurables, $ConfigFailRequiredCount required settings were not optimal, and $ConfigFailRecommendedCount recommended settings were not optimal."
      fi
   fi
else
   errmsg "No configs were checked. Something went wrong."
fi

#------------------------------------------------------------------------------
# FOR FUTURE ENHANCEMENT
# These '-n' checks are meaningless; they are here to suppress specific
# ShellCheck errors about unused variables.  These variables are intended
# for future use in a version that supports modifying configurables.
[[ -n "${SetWhen[server]:-}" ]] && true
[[ -n "${SuggestedValue[server]:-}" ]] && true
[[ -n "${ServerIDType[server]:-}" ]] && true
[[ "$NoOp" -eq 0 ]] && true
#------------------------------------------------------------------------------

# The exit code is a happy zero if all required checks passed AND there are no errors doing checks.
# See the terminate() function where this script really exits.
exit "$ExitCode"