ccheck.sh #26

#!/bin/bash
set -u

#==============================================================================
# Declarations and Environment
declare ThisScript=${0##*/}
declare ThisUser=
declare ThisHost=${HOSTNAME%%.*}
declare Version=2.3.2
declare Args="$*"
declare CmdLine="$0 $Args"

declare ServerType=
declare UserP4CONFIG=
declare P4DVersionString=
declare P4DMajorVersion=
declare P4DBuild=
declare P4=
declare -i ExitCode=0
declare -i ErrorCount=0
declare -i WarningCount=0
declare -i NoOp=1
declare -i Debug=${DEBUG:-0}
declare -i ConfigMatches=0
declare -i SilentMode=0
declare -i DoSecurityCheck=0
declare -i DoSSLCheck=0
declare -i EnableSSLAdvised=0
declare -i ApplyEasyFixes=0
declare -i EasyFixCount=0
declare -a EasyFixConfigurables
declare -a EasyFixValues
declare -i ApplyAllFixes=0
declare -i PlannedFixCount=0
declare -a PlannedFixConfigurables
declare -a PlannedFixAdvice
declare -a PlannedFixAutomated  # Set to 1 if Advice can be applied automatically, 0 if manual.
declare -i i=0 
declare SDPRoot="${SDP_ROOT:-/p4}"
declare SDPCommon="${SDPRoot}/common"
declare SDPCommonBin="${SDPCommon}/bin"
declare SDPCommonCfg="${SDPCommon}/config"
declare SDPCommonSiteCfg="${SDPCommon}/config"
declare SDPCommonLib="${SDPCommon}/lib"
declare Profile=prod
declare CfgFile="$SDPCommonCfg/configurables.cfg"
declare Log=
declare OldLogTimestamp=
declare LogTimestamp=
declare LogLink=
declare TmpFile=
declare TmpFile2=
declare SDPInstance=
declare SDPInstanceVars=
declare TimeoutDelay=5s
declare SuperUser=
declare Cmd=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare -i Verbosity=0
declare -i i=0

# Associative arrays indexed by configurable, capturing data loaded from
# /p4/common/config/configurables.cfg.
declare Configurable=
declare ConfigurableType=
declare -A CurrentValue
declare -A CurrentValueByDefault
declare -A ExpectedValue
declare -A CompareStyle
declare -A Optionality
declare -A ServerIDType
declare -A SetNotes
declare ThisServerIDType=

# Categories is indexed by configurable, and is loaded with data from
# 'p4 configure help'; not to be confused with 'p4 help configure'.
declare Category
declare -A Categories

# Summary Counters
declare -i ConfigCount=0
declare -i ConfigPassCount=0
declare -i ConfigFailCount=0
declare -i ConfigFailRequiredCount=0
declare -i ConfigFailRecommendedCount=0
declare -i ConfigFailOptionalCount=0
declare -i ConfigFixedCount=0

# Color support.
declare GREEN=
declare RED=
declare YELLOW=
declare RESET=

#==============================================================================
# SDP Library Functions

if [[ -d "$SDPCommonLib" ]]; then
   # shellcheck disable=SC1090 disable=SC1091
   source "$SDPCommonLib/logging.lib" ||\
      bail "Failed to load bash lib [$SDPCommonLib/logging.lib]. Aborting."
fi

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function msgn () { echo -n -e "$*"; }
function msg_green  { msg "${GREEN}$*${RESET}"; }
function msg_yellow { msg "${YELLOW}$*${RESET}"; }
function msg_red    { msg "${RED}$*${RESET}"; }

function dbg () { [[ "$Debug" -eq 0 ]] || echo -e "DEBUG: $*" >&2; }
function dbg2 () { [[ "$Debug" -ge 2 ]] && echo -e "DEBUG: $*" >&2; }
function errmsg () { msg_red "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg_yellow "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

# The cfgreqpass() function is called when required configuration settings are
# as expected, and cfgrecpass() is called when recommended settings are
# as expected. Both increment ConfigPassCount.
function cfgreqpass () { [[ "$Verbosity" -ne 0 ]] && msg_green "GREAT: $*"; ConfigPassCount+=1; }
function cfgrecpass () { [[ "$Verbosity" -ne 0 ]] && msg_green "GOOD: $*"; ConfigPassCount+=1; }

# The cfgreqfail() function is called when required configuration settings are
# not as expected, and cfgrecfail() is called when recommended settings are not
# as expected. Both increment ConfigFailCount and contribute to an unhappy exit
# code.
function cfgoptfail () { msg "INFO: $*"; ConfigFailOptionalCount+=1; }
function cfgreqfail () { msg_yellow "BAD: $*"; ConfigFailCount+=1; ConfigFailRequiredCount+=1; }
function cfgrecfail () { msg_yellow "WARN: $*"; ConfigFailCount+=1; ConfigFailRecommendedCount+=1; }

#------------------------------------------------------------------------------
# Function: at_least_size_compare ("$currentValue" "$expectedValue")
# Return true if size 1 is at least as big as size 2. Do the needful conversion
# of units (just numeric bytes, K, M, G, T, P, E, etc.) to compare correctly.
function at_least_size_compare () {
   local s1=${1:-}
   local s2=${2:-}
   local num1=
   local unit1=
   local num2=
   local unit2=
   local multiplier1=1
   local multiplier2=1
   local val1=
   local val2=

   # Normalize values to uppercase, so 10m treated as 10M.
   s1=${s1^^}
   s2=${s2^^}

   # If inputs are purely numeric, append a 'B' (bytes) suffix.
   [[ "$s1" =~ ^[0-9]+$ ]] && s1="${s1}B"
   [[ "$s2" =~ ^[0-9]+$ ]] && s2="${s2}B"

   if [[ ! "$s1" =~ ^[0-9]*([BKMGTPE]){1}$ || ! "$s2" =~ ^[0-9]*([BKMGTPE]){1}$ ]]; then
      errmsg "Cannot compare size value [$s1] with value [$s2]; the values are invalid."
      return 1
   fi

   # Extract the numeric part and unit from each value.
   [[ $s1 =~ ^([0-9]+)([BKMGTPE]) ]] && num1=${BASH_REMATCH[1]} && unit1=${BASH_REMATCH[2]}
   [[ $s2 =~ ^([0-9]+)([BKMGTPE]) ]] && num2=${BASH_REMATCH[1]} && unit2=${BASH_REMATCH[2]}

   # Convert units to base 10 multipliers.
   case $unit1 in
      B) multiplier1=1;;
      K) multiplier1=1000;;
      M) multiplier1=1000000;;
      G) multiplier1=1000000000;;
      T) multiplier1=1000000000000;;
      P) multiplier1=1000000000000000;;
      E) multiplier1=1000000000000000000;;
   esac

   case $unit2 in
      B) multiplier2=1;;
      K) multiplier2=1000;;
      M) multiplier2=1000000;;
      G) multiplier2=1000000000;;
      T) multiplier2=1000000000000;;
      P) multiplier2=1000000000000000;;
      E) multiplier2=1000000000000000000;;
   esac

   val1=$((num1 * multiplier1))
   val2=$((num2 * multiplier2))

   [[ "$val1" -ge "$val2" ]] && return 0
   return 1
}

#------------------------------------------------------------------------------
# If p4d is new enough (2024.1+) we can use it's features to do a better
# safety check to avoid having 'ccheck.sh' from setting a filesys.*.min
# value that instantly breaks things.
function safety_check_for_filesys_min() {
   local settingName="${1:-}"
   local newValue="${2:-}"
   local volume=
   local bytesFree=
   local niceUnit=

   if [[ -z "$settingName" || -z "$newValue" ]]; then
      errmsg "INTERNAL: safety_check_for_filesys_min(): Bad usage"
      return 1
   fi

   # shellcheck disable=SC2072
   if [[ ! "$P4D_VERSION" > "2024.1" ]]; then
      warnmsg "Skipping safety check; p4d version is too old to support needed 'p4 diskspace' features."
      return 0
   fi

   case "$settingName" in
      filesys.depot.min) volume=depot;;
      filesys.P4JOURNAL.min) volume=P4JOURNAL;;
      filesys.P4LOG.min) volume=P4LOG;;
      filesys.P4ROOT.min) volume=P4ROOT;;
      filesys.TEMP.min) volume=TEMP;;
      *)
         errmsg "INTERNAL: Cannot do disk space safety check for unknown setting $settingName; only filesys.{depot,P4JOURNAL,P4ROOT} are handled."
         return 1
      ;;
   esac

   dbg "Running: $P4 -ztag -F %freeBytes% diskspace $volume"
   bytesFree="$($P4 -ztag -F %freeBytes% diskspace "$volume")B"

   if [[ -n "$bytesFree" ]]; then
      dbg "Calling: at_least_size_compare ($bytesFree, $newValue)"
      if at_least_size_compare "$bytesFree" "$newValue"; then
         return 0
      else
         niceUnit=$(convert_size_values "$bytesFree")
         warnmsg "Refusing to set $settingName because 'p4 diskspace $volume' reports about $niceUnit available, which is less than $newValue required."
         return 1
      fi
   else
      errmsg "INTERNAL: safety_check_for_filesys_min(): Could not get volume for setting $settingName."
      return 1
   fi
}

#------------------------------------------------------------------------------
function convert_size_values() {
   local input=${1:-} target=${2:-}
   local s num unit pf bytes val pow lu

   # Normalize to uppercase, split off the unit
   s="${input^^}"
   num="${s%[BKMGTPE]}"            # strip trailing unit

   # shellcheck disable=SC2295
   unit="${s##${num}}"             # the unit letter

   if [[ -z "$num" || -z "$unit" ]]; then
      errmsg "INTERNAL: convert_size_values(): Bad Usage: convert_size_values <size><unit> [target_unit]"
      return 1
   fi

   # Map unit to power-of-1024 exponent
   case "$unit" in
      B) pf=0;;
      K) pf=1;;
      M) pf=2;;
      G) pf=3;;
      T) pf=4;;
      P) pf=5;;
      E) pf=6;;
      *) echo "Error: unknown unit '$unit'" >&2; return 1;;
   esac

   # Compute the raw byte count (rounded to nearest integer)
   bytes=$(awk "BEGIN { printf(\"%.0f\", $num * (1024**$pf)) }")

   if [[ -n $target ]]; then
      # Convert to a specific target unit
      local tt tu tf
      tt="${target^^}"
      tu="${tt: -1}"
      case "$tu" in
         B) tf=0;;
         K) tf=1;;
         M) tf=2;;
         G) tf=3;;
         T) tf=4;;
         P) tf=5;;
         E) tf=6;;
         *) echo "Error: unknown target unit '$target'" >&2; return 1;;
      esac

      val=$(awk "BEGIN {
         r = $bytes / (1024**$tf);
         if (r == int(r)) printf(\"%d\", r);
         else               printf(\"%.2f\", r);
      }")
      echo "${val}${tu}"
   else
      # Auto-select the best unit so the mantissa is 0â1023
      pow=$(awk "BEGIN {
         if ($bytes > 0) u = int(log($bytes)/log(1024));
         else            u = 0;
         if (u<0) u=0; if (u>6) u=6;
         print u;
      }")
      local units=(B K M G T P E)
      lu=${units[$pow]}
      val=$(awk "BEGIN {
         r = $bytes / (1024**$pow);
         if (r == int(r)) printf(\"%d\", r);
         else               printf(\"%.2f\", r);
      }")
      echo "${val}${lu}"
   fi
}

#------------------------------------------------------------------------------
# Function: run ($cmdAndArgs, $desc, $honorNoOp, $showOutput)
#
# Runs a command, with optional description, showing command line to execute
# and optionally also the output, and capturing and returning the exit code.
# Honors the global NoOp unless honorNoOp is 0.
#
# Input:
# $1 - Command and arguments to execute. Defaults to 'echo'.
# $2 - Optional message to display describing what the command is doing.
# $3 - Honor NoOp mode. If set to one, honor the NoOp variable -- so display
#      rather than execute commands.
# $4 - Numeric flag to show output; '1' indicates to show output, 0 to
#      suppress it.
#------------------------------------------------------------------------------
function run () {
   local cmdAndArgs="${1:-echo}"
   local desc="${2:-}"
   local honorNoOp="${3:-1}"
   local -i showOutput="${4:-1}"
   local -i cmdReturnCode=0
   local log

   [[ -n "$desc" ]] && msg "$desc"
   msg "Executing: $cmdAndArgs"
   if [[ "$honorNoOp" -eq 1 && "$NoOp" -eq 0 ]]; then
      log="$(mktemp "$P4TMP/run.${ThisScript%.sh}.XXXXXXXXXXX")"
      # shellcheck disable=SC2086
      eval $cmdAndArgs > "$log" 2>&1
      cmdReturnCode=$?
   else
      msg "NO_OP: Would run: $cmdAndArgs"
      cmdReturnCode=0

      # In NoOp mode, no command was executed, so there is
      # output to show.
      showOutput=0
   fi

   if [[ "$showOutput" -eq 1 ]]; then
      echo "EXIT_CODE: $cmdReturnCode" >> "$log"
      cat "$log"
   fi

   if [[ "$honorNoOp" -eq 0 && "$NoOp" -eq 0 ]]; then
      /bin/rm -f "$log"
   fi

   return "$cmdReturnCode"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   local style=${1:--h}
   local usageErrorMessage=${2:-}

   [[ -n "$usageErrorMessage" ]] && msg "\\n\\nUsage Error:\\n\\n$usageErrorMessage\\n\\n"

   msg "USAGE for $ThisScript v$Version:

$ThisScript [<SDPInstance>] [-p <Profile>] [-fix|-FIX] [-sec [-no_ssl]] [-p4config /path/to/.p4config] [-c <CfgFile>] [-y] [-v] [-d|-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script compares configurables set on the current server with best
	practices.  When used to check for security best practices, use the
	'-sec' option.

	The best practices are defined in a data file.  The default data file
	provided is:

	$SDPCommonCfg/configurables.cfg

	Optionally, if the '-fix' or '-FIX' options are used, this script can make
	changes to p4d, via 'p4 configure' commands, to bring it in line with best
	practices.  A preview of proposed changes is then displayed.  The '-y'
	option can then be used to proceed with the operation.

	This script is currently intended to run only on a commit server.  It
	may be updated in the future to provide further information for replica and
	edge servers.

OPTIONS:
 -p <Profile>
 	Specify a profile defined in the config file, such as 'demo' or 'hcc'.  A profile
	defines a set of expected configurable values that can differ from the expected
	values in other profiles.  For example, for a demo environment, the filesys.P4ROOT.min
	might have an expected value of 128M, while the expected value in a prod (production)
	profile might be 5G, and the same value might be 30G for 'prodent', the profile for
	production at large enterprise scale.

	The 'always' profile defines settings that always apply whether '-p' is specified
	or not. The profile specified with '-p' applies in addition to the 'always'
	configuration, adding to and possibly overriding settings from the 'always'
	configuration.

	The default profile is 'prod', the production profile.

	Specify the special value '-p none' to use only the settings defined in the
	'always' profile.

 -sec	Specify '-sec' to do a focused security check.

	This option also adds one additional check to ensure that the P4PORT value
	used is SSL-enabled (unless '-no_ssl' is specified).

 -no_ssl
	Specify '-no_ssl' with '-sec' to bypass the check for an SSL-enabled P4PORT.

 -p4config <P4CONFIG_File>
	Specify the path to a P4CONFIG file containing P4PORT, P4USER, P4TICKETS,
	and P4TRUST settings to connect as a super user to any P4 Server.

	This can be used with '-sec' on an SDP-managed server to check security
	settings of a non-SDP server.

	If the server targeted by the P4CONFIG settings is a non-SDP server the '-sec'
	option is not used, expect some errors related to not following best practices,
	some of which may be SDP-specific.

	Before using '-p4config' with this script, first create and test your P4CONFIG
	file using the 'p4' command line client with the '-E' option to exercise the
	settings.

	Start by creating the P4CONFIG file in $SDPCommonSiteCfg, a good location
	for site-local config files.

        mkdir -p $SDPCommonSiteCfg
	echo P4PORT=ssl:OtherServer:1666 > $SDPCommonSiteCfg/.p4config.OtherServer
	echo P4USER=bruno >> $SDPCommonSiteCfg/.p4config.OtherServer
	echo P4TICKETS=$SDPCommonSiteCfg/.p4tickets.OtherServer >> $SDPCommonSiteCfg/.p4config.OtherServer
	echo P4TRUST=$SDPCommonSiteCfg/.p4trust.OtherServer >> $SDPCommonSiteCfg/.p4config.OtherServer

	Next, establish trust (unless the other server is not SSL-enabled):

	p4 -E P4CONFIG=$SDPCommonSiteCfg/.p4config.OtherServer trust -y

 	Then login (which may require SSO authenticaiton if your other server is thusly
	configured):

	p4 -E P4CONFIG=$SDPCommonSiteCfg/.p4config.OtherServer login

	Ensure you are a super on the other server, and that Protections on the
	other server allows you to connect from your current machine.  You
	may need to modify Protections on the other server to get this to work:

	p4 -E P4CONFIG=$SDPCommonSiteCfg/.p4config.OtherServer protects -m

	Then give it a try, e.g.:

	$ThisScript -sec -p4config $SDPCommonSiteCfg/.p4config.OtherServer

 -c <CfgFile>
	Specify an alternate config file to define best practice configurables. This
	is intended primarily for testing. It can also be useful to define a site-local
	definition of best practices to compare against.  To use this option, first
	copy the default file to create a local copy in the $SDPCommonSiteCfg,
	e.g.
	
	cp -p /p4/common/config $SDPCommonSiteCfg/configurables.cfg

	Then reference it with '-c $SDPCommonSiteCfg/configurables.cfg'.

	WARNING: If you maintain a site-local copy of configurables.cfg, you will
	need to keep it current after SDP upgrades by manually merging in changes
	from the latest SDP version delivered with each release. Thus, using this
	option is discouraged.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to a log file
	pointed to by a symlink:

	\$LOGS/${ThisScript%.sh}.log

	The symlink is for convenience. It refers to the log from the most recent
	run where '-L' was not used.

	Each time this script is run, a new timestamped log is started, and
	the symlink updated to reference the new/latest log during startup.  Log
	files have timestamps that go to the second (or millisecond if needed)
	to differentiate logs.

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file. Using redirection operators like
	'> log' or '2>&1' are unnecessary, as is using 'tee' (though using 'tee'
	or redirects is safe and harmless).

 -fix	Specify -fix to take corrective action to resolve differences between
	current settings and the recommended/required values.  Optional settings
	are not affected by '-fix', only those indicated as Recommended or
	Required.

	When -fix is specified, this script determines on a per configurable basis
	whether it is safe to proceed immediately with the advised change, or if
	should be deferred until potentially disruptive impacts are understood.
	This determination is made based on the configuration file, which provides
	links go guidance documentation for configurables that are best changed
	with awareness of potential impact.  Such changes are displayed with '-fix',
	but require use of '-FIX' to process.

	As an example and a special case, the 'security' configurable will be
	changed to 4 with '-fix' if the starting value is 3, because that is deemed
	a low-risk change.  Changing the 'security' value to 4 if the starting
	value is 0-2 requires -FIX, as that is more likely to be impactful to
	users, and thus is best done with coordinated communications.

	In any case, if the security configurable is to be modified, additional
	guidance is provided regarding the potential impact to other p4d servers that
	access this server using the remote depot feature. Such access via the remote
	depot feature will cease to function when security is set to 4.

	This option previews advised changes by default. Use with -y to make changes.

 -FIX	Specify -FIX to make all changes that have been automated, even those that
	'-fix' would refuse to process immediately.

	In some cases even with -FIX, there may be follow up work to do. Follow up
	tasks will be indicated with 'TO DO:' comments in the output.

	The '-FIX' option implies '-fix'.

	This option previews advised changes by default. Use with -y to make changes.

 -y	Live operation mode.  By default, any commands that affect data, such as
	setting configurables, are displayed, but not executed.  With the '-y' option,
	commands may be executed.

 -d	Display debug messages.

 -D     Set extreme debugging verbosity using bash 'set -x' mode. Implies -d.

 -si	Silent Mode.  No output is displayed to the terminal (except for usage errors
	on startup). Output is captured in the log.  The '-si' cannot be used with
	'-L off'.

HELP OPTIONS:
 -h	Display short help message.
 -man	Display man-style help message.
 -V	Display script name and version.

GENERAL ADVICE and DISCLAIMER:
	This script is based on a data file that represents generalized best
	practices. This data file should be considered a source of information
	to be checked against other sources, such as the System Administration
	Guide and/or documentation on specific configurables found here:

	https://help.perforce.com/helix-core/server-apps/cmdref/current/Content/CmdRef/configurables.alphabetical.html

	Before acting on information provided by the script, and especially before
	using the '-fix' and '-FIX' options, be sure to review the output carefully.

	Contact Perforce Technical Support for guidance as needed.

EXAMPLES:
	Example 1: Basic Check, No Logging

	Check configurables with the default profile ('prod' for a commercial
	production server) and no logging:

	$ThisScript -L off

	Example 2: Alternate Profile

	Check configurables with the 'pub' profile (for a public/open source server):

	$ThisScript -p pub

	Example 3: Verbose comparison

	Check configurables with the 'demo' profile, doing a verbose comparison:

	$ThisScript -p demo -v

	Example 4: Security focused check

	Use the '-sec' option to report only security-related settings:

	$ThisScript -sec

	Example 5: Security focused check with fixes

	To do a security focused check making only non-disruptive fixes, use
	'-fix', and start with a preview by omitting the '-y' option:

	$ThisScript -sec -fix

	If the output is correct, append the '-y' to the command to make changes:

	$ThisScript -sec -fix -y

	Example 6:

	To do a security focused check, including potentially disruptive fixes,
	use '-FIX', and start with a preview by omitting the '-y' option:

	$ThisScript -sec -FIX

	Be sure to review the output carefully and read about potential impacts.
	Some configurables, such as 'auth.id'. require planning to change. For
	those configurables that require planning, additional guidance is
	provided if the script advises those settings be changed.  Read and heed
	the guidance.

	In the case of 'auth.id' specifically, this script will call 'p4login -v',
	'p4 login -service', and 'p4login -v -automation' if that configurable is
	set, account for the need to login again after setting 'auth.id'.  The
	script also displays a warning indicating that those commands should also
	be run manually on any other server machines in the fleet.

	If the output is correct, append the '-y' to the command to make changes:

	$ThisScript -sec -FIX -y

FUTURE ENHANCEMENTS:
	* Add multi-version support for backward compatibility. This version assumes
	  P4D 2024.2+ (though it may be useful for older versions).

FILES:
	The default configurables config file is: $CfgFile

	This file contains further documentation on the format of entries in
	the file.
"
   fi

   exit 2
}

#------------------------------------------------------------------------------
# Function: terminate
# shellcheck disable=SC2317
function terminate ()
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # For the exit code: If errors of any kind were reported, always report a
   # non-zero exit code. If there were no errors, we want a non-zero exit code
   # if there are any required configuration checks that failed AND that were
   # not subsequently fixed due to '-fix' and/or '-FIX'. Otherwise, exit with
   # a happy zero.
   if [[ "$ErrorCount" -ne 0 ]]; then
      ExitCode="$ErrorCount"
   elif ((ConfigFailRequiredCount > ConfigFixedCount)); then
      ExitCode=1
   else
      ExitCode=0
   fi
   msg "\\nExitCode: $ExitCode"

   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log\\n${H1}"

   # With the trap removed, exit.
   exit "$ExitCode"
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-V|-version|--version) msg "$ThisScript version $Version"; exit 0;;
      (-i) SDPInstance="$2"; shiftArgs=1;;
      (-sec) DoSecurityCheck=1; DoSSLCheck=1;;
      (-no_ssl) DoSSLCheck=0;;
      (-fix) ApplyEasyFixes=1;;
      (-FIX) ApplyEasyFixes=1; ApplyAllFixes=1;;
      (-p4config) UserP4CONFIG="$2"; shiftArgs=1;;
      (-p) Profile="$2"; shiftArgs=1;;
      (-c) CfgFile="$2"; shiftArgs=1;;
      (-L) Log="$2"; shiftArgs=1;;
      (-y|--yes) NoOp=0;;
      (-si) SilentMode=1;;
      (-v) Verbosity=1;;
      (-d) Debug=1;;
      (-d2) Debug=2;;
      (-D) Debug=2; set -x;; # Use bash 'set -x' extreme debug mode.
      (-*) usage -h "Unknown option ($1).";;
      (*)
         if [[ -z "$SDPInstance" ]]; then
            SDPInstance="$1"
         else
            usage -h "Extra parameter [$1] is unknown; SDP Instance is already set to [$SDPInstance]."
         fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usage -h "The '-si' option cannot be used with '-Log off'."

# If '-p4config' was specified, ensure the specified P4CONFIG file is readable
# and contains a P4PORT setting.
if [[ -n "$UserP4CONFIG" ]]; then
   if [[ -r "$UserP4CONFIG" ]]; then
      grep -q ^P4PORT= "$UserP4CONFIG" ||\
         usage -h "The P4CONFIG file [$UserP4CONFIG] specified with '-p4config' does not contain the required P4PORT setting."
   else
      usage -h "The P4CONFIG file [$UserP4CONFIG] specified with '-p4config' does not exist or is not readable."
   fi
fi

[[ -z "$SDPInstance" ]] && SDPInstance="${SDP_INSTANCE:-}"
[[ -z "$SDPInstance" ]] && usage -h "The SDP instance parameter is required unless SDP_INSTANCE is set. To set SDP_INSTANCE, do:\\n\\tsource $SDPCommonBin/p4_vars INSTANCE\\n\\nreplacing INSTANCE with your SDP instance name."

SDPInstanceVars="$SDPCommonCfg/p4_${SDPInstance}.vars"
[[ -r "$SDPInstanceVars" ]] || usage -h "The SDP instance specified [$SDPInstance] is missing Instance Vars file: $SDPInstanceVars"

# shellcheck disable=SC1090 disable=SC1091
source "$SDPCommonBin/p4_vars" "$SDPInstance" ||\
   bail "Could not do: source \"$SDPCommonBin/p4_vars\" \"$SDPInstance\""

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

# Detect support for colors
if [[ $SilentMode -eq 0 ]] \
  && command -v tput >/dev/null 2>&1 \
  && [[ -t 1 ]] \
  && [[ "$(tput colors)" -ge 8 ]]; then
  RED="$(tput setaf 1)" 
  GREEN="$(tput setaf 2)"
  YELLOW="$(tput setaf 3)"
  RESET="$(tput sgr0)"
else
  RED=; GREEN=; YELLOW=; RESET=
fi

# If the user specifies a log file file with '-L', write to the specified file.
# If no log was specified, create a default log file using a timestamp in the
# LOGS dir, and immediately update the symlink for the default log to point to
# it.
if [[ "$Log" != off ]]; then
   # If $Log is not yet defined, set it to a reasonable default.
   if [[ -z "$Log" ]]; then
      LogTimestamp=$(date +'%Y-%m-%d-%H%M%S')
      Log="$LOGS/${ThisScript%.sh}.${LogTimestamp}.log"
      # Make sure we have a unique log file. Prefer a human-readable timestamp
      # using hours/minutes/seconds. Append milliseconds if needed to ensure
      # a unique filename.
      while [[ -e "$Log" ]]; do
         LogTimestamp=$(date +'%Y-%m-%d-%H%M%S.%3N')
         Log="$LOGS/${ThisScript%.sh}.${LogTimestamp}.$i.log"
         i+=1
      done
   fi
   # The LogLink symlink has no timestamp. It points to the most recent log file.
   LogLink="$LOGS/ccheck.log"

   if [[ -e "$LogLink" ]]; then
      if [[ -L "$LogLink" ]]; then
         rm -f "$LogLink"
      else
         # If the name that should be a symlink is not a symlink, move it aside before
         # creating the symlink.
         OldLogTimestamp=$(get_old_log_timestamp "$LogLink")
         mv -f "$LogLink" "${LogLink%.log}.${OldLogTimestamp}.log" ||\
            bail "Could not move old log file aside; tried: mv -f \"$LogLink\" \"${LogLink%.log}.${OldLogTimestamp}.log\""
      fi
   fi

   touch "$Log" || bail "Couldn't touch new log file [$Log]."

   # Use a subshell so the 'cd' doesn't persist.
   ( cd "$LOGS"; ln -s "${Log##*/}" "${LogLink##*/}"; ) ||\
       bail "Couldn't initialize log symlink; tried: ln -s \"$Log\" \"$LogLink\""

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      if [[ -n "$GREEN" ]]; then
         exec > >( tee \
            >(sed -r \
            -e 's/\x1B\[[0-9;]*[a-zA-Z]//g' \
            -e 's/\x1B\(B//g' >>"$Log"))
      else
         exec > >(tee "$Log")
      fi
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Starting $ThisScript v$Version as $ThisUser@$ThisHost on $(date) with \\n$CmdLine"

msg "Comparing with Profile: [$Profile] loaded from: $CfgFile\\n${H2}"

# If '-p4config' was set, adapt P4PORT, P4USER, P4TICKETS, and P4TRUST values
# to match those.
if [[ -n "$UserP4CONFIG" ]]; then
   P4="p4 -E P4CONFIG=$UserP4CONFIG"
   export P4CONFIG="$UserP4CONFIG"
   # The 'p4 set' command will not process 'p4 -E P4CONFIG=x', so just use 'p4'
   # rather than $P4 for the 'p4 set' commands here.
   P4PORT=$(p4 set -q P4PORT)
   export P4PORT=${P4PORT##*=}
   P4USER=$(p4 set -q P4USER)
   export P4USER=${P4USER##*=}
   P4TICKETS=$(p4 set -q P4TICKETS)
   export P4TICKETS=${P4TICKETS##*=}
   P4TRUST=$(p4 set -q P4TRUST)
   export P4TRUST=${P4TRUST##*=}
   dbg "With user-provided P4CONFIG [$UserP4CONFIG]:\\nP4PORT=$P4PORT\\nP4USER=$P4USER\\nP4TICKETS=$P4TICKETS\\nP4TRUST=$P4TRUST\\n"

   # shellcheck disable=SC2155
   export P4ROOT=$($P4 -ztag -F %serverRoot% info -s)
   # Determine SDPInstance from P4ROOT value of target server.  This code to
   # extract SDPInstance and other settings will not work when '-p4config' is
   # used to target non-SDP servers. That's OK, since the main reason we
   # would target a non-SDP server is with '-sec', and the following details
   # don't apply to security configurables.
   SDPInstance=${P4ROOT#/p4/}
   SDPInstance=${SDPInstance%/root}
   export P4SERVER="p4_${SDPInstance}"
   P4LOG=$($P4 configure show P4LOG 2>&1|head -1)
   P4LOG=${P4LOG##*=}
   P4LOG=${P4LOG%% *}
   export LOGS=${P4LOG%/*}

   # Note: We don't have a reliable method to extract the KEEPLOGS SDP setting
   # from a server targeted with '-p4config', even if it is SDP.  This limitation
   # may result in some false-positive warnings related to serverlog config.

   dbg "With user-provided P4CONFIG [$UserP4CONFIG]:\\nP4ROOT=$P4ROOT\\nSDPInstance=$SDPInstance\\nP4SERVER=$P4SERVER\\nP4LOG=$P4LOG\\nLOGS=$LOGS\\n"
else
   P4="p4"
fi

ServerType=$($P4 -ztag -F %serverServices% info -s)

if [[ -n "$ServerType" ]]; then
   if [[ "$ServerType" == commit-server || "$ServerType" == standard ]]; then
      dbg "Verified: Running on a commit-server or standard server."
   else
      if [[ "$DoSecurityCheck" -eq 1 && "$ApplyEasyFixes" -eq 1 ]]; then
         bail "This must be on a server of type commit-server or standard if '-sec' and '-fix/-FIX' are used. Current server type is '$ServerType'."
      else
         warnmsg "This is intended to run on a server of type commit-server or standard, not $ServerType."
      fi
   fi
else
   if [[ "$DoSecurityCheck" -eq 1 && "$ApplyEasyFixes" -eq 1 ]]; then
      bail "Could not determine server type from 'p4 info'. This server type must be known if using '-sec' and '-fix/-FIX' are used."
   else
      warnmsg "Could not determine server type from 'p4 info'."
   fi
fi

# If the -p4config option is used, handle it here, after the standard SDP shell environment
# has been sourced (above).  If -p4config is used, we cannot use P4D_VERISON that was determined by
# parsing the output of 'p4d -V' using the p4d binary on the local machine (as is done when
# the standard p4_vars is sourced.  Instead, we run 'p4 info' against the p4d server defined in
# the user-provided P4CONFIG file and parse that output.
if [[ -n "$UserP4CONFIG" ]]; then
   P4DVersionString=$("$P4BIN" -ztag -F %serverVersion% info -s 2>/dev/null)
   [[ -n "$P4DVersionString" ]] ||\
      usage -h "Could not determine version using P4PORT in user-specified P4CONFIG file [$UserP4CONFIG]."
   P4DMajorVersion=$(echo "$P4DVersionString" | cut -d / -f 3)
   P4DBuild=$(echo "$P4DVersionString" | cut -d / -f 4 | cut -d ' ' -f 1)
   export P4D_VERSION="${P4DMajorVersion}.${P4DBuild}"
fi

# For environments that define P4SUPER, use that for P4USER rather than P4USER
# unless '-p4config' was used, in which case we use values set above from the
# server pointed to by P4CONFIG.
if [[ -z "$UserP4CONFIG" ]]; then
   SuperUser=$($P4 set -q P4SUPER)
   SuperUser=${SuperUser##*=}

   if [[ -n "$SuperUser" ]]; then
      export P4USER="$SuperUser"
   fi
fi

TmpFile=$(mktemp)

if [[ "$P4PORT" =~ ^ssl[46]*: ]]; then
   Cmd="timeout $TimeoutDelay $P4 trust -y"
   dbg "Running: $Cmd [P4PORT=$P4PORT]"
   $Cmd > "$TmpFile" 2>&1 ||\
      bail "Could not do: $Cmd\\n$(cat "$TmpFile")\\n"
fi

Cmd="timeout $TimeoutDelay $P4 info -s"
dbg "Running: $Cmd"
if $Cmd > "$TmpFile" 2>&1; then
   dbg "Verified: P4 Server is accessible."
   if [[ "$($P4 protects -m)" == "super" ]]; then
      dbg "Verified: super user access available."
   else
      bail "Could not verify super access [P4USER=$P4USER P4PORT=$P4PORT P4TICKETS=$P4TICKETS]."
   fi
else
   bail "Could not access P4 Server [P4PORT=$P4PORT] with: $Cmd\\n$(cat "$TmpFile")\\n"
fi

if [[ "$DoSecurityCheck" -eq 1 && "$DoSSLCheck" -eq 1 ]]; then
   ConfigCount+=1
   if [[ $P4PORT =~ ^ssl[46]*: ]]; then
      cfgreqpass "The P4PORT [$P4PORT] is SSL-enabled."
   else
      cfgreqfail "The P4PORT [$P4PORT] is NOT SSL-enabled."
      EnableSSLAdvised=1
   fi
fi

#------------------------------------------------------------------------------
# P4D's categories for each configurable (Security, Performance, etc.) are
# accessible via 'p4 configure help' if P4D is 2023.2+. If the P4D version
# is older, grep a list of security configurables from the config file
# instead.
dbg "Checking configurable categories."

# shellcheck disable=SC2072
if [[ "$P4D_VERSION" > "2023.2" ]]; then
   # Ask p4d for the list of configurables.
   $P4 -ztag -F "%Configurable%|%Category%" configure help > "$TmpFile" ||\
      bail "Could not get '$P4 configure help' output."
else
   # Extract list of security configurables from the config file.
   grep '^#SEC:' "$CfgFile" | sed -E -e 's|^#SEC:||g' > "$TmpFile"
fi

[[ "$Debug" -ge 2 ]] && dbg2 "FILE:\\n$(cat "$TmpFile")\\n\\n"

while read -r Line; do
   Configurable="${Line%%|*}"
   Category="${Line##*|}"
   Categories[$Configurable]="$Category"
   dbg2 "Configurable [$Configurable] has categories [$Category]."
done < "$TmpFile"

dbg "Comparing P4MASTER_ID=$P4MASTER_ID with SERVERID=$SERVERID."

# Determine whether the current ServerID is that of a commit, replica, standby, or edge.
### Use is_*() functions from backup_functions.sh, but avoid loading backup_functions.sh
### directly for now, due to compatibility issues with new bash template this script is
### based on. EDITME - Find a better way later.
# shellcheck disable=SC1091
if [[ "$P4MASTER_ID" == "$SERVERID" ]]; then
   ThisServerIDType=commit
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_replica "$SERVERID")" == YES ]]; then
   ThisServerIDType=replica
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_standby "$SERVERID")" == YES ]]; then
   ThisServerIDType=standby
elif [[ "$(source /p4/common/bin/backup_functions.sh; is_edge "$SERVERID")" == YES ]]; then
   ThisServerIDType=edge
else
   errmsg "Could not determine server type for ServerID [$SERVERID]."
fi

dbg "This ServerID [$SERVERID] is of type [$ThisServerIDType]."

dbg "Loading current configurable values."
$P4 -ztag -F "%Type%|%ServerName%|%Name%|%Value%" configure show allservers > "$TmpFile"
while read -r Line; do
   ConfigurableType="$(echo "$Line" | cut -d '|' -f1)"
   [[ "$ConfigurableType" == "configure" ]] || continue
   ServerName="$(echo "$Line"|cut -d '|' -f2)"

   # We are only concerned with configurables affecting the current ServerID.
   # That includes configurables set explicitly for the current ServerID,
   # as well as values set as global defaults with the 'any' config.
   [[ "$ServerName" == any || "$ServerName" == "$SERVERID" ]] || continue

   Configurable="$(echo "$Line"|cut -d '|' -f3)"

   # If a configurable is set both as a global default and explicitly
   # for the current ServerID, set the value to be that of the explicitly
   # set value. That overrides the global default effectively in p4d,
   # and we mirror that logic here.  We can't control whether the 'any'
   # or the explicit values appears first in the output of 'p4 configure
   # show allservers' (it depends where the ServerID name appears in
   # an listing by alphabetical order). So, ignore the 'any' value if
   # there already is a value defined for the configurable.

   [[ "$ServerName" == "any" && -n "${CurrentValue[$Configurable]:-}" ]] && continue

   currentValue="$(echo "$Line"|cut -d '|' -f4)"
   [[ -n "$currentValue" ]] || currentValue="unset"

   dbg "CurrentValue[$Configurable]=$currentValue"
   CurrentValue[$Configurable]="$currentValue"
done < "$TmpFile"

# Extract settings for the selected profile and the default profile.
# Normalize the profile to lowercase for searching in the config file.
grep '^always|' "$CfgFile" > "$TmpFile"

if [[ "$Profile" != none ]]; then
   TmpFile2=$(mktemp)
   grep "^${Profile,,}|" "$CfgFile" >> "$TmpFile2"
   if [[ -s "$TmpFile2" ]]; then
      cat "$TmpFile2" >> "$TmpFile" ||\
         bail "Could not append settings for profile $Profile to settings for default profile. Aborting."
      rm -f "$TmpFile2"
   else
      bail "No settings were found for profile [$Profile] in config file [$CfgFile]. Typo in the profile name? Aborting."
   fi
fi

#------------------------------------------------------------------------------
# Load best practices from the generated configuration, containing 'always'
# definitions and definitions for the selected profile. If '-sec' was used,
# exclude consideration for any configurable that isn't related to security.

# shellcheck disable=2094
while read -r Line; do
   Configurable="$(echo "$Line" | cut -d '|' -f 2)"

   if [[ "$DoSecurityCheck" -eq 1 ]]; then
      if [[ "${Categories[$Configurable]:-}" =~ Security ]]; then
         dbg "Considering security-related configurable: $Configurable"
      else
         dbg "Ignoring configurable unrelated to security: $Configurable"
         continue
      fi
   fi

   expectedValue="$(echo "$Line" | cut -d '|' -f 3)"
   compareStyle="$(echo "$Line" | cut -d '|' -f 4)"
   optionality="$(echo "$Line" | cut -d '|' -f 5)"
   serverIDType="$(echo "$Line" | cut -d '|' -f 6)"
   setNotes="$(echo "$Line" | cut -d '|' -f 7)"

   # If the setNotes value is Standard, apply the usual documentation URL for
   # configurables.
   if [[ "$setNotes" == Standard ]]; then
      setNotes="https://help.perforce.com/helix-core/server-apps/cmdref/current/Content/CmdRef/configurables.alphabetical.html#$Configurable"
   fi

   # Do substitutions for expectedValue, e.g. replacing SDP Instance.
   expectedValue="${expectedValue/__SDP_INSTANCE__/$SDPInstance}"
   expectedValue="${expectedValue/__P4ROOT__/$P4ROOT}"
   expectedValue="${expectedValue/__P4SERVER__/$P4SERVER}"
   expectedValue="${expectedValue/__KEEPLOGS__/$KEEPLOGS}"
   expectedValue="${expectedValue/__LOGS__/$LOGS}"

   currentValue="${CurrentValue[$Configurable]:-}"

   # For values that are not set explicitly, ask p4d for the current value,
   # which will be the default value.  If that has no value, use the value
   # 'unset'.
   if [[ -z "$currentValue" ]]; then
      dbg "No value explicitly set for $Configurable; checking default."
      currentValue="$($P4 -ztag -F %Value% configure show "$Configurable")"
      [[ -n "$currentValue" ]] || currentValue="unset"
      CurrentValue[$Configurable]="$currentValue"
      CurrentValueByDefault[$Configurable]=1
   fi

   dbg "C=[$Configurable] EV=[$expectedValue] CV=[$currentValue] CS=[$compareStyle] O=[$optionality] ST=[$serverIDType] SN=[$setNotes]"
   ExpectedValue[$Configurable]="$expectedValue"
   CompareStyle[$Configurable]="$compareStyle"
   Optionality[$Configurable]="$optionality"
   ServerIDType[$Configurable]="$serverIDType"
   SetNotes[$Configurable]="$setNotes"
done < "$TmpFile"

rm -f "$TmpFile"

#------------------------------------------------------------------------------
# For all configurables for which we have defined/expected values, collect
# current values (even of those are default values) and compare against best
# defined base practices.
for Configurable in "${!ExpectedValue[@]}"; do
   ConfigCount+=1
   dbg "Configurable: [$Configurable]\\n  EValue=[${ExpectedValue[$Configurable]}]\\n  CValue=[${CurrentValue[$Configurable]:-}]\\n  CompareStyle=[${CompareStyle[$Configurable]}]\\n  Optionality=[${Optionality[$Configurable]}]\\n  SetNotes=[${SetNotes[$Configurable]}]\\n"

   # Determine if CompareStyle loaded from config file is known/handled. While
   # CamelCase is preferred in the config file, normalize to uppercase for comparison
   # to be forgiving case variations (case-insensitive).
   case "${CompareStyle[$Configurable]^^}" in
      (ATLEAST) compareStyle=AtLeast;;
      (NOMORETHAN) compareStyle=NoMoreThan;;
      (CONTAINS) compareStyle=Contains;;
      (EXACT) compareStyle=Exact;;
      (SET) compareStyle=Set;;
      (UNSET) compareStyle=Unset;;
      (*)
         errmsg "Unknown CompareStyle [${CompareStyle[$Configurable]}] for configurable [$Configurable]. Treating as Exact."
         compareStyle=Exact
      ;;
   esac

   # Match expected vs. actual using the compare style.
   ConfigMatches=0
   expectedValue="${ExpectedValue[$Configurable]}"
   currentValue="${CurrentValue[$Configurable]:-}"

   case "$compareStyle" in
      (AtLeast)
         # For a numeric compare, change 'unset' to '0'.
         [[ "$currentValue" == unset ]] && currentValue=0
         if [[ "$currentValue" =~ ^[0-9]+$ && "$expectedValue" =~ ^[0-9]+$ ]]; then
            [[ "$currentValue" -ge "$expectedValue" ]] && ConfigMatches=1
         else
            # If the numbers aren't simple integers, try a size compare with sizes like 20M,
            # 2G, etc.  This function will display an appropriate error message if the
            # values are not valid for comparison.
            at_least_size_compare "$currentValue" "$expectedValue" && ConfigMatches=1
         fi
      ;;
      (NoMoreThan)
         # For a numeric compare, change 'unset' to '0'.
         [[ "$currentValue" == unset ]] && currentValue=0
         if [[ "$currentValue" =~ ^[0-9]+$ && "$expectedValue" =~ ^[0-9]+$ ]]; then
            [[ "$currentValue" -le "$expectedValue" ]] && ConfigMatches=1
         else
            # If the numbers aren't simple integers, try a size compare with sizes like 20M,
            # 2G, etc.  This function will display an appropriate error message if the
            # values are not valid for comparison.
            # Use the 'at_least' comparison function as above, but swap the
            # arguments to get the desired NoMoreThan comparison.
            at_least_size_compare "$expectedValue" "$currentValue" && ConfigMatches=1
         fi
      ;;
      (Contains)
         [[ $currentValue =~ $expectedValue ]] && ConfigMatches=1
      ;;
      (Exact)
         [[ "$currentValue" == "$expectedValue" ]] && ConfigMatches=1

      # Special case for 'security' counter. If we expect a '0', that matches 'unset'.
      [[ "$Configurable" == "security" && "$expectedValue" == "0" && "$currentValue" == "unset" ]] && \
            ConfigMatches=1
      ;;
      (Set) # If any value is set at all, it is considered to match for 'Set' compare.
         [[ -n "$currentValue" && "$currentValue" != unset ]] && ConfigMatches=1
      ;;
      (Unset) # If no value is set at all, it is considered to match for 'Unset' compare.
         if [[ "$currentValue" == unset || -n "${CurrentValueByDefault[$Configurable]:-}" ]]; then
            ConfigMatches=1
         fi
      ;;
   esac

   if [[ "${Optionality[$Configurable]}" == "Required" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgreqpass "Value for configurable [$Configurable] is at least [$expectedValue], as is required.";;
            (NoMoreThan) cfgreqpass "Value for configurable [$Configurable] is no more than [$expectedValue], as is required.";;
            (Contains) cfgreqpass "Value for configurable [$Configurable] contains required text [$expectedValue].";;
            (Exact) cfgreqpass "Value for configurable [$Configurable] matches required value [$expectedValue] exactly.";;
            (Set) cfgreqpass "A value for configurable [$Configurable] is set, as required. Value is [$currentValue].";;
            (Unset) cfgreqpass "The configurable [$Configurable] is unset (i.e. not explicitly set), exactly as is required. Default value is [$currentValue].";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgreqfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is required."
               # The 'security' configurable requires special handling. Except for public
               # servers (where security may be 0), this will be configured as an 'AtLeast'
               # comparison.
               if [[ "$Configurable" == security ]]; then
                  if [[ "$currentValue" -eq 3 ]]; then
                     EasyFixConfigurables[EasyFixCount]="$Configurable"
                     EasyFixValues[EasyFixCount]="$expectedValue"
                     EasyFixCount+=1
                  else # If security is 0-2, it may be more impactful to set.
                     SetNotes[$Configurable]="https://help.perforce.com/helix-core/server-apps/p4sag/current/Content/P4SAG/security-levels.html"
                     PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                     PlannedFixAdvice[PlannedFixCount]="$P4 -s configure set $Configurable=\"$expectedValue\""
                     PlannedFixAutomated[PlannedFixCount]=1
                     PlannedFixCount+=1
                  fi
               elif [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (NoMoreThan)
               cfgreqfail "Value for configurable [$Configurable] is [$currentValue]; it is required to be no more than [$expectedValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure set $Configurable=$expectedValue"
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (Contains)
               cfgreqfail "Value for configurable [$Configurable] does not contain required text [$expectedValue]."
               PlannedFixConfigurables[PlannedFixCount]="$Configurable"
               PlannedFixAdvice[PlannedFixCount]="Set the configurable '$Configurable' to a value that contains '$expectedValue'."
               PlannedFixAutomated[PlannedFixCount]=0
               PlannedFixCount+=1
            ;;
            (Exact)
               cfgreqfail "Value for configurable [$Configurable] does not exactly match required value [$expectedValue]; value is: [$currentValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (Set)
               cfgreqfail "A value for configurable [$Configurable] is not set; it is required to have a value."
               if [[ "$Configurable" == auth.id ]]; then
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure set $Configurable=\"p4_${P4MASTER_ID}\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               else
                  if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                     EasyFixConfigurables[EasyFixCount]="$Configurable"
                     EasyFixValues[EasyFixCount]="$expectedValue"
                     EasyFixCount+=1
                  else
                     PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                     PlannedFixAdvice[PlannedFixCount]="The configurable '$Configurable' should be set so it has a value."
                     PlannedFixAutomated[PlannedFixCount]=0
                     PlannedFixCount+=1
                  fi
               fi
            ;;
            (Unset)
               cfgreqfail "The configurable [$Configurable] is required to be unset (i.e. not explicitly set), but it is explicitly set to [$currentValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="UNSET"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure unset $Configurable"
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
         esac
      fi
   elif [[ "${Optionality[$Configurable]}" == "Recommended" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgrecpass "Value for configurable [$Configurable] is [$currentValue], which is at least [$expectedValue], as is recommended.";;
            (NoMoreThan) cfgrecpass "Value for configurable [$Configurable] is [$currentValue], which is no more than [$expectedValue], as is recommended.";;
            (Contains) cfgrecpass "Value for configurable [$Configurable] contains recommended text [$expectedValue].";;
            (Exact) cfgrecpass "Value for configurable [$Configurable] matches recommended value [$expectedValue] exactly.";;
            (Set) cfgrecpass "A value for configurable [$Configurable] is set, as recommended.";;
            (Unset) cfgrecpass "The configurable [$Configurable] is unset (i.e. not explicitly set), exactly as is recommended. Default value is [$currentValue].";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgrecfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is recommended."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (NoMoreThan)
               cfgrecfail "Value for configurable [$Configurable] is [$currentValue]; it is recommended to be no more than [$expectedValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (Contains)
               cfgrecfail "Value for configurable [$Configurable] does not contain recommended text [$expectedValue]."
               PlannedFixConfigurables[PlannedFixCount]="$Configurable"
               PlannedFixAdvice[PlannedFixCount]="TO DO: Set a value for '$Configurable' that contains '$expectedValue'."
               PlannedFixAutomated[PlannedFixCount]=0
               PlannedFixCount+=1
            ;;
            (Exact)
               cfgrecfail "Value for configurable [$Configurable] does not exactly match recommended value [$expectedValue]; value is: [$currentValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (Set)
               cfgrecfail "A value for configurable [$Configurable] is not set; setting a value is recommended."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="$expectedValue"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure set $Configurable=\"$expectedValue\""
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
            (Unset)
               cfgrecfail "The configurable [$Configurable] is recommended to be unset (i.e. not explicitly set), but it is explicitly set to [$currentValue]."
               if [[ "${SetNotes[$Configurable]:-None}" == None ]]; then
                  EasyFixConfigurables[EasyFixCount]="$Configurable"
                  EasyFixValues[EasyFixCount]="UNSET"
                  EasyFixCount+=1
               else
                  PlannedFixConfigurables[PlannedFixCount]="$Configurable"
                  PlannedFixAdvice[PlannedFixCount]="$P4 -s configure unset $Configurable"
                  PlannedFixAutomated[PlannedFixCount]=1
                  PlannedFixCount+=1
               fi
            ;;
         esac
      fi
   elif [[ "${Optionality[$Configurable]}" == "Optional" ]]; then
      if [[ "$ConfigMatches" -eq 1 ]]; then
         case "$compareStyle" in
            (AtLeast) cfgrecpass "Value for configurable [$Configurable] is [$currentValue], which is at least [$expectedValue], as is suggested.";;
            (NoMoreThan) cfgrecpass "Value for configurable [$Configurable] is [$currentValue], which is no more than [$expectedValue], as is suggested.";;
            (Contains) cfgrecpass "Value for configurable [$Configurable] contains suggested text [$expectedValue].";;
            (Exact) cfgrecpass "Value for configurable [$Configurable] matches suggested value [$expectedValue] exactly.";;
            (Set) cfgrecpass "A value for configurable [$Configurable] is set, as suggested.";;
            (Unset) cfgrecpass "The configurable [$Configurable] is unset (i.e. not explicitly set), exactly as is suggested. Default value is [$currentValue].";;
         esac
      else
         case "$compareStyle" in
            (AtLeast)
               cfgoptfail "Value for configurable [$Configurable] is [$currentValue], which is not at least [$expectedValue], as is suggested."
            ;;
            (NoMoreThan)
               cfgoptfail "Value for configurable [$Configurable] is [$currentValue]; which is suggested to be no more than [$expectedValue]."
            ;;
            (Contains)
               cfgoptfail "Value for configurable [$Configurable] does not contain suggested text [$expectedValue]."
            ;;
            (Exact)
               cfgoptfail "Value for configurable [$Configurable] does not exactly match suggested value [$expectedValue]; value is: [$currentValue]."
            ;;
            (Set)
               cfgoptfail "A value for configurable [$Configurable] is not set; setting a value is suggested."
            ;;
            (Unset)
               cfgoptfail "The configurable [$Configurable] is suggested to be unset (i.e. not explicitly set), but it is explicitly set to [$currentValue]."
            ;;
         esac
      fi
   else
      errmsg "Unknown optionality [${Optionality[$Configurable]}]; should be 'Required' or 'Recommended'."
   fi
done

#------------------------------------------------------------------------------
# Do some special extra checks that are not dependent on the config file.
if [[ "$DoSecurityCheck" -eq 0 ]]; then
   # Check that client.readonly.dir is not an absolute path.
   Configurable=client.readonly.dir
   ConfigCount+=1
   if [[ "${CurrentValue[$Configurable]:-}" == /* ]]; then
      cfgreqfail "Value for configurable $Configurable is [${CurrentValue[$Configurable]:-}]. This references an absolute path, but is required not to be an absolute path."
      PlannedFixConfigurables[PlannedFixCount]="$Configurable"
      PlannedFixAdvice[PlannedFixCount]="TO DO: Set $Configurable to \"${ExpectedValue[$Configurable]:-}\" in coordination with an rsync of data to that folder under P4ROOT."
      PlannedFixAutomated[PlannedFixCount]=0
      PlannedFixCount+=1
   else
      cfgreqpass "The client.readonly.dir value is not an absolute path (and it should not be)."
   fi

   # Check that client.sendq.dir is set to the same as client.readonly.dir.
   Configurable=client.sendq.dir
   ConfigCount+=1
   if [[ -n "${CurrentValue[client.readonly.dir]:-}" && "${CurrentValue[client.sendq.dir]:-}" != "${CurrentValue[client.readonly.dir]:-}" ]]; then
      cfgrecfail "Value for configurable client.sendq.dir does not match client.readonly.dir. It is recommended to match."
      PlannedFixConfigurables[PlannedFixCount]="$Configurable"
      PlannedFixAdvice[PlannedFixCount]="TO DO: Set $Configurable to \"${ExpectedValue[client.readonly.dir]:-}\" in coordination with a service restart."
      PlannedFixAutomated[PlannedFixCount]=0
      PlannedFixCount+=1
   else
      cfgrecpass "The client.sendq.dir has the same value as client.readonly.dir, as is recommended."
   fi
fi

#------------------------------------------------------------------------------
# Display the summary.
msg "\\nSummary:
   Configs Checked:               $ConfigCount
   Configs GOOD/GREAT:            $ConfigPassCount
   Configs BAD/WARN (Total):      $ConfigFailCount
      Configs BAD (Required):       $ConfigFailRequiredCount
      Configs WARN (Recommended):   $ConfigFailRecommendedCount

   Configs INFO:                  $ConfigFailOptionalCount (not counted for grading)."
if [[ "$ErrorCount" -ne 0 ]]; then
   msg "\\n   ERRORS:                        $ErrorCount (The existence of any errors at all indicates an inability to fully check configurable values)."
fi

msgn "\\nResult: "
ExitCode=$((ErrorCount+ConfigFailRequiredCount))

if [[ "$ConfigCount" -ne 0 ]]; then
   if [[ "$ConfigFailCount" -eq 0 ]]; then
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg_green "PASS with Grade A: All $ConfigCount checked configurables pass, and there are no errors."
      else
         msg_yellow "FAIL: There were $ErrorCount errors attempting to check configurables.  All $ConfigCount checked configurables pass."
      fi
   elif [[ "$ConfigFailRequiredCount" -eq 0 ]]; then
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg_green "PASS with Grade B: Of $ConfigCount configurables checked, all required values pass, but $ConfigFailRecommendedCount recommended settings were not optimal."
      else
         msg_yellow "FAIL: There were $ErrorCount errors attempting to check configurables.  Of $ConfigCount configurables checked, all required values pass, but $ConfigFailRecommendedCount recommended settings were not optimal."
      fi
   else
      if [[ "$ErrorCount" -eq 0 ]]; then
         msg_yellow "FAIL: Of $ConfigCount configurables, $ConfigFailRequiredCount required settings were not optimal, and $ConfigFailRecommendedCount recommended settings were not optimal."
      else
         msg_yellow "FAIL: There were $ErrorCount errors attempting to check configurables. Of $ConfigCount configurables, $ConfigFailRequiredCount required settings were not optimal, and $ConfigFailRecommendedCount recommended settings were not optimal."
      fi
   fi
else
   errmsg "No configs were checked. Something went wrong."
fi

# If '-fix' or '-FIX' were specified and there were hard errors, abort immediately.
if [[ "$ErrorCount" -ne 0 ]]; then
   if [[ "$ApplyAllFixes" -eq 1 ]]; then
      bail "Aborting before attempting fixes requested with '-FIX' due to errors attempting to determine configuration status."
   elif [[ "$ApplyEasyFixes" -eq 1 ]]; then
      bail "Aborting before attempting fixes requested with '-fix' due to errors attempting to determine configuration status."
   fi
fi

if [[ "$ApplyEasyFixes" -eq 1 ]]; then
   msg "${H2}\\nApplying fixes that can be applied non-disruptively."

   if [[ "$EasyFixCount" -gt 0 ]]; then
      for ((i=0; i < ${#EasyFixConfigurables[@]}; i++)); do
         Configurable="${EasyFixConfigurables[i]}"
         if [[ "${EasyFixValues[i]}" != UNSET ]]; then
            if [[ "$Configurable" =~ ^filesys.(depot|P4JOURNAL|P4LOG|P4ROOT|TEMP).min$ ]]; then
               safety_check_for_filesys_min "$Configurable" "${EasyFixValues[i]}" || continue
            fi
            if run "$P4 -s configure set $Configurable=${EasyFixValues[i]}"; then
               ConfigFixedCount+=1
            else
               errmsg "Error setting '$Configurable' to '${EasyFixValues[i]}'."
            fi
         else
            if run "$P4 -s configure unset $Configurable"; then
               ConfigFixedCount+=1
            else
               errmsg "Error unsetting '$Configurable'."
            fi
         fi
            [[ "$Configurable" == security ]] && \
               warnmsg "Because security was just set to 4, if paths on this p4d server are referred to using remote depots from other p4d servers, the remote depots on those other servers will cease to function. To fix, establish a serviceUser to be used for remote depots, e.g. svc_remote, and enable that user in Protections with read access coming from the IP of other p4d server machines to required paths on this server."
      done
   else
      msg "There are no fixes that can be applied non-disruptively."
   fi

   if [[ "$ApplyAllFixes" -eq 1 ]]; then
      msg "${H2}\\nApplying fixes that should be planned to avoid potential disruption.\\n"

      if [[ "$EnableSSLAdvised" -eq 1 ]]; then
         PlannedFixConfigurables[PlannedFixCount]="P4PORT"
         PlannedFixAdvice[PlannedFixCount]="TO DO: The SSL feature should be enabled for all P4PORT values on servers and clients in the topology."
         PlannedFixAutomated[PlannedFixCount]=0
         PlannedFixCount+=1
      fi

      if [[ "$PlannedFixCount" -gt 0 ]]; then
         for ((i=0; i < ${#PlannedFixConfigurables[@]}; i++)); do
            Configurable="${PlannedFixConfigurables[i]}"
            if [[ "${SetNotes[$Configurable]:-None}" != None ]]; then
               msg "Due to -FIX, setting '$Configurable'. Guidance: ${SetNotes[$Configurable]}"
            else
               msg "Due to -FIX, setting '$Configurable'"
            fi

            if [[ "${PlannedFixAutomated[i]}" == 1 ]]; then
               # Last-minute safety check for setting filesys.*.min settings.
               if [[ "$Configurable" =~ ^filesys.(depot|P4JOURNAL|P4LOG|P4ROOT|TEMP).min$ ]]; then
                  safety_check_for_filesys_min "$Configurable" "${ExpectedValue[$Configurable]}" || continue
               fi

               if run "${PlannedFixAdvice[i]}"; then
                  ConfigFixedCount+=1
                  # Limit disruption caused by setting auth.id.
                  if [[ "$Configurable" == auth.id ]]; then
                     run "p4login -v -L off" "Because auth.id was set, calling p4login on host $ThisHost as: p4login -v -L off." ||\
                        errmsg "Error calling: p4login -v -L off"
                     run "p4login -v -service -v -L off" "Because auth.id was set, calling p4login on host $ThisHost as: p4login -v -service -L off." ||\
                        errmsg "Error calling: p4login -v -service -L off"
                     run "p4login -v -automation -v -L off" "Because auth.id was set, calling p4login on host $ThisHost as: p4login -v -automation -L off." ||\
                        errmsg "Error calling: p4login -v -automation -L off"

                     warnmsg "Because auth.id was set, run 'p4login -v', 'p4login -v -service', and 'p4login -v -automation' on all other p4d server machines associated with SDP instance $SDPInstance."
                  fi

                  [[ "$Configurable" == security ]] && \
                     warnmsg "Because security was just set to 4, if paths on this p4d server are referred to using remote depots from other p4d servers, the remote depots on those other servers will cease to function. To fix, establish a serviceUser to be used for remote depots, e.g. svc_remote, and enable that user in Protections with read access coming from the IP of other p4d server machines to required paths on this server."
               else
                  errmsg "Error implementing best practice for ${PlannedFixConfigurables[i]}."
               fi
            else
               msg "${PlannedFixAdvice[i]}"
            fi
         done
      else
         msg "There are no fixes to apply that would be non-disruptive."
      fi
   else
      msg "${H2}\\nDisplaying fixes that should be planned to avoid potential disruption.\\n"

      if [[ "$PlannedFixCount" -gt 0 ]]; then
         for ((i=0; i < ${#PlannedFixConfigurables[@]}; i++)); do
            ConfigFixedCount+=1
            Configurable="${PlannedFixConfigurables[i]}"
            [[ "${SetNotes[$Configurable]:-None}" != None ]] && \
               msg "Before setting '$Configurable', be aware: ${SetNotes[$Configurable]}"

            if [[ "${PlannedFixAutomated[i]}" == 1 ]]; then
               # Prepend 'TO DO:' to commands that could be automated with '-FIX'.
               msg "TO DO: ${PlannedFixAdvice[i]}\\n"
               [[ "$Configurable" == auth.id ]] && \
                  warnmsg "When auth.id is set, be prepared to run 'p4login -v', 'p4login -v -service', and 'p4login -v -automation' on all other p4d server machines associated with SDP instance $SDPInstance. This script will execute these commands when auth.id is set on the current machine, $ThisHost."
               [[ "$Configurable" == security ]] && \
                  warnmsg "When security is set to 4, if paths on this p4d server are referred to using remote depots from other p4d servers, the remote depots on those other servers will cease to function. To fix, establish a serviceUser to be used for remote depots, e.g. svc_remote, and enable that user in Protections with read access coming from the IP of other p4d server machines to required paths on this server."
            else
               # The non-automated messages are already prefixed with 'TO DO:'.
               msg "${PlannedFixAdvice[i]}\\n"
            fi
         done
      else
         msg "There are no fixes to apply that would be non-disruptive."
      fi
   fi
fi

#------------------------------------------------------------------------------
# FOR FUTURE ENHANCEMENT
# These '-n' checks are meaningless; they are here to suppress specific
# ShellCheck errors about unused variables.  These variables are intended
# for future use in a version that supports modifying configurables.
[[ -n "${ServerIDType[server]:-}" ]] && true

#------------------------------------------------------------------------------
# See the terminate() function where this script really exits.
exit "$ExitCode"