verify_sdp.sh #70

  • //
  • guest/
  • perforce_software/
  • sdp/
  • dev/
  • Server/
  • Unix/
  • p4/
  • common/
  • bin/
  • verify_sdp.sh
  • View
  • Commits
  • Open Download .zip Download (84 KB)
#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

# verify_sdp.sh
# Verifies SDP structure and environment.

#==============================================================================
# Declarations and Environment
set -u

declare Version=5.34.0
declare ThisScript="${0##*/}"
declare ThisHost=${HOSTNAME%%.*}
declare SDPInstallRoot="/p4"
declare SDPCommon="$SDPInstallRoot/common"
declare SDPCommonBin="$SDPCommon/bin"
declare SDPCommonCfg="$SDPCommon/config"
declare SDPEnvFile="$SDPCommonBin/p4_vars"
declare InstanceBinDir=
declare SDPInstance=
declare SDPOwner=
declare CCheckProfile=
declare CCheckCmd=
declare CmdArgs="$*"
declare CmdLine="$0 $CmdArgs"
declare SDPVersionA=
declare SDPVersionB=
declare SDPVersionC=
declare -i ServerOnline=0
declare -i ErrorCount=0
declare -i WarningCount=0
declare -i CheckCount=0
declare -i SilentMode=0
declare -i Debug=${SDP_DEBUG:-0}
declare -i ShowSkippingIndicators=1
declare -i ShowErrorSummary=1

# Tests that can be skipped with '-skip', converted to warnings with '-warn',
# added with '-extra', or specified with '-only'.
declare -i DoCrontabTest=1
declare -i DoCrontabTestWarn=0
declare -i DoLicenseTest=1
declare -i DoLicenseTestWarn=0
declare -i DoVersionTest=1
declare -i DoVersionTestWarn=0
declare -i DoExcessBinaryTest=1
declare -i DoExcessBinaryTestWarn=0
declare -i DoInitCompareTest=1
declare -i DoInitCompareTestWarn=0
declare -i DoCommitIDTest=1
declare -i DoCommitIDTestWarn=0
declare -i DoOfflineDBTest=1
declare -i DoOfflineDBTestWarn=0
declare -i DoOwnerChecks=1
declare -i DoOwnerChecksWarn=0
declare -i DoP4ROOTTest=1
declare -i DoP4ROOTTestWarn=0
declare -i DoPasswordChecks=1
declare -i DoPasswordChecksWarn=0
declare -i DoP4TFilesTest=1
declare -i DoP4TFilesTestWarn=0
declare -i DoConfigurablesCheck=0
declare -i DoSecurityConfigurablesCheck=0
declare -i DoCommitDefinedTest=0
declare -i DoServerTypeKnownTest=0
declare -i DoSystemdConfigTest=0
declare -i DoRemoteDisabledTest=0
declare -i DoProtectionsTest=0
declare SecuritySetting=
declare CleartextPasswordFile=
declare EncryptedPasswordFile=

declare -i ExcessServerBinariesFound=0
declare -i P4DServer=0
declare -i P4BrokerServer=0
declare -i P4ProxyServer=0
declare -i RunIfCount=0
declare -i SystemdServicesCheckedCount=0
declare ExtraTestList=
declare SkipTestList=
declare WarnTestList=
declare OnlyTestList=
declare LinkTarget=
declare ExpectedTarget=
declare ThisUser=
# Minimum ticket duration is 31 days.
declare MinTicketExpiration=$((31*60*60*24))
declare TicketExpiration=
declare LicenseInfo=
declare LicenseExpiration=
declare CurrentTime=
declare ExpirationTime=
declare TimeDiff=
declare DaysDiff=
declare LicenseDaysExpirationAlert=21
declare LinkP4ROOT=
declare LinkOfflineDB=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare BadLog=
declare Log="Unset"
export P4TMP="Unset"
declare TmpFile=
declare TmpFile2=
declare P4DInitTemplate="$SDPInstallRoot/common/etc/init.d/p4d_instance_init.template"
declare P4BrokerInitTemplate="$SDPInstallRoot/common/etc/init.d/p4broker_instance_init.template"
declare P4ProxyInitTemplate="$SDPInstallRoot/common/etc/init.d/p4p_instance_init.template"
declare P4DServiceTemplate="$SDPInstallRoot/common/etc/systemd/system/p4d_N.service.t"
declare P4BrokerServiceTemplate="$SDPInstallRoot/common/etc/systemd/system/p4broker_N.service.t"
declare P4ProxyServiceTemplate="$SDPInstallRoot/common/etc/systemd/system/p4p_N.service.t"

# The following variables are exported and assigned in set_vars() in backup_functions.sh.
declare P4DInitScript=
declare P4DRef=

declare P4DSystemdServiceFile=
declare P4BrokerInitScript=
declare P4BrokerRef=
declare P4ProxyInitScript=
declare P4ProxyRef=
declare P4BrokerSystemdServiceFile=
declare P4ProxySystemdServiceFile=

#==============================================================================
# Local Functions

# Note: This script does not use SDP library files, as its purpose is to
# verify the integrity of an SDP installation.  Thus, it has its own
# self-contained versions of some functions for which similar versions
# would normally be sourced from files in /p4/common/lib, like libcore.sh.

function msg () { echo -e "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# This function takes as input an SDP version string, and returns a version
# id of the form YYYY.N.CL, where YYYY is the year, N is an incrementing
# release ID with a given year, and CL is a changelist identifier. The
# YYYY.N together comprise the major version, often shortened to YY.N, e.g.
# r20.1 for the 2020.1 release.
#
# The full SDP Version string looks something like this:
# Rev. SDP/MultiArch/2019.3/26494 (2020/04/23).
#
# This function parses that full string and returns a value like: 2019.3.26494
function get_sdp_version_from_string () {
   local versionString="${1:-}"
   local version=
   version="20${versionString##*/20}"
   version="${version%% *}"
   version="${version/\//.}"

   [[ "$version" == "20" || "$version" == "200" ]] && version="Unknown"
   echo "$version"
}

#------------------------------------------------------------------------------
# Function: run ($cmd, $desc, $showOutput)
#
# Runs a command, with optional description, showing command line to execute
# and optionally also the output, and capturing and returning the exit code.
#
# Input:
# $1 - Command and arguments to execute. Defaults to 'echo'.
# $2 - Optional message to display describing what the command is doing.
# $3 - Numeric flag to show output; '1' indicates to show output, 0 to
#      suppress it.
#------------------------------------------------------------------------------
function run () {
   local cmd="${1:-echo}"
   local desc="${2:-}"
   local -i showOutput="${3:-1}"
   local -i exitCode=
   local log

   log="$(mktemp "$P4TMP/run.XXXXXXXXXXX")"

   [[ -n "$desc" ]] && msg "$desc"
   msg "Executing: $cmd"
   $cmd > "$log" 2>&1
   exitCode=$?

   if [[ "$showOutput" -eq 1 ]]; then
      echo "EXIT_CODE: $exitCode" >> "$log"
      cat "$log"
   fi

   /bin/rm -f "$log"
   return "$exitCode"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
   fi

   echo "USAGE for verify_sdp.sh v$Version:

verify_sdp.sh [<instance>] [-online] [{-skip,-warn,-extra,-only} <test>[,<test2>,...]] [-skip_summary] [-c|-csec] [-si] [-L <log>|off ] [-d|-D]

   or

verify_sdp.sh -h|-man
"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:

	This script verifies the current SDP setup for the specified instance,
	and also performs basic health checks of configured servers.

	This uses the SDP instance bin directory /p4/N/bin to determine
	what server binaries (p4d, p4broker, p4p) are expected to be configured
	on this machine.

	Existence of the '*_init' script indicates the given binary is
	expected. For example, for instance 1, if /p4/1/bin/p4d_1_init
	exists, a p4d server is expected to run on this machine.

	Checks may be executed or skipped depending on what servers are
	configured. For example, if a p4d is configured, the \$P4ROOT/server.id
	file should exist. If p4p is configured, the 'cache' directory
	should exist.

OPTIONS:
 <instance>
	Specify the SDP instances.  If not specified, the SDP_INSTANCE
	environment variable is used instead.  If the instance is not
	defined by a parameter and SDP_INSTANCE is not defined,
	exits immediately with an error message.

 -online
	Online mode.  Does additional checks that expect p4d, p4broker,
	and/or p4p to be online. Any servers for which there are
	*_init scripts in the Instance Bin directory are checked.  An
	error is reported if p4d is expected to be online and is not;
	warnings are displayed if p4broker or p4p are not online.
	The Instance Bin directory is the /p4/N/bin directory, where N
	is the SDP instance name.

 -c	Specify '-c' to call ccheck.sh to compare configurables, using
	the default config file: $P4CCFG/configurables.cfg

	See 'ccheck.sh -man' for more information.

	This option can only be used in Online mode; if '-c' is specified,
	'-online' is implied.

 -csec
	Specify '-csec' to call ccheck.sh checking only for security
	settings.

	This option can only be used in Online mode; if '-csec' is
	specified, '-online' is implied.

 -skip <test>[,<test2>,...]

	Specify a comma-delimited list of named tests to skip.
	
	Valid test names are:

	* cron|crontab: Skip crontab check. Use this if you do not expect crontab to
	be configured, perhaps if you use a different scheduler.
	* excess: Skip checks for excess copies of p4d/p4p/p4broker in PATH.
	* init: Skip compare of init scripts w/templates in /p4/common/etc/init.d
	* license: Skip license related checks.
	* commitid: Skip check ensuring ServerID of commit starts with 'commit' or 'master'.
	* masterid: Synonym for commitid.
	* offline_db: Skip checks that require a healthy offline_db.
	* owner: Skip checks that require ownership of specific files/folders.
	* p4root: Skip checks that require healthy P4ROOT db files.
	* p4t_files: Skip checks for existence of P4TICKETS and P4TRUST files.
	* passwd|password: Skip SDP password checks.
	* version: Skip version checks.

	As an alternative to using the '-skip' option, the shell environment
	variable VERIFY_SDP_SKIP_TEST_LIST can be set to a comma-separated
	list of named tests to skip.  Using the command line parameter is the
	best choice for temporarily skipping tests, while specifying the
	environment variable is better for making permanent exceptions (e.g.
	always excluding the crontab check if crontabs are not used at this
	site).  The variable should be set in $SDPCommonCfg/p4_N.vars.

	If the '-skip' option is provided, the VERIFY_SDP_SKIP_TEST_LIST
	variable is ignored (not appended to). So it may make sense to
	reference the variable on the command line. For example, if the
	value of the variable is 'crontab', to skip crontab and license
	checks, you could specify:

	-skip \$VERIFY_SDP_SKIP_TEST_LIST,license

	The '-skip' option can be used with '-warn' and '-extra', but is
	mutually exclusive with '-only'.

 -warn <test>[,<test2>,...]
	Specify a comma-delimited list of named tests that will be reported
	as warnings rather than errors.

	The list of valid test names as the same as for the '-skip' option.

	As an alternative to using the '-warn' option, the shell environment
	variable VERIFY_SDP_WARN_TEST_LIST can be set to a comma-separated
	list of name tests to skip.  Using the command line parameter is the
	best choice for temporarily converting errors to warnings, while
	specifying the environment variable is better for making the
	conversion to warnings permanent.  The variable should be set in
	$SDPCommonCfg/p4_N.vars file.

	If the '-warn' option is provided, the VERIFY_SDP_WARN_TEST_LIST
	variable is ignored (not appended to). So it may make sense to
	reference the variable on the command line. For example, if the
	value of the variable is 'crontab', to convert to warnings for
	crontab and excess binaries tests, you could specify:

	-warn \$VERIFY_SDP_WARN_TEST_LIST,excess

	The '-warn' option can be used with '-skip' and '-extra', but is
	mutually exclusive with '-only'.

 -extra <test>[,<test2>,...]

	Some tests are not executed by default, but are instead invoked
	only on request with the '-extra' option.  The following tests
	are executed if specified with the '-extra' option:

	* commit_defined
	* protections
	* remote_disabled
	* server_type_known
	* systemd_config
	* all (short for all of the above)

	commit_defined: Do a test to check defined server specs, and
	ensure that exactly one server spec has a 'Services' field value
	of 'commit-server', and that no server specs are defined with a
	'Services' field value of 'standard' (the obsolete predecessor to
	'commit-server').  This test requires p4d to be online and thus
	implies '-online'.

	protections: Do sanity check on the Protections table:
	* Ensure the last line of the Protections table starts with:
	'super user <P4USER>'.

	remote_disabled: This test ensures the legacy built-in user'remote'
	is disabled.  Any one of the following must be true to pass this
	check:
	* P4D version is 2025.1+
	* security=4
	* A 'p4 protects -u remote' lists nothing but exclusionary mappings.

	server_type_known: Do a test to confirm that exactly one of
	run_if_master.sh, run_if_replicas.sh, and run_if_edge.sh returns
	true. If 0 or more than 1 are true, report that as an error.

	systemd_config: Check the systemd service configuration. This
	checks that:
	* Appropriate system services (p4d/p4broker/p4p) are defined
	corresponding init scripts existing in the Instance Bin directory,
	/p4/N/bin.
	* The p4d_N service, if defined, matches the template.
	* The p4broker_N service, if defined, matches the template.
	* The p4p_N service, if defined, matches the template.

	The '-extra' option can be used with '-skip' and '-warn', but is
	mutually exclusive with '-only'.

 -only <test>[,<test2>,...]
	Use the '-only' option to execute only specified tests.  If this
	option is used, even tests that cannot be skipped with '-skip'
	and thus are usually always executed are not executed.  This
	option is primarily intended to support testing of verify_sdp.sh.

	Only a limited set of tests can be specified with '-only', including:

	* crontab
        * commit_defined
	* commitid
	* excess
        * protections
        * remote_disabled
        * server_type_known
        * systemd_config
	* version

	The '-only' option is mutually exclusive with '-skip', '-warn',
	and '-extra'.

 -skip_summary
	By default, if any errors or warnings are displayed in the output,
	a summary of those errors appears at the end.  Specify this
	option to avoid displaying the summary at the end.

 -si	Silent mode, useful for cron operation.  Both stdout and stderr
	are still captured in the log.  The '-si' option cannot be used
	with '-L off'.

 -L <log>
	Specify the log file to use.  The default is /p4/N/logs/verify_sdp.log
	The special value 'off' disables logging to a file.

	Note that '-L off' and '-si' are mutually exclusive.

 -d	Enabled debug messages.

 -D	Set extreme debugging verbosity using bash 'set -x' mode. Implies -d.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	Example 1: Typical usage:

	This script is typically called after SDP update with only the instance
	name or number as an argument, e.g.:

	verify_sdp.sh 1

	Example 2: Skipping some checks.

	verify_sdp.sh 1 -skip crontab

	Example 3: Automation Usage

	If used from automation that is already doing its own logging, use -L off:

	verify_sdp.sh 1 -L off

	Example 4: Include online checks

	This examples relies on the \$SDP_INSTANCE variable rather than passing
	the <instance> parameter.

	verify_sdp.sh -online

	Example 5: Thorough Usage

	Run all available tests (-online is implied):

	verify_sdp.sh -extra all

LOGGING:
	This script generates a log file and also displays it to stdout at the
	end of processing.  By default, the log is:
	/p4/N/logs/verify_sdp.log.

	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.

	If the '-si' (silent) flag is used, the log is generated, but its
	contents are not displayed to stdout at the end of processing.

EXIT CODES:
	An exit code of 0 indicates no errors were encountered attempting to
	perform verifications, and that all checks verified cleanly.
"

   fi

   exit 2
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   #shellcheck disable=SC2317
   trap - EXIT SIGINT SIGTERM

   #shellcheck disable=SC2317
   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log\\n${H1}\\n"

   # With the trap removed, exit.
   #shellcheck disable=SC2317
   exit "$ErrorCount"
}

#------------------------------------------------------------------------------
# Function: do_preflight_checks ($instance)
#
# If preflight checks fail, further tests are aborted. Failure of the very
# basic preflight checks is an indication that the SDP structure is in
# need of repair.
#
# Sample Usage:
# do_preflight_checks "$SDPInstance" ||\
#    bail "Preflight checks failed. Aborting further checks."
#------------------------------------------------------------------------------
function do_preflight_checks () {

   local instance="${1:-}"
   local toolsList="awk date file grep head id ls sort tail tee which"

   msg "$H2\\nDoing preflight sanity checks."
   msg "Preflight Check: Ensuring these utils are in PATH: $toolsList"

   for tool in $toolsList; do
      CheckCount+=1
      [[ -z "$(command -v "$tool")" ]] && \
         errmsg "Tool '$tool' not in PATH."
   done

   [[ $ErrorCount -eq 0 ]] || return 1

   msg "Verified: Essential tools are in the PATH."

   msg "Preflight Check: cd $SDPCommonBin"

   CheckCount+=1
   if cd "$SDPCommonBin"; then
      cd "$OLDPWD" || bail "Failed to cd to $OLDPWD. Aborting."
   else
      errmsg "Could not cd to: $SDPCommonBin"
      return 1
   fi

   msg "Verified: cd works to: $SDPCommonBin"

   msg "Preflight Check: Checking current user owns $SDPCommonBin"

   # shellcheck disable=SC2012
   SDPOwner="$(stat --format=%U "$SDPCommonBin")"
   ThisUser="$(id -n -u)"

   CheckCount+=1
   if [[ "$ThisUser" == "$SDPOwner" ]]; then
      msg "Verified: Current user [$ThisUser] owns $SDPCommonBin"
   else
      errmsg "Current user [$ThisUser] does not own $SDPCommonBin. This most likely means this script is running as the wrong user.  It could also mean the $SDPCommonBin directory is not owned by the correct owner, which should be the OS account under which the p4d process runs."
      return 1
   fi

   msg "Preflight Check: Checking /p4 and /p4/<instance> are local dirs."
   if ! check_local_instance_home_dir "$instance"; then
      errmsg "The SDP /p4 and /p4/<instance> dirs are NOT local."
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: check_file ($file, $errMsg, $warningOnly)
#
# Checks for existence of a file. Returns 0 if it exists, 1 otherwise.
#
# Inputs:
# $1 - $file path to check. Required.
# $2 - Optional error message to display if file is missing.
#      Default: "Missing file [$file]."
# $3 - $warningOnly. If 0, an error is displayed if the file does not exist.
#      If 1, a warning is displayed instead of an error.  Default is 0.
#
# Allows optional custom error message describing the file, to be displayed if
# the file is missing.  Default error message is "Missing file [FILE]."
#------------------------------------------------------------------------------
function check_file () {
   local file=$1
   local errMsg=${2:-Missing file}
   local -i warningOnly=${3:-0}
   CheckCount+=1
   msg "Checking existence of file $file"
   if [[ "$warningOnly" -eq 0 ]]; then
      [[ -f "$file" ]] && return 0
      errmsg "$errMsg: [$file]."
   else
      [[ -f "$file" ]] && return 0
      warnmsg "$errMsg: [$file]."
   fi
   return 1
}

#------------------------------------------------------------------------------
# Function: check_file_x ($file, $errMsg, $warningOnly)
#
# Checks existence of a file with executable bit set. Returns 0 if it exists
# and is executable, 1 otherwise.
#
# Inputs:
# $1 - $file path to check. Required.
# $2 - Optional error message to display if file is missing.
#      Default: "Missing not executable [$file]."
# $3 - $warningOnly. If 0, an error is displayed if the file does not exist.
#      If 1, a warning is displayed instead of an error.  Default is 0.
#
# Allows optional custom error message describing the file, to be displayed if
# the file is missing.  Default error message is "File not executable [FILE]."
#------------------------------------------------------------------------------
function check_file_x () {
   local file=$1
   local errMsg=${2:-File not executable}
   local -i warningOnly=${3:-0}
   CheckCount+=1

   msg "Checking executable file $file"
   [[ -x "$file" ]] && return 0

   if [[ "$warningOnly" -eq 0 ]]; then
      errmsg "$errMsg: [$file]."
   else
      warnmsg "$errMsg: [$file]."
   fi

   return 1
}

#------------------------------------------------------------------------------
# Function: check_file_dne ($file, $errMsg, $warningOnly)
#
# Confirm that a specific file does not exist, e.g. a semaphore file.  If the
# specified files does not exist, return 0, or 1 if it exists. This is the
# opposite of check_file().
#
# Inputs:
# $1 - $file path to check. Required.
# $2 - Optional error message to display if file exists.
#      Default: "This file should not exist: [$file]."
# $3 - $warningOnly. If 0, an error is displayed if the file does not exist.
#      If 1, a warning is displayed instead of an error.  Default is 0.
#
#
# Allows optional custom error message describing the file, to be displayed if
# the file is found.  Default error message is "This file should not exist: [FILE]."
#------------------------------------------------------------------------------
function check_file_dne () {
   local file=$1
   local errMsg=${2:-This file should not exist}
   local -i warningOnly=${3:-0}
   CheckCount+=1

   msg "Confirming this file does not exist: $file"
   [[ ! -f "$file" ]] && return 0

   if [[ "$warningOnly" -eq 0 ]]; then
      errmsg "$errMsg: [$file]."
   else
      warnmsg "$errMsg: [$file]."
   fi

   return 1
}

#------------------------------------------------------------------------------
# Function: check_file_is_shell_script ($file)
#
# Confirm that a specific file exists, and is a regular file (not a symlink),
# and is a shell script rather than a binary.
#
# Inputs:
# $1 - $file path to check. Required.
# $2 - Optional error message to display if file exists.
#      Default: "This file must be a shell script: [$file]."
# $3 - $warningOnly. If 0, an error is displayed if the file does not exist,
#      is a symlink, or is not a shell script.  If 1, a warning is displayed
#      instead of an error.  Default is 0.
#
# Allows optional custom error message describing the file, to be displayed if
# the file is not a shell script.  Default error message is:
# "This file should be a script: [FILE]."
#------------------------------------------------------------------------------
function check_file_is_shell_script () {
   local file=$1
   local errMsg=${2:-This file must be a shell script}
   local -i warningOnly=${3:-0}
   local fileType=

   msg "Confirming this file is a script: $file"
   CheckCount+=1
   if [[ -e "$file" ]]; then
      msg "Verified: File [$file] exists."
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsg: [$file]. It does not exist."
      else
         warnmsg "$errMsg: [$file]. It does not exist."
      fi
      return 1
   fi

   CheckCount+=1
   if [[ ! -L "$file" ]]; then
      msg "Verified: File [$file] is not a symlink."
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsg: [$file]. It is a symlink."
      else
         warnmsg "$errMsg: [$file]. It is a symlink."
      fi
      return 1
   fi

   CheckCount+=1
   fileType="$(file "$file" 2>&1)"
   if [[ "${fileType,,}" == *"shell script"* ]]; then
      msg "Verified: File [$file] is a shell script."
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsg: [$file]. Type is: $fileType"
      else
         warnmsg "$errMsg: [$file]. Type is: $fileType"
      fi
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: check_configurable ($instance, $configurable, $scope, $expectedVal, $errMsg1, $errMsg2, $warningOnly)
#
# Check that a configurable is set, and optionally check that it is set to
# an expected value.
#
# Inputs:
# $1 - SDP Instance. Required.
# $2 - Configurable name. Required.
# $3 - Configurable scope/ServerID, as per 'p4 help configure'.  The default
#      is "any", meaning what it means with 'p4 configure set', i.e. that the
#      configurable is a global default.  The special value 'ALL' can
#      also be supplied parameter, which is has the special meaning of checking
#      if the configurable is defined for any ServerID, including the 'any'
#      value.  The value returned is that of the first setting encountered.
# $4 - Expected value of configurable. Optional. If defined, an additional check is
#      done, checking the current value against the expected value.  Optionally,
#      the special value UNDEF can be used, which reverses the exit code, such
#      that a happy zero is returned only if the value is not set.
# $5 - Optional error message to display if no value is defined.  See code
#      below for the default message.
# $6 - Optional error message to display if a value is defined but does not
#      match the expected value.  See code below for the default message.
# $7 - $warningOnly. If 0, an error is displayed if the configurable is not
#      defined or does not have the expected value.
#      If 1, a warning is displayed instead of an error.  Default is 0.
#
# Return Codes:
# 1 - Verifications failed.
# 0 - Verifications passed.
# 
# Sample Usage: 
# check_configurable "$SDPInstance" journalPrefix
#
# check_configurable "$SDPInstance" journalPrefix any "$CHECKPOINTS/$P4SERVER"
#
# check_configurable "$SDPInstance" journalPrefix any "$CHECKPOINTS/$P4SERVER" ||\
#   bail "Yikes, journalPrefix is not set, all bets are off. Aborting."
#------------------------------------------------------------------------------
function check_configurable () {
   local instance="$1"
   local configurable="$2"
   local scope="${3:-any}"
   local expectedValue="${4:-NoExpectedValue}"
   local errMsgMissing="${5:-No value defined}"
   local errMsgBadValue="${6:-Value does not match what is expected}"
   local -i warningOnly=${7:-0}
   local detectedScope=
   local value=

   # If skipping P4ROOT tests, don't bother with configurable checks, as
   # they require P4ROOT.
   [[ "$DoP4ROOTTest" -eq 1 ]] || return 0

   CheckCount+=1

   if [[ ! -r "$P4ROOT"/db.config ]]; then
      warnmsg "Skipping check for configurable $configurable; no db.config."
      return 1
   fi

   copy_jd_table "db.config" "$P4ROOT"
   if [[ "$scope" != "ALL" ]]; then
      # shellcheck disable=SC2154
      value=$($P4DBIN -r "$JDTmpDir" -cshow | grep "^${scope}: ${configurable} = ")
   else
      # shellcheck disable=SC2154
      value=$($P4DBIN -r "$JDTmpDir" -cshow | grep ": ${configurable} = " | head -1)
      detectedScope="$value"
      value=${value##* = }
      detectedScope="${detectedScope%%:*}"
   fi
   remove_jd_tables

   if [[ "$expectedValue" != "UNDEF" ]]; then
      if [[ -n "$value" ]]; then
         value=${value##* = }
         if [[ "$scope" != "ALL" ]]; then
            msg "Verified: Configurable ${scope}:${configurable} is defined."
         else
            msg "Verified: Configurable ${configurable} is defined at least once."
         fi
      else
         if [[ "$warningOnly" -eq 0 ]]; then
            errmsg "$errMsgMissing for configurable [${scope}:${configurable}]."
         else
            warnmsg "$errMsgMissing for configurable [${scope}:${configurable}]."
         fi
         return 1
      fi
   else
      if [[ -n "$value" ]]; then
         if [[ "$scope" != "ALL" ]]; then
            if [[ "$warningOnly" -eq 0 ]]; then
               errmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${scope} of: ${value}"
            else
               warnmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${scope} of: ${value}"
            fi
            return 1
         else
            if [[ "$warningOnly" -eq 0 ]]; then
               errmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${detectedScope} of: ${value} (and possibly for other ServerIDs)."
            else
               warnmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${detectedScope} of: ${value} (and possibly for other ServerIDs)."
            fi
            return 1
         fi
      else
         if [[ "$scope" != "ALL" ]]; then
            msg "Verified: Configurable ${scope}:${configurable} is undefined (as expected)."
         else
            msg "Verified: Configurable ${configurable} is undefined (as expected) for all ServerID values."
         fi
      fi
   fi

   [[ "$expectedValue" == "NoExpectedValue" ]] && return 0

   CheckCount+=1

   if [[ "$expectedValue" != "UNDEF" ]]; then
      if [[ "$value" == "$expectedValue" ]]; then
         msg "Verified: Configurable ${scope}:${configurable} has expected value [$value]."
      else
         if [[ "$warningOnly" -eq 0 ]]; then
            errmsg "$errMsgBadValue for variable [${scope}:${configurable}]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
         else
            warnmsg "$errMsgBadValue for variable [${scope}:${configurable}]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
         fi
         return 1
      fi
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: check_env_var ($instance, $var, $expectedval, $msg1, $msg2, $warningOnly)
#
# Check that a shell environment variable is set when sourcing the SDP
# environment. Optionally checks that variables are set to expected values.
#
# Inputs:
# $1 - SDP Instance. Required.
# $2 - Variable name. Required.
# $3 - Expected value of variable. Optional. If defined, an additional check is
#      done, checking the current value against the expected value.
# $4 - Optional error message to display if no value is defined.  See code
#      below for the default message.
# $5 - Optional error message to display if a value is defined but does not match
#      the expected value.  See code below for the default message.
# $6 - $warningOnly. If 0, an error is displayed if the environment variable
#      is not set or does not match the expected value.
#      If 1, a warning is displayed instead of an error.  Default is 0.
# 
# Return Codes:
# 1 - Verifications failed.
# 0 - Verifications passed.
# Sample Usage: 
# check_env_var $SDPInstance P4JOURNAL "/p4/$SDPInstance/logs/journal"
#
# check_env_var $SDPInstance P4JOURNAL "/p4/$SDPInstance/logs/journal" ||\
#   bail "Yikes, P4JOURNAL is not set, all bets are off. Aborting."
#------------------------------------------------------------------------------
function check_env_var () {
   local instance="$1"
   local var="$2"
   local expectedValue="${3:-NoExpectedValue}"
   local errMsgMissing="${4:-No value defined}"
   local errMsgBadValue="${5:-Value does not match what is expected}"
   local -i warningOnly=${6:-0}
   local value=
   CheckCount+=1

   eval unset "${var}"
   # shellcheck disable=SC1090
   source "$SDPEnvFile" "$instance"

   set +u
   if [[ -n "$(eval echo \$"${var}")" ]]; then
      msg "Verified: Variable ${var} is defined."
      set -u
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsgMissing for variable [$var]."
      else
         warnmsg "$errMsgMissing for variable [$var]."
      fi
      set -u
      return 1
   fi

   [[ "$expectedValue" == "NoExpectedValue" ]] && return 0

   CheckCount+=1
   value="$(eval echo \$"${var}")"

   if [[ "$value" == "$expectedValue" ]]; then
      msg "Verified: Variable ${var} has expected value [$value]."
   else
      if [[ "$warningOnly" -eq 0 ]]; then
         errmsg "$errMsgBadValue for variable [$var]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
      else
         warnmsg "$errMsgBadValue for variable [$var]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
      fi
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: check_local_instance_home_dir ($instance)
#
# Check that the '/p4' directory and the instance home directory '/p4/N' are
# local directories on the root volume, per SDP structural intent.
#
# Inputs:
# $1 - SDP Instance. Required.
#
# Return Codes:
# 0 - Verifications were able to at least run; ErrorCount is incremented
#     if tests fail.
# 1 - Verifications could not even complete. This is a pre-flight failure.
#
# This increments globals CheckCount and possibly ErrorCount.
#
# Sample Usage: 
# check_local_instance_home_dir "$SDPInstance" ||\
#    bail "Error checking p4dir and/or local instance home dir."
#------------------------------------------------------------------------------
function check_local_instance_home_dir () {
   local instance="$1"
   local p4Dir="/p4"
   local p4HomeDir="$p4Dir/$instance"

   CheckCount+=1
   if [[ "$P4HOME" == "$p4HomeDir" ]]; then
      msg "Verified: P4HOME has expected value: $p4HomeDir"
   else
      errmsg "P4HOME has unexpected value: $p4HomeDir"
   fi

   CheckCount+=1
   if [[ -L "$p4HomeDir" ]]; then
      errmsg "This is a symlink; it should be a local directory: $p4HomeDir"
   else
      msg "Verified: This P4HOME path is not a symlink: $p4HomeDir"
   fi

   CheckCount+=1
   if cd "$p4Dir"; then
      msg "Verified: cd to $p4Dir OK."
      CheckCount+=1
      if [[ "$(pwd -P)" == "$p4Dir" ]]; then
         msg "Verified: Dir $p4Dir is a local dir."
      else
         errmsg "Dir $p4Dir is NOT a local dir."
      fi
      cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."

      CheckCount+=1
      if cd "$p4HomeDir"; then
         msg "Verified: cd to $p4HomeDir OK."
         CheckCount+=1
         if [[ "$(pwd -P)" == "$p4HomeDir" ]]; then
            msg "Verified: P4HOME dir $p4HomeDir is a local dir."
         else
            errmsg "P4HOME dir $p4HomeDir is NOT a local dir."
         fi
         cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."
      else
         errmsg "Could not cd to $p4HomeDir."
         return 1
      fi
   else
      errmsg "Could not cd to $p4Dir."
      return 1
   fi

   return 0
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-online) ServerOnline=1;;
      (-extra) ExtraTestList="$2"; shiftArgs=1;;
      (-only) OnlyTestList="$2"; shiftArgs=1;;
      (-skip) SkipTestList="$2"; shiftArgs=1;;
      (-warn) WarnTestList="$2"; shiftArgs=1;;
      (-skip_summary) ShowErrorSummary=0;;
      (-c)
         ServerOnline=1
         DoConfigurablesCheck=1
      ;;
      (-csec)
         ServerOnline=1
         DoConfigurablesCheck=1
         DoSecurityConfigurablesCheck=1
      ;;
      (-si) SilentMode=1;;
      (-L) Log="$2"; shiftArgs=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Debug; use bash 'set -x' mode.
      (-*) usage -h "Unknown command line option ($1).";;
      (*)
         if [[ -z "$SDPInstance" ]]; then
            SDPInstance="$1"
            export SDP_INSTANCE="$SDPInstance"
         else
            usage -h "Unknown parameter ($1); <instance> parameter already set to '$SDPInstance'."
         fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

# If no <instance> parameter was passed in, try using $SDP_INSTANCE from the
# environment.
[[ -n "$SDPInstance" ]] || SDPInstance="${SDP_INSTANCE:-UnsetSDPInstance}"
[[ "$SDPInstance" == "UnsetSDPInstance" ]] && \
   usage -h "Missing <instance> parameter. The <instance> must be given as a parameter to this script, or else the \$SDP_INSTANCE environment variable defined.  It can be set by doing:\\n\\n\\tsource $SDPEnvFile <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n"

[[ -n "$SkipTestList" && -n "$OnlyTestList" ]] && \
   usage -h "The '-skip' and '-only' options are mutually exclusive."
[[ -n "$WarnTestList" && -n "$OnlyTestList" ]] && \
   usage -h "The '-warn' and '-only' options are mutually exclusive."
[[ -n "$ExtraTestList" && -n "$OnlyTestList" ]] && \
   usage -h "The '-extra' and '-only' options are mutually exclusive."

[[ "$SilentMode" -eq 1 && "$Log" == "off" ]] && \
   usage -h "Cannot use '-si' with '-L off'."

#==============================================================================
# Main Program

# shellcheck disable=SC1090
source "$SDPEnvFile" "$SDPInstance" ||\
   bail "Failed to load SDP environment for instance $SDPInstance."

# shellcheck disable=SC1090 disable=SC1091
source "$P4CBIN/backup_functions.sh" ||\
   bail "Failed to load backup_functions.sh."

[[ "${OSUSER:-Unset}" == "Unset" ]] &&\
   bail "The critical OSUSER setting is not defined in $SDPEnvFile. Aborting."

# If this verify_sdp.sh script is called by root, change user to OSUSER.
if [[ $(id -u) -eq 0 ]]; then
   exec su - "$OSUSER" -c "$SDPCommonBin/${0##*/} $CmdArgs"
elif [[ $(id -u -n) != "${OSUSER:-UnknownOSUSER}" ]]; then
   bail "${0##*/} can only be run by root or $OSUSER"
fi

trap terminate EXIT SIGINT SIGTERM

if [[ -n "$OnlyTestList" ]]; then
   DoCrontabTest=0
   DoLicenseTest=0
   DoVersionTest=0
   DoExcessBinaryTest=0
   DoInitCompareTest=0
   DoCommitIDTest=0
   DoOfflineDBTest=0
   DoOwnerChecks=0
   DoP4ROOTTest=0
   DoPasswordChecks=0
   DoP4TFilesTest=0
   DoConfigurablesCheck=0
   DoCommitDefinedTest=0
   DoServerTypeKnownTest=0
   DoSystemdConfigTest=0
   DoRemoteDisabledTest=0
   DoProtectionsTest=0
   ServerOnline=0
   SkipTestList=
   WarnTestList=
   ExtraTestList=
   export VERIFY_SDP_SKIP_TEST_LIST=
   export VERIFY_SDP_WARN_TEST_LIST=
   [[ "$Debug" -eq 0 ]] && ShowSkippingIndicators=0

   for test in $(echo "$OnlyTestList" | tr ',' ' '); do
      case "$test" in
         (commitid|masterid) DoCommitIDTest=1;;
         (cron|crontab) DoCrontabTest=1;;
         (excess) DoExcessBinaryTest=1;;
         (init) DoInitCompareTest=1;;
         (license) DoLicenseTest=1;;
         (offline_db) DoOfflineDBTest=1;;
         (owner) DoOwnerChecks=1;;
         (p4root) DoP4ROOTTest=1;;
         (p4t_files) DoP4TFilesTest=1;;
         (passwd|password) DoPasswordChecks=1;;
         (version) DoVersionTest=1;;
         # We don't set ServerOnline=1 for tests specified with '-only', as these
         # may be running in a test suite that uses only a 'p4 init' server rather
         # than a true SDP server to test explicit code paths.
         (commit_defined) DoCommitDefinedTest=1;;
         (remote_disabled) DoRemoteDisabledTest=1;;
         (protections) DoProtectionsTest=1;;
         (server_type_known) DoServerTypeKnownTest=1;;
         (systemd_config) DoSystemdConfigTest=1;;
         (*) errmsg "Invalid test name specified with '-only': $test";;
      esac
   done
fi

if [[ -z "$SkipTestList" && -n "${VERIFY_SDP_SKIP_TEST_LIST:-}" ]]; then
   SkipTestList="$VERIFY_SDP_SKIP_TEST_LIST"
fi

if [[ -z "$WarnTestList" && -n "${VERIFY_SDP_WARN_TEST_LIST:-}" ]]; then
   WarnTestList="$VERIFY_SDP_WARN_TEST_LIST"
fi

if [[ -n "$SkipTestList" ]]; then
   for test in $(echo "$SkipTestList" | tr ',' ' '); do
      case "$test" in
         (commitid|masterid) DoCommitIDTest=0;;
         (cron|crontab) DoCrontabTest=0;;
         (excess) DoExcessBinaryTest=0;;
         (init) DoInitCompareTest=0;;
         (license) DoLicenseTest=0;;
         (offline_db) DoOfflineDBTest=0;;
         (owner) DoOwnerChecks=0;;
         (p4root) DoP4ROOTTest=0;;
         (p4t_files) DoP4TFilesTest=0;;
         (passwd|password) DoPasswordChecks=0;;
         (version) DoVersionTest=0;;
         (*) errmsg "Invalid test name specified with '-skip': $test";;
      esac
   done
fi

if [[ -n "$WarnTestList" ]]; then
   for test in $(echo "$WarnTestList" | tr ',' ' '); do
      case "$test" in
         (commitid|masterid) DoCommitIDTestWarn=1;;
         (cron|crontab) DoCrontabTestWarn=1;;
         (excess) DoExcessBinaryTestWarn=1;;
         (init) DoInitCompareTestWarn=1;;
         (license) DoLicenseTestWarn=1;;
         (offline_db) DoOfflineDBTestWarn=1;;
         (owner) DoOwnerChecksWarn=1;;
         (p4root) DoP4ROOTTestWarn=1;;
         (p4t_files) DoP4TFilesTestWarn=1;;
         (passwd|password) DoPasswordChecksWarn=1;;
         (version) DoVersionTestWarn=1;;
         (*) errmsg "Invalid test name specified with '-warn': $test";;
      esac
   done
fi

if [[ -n "$ExtraTestList" ]]; then
   for test in $(echo "$ExtraTestList" | tr ',' ' '); do
      case "$test" in
         (commit_defined) DoCommitDefinedTest=1; ServerOnline=1;;
         (server_type_known) DoServerTypeKnownTest=1;;
         (systemd_config) DoSystemdConfigTest=1;;
         (remote_disabled) DoRemoteDisabledTest=1;;
         (protections) DoProtectionsTest=1;;
         (all)
            ServerOnline=1
            DoCommitDefinedTest=1
            DoServerTypeKnownTest=1
            DoSystemdConfigTest=1
            DoRemoteDisabledTest=1
            DoProtectionsTest=1
         ;;
         (*) errmsg "Invalid test name specified with '-extra': $test";;
      esac
   done
fi

# Logs should be defined to /p4/N/logs after sourcing the environment
# file above; default to /tmp for cases of incomplete environment where
# LOGS is not defined.
export LOGS="${LOGS:-/tmp}"

[[ "$Log" == "Unset" ]] && Log="${LOGS}/verify_sdp.log"

if [[ "$Log" != "off" ]]; then
   if [[ -f "$Log" ]]; then
      if [[ ! -w "$Log" ]]; then
         BadLog="$Log"
         Log="off"
         bail "Existing log file [$BadLog] is not writable. Aborting."
      fi
      rotate_log_file "$Log" ".gz"
   else
      if [[ ! -d "${LOGS}" ]]; then
         Log="off"
         bail "Logs directory [$LOGS] is not writable. Aborting."
      fi
   fi

   if ! touch "$Log"; then
      BadLog="$Log"
      Log="off"
      bail "Couldn't touch log file [$BadLog]. Aborting."
   fi

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log"
fi

[[ "$P4TMP" != "Unset" && -d "$P4TMP" && -w "$P4TMP" ]] ||\
   bail "SDP environment must define required P4TMP variable. Value must be a directory that is writable; value is: $P4TMP"

msg "$ThisScript v$Version Starting SDP verification on host $ThisHost at $(date +'%a %Y-%m-%d %H:%M:%S %Z') with this command line:\\n$CmdLine"

InstanceBinDir="$SDPInstallRoot/$SDPInstance/bin"

if [[ -z "$OnlyTestList" ]]; then
   msg "\\nIf you have any questions about the output from this script, contact support@perforce.com."

   do_preflight_checks "$SDPInstance" ||\
      bail "Preflight checks failed. Aborting further checks."

   msg "${H2}\\nChecking environment variables."
   check_env_var "$SDPInstance" SDP_INSTANCE
   check_env_var "$SDPInstance" P4ROOT "/p4/$SDPInstance/root"
   check_env_var "$SDPInstance" P4JOURNAL "/p4/$SDPInstance/logs/journal"
   check_env_var "$SDPInstance" P4MASTER_ID
   check_env_var "$SDPInstance" P4MASTERHOST
   check_env_var "$SDPInstance" P4MASTERPORT
   check_env_var "$SDPInstance" SERVERID
fi

if [[ "$DoPasswordChecks" -eq 1 ]]; then
   if check_env_var "$SDPInstance" SDP_ADMIN_PASSWORD_FILE "$P4CCFG/.p4passwd.$P4SERVER.admin" $DoPasswordChecksWarn; then
      CleartextPasswordFile="$SDP_ADMIN_PASSWORD_FILE"
      EncryptedPasswordFile="${SDP_ADMIN_PASSWORD_FILE}.enc"

      msg "Checking existence of encrypted SDP admin password file: $EncryptedPasswordFile"
      CheckCount+=1

      if [[ -r "$EncryptedPasswordFile" ]]; then
         msg "Verified: Encrypted SDP admin password file exists."
      else
         CheckCount+=1
         msg "Checking existence of cleartext SDP admin password file: $CleartextPasswordFile"
         if [[ -r "$CleartextPasswordFile" ]]; then
            warnmsg "Encrypted SDP admin password file does not exist: $EncryptedPasswordFile"
            msg "Verified: Cleartext SDP admin password file exists."
         else
            if [[ "$DoPasswordChecksWarn" -eq 1 ]]; then
               warnmsg "SDP admin password file does not exist in either encrypted or cleartext forms; neither $EncryptedPasswordFile nor $CleartextPasswordFile exist."
            else
               errmsg "SDP admin password file does not exist in either encrypted or cleartext forms; neither $EncryptedPasswordFile nor $CleartextPasswordFile exist."
            fi
         fi
      fi
   fi
else
   [[ "$ShowSkippingIndicators" -eq 1 ]] && \
      msg "Skipping password file check per '-skip'."
fi

if [[ -z "$OnlyTestList" ]]; then
   msg "${H2}\\nRunning standard checks typically called within SDP scripts."
   CheckCount+=1
   check_vars
   set_vars
fi

if [[ -z "$OnlyTestList" ]]; then
   msg "${H2}\\nChecking *_init scripts in instance bin dir [$InstanceBinDir] to see what servers are configured on this machine."

   if [[ -e "$P4DInitScript" ]]; then
      msg "A p4d server is here."
      P4DServer=1
      check_file_x "$P4DInitScript"

      check_dirs
      CheckCount+=1

      if [[ "$DoOwnerChecks" -eq 1 ]]; then
         if [[ "$DoOwnerChecksWarn" -eq 0 ]]; then
            run "$P4CBIN/check_dir_ownership.sh $P4ROOT $OSUSER unset 1" ||\
               errmsg "Ownership check for $P4ROOT failed."
         else
            run "$P4CBIN/check_dir_ownership.sh $P4ROOT $OSUSER unset 1" ||\
               warnmsg "Ownership check for $P4ROOT failed."
         fi
      fi

      if [[ -r "$P4DInitTemplate" ]]; then
         msg "Verified: P4D init script template exists: $P4DInitTemplate"
         if [[ "$DoInitCompareTest" -eq 1 ]]; then
            CheckCount+=1
            TmpFile=$(mktemp)
            TmpFile2=$(mktemp)
            grep -v -e '^#' -e '^[[:space:]]*$' "$P4DInitScript" > "$TmpFile"
            sed -e "s/REPL_SDP_INSTANCE/${SDP_INSTANCE}/g" "$P4DInitTemplate" | grep -v -e '^#' -e '^[[:space:]]*$' > "$TmpFile2"

            if diff -q "$TmpFile" "$TmpFile2" > /dev/null; then
               msg "Verified: P4D init script contents are OK."
            else
               if [[ "$DoInitCompareTestWarn" -eq 1 ]]; then
                  warnmsg "P4D init script contents are not as expected:\\n== Expected Contents (Trimmed) ==\\n$(cat "$TmpFile2")\\n== Actual Contents (Trimmed) ==\\n$(cat "$TmpFile")\\n"
               else
                  errmsg "P4D init script contents are not as expected:\\n== Expected Contents (Trimmed) ==\\n$(cat "$TmpFile2")\\n== Actual Contents (Trimmed) ==\\n$(cat "$TmpFile")\\n"
               fi
            fi
            rm -f "$TmpFile" "$TmpFile2"
         else
            [[ "$ShowSkippingIndicators" -eq 1 ]] && \
               msg "Skipping P4D init script compare per '-skip'."
         fi
      else
         errmsg "P4D init script template does not exist: $P4DInitTemplate"
      fi

      CheckCount+=1
      if [[ -e "$P4DRef" ]]; then
         ExpectedTarget="/p4/common/bin/p4d_${SDP_INSTANCE}_bin"
         if [[ -L "$P4DRef" ]]; then
            CheckCount+=1
            LinkTarget=$(readlink "$P4DRef")

            if [[ "$LinkTarget" == "$ExpectedTarget" ]]; then
               msg "Verified: Symlink target for $P4DRef is correct ($LinkTarget)."
            else
               errmsg "P4D Instance Symlink target value is wrong:\\Expected: $ExpectedTarget\\nActual: $LinkTarget\\n\\n"
            fi
         elif [[ -f "$P4DRef" ]]; then
            CheckCount+=1
            # For case-insensitive instances, /p4/N/bin/p4d_N is a script rather
            # than a symlink, but still references a target in /p4/common/bin.
            LinkTarget=$(grep ^P4D= "$P4DRef" | cut -d '=' -f 2)

            if [[ "$LinkTarget" == "$ExpectedTarget" ]]; then
               msg "Verified: Target for P4D= in $P4DRef is correct ($LinkTarget)."
            else
               errmsg "P4D Instance P4D= target value in $P4DRef is wrong:\\nExpected: $ExpectedTarget\\nActual: $LinkTarget\\n\\nTo fix this error, run these commands:\\n\\tmv -f $P4DRef ${P4DRef}.junk\\n\\techo '#!/bin/bash' > $P4DRef\\n\\techo 'P4D=/p4/common/bin/p4d_${SDP_INSTANCE}_bin' >> $P4DRef\\n\\techo 'exec \$P4D -C1 \"\$@\"' >> $P4DRef\\n\\tchmod +x $P4DRef\\n\\n"
            fi
         else
            errmsg "Element $P4DRef exists but is neither a file or symlink."
         fi
      else
         errmsg "A p4d server is here, but $P4DRef does not exist."
      fi
   fi

   if [[ -e "$P4BrokerInitScript" ]]; then
      msg "A p4broker server is here."
      P4BrokerServer=1
      check_file_x "$P4BrokerInitScript"

      CheckCount+=1
      if [[ -r "$P4BrokerInitTemplate" ]]; then
         msg "Verified: P4Broker init script template exists: $P4BrokerInitTemplate"
         if [[ "$DoInitCompareTest" -eq 1 ]]; then
            CheckCount+=1
            TmpFile=$(mktemp)
            TmpFile2=$(mktemp)
            grep -v -e '^#' -e '^[[:space:]]*$' "$P4BrokerInitScript" > "$TmpFile"
            sed -e "s/REPL_SDP_INSTANCE/${SDP_INSTANCE}/g" "$P4BrokerInitTemplate" | grep -v -e '^#' -e '^[[:space:]]*$' > "$TmpFile2"

            if diff -q "$TmpFile" "$TmpFile2" > /dev/null; then
               msg "Verified: P4Broker init script contents are OK."
            else
               if [[ "$DoInitCompareTestWarn" -eq 1 ]]; then
                  warnmsg "P4Broker init script contents are not as expected:\\n== Expected Contents (Trimmed) ==\\n$(cat "$TmpFile2")\\n== Actual Contents (Trimmed) ==\\n$(cat "$TmpFile")\\n"
               else
                  errmsg "P4Broker init script contents are not as expected:\\n== Expected Contents (Trimmed) ==\\n$(cat "$TmpFile2")\\n== Actual Contents (Trimmed) ==\\n$(cat "$TmpFile")\\n"
               fi
            fi
            rm -f "$TmpFile" "$TmpFile2"
         else
            [[ "$ShowSkippingIndicators" -eq 1 ]] && \
               msg "Skipping P4Broker init script compare per '-skip'."
         fi
      else
         errmsg "P4Broker init script template does not exist: $P4BrokerInitTemplate"
      fi

      CheckCount+=1
      if [[ -e "$P4BrokerRef" ]]; then
         ExpectedTarget="/p4/common/bin/p4broker_${SDP_INSTANCE}_bin"
         if [[ -L "$P4BrokerRef" ]]; then
            CheckCount+=1
            LinkTarget=$(readlink "$P4BrokerRef")

            if [[ "$LinkTarget" == "$ExpectedTarget" ]]; then
               msg "Verified: Symlink target for $P4BrokerRef is correct ($LinkTarget)."
            else
               errmsg "P4Broker Instance Symlink target value is wrong:\\Expected: $ExpectedTarget\\nActual: $LinkTarget\\n\\n"
            fi
         else
            errmsg "$P4BrokerRef exists but is not a symlink."
         fi
      else
         errmsg "A p4broker server is here, but $P4BrokerRef does not exist."
      fi
   fi

   if [[ -e "$P4ProxyInitScript" ]]; then
      msg "A p4p server is here."
      P4ProxyServer=1
      check_file_x "$P4ProxyInitScript"

      CheckCount+=1
      if [[ -r "$P4ProxyInitTemplate" ]]; then
         msg "Verified: P4Proxy init script template exists: $P4ProxyInitTemplate"
         if [[ "$DoInitCompareTest" -eq 1 ]]; then
            CheckCount+=1
            TmpFile=$(mktemp)
            TmpFile2=$(mktemp)
            grep -v -e '^#' -e '^[[:space:]]*$' "$P4ProxyInitScript" > "$TmpFile"
            sed -e "s/REPL_SDP_INSTANCE/${SDP_INSTANCE}/g" "$P4ProxyInitTemplate" | grep -v -e '^#' -e '^[[:space:]]*$' > "$TmpFile2"

            if diff -q "$TmpFile" "$TmpFile2" > /dev/null; then
               msg "Verified: P4Proxy init script contents are OK."
            else
               if [[ "$DoInitCompareTestWarn" -eq 1 ]]; then
                  warnmsg "P4Proxy init script contents are not as expected:\\n== Expected Contents (Trimmed) ==\\n$(cat "$TmpFile2")\\n== Actual Contents (Trimmed) ==\\n$(cat "$TmpFile")\\n"
               else
                  errmsg "P4Proxy init script contents are not as expected:\\n== Expected Contents (Trimmed) ==\\n$(cat "$TmpFile2")\\n== Actual Contents (Trimmed) ==\\n$(cat "$TmpFile")\\n"
               fi
            fi
            rm -f "$TmpFile" "$TmpFile2"
         else
            [[ "$ShowSkippingIndicators" -eq 1 ]] && \
               msg "Skipping P4Proxy init script compare per '-skip'."
         fi
      else
         errmsg "P4Broker init script template does not exist: $P4BrokerInitTemplate"
      fi

      CheckCount+=1
      if [[ -e "$P4ProxyRef" ]]; then
         ExpectedTarget="/p4/common/bin/p4p_${SDP_INSTANCE}_bin"
         if [[ -L "$P4ProxyRef" ]]; then
            CheckCount+=1
            LinkTarget=$(readlink "$P4ProxyRef")

            if [[ "$LinkTarget" == "$ExpectedTarget" ]]; then
               msg "Verified: Symlink target for $P4ProxyRef is correct ($LinkTarget)."
            else
               errmsg "P4Proxy Instance Symlink target value is wrong:\\Expected: $ExpectedTarget\\nActual: $LinkTarget\\n\\n"
            fi
         else
            errmsg "$P4ProxyRef exists but is not a symlink."
         fi
      else
         errmsg "A p4p server is here, but $P4ProxyRef does not exist."
      fi
   fi

   if [[ $((P4DServer+P4BrokerServer+P4ProxyServer)) -eq 0 ]]; then
      CheckCount+=1
      errmsg "No servers (p4d, p4p, p4broker) are configured."
   fi

   msg "${H2}\\nConfirming simply-named p4/p4d/p4broker/p4p in $P4CBIN are shell scripts."
   for b in p4 p4d p4broker p4p; do
      check_file_is_shell_script "$P4CBIN/$b"
   done

   if [[ "$P4DServer" -eq 1 ]]; then
      # Check whether P4ROOT db files exist
      if [[ "$DoP4ROOTTest" -eq 1 ]]; then
         msg "${H2}\\nChecking for a few database files in P4ROOT."
         for file in db.counters db.domain db.config; do
            check_file "$P4ROOT/$file" "Expected database file doesn't exist." $DoP4ROOTTestWarn
         done
      fi

      # Check whether offline_db files exist
      if [[ "$DoOfflineDBTest" -eq 1 ]]; then
         msg "${H2}\\nChecking for a few database files in offline_db."
         for file in db.counters db.domain db.config; do
            check_file "$OFFLINE_DB/$file" "Expected database file doesn't exist." $DoOfflineDBTestWarn
         done
      fi
   fi

   msg "${H2}\\nChecking for existence of key files."
   check_file "$P4BIN" "The p4 binary (or symlink) doesn't exist."

   if [[ "$DoP4TFilesTest" -eq 1 ]]; then
      check_file "$P4TICKETS" "The P4TICKETS file doesn't exist." \
         $DoP4TFilesTestWarn

      if [[ "$P4PORT" =~ ^ssl[46]*: ]]; then
         check_file "$P4TRUST" "The P4TRUST file doesn't exist and SSL is enabled." \
            $DoP4TFilesTestWarn
      fi
   fi

   if [[ "$P4DServer" -eq 1 ]]; then
      if [[ "$DoP4ROOTTest" -eq 1 ]]; then
         check_file "$P4ROOT/server.id" \
            "The required $P4ROOT/server.id file is missing." $DoP4ROOTTestWarn
         check_file_dne "$P4ROOT/P4ROOT_not_usable.txt" \
            "P4ROOT is not in a usable state." $DoP4ROOTTestWarn
      fi

      if [[ "$DoOfflineDBTest" -eq 1 ]]; then
         check_file "$OFFLINE_DB/offline_db_usable.txt" \
            "Offline database not in a usable state." $DoOfflineDBTestWarn
         check_file_dne "$OFFLINE_DB/P4ROOT_not_usable.txt" \
            "Offline database has P4ROOT_not_usable.txt file." $DoOfflineDBTestWarn
      fi

      check_file_dne "$P4ROOT/statepullL" \
         "The file \$P4ROOT/statepullL exists. Remove this failover remnant" 1

      check_file_dne "$P4ROOT/statefailover" \
         "The file \$P4ROOT/statefailover exists. Remove this failover remnant" 1

      if [[ "$DoLicenseTest" -eq 1 && "$DoP4ROOTTest" -eq 1 ]]; then
         msg "${H2}\\nLicense Checks."

         #--------------------------------------------------------------------------
         # A sample license line looks like this:
         # License: Perforce Battle School 28 users (support ends 2020/06/01) (expires 2020/06/01)
         # Existence of 'support ends' indicates the license is valid.
         # Existence of 'expires' indicates a temp or subscription license.
         #--------------------------------------------------------------------------
         if check_file "$P4ROOT/license" "The \$P4ROOT/license file doesn't exist" 1; then
            LicenseInfo=$($P4DBIN -V | grep '^License:')
            CheckCount+=1
            if [[ -n "$LicenseInfo" ]]; then
               if [[ "$LicenseInfo" == *" expired "* ]]; then
                  if [[ "$DoLicenseTestWarn" -eq 1 ]]; then
                     warnmsg "The license is expired."
                  else
                     errmsg "The license is expired."
                  fi
               elif [[ "$LicenseInfo" == *"expires"* ]]; then
                  LicenseExpiration=${LicenseInfo##*(expires }
                  LicenseExpiration=${LicenseExpiration%%)*}

                  # Check License
                  CheckCount+=1
                  CurrentTime=$(date +%s 2>/dev/null)
                  ExpirationTime=$(date +%s --date "$LicenseExpiration" 2>/dev/null)
                  if [[ -n "$CurrentTime" && -n "$ExpirationTime" ]]; then
                     TimeDiff=$((ExpirationTime-CurrentTime))
                     DaysDiff=$((TimeDiff/(3600*24)))

                     msg "Info: License expires on $LicenseExpiration (in $DaysDiff days)."
                     if [[ "$DaysDiff" -le "$LicenseDaysExpirationAlert" ]]; then
                        if [[ "$DoLicenseTestWarn" -eq 1 ]]; then
                           warnmsg "License will expire within $LicenseDaysExpirationAlert days."
                        else
                           errmsg "License will expire within $LicenseDaysExpirationAlert days."
                        fi
                     fi
                  else
                     [[ "$ShowSkippingIndicators" -eq 1 ]] && \
                        msg "Skipping license check due to incompatible 'date' utility on this OS."
                  fi
               elif [[ "$LicenseInfo" == *"support ends"* ]]; then
                  msg "Info: License is perpetual."
               fi
            else
               if [[ "$DoLicenseTestWarn" -eq 1 ]]; then
                  warnmsg "Could not determine license info from license file."
               else
                  errmsg "Could not determine license info from license file."
               fi
            fi
         fi
      else
         [[ "$ShowSkippingIndicators" -eq 1 ]] && \
            msg "Skipping license check per '-skip'."
      fi
   fi

   if [[ "$P4DServer" -eq 1 && "$DoP4ROOTTest" -eq 1 ]]; then
      msg "${H2}\\nChecking configurables values."

      msg "This next test checks that P4JOURNAL is defined only as a shell environment variable and is not set in db.config.\\n"
      check_configurable "$SDP_INSTANCE" P4JOURNAL ALL UNDEF
      check_configurable "$SDP_INSTANCE" journalPrefix any "$CHECKPOINTS/$P4SERVER"
      check_configurable "$SDP_INSTANCE" server.depot.root any "$DEPOTS"
   fi

   if [[ "$P4DServer" -eq 1 ]]; then

      if [[ "$DoP4ROOTTest" -eq 1 ]]; then
         msg "${H2}\\nChecking P4ROOT symlinks."
         CheckCount+=1
         LinkP4ROOT="$(readlink "$P4ROOT")"
         if [[ "$LinkP4ROOT" == *"/db1" || "$LinkP4ROOT" == *"/db2" ]]; then
            msg "Verified: Symlink for P4ROOT points to a db1 or db2 path."
         else
            errmsg "Symlink for P4ROOT does not point to a db1 or db2 path."
         fi
      fi

      if [[ "$DoOfflineDBTest" -eq 1 ]]; then
         msg "${H2}\\nChecking offline_db symlinks."
         CheckCount+=1
         LinkOfflineDB="$(readlink "$OFFLINE_DB")"
         if [[ "$LinkOfflineDB" == *"/db1" || "$LinkOfflineDB" == *"/db2" ]]; then
            msg "Verified: Symlink for offline_db points to a db1 or db2 path."
         else
            errmsg "Symlink for offline_db does not point to a db1 or db2 path."
         fi
      fi

      if [[ "$DoP4ROOTTest" -eq 1 && "$DoOfflineDBTest" -eq 1 ]]; then
         CheckCount+=1
         if [[ "$LinkP4ROOT" != "$LinkOfflineDB" ]]; then
            msg "Verified: Symlinks for P4ROOT and offline_db do not point to the same target."
         else
            errmsg "Symlinks for P4ROOT and offline_db point to the same target."
         fi
      fi
   fi

   msg "${H2}\\nChecking for standard symlink/dir structure."

   CheckCount+=1
   if cd "$P4HOME"; then
      CheckCount+=1
      if [[ -d "$PWD/bin" ]]; then
         msg "Verified: $PWD/bin is a regular directory."
      else
         warnmsg "$PWD/bin is not a regular directory."
      fi

      linkList="logs tmp"
      [[ "$P4DServer" -eq 1 ]] && linkList+=" checkpoints depots root offline_db"
      [[ "$P4ProxyServer" -eq 1 ]] && linkList+=" cache"

      for link in $linkList; do
         CheckCount+=1
         if [[ -L "$PWD/$link" ]]; then
            msg "Verified: $PWD/$link is a symlink."

            CheckCount+=1
            targetDir=$(readlink "$link")
            if [[ -d "$targetDir" ]]; then
               msg "Verified: Target for $PWD/$link is a directory."
            else
               # If we are on a server that runs both 'p4p' and 'p4d', report proxy config
               # issues as Warnings rather than Errors.
               if [[ "$P4DServer" -eq 1 && "$link" == "cache" ]]; then
                  warnmsg "Target for symlink $PWD/$link, $targetDir, is not a directory."
               else
                  errmsg "Target for symlink $PWD/$link, $targetDir, is not a directory."
               fi
            fi
         else
            # If we are on a server that runs both 'p4p' and 'p4d', report proxy config
            # issues as Warnings rather than Errors.
            if [[ "$P4DServer" -eq 1 && "$link" == "cache" ]]; then
               warnmsg "$PWD/$link is not a symlink."
            else
               errmsg "$PWD/$link is not a symlink."
            fi
         fi
      done

      CheckCount+=1
      if [[ -L "$SDPCommon" ]]; then
         msg "Verified: $SDPCommon is a symlink."
      else
         errmsg "Path $SDPCommon is not a symlink."
      fi

      CheckCount+=1
      if [[ -L "/p4/sdp" ]]; then
         msg "Verified: /p4/sdp is a symlink."
      else
         errmsg "Path /p4/sdp is not a symlink."
      fi

      cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."
   else
      errmsg "Could not cd to $P4HOME." 
   fi

   if [[ "$ServerOnline" -eq 1 ]]; then
      msg "${H2}\\nDoing check that require server to be running, due to '-online'."

      if [[ "$P4DServer" -eq 1 ]]; then
         CheckCount+=1
         if run "$P4BIN -s info -s" "Checking 'p4 -s info -s'" 1; then
            msg "Verified: 'p4 -s info -s' output is nominal."
         else
            errmsg "Could not verify the p4d server is online (P4PORT=$P4PORT)."
         fi

         CheckCount+=1
         if run "$P4BIN -s -p $P4MASTERPORT info -s" "Checking 'p4 -s info -s'" 1; then
            msg "Verified: 'p4 -s -p $P4MASTERPORT info -s' output is nominal."
         else
            errmsg "Could not verify the p4d server is online (P4MASTERPORT=$P4MASTERPORT)."
         fi

         CheckCount+=1
         if run "$P4CBIN/p4login" "Login check." 0; then
            msg "Verified: Login OK."
         else
            errmsg "Login as P4USER $P4USER to P4PORT $P4PORT could not be verified."
         fi

         CheckCount+=1
         if run "$P4CBIN/p4login -p $P4MASTERPORT" "Login check using P4MASTERPORT." 0; then
            msg "Verified: Login OK using P4MASTERPORT."
         else
            errmsg "Login as P4USER $P4USER to P4MASTERPORT $P4MASTERPORT could not be verified."
         fi

         TicketExpiration=$($P4BIN -ztag -F %TicketExpiration% login -s 2>/dev/null)
         CheckCount+=1
         if [[ -n "$TicketExpiration" ]]; then
            msg "Verified: User '$P4USER' exists and has a valid ticket."
            CheckCount+=1
            if [[ "$TicketExpiration" -ge "$MinTicketExpiration" ]]; then
               msg "Verified: Ticket expiration [$((TicketExpiration/86400)) days $((TicketExpiration%86400/3600)) hours $((TicketExpiration%3600/60)) minutes $((TicketExpiration%60)) seconds] is at least [$((MinTicketExpiration/86400)) days $((MinTicketExpiration%86400/3600)) hours $((MinTicketExpiration%3600/60)) minutes $((MinTicketExpiration%60)) seconds]."
            else
               warnmsg "Ticket expiration [$((TicketExpiration/86400)) days $((TicketExpiration%86400/3600)) hours $((TicketExpiration%3600/60)) minutes $((TicketExpiration%60)) seconds] is less than the minimum required [$((MinTicketExpiration/86400)) days $((MinTicketExpiration%86400/3600)) hours $((MinTicketExpiration%3600/60)) minutes $((MinTicketExpiration%60)) seconds]."
            fi
         else
            errmsg "No ticket for user '$P4USER' available."
         fi

         AccessLevel=$($P4BIN protects -m 2>/dev/null)

         CheckCount+=1
         if [[ -n "$AccessLevel" ]]; then
            if [[ "$AccessLevel" == super ]]; then
               msg "Verified: Access level for P4USER '$P4USER' is super, as required."
            else
               errmsg "User access level for P4USER $P4USER is [$AccessLevel]; it must be super."
            fi
         else
            errmsg "Could not determine access level with: $P4BIN protects -m"
         fi

         if [[ "$DoVersionTest" -eq 1 ]]; then
            CheckCount+=1
            LinkP4DVersion=$("$P4DBIN" -V | grep ^Rev)
            LinkP4DVersion=${LinkP4DVersion#Rev. }
            LinkP4DVersion=${LinkP4DVersion%\.}
            LiveP4DVersion=$("$P4BIN" -ztag -F %serverVersion% info -s)

            if [[ "$LinkP4DVersion" == "$LiveP4DVersion" ]]; then
               msg "Verified: Live running p4d version matches expected version [$LinkP4DVersion]."
            else
               if [[ "$DoVersionTestWarn" -eq 0 ]]; then
                  errmsg "Live running p4d version [$LiveP4DVersion] does not match expected version [$LinkP4DVersion]."
               else
                  warnmsg "Live running p4d version [$LiveP4DVersion] does not match expected version [$LinkP4DVersion]."
               fi
            fi
         else
            [[ "$ShowSkippingIndicators" -eq 1 ]] && \
               msg "Skipping live p4d version check per '-skip'."
         fi
      fi

      if [[ "$P4BrokerServer" -eq 1 ]]; then
         CheckCount+=1
         if run "$P4BIN -s -p $P4BROKERPORT info -s" "Checking 'p4 -s -p $P4BROKERPORT info -s'" 1; then
            msg "Verified: 'p4 -s -p $P4BROKERPORT info -s' output is nominal."
         else
            warnmsg "Could not verify the p4broker server is online (P4PORT=$P4BROKERPORT)."
         fi

         if [[ "$DoVersionTest" -eq 1 ]]; then
            CheckCount+=1
            LinkP4BrokerVersion=$("$P4BROKERBIN" -V | grep ^Rev)
            LinkP4BrokerVersion=${LinkP4BrokerVersion#Rev. }
            LinkP4BrokerVersion=${LinkP4BrokerVersion%% (*}
            LiveP4BrokerVersion=$("$P4BIN" -p "$P4BROKERPORT" -ztag -F %brokerVersion% info -s)

            if [[ "$LinkP4BrokerVersion" == "$LiveP4BrokerVersion" ]]; then
               msg "Verified: Live running p4broker version matches expected version [$LinkP4BrokerVersion]."
            else
               if [[ -n "$LiveP4BrokerVersion" ]]; then
                  warnmsg "Live running p4broker version [$LiveP4BrokerVersion] does not match expected version [$LinkP4BrokerVersion]."
               else
                  warnmsg "Could not determine live running p4broker version and '-online' was specified."
               fi
            fi
         else
            [[ "$ShowSkippingIndicators" -eq 1 ]] && \
               msg "Skipping live p4broker version check per '-skip'."
         fi
      fi

      if [[ "$P4ProxyServer" -eq 1 ]]; then
         if run "$P4BIN -s -p $PROXY_PORT info -s" "Checking 'p4 -s -p $PROXY_PORT info -s'" 1; then
            msg "Verified: 'p4 -s -p $PROXY_PORT info -s' output is nominal."
         else
            warnmsg "Could not verify the p4p server is online (P4PORT=$PROXY_PORT)."
         fi

         if [[ "$DoVersionTest" -eq 1 ]]; then
            CheckCount+=1
            LinkP4ProxyVersion=$("$P4PBIN" -V | grep ^Rev)
            LinkP4ProxyVersion=${LinkP4ProxyVersion#Rev. }
            LinkP4ProxyVersion=${LinkP4ProxyVersion%\.}
            LiveP4ProxyVersion=$("$P4BIN" -p "$PROXY_PORT" -ztag -F %proxyVersion% info -s)

            if [[ "$LinkP4ProxyVersion" == "$LiveP4ProxyVersion" ]]; then
               msg "Verified: Live running p4p version matches expected version [$LinkP4ProxyVersion]."
            else
               if [[ -n "$LiveP4ProxyVersion" ]]; then
                  warnmsg "Live running p4p version [$LiveP4ProxyVersion] does not match expected version [$LinkP4ProxyVersion]."
               else
                  warnmsg "Could not determine live running p4p version and '-online' was specified."
               fi
            fi
         else
            [[ "$ShowSkippingIndicators" -eq 1 ]] && \
               msg "Skipping live p4p version check per '-skip'."
         fi
      fi
   fi
fi

if [[ "$DoCrontabTest" -eq 1 ]]; then
   msg "${H2}\\nChecking crontab for user $USER."
   CheckCount+=1
   TmpFile="$(mktemp "$P4TMP/crontab.XXXXXXXXXXX")"
   if crontab -l | grep -v '^#' > "$TmpFile"; then
      if grep -q -E '/(daily_checkpoint.sh|replica_status.sh|p4pcm.pl|broker_rotate.sh) ' "$TmpFile"; then
         msg "Verified: Crontab for user $USER passed basic sanity check."
      elif [[ "$DoCrontabTestWarn" -eq 1 ]]; then
         warnmsg "Crontab for user $USER did not pass basic sanity check; missing call to daily_checkpoint.sh or replica_status.sh"
      else
         errmsg "Crontab for user $USER did not pass basic sanity check; missing call to daily_checkpoint.sh or replica_status.sh"
      fi
   else
      errmsg "Could not capture crontab."
   fi
   rm -f "$TmpFile"
else
   [[ "$ShowSkippingIndicators" -eq 1 ]] && \
      msg "Skipping crontab check per '-skip'."
fi

if [[ "$DoVersionTest" -eq 1 ]]; then
   msg "${H2}\\nChecking SDP Version Id (current and legacy methods)."

   CheckCount+=1
   if [[ -r /p4/sdp/Version ]]; then
      SDPVersionA=$(get_sdp_version_from_string "$(cat /p4/sdp/Version)")
   else
      SDPVersionA="Unknown"
      errmsg "Missing SDP Version file: /p4/sdp/Version"
   fi

   CheckCount+=1
   SDPVersionB=$(get_sdp_version_from_string "$SDP_VERSION")
   if [[ "$SDPVersionA" == "$SDPVersionB" ]]; then
      msg "SDP Version from $SDPEnvFile matches."
   else
      errmsg "SDP Version from $SDPEnvFile isn't set or doesn't match /p4/sdp/Version:\\n[$SDPVersionB] vs. [$SDPVersionA]"
   fi

   if [[ "$ServerOnline" -eq 1 ]]; then
      CheckCount+=1
      SDPVersionC=$(get_sdp_version_from_string "$("$P4BIN" counter SDP_VERSION)")

      if [[ "$SDPVersionA" == "$SDPVersionC" ]]; then
         msg "SDP Version from 'p4 counter SDP_VERSION' matches."
      else
         if [[ "$SDPVersionC" == "Unknown" ]]; then
            warnmsg "SDP Version from 'p4 counter SDP_VERSION' is not set. To fix:\\n\\tsource /p4/common/bin/p4_vars $SDP_INSTANCE\\n\\tp4 counter SDP_VERSION \"\$(cat /p4/sdp/Version)\"\\n\\tp4 counter SDP_DATE \"\$(date +'%Y-%m-%d')\""
         else
            warnmsg "SDP Version from 'p4 counter SDP_VERSION' doesn't match /p4/sdp/Version:\\n[$SDPVersionC] vs. [$SDPVersionA].  To fix:\\n\\tsource /p4/common/bin/p4_vars $SDP_INSTANCE\\n\\tp4 counter SDP_VERSION \"\$(cat /p4/sdp/Version)\"\\n\\tp4 counter SDP_DATE \"\$(date +'%Y-%m-%d')\""
         fi
      fi

      # If '-c' or '-csec' were specified, call ccheck.sh with the default profile.
      # For now, SDP_CCHECK_PROFILE is undoc; it may be added to Instance Vars later.
      # As undoc, it can be using the standard mechanism for adding undoc values,
      # e.g.:
      #   mkdir -p /p4/common/site/config
      #   echo 'export SDP_CCHECK_PROFILE=demo' >> /p4/common/site/config/$P4SERVER.vars
      if [[ "$DoConfigurablesCheck" -eq 1 ]]; then
         CCheckProfile=${SDP_CCHECK_PROFILE:-}
         CCheckCmd="$P4CBIN/ccheck.sh -L off"
         [[ -n "$CCheckProfile" ]] && CCheckCmd+=" -p $CCheckProfile"
         # If '-csec' was specified, appent '-sec' to the call to ccheck.sh.
         [[ "$DoSecurityConfigurablesCheck" -eq 1 ]] && \
            CCheckCmd+=" -sec"

         msg "\\nCalling ccheck.sh to compare configurables against best practices.\\nBEGIN CCHECK\\n"
         if eval "$CCheckCmd"; then
            msg "\\nEND CCHECK\\n"
            msg "Configurables pass the checks."
         else
            msg "\\nEND CCHECK\\n"
            errmsg "Configurables failed the checks."
         fi
      fi
   fi
else
   [[ "$ShowSkippingIndicators" -eq 1 ]] && msg \
      "Skipping version checks per '-skip'."
fi

if [[ "$DoExcessBinaryTest" -eq 1 ]]; then
   msg "${H2}\\nChecking for Helix executables outside $SDPCommonBin in PATH."
   CheckCount+=1

   # Check for excess Helix server binaries in PATH, and classify that as an error.
   for exe in p4d p4broker p4p; do
      # shellcheck disable=SC2230 disable=2164
      for exeInPath in $(cd "$LOGS"; which -a "$exe" | sort -u); do
         dir="${exeInPath%/*}"
         if [[ "$dir" != *"$P4CBIN" && "$dir" != "." ]]; then
            if [[ "$DoExcessBinaryTestWarn" -eq 1 ]]; then
               warnmsg "Executable $exe found outside $SDPCommonBin in PATH: $dir"
               ExcessServerBinariesFound=1
            else
               errmsg "Executable $exe found outside $SDPCommonBin in PATH: $dir"
               ExcessServerBinariesFound=1
            fi
         fi
      done
   done

   # Check for excess Helix client binary in PATH, and classify that as a warning.
   # shellcheck disable=SC2230 disable=2164
   for exeInPath in $(cd "$LOGS"; which -a p4 | sort -u); do
      if [[ "$dir" != *"$P4CBIN" && "$dir" != "." ]]; then
         warnmsg "Executable 'p4' found outside $SDPCommonBin in PATH: ${exeInPath%/*}"
      fi
   done

   if [[ "$ExcessServerBinariesFound" -eq 0 ]]; then
      msg "Verified: No excess Helix server binaries found outside $SDPCommonBin."
   fi
else
   [[ "$ShowSkippingIndicators" -eq 1 ]] && \
      msg "Skipping excess binary checks per '-skip'."
fi

if [[ "$DoCommitIDTest" -eq 1 ]]; then
   msg "${H2}\\nChecking that P4MASTER_ID value starts with 'commit' or 'master'."
   CheckCount+=1
   if [[ "$P4MASTER_ID" == "commit"* ]]; then
      msg "Verified: The P4MASTER_ID value starts with 'commit'."
   elif [[ "$P4MASTER_ID" == "master"* ]]; then
      msg "Verified: The P4MASTER_ID value starts with 'master'."
   else
      if [[ "$DoCommitIDTestWarn" -eq 1 ]]; then
         warnmsg "The P4MASTER_ID value ($P4MASTER_ID) does not start with 'commit' or 'master'."
      else
         errmsg "The P4MASTER_ID value ($P4MASTER_ID) does not start with 'commit' or 'master'."
      fi
   fi
fi

if [[ "$DoCommitDefinedTest" -eq 1 ]]; then
   msg "${H2}\\nChecking server specs for singleton 'commit-server' and no 'standard'."
   CheckCount+=1

   TmpFile=$(mktemp)

   if run "$P4BIN -ztag -F %ServerID%:%Services% servers" "Gathering server specs." 1 > "$TmpFile" 2>&1; then
      [[ "$Debug" -eq 0 ]] || cat "$TmpFile"
      if [[ "$(grep -c :commit-server "$TmpFile")" == 1 ]]; then
         msg "Verified: Exactly one server spec with Services 'commit-server' exists: $(grep :commit-server "$TmpFile")"
      else
         errmsg "No server spec exists with Services 'commit-server'."
      fi

      CheckCount+=1
      if [[ "$(grep -c :standard "$TmpFile")" == 0 ]]; then
         msg "Verified: No server specs with Services 'standard' exist."
      else
         errmsg "One or more server specs with Services 'standard' exist; none should: $(grep :standard "$TmpFile")"
      fi
   else
      errmsg "Failed to get 'p4 servers' info."
   fi
   rm -f "$TmpFile"
fi

if [[ "$DoServerTypeKnownTest" -eq 1 ]]; then
   msg "${H2}\\nChecking exactly one of run_if_{master,replica,edge}.sh is on."

   TmpFile=$(mktemp)
   CheckCount+=1

   if [[ "$(run_if_master.sh "$SDP_INSTANCE" echo YES)" == "YES" ]]; then
      msg "A call to run_if_master.sh indicates Yes."
      RunIfCount+=1
   else
      msg "A call to run_if_master.sh indicates No."
   fi

   if [[ "$(run_if_replica.sh "$SDP_INSTANCE" echo YES)" == "YES" ]]; then
      msg "A call to run_if_replica.sh indicates Yes."
      RunIfCount+=1
   else
      msg "A call to run_if_replica.sh indicates No."
   fi

   if [[ "$(run_if_edge.sh "$SDP_INSTANCE" echo YES)" == "YES" ]]; then
      msg "A call to run_if_edge.sh indicates Yes."
      RunIfCount+=1
   else
      msg "A call to run_if_edge.sh indicates No."
   fi

   if [[ $RunIfCount -eq 1 ]]; then
      msg "Verified: Exactly one of the run_if_{master,replica,edge}.sh indicates Yes."
   else
      errmsg "Exactly one of the run_if_{master,replica,edge}.sh should indicate Yes, but $RunIfCount do."
   fi

   rm -f "$TmpFile"
fi

if [[ "$DoSystemdConfigTest" -eq 1 ]]; then
   msg "${H2}\\nChecking that systemd services are properly configured for init script in $InstanceBinDir."
   TmpFile=$(mktemp)
   TmpFile2=$(mktemp)

   P4DInitScript="$InstanceBinDir/p4d_${SDPInstance}_init"
   if [[ -e "$P4DInitScript" ]]; then
      SystemdServicesCheckedCount+=1
      CheckCount+=1
      P4DSystemdServiceFile="p4d_${SDPInstance}.service"
      msg "Checking $P4DSystemdServiceFile configuration."

      if timeout 10s systemctl cat "$P4DSystemdServiceFile" > "$TmpFile"; then
         # Echo an extra first line to match the way 'systemctl cat' displays
         # services files so diffs show only meaningful differences.
         echo "# /etc/systemd/system/${P4DSystemdServiceFile}" > "$TmpFile2"
         if sed -E -e "s|__INSTANCE__|$SDPInstance|g" -e "s|__OSUSER__|$OSUSER|g" "$P4DServiceTemplate" >> "$TmpFile2"; then
            CheckCount+=1
            if diff -q "$TmpFile" "$TmpFile2"; then
               msg "Verified: Systemd service file $P4DSystemdServiceFile matches template $P4DServiceTemplate"
            else
               errmsg "Systemd service file $P4DSystemdServiceFile does not match template. Diffs:"
               diff "$TmpFile" "$TmpFile2"
            fi
         else
            errmsg "Could not determine expected service file from template $P4DServiceTemplate"
         fi
      else
         errmsg "Could not do: systemctl cat $P4DSystemdServiceFile"
      fi
   fi

   P4BrokerInitScript="$InstanceBinDir/p4broker_${SDPInstance}_init"
   if [[ -e "$P4BrokerInitScript" ]]; then
      SystemdServicesCheckedCount+=1
      CheckCount+=1
      P4BrokerSystemdServiceFile="p4broker_${SDPInstance}.service"
      msg "Checking $P4BrokerSystemdServiceFile configuration."

      if timeout 10s systemctl cat "$P4BrokerSystemdServiceFile" > "$TmpFile"; then
         # Echo an extra first line to match the way 'systemctl cat' displays
         # services files so diffs show only meaningful differences.
         echo "# /etc/systemd/system/${P4BrokerSystemdServiceFile}" > "$TmpFile2"
         if sed -E -e "s|__INSTANCE__|$SDPInstance|g" -e "s|__OSUSER__|$OSUSER|g" "$P4BrokerServiceTemplate" >> "$TmpFile2"; then
            CheckCount+=1
            if diff -q "$TmpFile" "$TmpFile2"; then
               msg "Verified: Systemd service file $P4BrokerSystemdServiceFile matches template $P4BrokerServiceTemplate"
            else
               errmsg "Systemd service file $P4BrokerSystemdServiceFile does not match template. Diffs:"
               diff "$TmpFile" "$TmpFile2"
            fi
         else
            errmsg "Could not determine expected service file from template $P4BrokerServiceTemplate"
         fi
      else
         errmsg "Could not do: systemctl cat $P4BrokerSystemdServiceFile"
      fi
   fi

   P4ProxyInitScript="$InstanceBinDir/p4p_${SDPInstance}_init"
   if [[ -e "$P4ProxyInitScript" ]]; then
      SystemdServicesCheckedCount+=1
      CheckCount+=1
      P4ProxySystemdServiceFile="p4p_${SDPInstance}.service"
      msg "Checking $P4ProxySystemdServiceFile configuration."

      if timeout 10s systemctl cat "$P4ProxySystemdServiceFile" > "$TmpFile"; then
         # Echo an extra first line to match the way 'systemctl cat' displays
         # services files so diffs show only meaningful differences.
         echo "# /etc/systemd/system/${P4ProxySystemdServiceFile}" > "$TmpFile2"
         if sed -E -e "s|__INSTANCE__|$SDPInstance|g" -e "s|__OSUSER__|$OSUSER|g" "$P4ProxyServiceTemplate" >> "$TmpFile2"; then
            CheckCount+=1
            if diff -q "$TmpFile" "$TmpFile2"; then
               msg "Verified: Systemd service file $P4ProxySystemdServiceFile matches template $P4ProxyServiceTemplate"
            else
               errmsg "Systemd service file $P4ProxySystemdServiceFile does not match template. Diffs:"
               diff "$TmpFile" "$TmpFile2"
            fi
         else
            errmsg "Could not determine expected service file from template $P4ProxyServiceTemplate"
         fi
      else
         errmsg "Could not do: systemctl cat $P4ProxySystemdServiceFile"
      fi
   fi

   if [[ "$SystemdServicesCheckedCount" -ge 1 ]]; then
      msg "Verified: There was at least 1 systemd service to check; $SystemdServicesCheckedCount were checked."
   else
      errmsg "There are no systemd services corresponding to init scripts in $InstanceBinDir."
   fi

   rm -f "$TmpFile" "$TmpFile2"
fi

if [[ "$DoRemoteDisabledTest" -eq 1 ]]; then
   msg "${H2}\\nChecking that legacy built-in user 'remote' is disabled."
   CheckCount+=1

   # shellcheck disable=SC2072
   if [[ "$P4D_VERSION" > "2025.1" ]]; then
      msg "Verified: The 'remote' user is disabled; P4D is 2025.1+."
   else
      SecuritySetting="$($P4DBIN -cshow|grep '^any: security'|cut -d ' ' -f 4)"
      if ((SecuritySetting > 3)); then
         msg "Verified: The 'remote' user is disabled; security is $SecuritySetting."
      else
         TmpFile=$(mktemp)
         TmpFile2=$(mktemp)
         if $P4BIN protects -u remote >  "$TmpFile"; then
            grep -v -- -// "$TmpFile" > "$TmpFile2"
            if [[ -s "$TmpFile2" ]]; then
               errmsg "The 'remote' user is not disabled; it has this access:"
               cat "$TmpFile2"
            else
               msg "Verified: The 'remote' user is disabled with Protections."
            fi
         else
            errmsg "Could not extract Protections for remote user with 'p4 protect -u remote'."
         fi

         rm -f "$TmpFile" "$TmpFile2"
      fi
   fi
fi

if [[ "$DoProtectionsTest" -eq 1 ]]; then
   msg "${H2}\\nDoing sanity check on Protections."
   CheckCount+=1

   TmpFile=$(mktemp)
   TmpFile2=$(mktemp)
   if $P4BIN protect -o > "$TmpFile"; then
      tail -2 "$TmpFile" | head -1 > "$TmpFile2"
      if grep -q "super user $P4USER " "$TmpFile2"; then
         msg "Verified: Last line of Protections references 'super user $P4USER'."
      else
         errmsg "Last line of Protections table does not reference 'super user $P4USER'. It may still work but is not in alignment with best practices for Protections table management."
      fi
   else
      errmsg "Could not extract Protections table with 'p4 protect -o'."
   fi

   rm -f "$TmpFile" "$TmpFile2"
fi

# TODO:
# - Ensure checkpoints dir contains a checkpoint or two.
# - service password verified as set for a replica (value doesn't matter, it just needs to be set).
# - Add flag to check less-critical SDP configurables, and generate
#   warnings (rather than errors) if they are not set as expected,
#   using SDP configure_new_servers.sh script as a guide.

if [[ "$ErrorCount" -eq 0 && "$WarningCount" -eq 0 ]]; then
   msg "\\n${H1}\\n\\nALL CLEAN: $CheckCount verifications completed OK."
elif [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\n${H1}\\n\\nNO ERRORS: $CheckCount verifications completed, with $WarningCount warnings detected."

   if [[ "$ShowErrorSummary" -eq 1 ]]; then
      msg "\\nSummary of warnings reported above:"
      grep ^Warning: "$Log"
   fi
else
   msg "\\n${H1}\\n\\nVerifications completed, with $ErrorCount errors and $WarningCount warnings detected in $CheckCount checks."

   if [[ "$ShowErrorSummary" -eq 1 ]]; then
      msg "\\nSummary of errors and warnings reported above:"
      grep -E '^(Error|Warning):' "$Log"
   fi
fi

# See the terminate() function, which is really where this script exits.
exit "$ErrorCount"
# Change User Description Committed
#70 31743 C. Thomas Tyler New logic to check for existence of an encrypted password file.

If the encrypted file exists, skip the check for the cleartext file.

If the encrypted file does not exist, check for the cleartext file. If
found, display a warning that the encrypted file does not exist,
but indicate that the cleartext file does. This is a nudge to start
using encrypted files.

If neither files is found, display an error (unless warning only mode
was explicitly requested with '-warn passwd').

This is related to SDP-641.

#review
#69 31561 C. Thomas Tyler Doc tweaks for verify_sdp.sh.
#68 31558 C. Thomas Tyler verify_sdp.sh: Added 2 new '-extra' checks, remote_disabled and protections.

#review-31559
#67 31500 C. Thomas Tyler verify_sdp.sh now has '-csec' option to do a focused security check.

The verify_sdp.sh script already has a '-c' option to do a full
configurables check by calling 'ccheck.sh'; with the new '-csec' it
call 'ccheck.sh' with its own new '-sec' option to check only security
related configurables.
#66 31444 C. Thomas Tyler Added shorthand '-extra all' option, short for specifying all available
'extra' tests.  Added more examples.

Adjusted test count for systemd config checks.

Fixed various spelling errors with aspell.
#65 31442 C. Thomas Tyler Added new '-extra' option 'systemd_config', to check systemd service
configurations. Checks include:
* Ensure systemd services exist for each SDP init script in the
Instance Bin dir (/p4/N/bin).
* Ensure at least one systemd service exists (p4d/p4broker/p4p).
* Ensure systemd unit files match the SDP templates.

Among other things, this will detect whether the systemd service file
for p4d is using AmbientCapabilities.

#review-31443
#64 31363 C. Thomas Tyler In verify_sdp.sh, added checks to ensure defined P4USER exists, is
a super, and has a long-term ticket.  If the users does not exist
or does not have super access, and error is generated. If the ticket
duration is too short, a warning is given.
#63 31329 C. Thomas Tyler Added summary of errors and warnings at the end.
This summary is displayed
by default. A '-skip_summary' option has been added to suppress the summary.

Added checks for existence of statepullL and statefailover files in P4ROOT
on p4d servers.  A warning is reported if these files exist, advising the
user to remove these failover remnant files.

Addressed some issues reported by shellcheck.

#review-31330 @robert_cowham
#62 31170 C. Thomas Tyler In verify_sdp.sh, added '-extra' to specify tests not normally run by
default, but which can be called on request.  Two additional tests
added are:

* commit_defined
* server_type_known

Fixes:
* SDP-1187
* SDP-1188

#review-31171
#61 31028 C. Thomas Tyler In verify_sdp.sh, changed call to check_dir_ownership.sh to use faster
test, matching what is required by p4d_base to start the service.
#60 31018 C. Thomas Tyler Renamed unreleased new script, check_dir_perms.sh -> check_dir_ownership.sh.
Fixed exit code in this new script.

Modified p4d_base to call this script as a preflight check when starting p4d
(regular start only, not force_start).  It will refuse to start p4d if there
are any files in P4ROOT owned by a user other than the defined OSUSER. This
should be helpful in finding things like a root-owned state.xu file.

Fixes SDP-1119.

#review-31019
#59 30979 C. Thomas Tyler Eliminated buildup of temp dirs, e.g.
/tmp/tmp.XXXXXXXXXX.

Added remove_jd_tables() function and calls to it to prevent buildup of new
cruft.

Modified remove_old_logs() to cleanup cruft created previously.

#review-30980 @robert_cowham
#58 30933 C. Thomas Tyler Added check for proper ownership of all files in P4ROOT.

Added separate utility for checking ownership (and optionally group) of all files
in a given path.

Fixes SDP-1119.

#review-30934
#57 30793 C. Thomas Tyler Expanded crontab check to allow for a standalone broker install.
#56 30270 Robert Cowham Fix shellcheck warnings and use of copy_jd_table
#55 30267 Robert Cowham Copy files to be dumped via p4d -jd to tmp dir first
to avoid locks on P4ROOT (or offline_db)

SDP-1087
#54 30131 C. Thomas Tyler In verify_sdp.sh:
* The P4MASTER_ID check now accepts 'commit' or 'master'.
* The 'masterid' skippable check is now synonymous with 'commitid'

#review-30132
#53 30022 C. Thomas Tyler Minor doc clarification in verify_sdp.sh
#52 30021 C. Thomas Tyler Enhanced docs for ccheck.sh script.
Added examples.

Added '-c' option to verify_sdp.sh to do configurables check.
#51 29121 C. Thomas Tyler Fixed dependency on cwd if dot is in PATH in check for exes.
#review-29003
#50 29100 C. Thomas Tyler Removed requirement for systemd *.service files to have open perms
in *_base scripts and templates.

Removed checks for open perms on *.service files in verify_sdp.sh.

Fixed minor ShellCheck compliance issue.

See also: HI-101: https://swarm.workshop.perforce.com/jobs/HI-101

[Submitting, then re-opening for post-commit final review].
#49 28815 C. Thomas Tyler Enhanced error message in verify_sdp.sh if p4d_N script is malformed.

The /p4/N/bin/p4d_N only exists as a script for case-insensitive
servers.  Normally this file is generated (by mkdirs.sh), but in
some DIY installations it may be handcrafted, and possibly wrong.

This changes improves the error message in that scenario, such that
the user has enough information to correct the problem.

This also fixes a funky formatting issue when the error occurred.

#review-28816
#48 28771 C. Thomas Tyler Changed email address for Perforce Support.

#review-28772 @amo @robert_cowham
#47 28641 C. Thomas Tyler Tweaked scripts to support IPv6 SSL prefixes.

Added test script to test bash code snippets. First test is the
snippet to check if SSL is enabled, and if so get the SSL prefix.
#46 28553 C. Thomas Tyler Fixed issue file type detection (shell script vs.
binary) did
not work on some Linux platforms, due to slightly different output
from 'file' on shell scripts on different platforms.  Type
detection logic is now more robust.

The test suite will catch this when we add support for
Rocky Linux 8.

Also corrected a copy/paste error in an error message and
enhanced code comments.

This is a bug fix to a new, not-yet-released feature.

#review-28554
#45 28423 C. Thomas Tyler verify_sdp.sh v5.21.0:

Added check that simply named files p4, p4d, p4p, and p4broker in
/p4/common/bin are indeed shell scripts, not binaries.

Also silenced configurable checks and some other checks that should
be skipped if '-skip p4root' is specified.

#review-28424 @russell_jackson
#44 28421 C. Thomas Tyler verify_sdp.sh v5.20.0:
* New checks: /p4/N/bin/p4{d,p,broker}_N need correct symlink target,
and must exist if the corresponding _init script exists.

For p4d, it can be a symlink (for a case-sensitive instance) or script
(for a case-insensitive instance to pass the C1 flag). Either way the
target is checked.

These checks cannot be skipped or converted to warnings.

* Added check that /p4/N/bin/p4{d,p,broker}_N_init scripts have content
that matches templates. This can be skipped with '-skip' or reported as
mere warnings (with '-warn') with a new and documented 'init' category
of test skipping/warning.

#review-28422
#43 28250 C. Thomas Tyler SDP Version checks for 'p4 counter SDP_VERSION' downgraded to warning.
#42 28134 C. Thomas Tyler Added '-warn' option to verify_sdp.sh to convert errors to warnings.

This option takes a list of named tests, using the same set of tests
names as used with the existing '-skip' option.  Rather than skipping
the tests entirely, with '-warn' wany would-be errors are instead
reported as warnings.  Warnings will not cause a non-zero exit
status.

A new environment variable is added to go with the new option.  Similar
to how the existing '-skip' option has a corresponding
VERIFY_SDP_SKIP_TEST_LIST setting the Instance Vars file, the -warn
option has a corresponding $VERIFY_SDP_WARN_TEST_LIST.
#41 28047 C. Thomas Tyler Corrected counter name re: SDP_VERSION counter, and added
supplemental information on how to fix it.
#40 27961 C. Thomas Tyler Corrected typo in error message in verify_sdp.sh.
#39 27918 C. Thomas Tyler Corrected typo in a comment, and minor code style tweaks.

This is a non-functional change to verify_sdp.sh.
#38 27916 C. Thomas Tyler Fixed bug where verify_sdp.sh can wrongly give 'Current user ...
does not own'
error if started from a directory other than one owned by defined OSUSER.

Fixed formatting error.

Added quotes in 'case' per ShellCheck warning.
#37 27750 C. Thomas Tyler upgrade.sh v4.6.9:
* Fixed issue where where patch-only upgrades of instances after the first
in a multi-instance environment are skipped.
* Corrected error message for scenario where downgrades are attempted; the
logic was correct but error message was confusing.

verify_sdp.sh v5.17.3:
* Extended '-skip version' meaning to also skip new live binary version
comparison checks.

Related updates:
* A call to verify_sdp.sh in the switch_db_files() function in backup_functions.sh
now skips the version check.
* A call to daily_checkpoint.sh now skips the version check.

#review-27743
#36 27746 C. Thomas Tyler verify_sdp.sh v5.17.2:
* Downgraded more broker/proxy errors to warnings.
* Added warning message for case where '-online' is specified and there
is no live running p4p/p4broker (and they are defined as running run per
existence of Instance Bin init scripts).
#35 27745 C. Thomas Tyler verify_sdp.sh v5.17.1:
* Added missing check for 'p4 info' against proxy port.
* New -online checks for p4broker/p4p downgraded to Warnings.
#34 27742 C. Thomas Tyler verify_sdp.sh v5.17.0:
* Enhanced -online to also work for p4broker and p4p.
* Added comparison check for live running vs. symlink'd version of servers
(p4d, p4broker, and p4p).  This is a part of the solution for SDP-625.
* When checking for symlinks, also added checks that symlink target dirs exist.
* For the special case where 'p4d' and 'p4p' are running on the same machine
  (according to the Instance Bin init scripts), proxy misconfiguration issues
  such as a missing /p4/N/cache symlink or a missing /hxdepots/p4/N/cache
  directory are reported as Warnings rather than Errors.
#33 27737 C. Thomas Tyler In verify_sdp.sh, made check for SDP password file a skippable check.

This also skips the check for SDP_ADMIN_PASSWORD_FILE variable.

See: SDP-641: Get SDP to work without a cleartext password file.

#review-27732
#32 27731 C. Thomas Tyler Fixed logic bug that prevented check for existence of password file.
#31 27723 C. Thomas Tyler In verify_sdp.sh, handle scenario where there are no p4*_N.service files
at all. Also, imrpove error messages to advise on necessary fix.
#30 27722 C. Thomas Tyler Refinements to @27712:
* Resolved one out-of-date file (verify_sdp.sh).
* Added missing adoc file for which HTML file had a change (WorkflowEnforcementTriggers.adoc).
* Updated revdate/revnumber in *.adoc files.
* Additional content updates in Server/Unix/p4/common/etc/cron.d/ReadMe.md.
* Bumped version numbers on scripts with Version= def'n.
* Generated HTML, PDF, and doc/gen files:
  - Most HTML and all PDF are generated using Makefiles that call an AsciiDoc utility.
  - HTML for Perl scripts is generated with pod2html.
  - doc/gen/*.man.txt files are generated with .../tools/gen_script_man_pages.sh.

#review-27712
#29 27719 C. Thomas Tyler Fixed issue preventing systemd service checks from working.

Expanded scope of systemd checks to include any services related to
the instance based on namving convention (p4*_N*.service), in addition
to the checks for the 3 standard services (p4d_N/p4broker_N/p4p_N).
This would include, for example, p4dtg_1.

#review-27720
#28 27684 C. Thomas Tyler Fixed invalid reporting of crontab issue if run on proxy host.

#review-27685
#27 27412 C. Thomas Tyler Refined logic for test skipping.
#26 27410 C. Thomas Tyler Added 'p4root' to list of tests that can be skipped.
#25 27403 C. Thomas Tyler verify_sdp.sh v5.14.0:
* Added check for P4TRUST file if P4PORT starts with ssl:.
* Added '-skip' option to avoid check for P4TICKETS and P4TRUST files.
#24 27372 C. Thomas Tyler Added 'sort' to list of required utils in early preflight checks.
#23 27367 C. Thomas Tyler verify_sdp.sh v5.13.1:
* Refined 'excess binaries in path' check ignore '.'
* Dedupe 'excess binaries in path' via 'sort -u'.
* Removed obsolete doc ref comment tags (non-functional change).

#review-27368
#22 27348 C. Thomas Tyler Added option to skip checks requiring a healty offline_db.
#21 27207 C. Thomas Tyler upgrade.sh v4.3.0:
* Greatly enhanced documentation and examples.
* Before doing second journal rotation:
  - Added wait for 'p4 storage -w' (for 'to or thru' P4D 2019.1).
  - Added wait for 'p4 upgrades|grep -v completed' (for P4D 2020.2+)
* Added check for whether p4broker and p4p were online at the start
of upgrade processing. Only start those services that were running at
the beginning of processing are now started after the binaries and
symlinks are updated.  For the broker, only the broker with the default
configuration is stopped and started; DFM brokers are ignored by this
script (thus making this script compaitble with using DFM brokers).
* Fixed bug where '-c' (Protections table comment conversion) would
have failed due to 'p4d' addition of 'Update:' field to the Protections
table. Also generally enhanced logic to convert Protections table
comments.
* Added support for operation on proxy-only and broker-only hosts.
Processing of upgrades for p4d occur only if /p4/N/bin/p4d_N_init
script exists on the machine.
* Refined lexigraphical P4D version comparsion checks.

verify_sdp.sh v5.12.0:
* Added support for proxy-only and broker-only hosts.  The
existence of a *_init script in the instance bin dir for any of the
Helix server binaries p4d/p4p/p4broker indicate they are configured,
determining what tests are executed or skipped.
* Added check_file_x() function to check for execute bit on files.

In backup_functions.sh, fixed is_server_up() to avoid displaying
output.

#review-27208
#20 27204 C. Thomas Tyler Fixed error where 'p4' client was reported as 'p4p' if found in PATH.
#19 27191 C. Thomas Tyler Fixed typo.
#18 27189 C. Thomas Tyler verify_sdp.sh v5.11.0:
* Added check for existance of $P4ROOT/server.id file.
* Added optional check that P4MASTER_ID value starts with 'master'; this
can be disabled with '-skip masterid'.

#review-27190
#17 27170 C. Thomas Tyler Fixed typos in docs; no functional changes.
#16 27169 C. Thomas Tyler Fixed typo in error message.
Thanks @roadkills_r_us!
#15 27099 C. Thomas Tyler Eliminated extra /p4/Version file.

The standard for determining the SDP version for 2020.1 and forward
will be the file /p4/sdp/Version.  This is more clear.  Job SDP-564
evolved slightly (per DevNotes in the job). The goal remains the same,
to standardize the method of determining the SDP version for SDP
2020.1+.

It was deemed that having an extra copy in /p4/Version will not help
with that, and instead would introduce more failure modes and
possibilities for out-of-sync files.

This does mean the 'tarball extraction' sdp folder that is symlinked
to from /p4/sdp is now a critical part of the SDP installation. This
is normal and as documented, though there have been cases where
SDP is copied from one machine to another in some incomplete way,
e.g. rysnc of /p4/common but not /hxdepots/sdp and the symlink to it
from /p4/sdp. However, the verify_sdp.sh will catch that form of
misconfiguration.

#review-27100
#14 27082 C. Thomas Tyler verfiy_sdp.sh v5.9.2: Tweaked check for excess binaries in path to
differentiate client (p4) from server (p4d/p4broker/p4p), Report
excess servers as an error, and excess client as a warning (and
thus still allowing a happy zero exit code).  This is needed for
compatibility with H4G.
#13 27064 C. Thomas Tyler Fixed issue where 'source p4_vars' hangs if load_checkpoint.sh is running.

Added new semaphore file, $P4ROOT/P4ROOT_not_usable.txt.  This is used in
a way similar to 'offline_db_usable.txt' in the offline_db, except that this
file only exists when the databases in P4ROOT are not usable. This is the
opposite of how offline_db_usable.txt works, because P4ROOT is expected to
be usable 99.9% fo the time.  p4d_base will refuse to start p4d if this file
exists, protecting against possible operator errors (like trying to start
p4d when a checkpoint is still loading).

Added check_file_dne() function to verify_sdp.sh to confirm a named file does not exist.
Added checks in verify_sdp.sh that P4ROOT_not_usable.txt does not exist in P4ROOT
or offline_db.

Modified switch_db_files() (called by refresh_P4ROOT_from_offline_db.sh) to properly
use the new P4ROOT_not_usable.txt safety file.

Fixed bugs in p4d_base that could cause p4d_init.log to be overwritten if error output
was generated.

Removed call to 'backup_functions.sh' in p4d_base, as on balance it added more complexity
than needed.

#review-27065
#12 26982 C. Thomas Tyler mkdirs.sh v4.1.0:
* Accounted for directory structure change of Maintenance to Unsupported.
* Added standard command line processing with '-h' and '-man' doc flags,
and other flags (all documented).
* Added in-code docs and updated AsciiDoc.
* Enhanced '-test' mode to simulate /hx* mounts.
* Enhanced preflight testing, and fixed '-test' mode installs.
* Added support for installing to an alternate root directory.
* Added '-s <ServerID>' option to override REPLICA_ID.
* Added '-S <TargetServerID>' used for replicas of edge servers.
* Added '-t <server_type>' option to override SERVER_TYPE.
* Added '-M' option to override mount points.
* Added '-f' fast option to skip big chown/chmod commands, and
moved those commands near the end as well.

verify_sdp.sh v5.9.0:
* Added check for /p4/Version file, and checked that other legacy
SDP methods of checking version
* Added sanity check for crontab.
* Added 'test skip' mechanism to skip certain tests:
 - crontab: Skip crontab check. Use this if you do not expect crontab to be configured, perhaps if a different scheduler is used.
 - license: Skip license related checks.
 - version: Skip version checks.
 - excess: Skip checks for excess copies of p4d/p4p/p4broker in PATH.
* Added VERIFY_SDP_SKIP_TEST_LIST setting ton instance_vars.template,
to define a standard way to have verify_sdp.sh always skip certain
tests for a site.
* Extended '-online' checks to check for bogus P4MASTERPORT, a common
config error.

Update test_SDP.py:
* Adjusted test suite to account for various changes in mkdirs.sh.
* Added 'dir' parameter to run_cmd() and sudo_cmd(), to run a
command from a specified directory (as required to test new
mkdirs.sh)
* Added check_links() similar to existing check_dirs() function.

=== Upgrade Process Changes ===

Made /p4/common/bin/p4d/p4/p4broker/p4p shell script rather than binary.

This changes the way SDP new binaries are staged for upgrade.  For
safety, exes are now staged to a director outside the PATH, the
/p4/sdp/exes folder. A new 'get_latest_exes.sh' script simplifies
the task of pulling executables from the Perforce FTP server. This
can be used 'as is' for environments with outbound internet access,
and is useful in any case to describe now to acquire binaries.

This addresses an issue where a p4d binary staged for a future
upgrade might be called before the actual upgrade is performed.

upgrade.sh v4.0.0:
* All preflight checks are now done first. Added '-p' to abort after preflight.
* Added '-n' to show what would be done before anything is executed.
* Minimalist logic to start/stop only servers that are upgrade, and apply
upgrades only as needed.
* Staging of exes for upgrade is now separate from /p4/common/bin
* Improved in-code docs, added '-h' and '-man' options.
* Retained pre/post P4D 2019.1 upgrade logic.
#11 26814 C. Thomas Tyler Skip license check on platforms for which 'date' utility is
incompatible (e.g. Mac).
#10 26718 Robert Cowham Rename P4MASTER to P4MASTERHOST for clarity with comments in:
- mkdirs.cfg/mkdirs.sh
- p4_<instance>.vars
- other files which reference
Remove unnecessary sed for p4p.template
#9 26637 Robert Cowham Include script help within doc
Requires a couple of tags in the scripts themselves.
#8 26422 C. Thomas Tyler Fixed issue where checks cause db.config to be created in otherwise empty P4ROOT.
Fixed typo in declaration.
#7 26400 C. Thomas Tyler Added refresh_P4ROOT_from_offline_db.sh.

Updated backup_functions.sh to support functionality for db refresh.

Upgrade start_p4d() and stop_p4d() to use systemd if available, else
use the underlying SysV init scripts.

Updated verify_sdp.sh to be called from other scripts (sans
its own logging).  Added many checks to verify_sdp.sh to
support P4ROOT/offline_db swap.

Logic in P4ROOT/offline_db swap is more careful about what gets
swapped.

Added start_p4broker() and stop_p4broker() that behave similarly.

More shellcheck compliance.

#review-26401
#6 26387 C. Thomas Tyler verify_sdp.sh v5.5.1:
* Removed a redundant check.
* Cleaned up some output issues.
#5 26385 C. Thomas Tyler Added license expiration check.

Added new 'warning' indicator, so some things may be reported as warnings
rather than errors.

Fixed bug in not-released dev version handling su/exec when started
as root.

Enhanced verify_sdp.sh to support being called by other scripts by
giving a reliable exit code.  Errors detected result in a non-zero
exit code; warnings do not.  This is in support of SDP-444, but does
not complete that job.

Maintained shellcheck compliance.

#review-26386
#4 26074 C. Thomas Tyler Added missing flags in usage (-h) showing in '-man'.
No functional change.
#3 25372 C. Thomas Tyler Various updates in verify_sdp.sh v5.3.0:
* Passes shellcheck v.0.6.0 check; which drove various minor
code changes, e.g. using bash built-in 'command -v' rather than
system utility 'which', and others.
* Improved usage of mktemp util to use P4TMP in a template, to
avoid SELinux issues.
* Fixed issue where early calls to bail() resulted in output
not being displayed.
* Bullet-proofed for scenarios where script is called by the
wrong user, or called by correct user after first calling by
wrong user and log ownership is wrong, or user can't write log
file, or P4TMP not defined, etc.
* Clarified TODO comments.
#2 25206 C. Thomas Tyler Removed logic that uses 'p4d -cset' to force the value for
P4JOURNAL, and also automatic journal rotation on server
startup.

Added related logic to verify_sdp.sh to ensure there is
one source of truth for the P4JOURNAL definition.

=== On Journal Rotation at Server Startup ===

The goal with journal rotation on server stratup is noble, to
make it so any potential journal corruption *always* appears
at the end of a numbered journal file, rather than being in
the middle of the active journal.  This can make it easier and
faster to recover from journal corruption caused by sudden power
loss, kernel panic, a p4d bug/crash, etc.

However, the implementation causes problems (noted below).

=== On Forcing P4JOURNAL ===

The goal of forcing the value of P4JOURNAL via db.config
is also noble, in that having a value anything other than
the SDP standard can really wreak havoc with things.  This
is generally not an issue in a 'fresh' SDP install, but can
be an issue (wreak havoc!) in cases where 'p4 configure' was
used to set a value for P4JOURNAL that conflicts with the
value defined by the SDP environment mechanism, which is in
turn passed to 'p4d' on the command line.  Even if the value
defined differently, it should be set in to exactly one value,
and exactly one mechanism.

The current implementation causes problems (noted below).

== Problems with setting P4JOURNAL in db.config ==

1. Things  Break

The forced P4JOURNAL set via 'p4d -cset' causes a mild form of
journal corruption that breaks 'standby' replicas using
journalcopy, as this type of replica is extremely sensitive to
the contents of every byte in the journal file, and doesn't
allow for use of 'p4d -cset' to modify the P4JOURNAL file.

While it does not cause any actual loss of data, it does require
manual reset to fix things.  In the case of a site-wide topology
with a mandatory standby replica, it causes global replication to
stall.

2. Not our Place (not the place of SDP scripts)

Based on the above and taking a step back, I think this
script behavior of forcing a back-door journal rotation is simply
too intrusive for what SDP scritps should be allowed to do.
They live to have some understanding of p4d workings, but shoulnd't
pretend to have too much insight into the inner workings of p4d.

== Problem with Always-On Journal Rotation on Start ==

1. What the wah?

This confuses admins by incrementing the journal counter
unexpectedly.  In Battle School training classes, for example,
students (Perforce admins) are confused by seemingly random
journal incrementing.  While this could be documented and trained
for, it violates the principal of least surprise, and is not
typical 'p4d' beavhior.

2. Always vs. Rare

It rotates the journal even when there is no corruption,
which of course 99.99999% or more of the time at any given site.
Anyone who has been through a corruption scenario is happy to
have the corruption at the end rather than in the middle of a
journal file -- as noted, the intent here is noble.  But before
we do any journal rotations, we should detect whether there is
corruption.  Turns out we have a means to detect journal corruption
at the end of the current/active journal file, and should employ
such detection and handle it in some approrpaite manner, e.g.
by expanding the 'force_start' logic in this p4d_base init
script.

Journal corrption detection and preliminary handling may be added
in a future SDP release.  When the journal is truly corrupted,
global replication will stall in any case, so measure like journal
file rotation may be called for in that scenario.

3. Accelerated Deletion of Backups

Increased journal counter rotations result in unexpectedly
fast removal of backups.  Admins are used to thinking that
roughly, "one journal rotation is roughly one day."  Settings
like KEEPLOGS, KEEPCKPS, and KEEPJNLS trigger off the number
of journal rotatations, not the number of actual calendar days.

Now, I think it's OK that journal rotations and days don't
match precisely.  In a typical "big deal" maintenance window,
for example, there might be an additional 1-3 journal rotations
induced by extra checkpoints or journals being created over the
course of a maintenance activity.  But there may be a dozen more
'p4d' restarts during sanity testing and playing around with things.

With the current logic, each restart causes another journal rotation.
By the end fo the weekend, your next call to daily_checkpoint might
remove more of your recent backups than you'd like or expect. (A
long standing safety feature always preserves the last few, but still
we don't want to delete more than desired.)

=== Foor for Thougt: KEEP* = numbrer of days? ===

Making it so KEEPLOGS/KEEPJNLS/KEEPCKPS mean literally number
of days rather than journal rotations is worthy of consideration.
That's beyond the scope of this change though.

#review @robert_cowham @josh
#1 24804 C. Thomas Tyler Terminology tweak, 'validate' -> 'verify'.

#review @robert_cowham
//guest/perforce_software/sdp/dev/Server/Unix/p4/common/bin/validate_sdp.sh
#3 24534 C. Thomas Tyler Various enhancements and internal refactoring for validate_sdp.sh.

Added important check that /p4/N is a dir, not a symlink.

#review-24532 @robert_cowham
#2 24356 C. Thomas Tyler Enhancements to validate_sdp.sh:
* Added simple bold ALL CLEAN message to look for.
* Added check_env_var() function to check shell environment
  variables, with some calls to it.
* Added check_configurable() to check for configurables.  This
  is implemented using back-door check methodology (using
  'p4d_N -cshow') so values can be checked with P4D offline.
  This replaced stub function check_var().
* Removed stub function check_configurables().  It's easier
  to understand if all checks in the Main section of the code.
* Changed so checks requiring p4d to be online are not done
  by default; added '-online' flag to run those tests.  This
  is because I anticpate typical usage of the validator to
  be requiring it to be report ALL CLEAN before starting
  P4D after a server upgrade.
* Added check for new $SDP_ADMIN_PASSWORD_FILE variable.
* Added check admin password file pointed to by $SDP_ADMIN_PASSWORD_FILE.
* Added errmsg() function, with corresponding tweak to bail().
* Consolidated Log an LOGIFLE to just LOGFILE.
* Removed a few items from TOOD comments that got done.
* Made a few tweaks for style normalization:
  - Functions are lowercase with undescore separators.
  - Functions vars are lowercase-initiated camelCase.
  - Indentation: 3 spaces for functions/loops/etc.
* Added run() function replacing cmd() stub function.
       * Enhanced p4login check.
* Added comment noting why this script uses self-contained
  copies of functions defined in other SDP files in /p4/common/lib.
* And other things.

Warning: In the short run, this may fail tests as the new
SDP_ADMIN_PASSWORD_FILE variable is also pending review.

#review @robert_cowham
#1 23640 Robert Cowham Super basic validation - placeholder for many more tests to come!