verify_sdp.sh #4

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

# verify_sdp.sh
# Verifies SDP structure and environment.

#==============================================================================
# Declarations and Environment

export VS_SDP_P4CBIN=/p4/common/bin
export VS_SDP_ENV="$VS_SDP_P4CBIN/p4_vars"
export SDP_INSTANCE="${SDP_INSTANCE:-UnsetSDPInstance}"
export VS_SDP_OWNER

declare Version=5.6.0
declare ThisScript="${0##*/}"
declare CmdArgs="$*"
declare CmdLine="$0 $CmdArgs"
declare -i ServerOnline=0
declare -i ErrorCount=0
declare -i WarningCount=0
declare -i CheckCount=0
declare -i ExitCode=0
declare -i ShowLog=1
declare BadLog=
declare ThisUser=
declare LicenseInfo=
declare LicenseExpiration=
declare CurrentTime=
declare ExpirationTime=
declare TimeDiff=
declare DaysDiff=
declare LicenseDaysExpireWarning=14
declare LinkP4ROOT=
declare LinkeOfflineDB=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare Log=Unset
export P4TMP=Unset

#==============================================================================
# Local Functions

# Note: This script does not use SDP library files, as its purpose is to
# verify the integrity of an SDP installation.  Thus, it has its own
# self-contained versions of some functions for which similar versions
# would normally be sourced from files in /p4/common/lib, like libbcore.sh.

# Micro-functions, small or one-liners used to avoid external dependencies,
# which is critical for this particular verify_sdp.sh script.
function msg () { if [[ "$Log" != off ]]; then echo -e "$*" >> "$Log"; else echo -e "$*"; fi; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () {
   errmsg "${1:-Unknown Error}"
   [[ "$ShowLog" -eq 1 && -s "$Log" ]] && cat "$Log"
   exit "${2:-1}"
}

#------------------------------------------------------------------------------
# Function: run ($cmd, $desc, $shoutOutput)
#
# Runs a command, with optional description, showing command line to execute
# and optionally also the output, and capturing and returning the exit code.
#
# Input:
# $1 - Command and arguments to execute. Defaults to 'echo'.
# $2 - Optional message to display describing what the command is doing.
# $3 - Numeric flag to show output; '1' indicates to show output, 0 to
#      suppress it.
#------------------------------------------------------------------------------
function run () {
   local cmd="${1:-echo}"
   local desc="${2:-}"
   local -i showOutput="${3:-1}"
   local -i exitCode=
   local log

   log="$(mktemp "$P4TMP/run.XXXXXXXXXXX")"

   [[ -n "$desc" ]] && msg "$desc"
   msg "Executing: $cmd"
   $cmd > "$log" 2>&1
   exitCode=$?

   if [[ "$showOutput" -eq 1 ]]; then
      echo "EXIT_CODE: $exitCode" >> "$log"
      cat "$log" >> $Log
   fi

   /bin/rm -f "$log"
   return "$exitCode"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
   fi

   echo "USAGE for verify_sdp.sh v$Version:

verify_sdp.sh [<instance>] [-online] [-si] [-L <log>|off ] [-D]

   or

verify_sdp.sh -h|-man
"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:

	This script verifies the current SDP setup for the specified instance.

	Useful if you change anything, particularly after an SDP upgrade.

OPTIONS:
 <instance>
	Specify the SDP instances.  If not specified, the SDP_INSTANCE
	environment variable is used instead.  If the instance is not
	defined by a parameter and SDP_INSTANCE is not defined,
	exits immediately with an error message.

 -online
	Online mode.  Does additional checks that require P4D to be online.

 -si	Silent mode.  Does not display stdout at the end of processing.

 -L <log>
	Specify the log file to use.  The default is /p4/N/logs/verify_sdp.log
	The special value 'off' disables logging to a file.

 -D	Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	This script is typically called after SDP update with only the instance
	name or number as an argument, e.g.:

	verify_sdp.sh 1

	Or, if used from automation already doing its own logging, use -L off:

	verify_sdp.sh 1 -L off

LOGGING:
	This script generates a log file and also displays it to stdout at the
	end of processing.  By default, the log is:
	/p4/N/logs/verify_sdp.log.

	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.

	If the '-si' (silent) flag is used, the log is generated, but its
	contents are not displayed to stdout at the end of processing.

EXIT CODES:
	An exit code of 0 indicates no errors were encounted attempting to
	perform verifications, and that all checks verified cleanly.
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function: do_preflight_checks ($instance)
#
# If preflight checks fail, further tests are aborted. Failure of the very
# basic preflight checks is an indication that the SDP structure is in
# need of repair.
#
# Sample Usage:
# do_preflght_checks "$SDP_INSTANCE" ||\
#    bail "Preflight checks failed. Aborting further checks."
#------------------------------------------------------------------------------
function do_preflight_checks () {

   local instance="${1:-}"
   local toolsList="date ls grep awk id head tail"

   msg "$H2\\nDoing preflight sanity checks."
   msg "Preflight Check: Ensuring these utils are in PATH: $toolsList"

   for tool in $toolsList; do
      CheckCount+=1
      [[ -z "$(command -v "$tool")" ]] && \
         errmsg "Tool '$tool' not in PATH."
   done

   [[ $ErrorCount -eq 0 ]] || return 1

   msg "Verified: Essential tools are in the PATH."

   msg "Preflight Check: cd $VS_SDP_P4CBIN"

   CheckCount+=1
   if cd "$VS_SDP_P4CBIN" >> $Log 2>&1; then
      cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."
   else
      errmsg "Could not cd to: $VS_SDP_P4CBIN"
      return 1
   fi

   msg "Verified: cd works to: $VS_SDP_P4CBIN"

   msg "Preflight Check: Checking current user owns $VS_SDP_P4CBIN"

   # shellcheck disable=SC2012
   VS_SDP_OWNER="$(ls -ld . | awk '{print $3}')"
   ThisUser="$(id -n -u)"

   CheckCount+=1
   if [[ "$ThisUser" == "$VS_SDP_OWNER" ]]; then
      msg "Verified: Current user [$ThisUser] owns $VS_SDP_P4CBIN"
   else
      errmsg "Current user [$ThisUser] does not own $VS_SDP_P4CBIN. This most likely means this script is running as the wrong user.  It could also mean the $VS_SDP_P4CBIN directory is not owned by by the correct owner, which should be the OS account under which the p4d process runs."
      return 1
   fi

   msg "Preflight Check: Checking /p4 and /p4/<instance> are local dirs."
   if ! check_local_instance_home_dir "$instance"; then
      errmsg "The SDP /p4 and /p4/<instance> dirs are NOT local."
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: check_file ($file, $errMsg, $warningOnly)
#
# Checks for existance of a file, returns 0 if it exists, 1 otherwise.
# Allows optional custom error message describing the file, to be displayed if
# the file is missing.  Default error message is "Missing file [FILE]."
#------------------------------------------------------------------------------
function check_file () {
   local file=$1
   local errMsg=${2:-Missing file}
   local warningOnly=${3:-0}
   CheckCount+=1
   msg "Checking existence of file $file"
   if [[ $warningOnly -eq 0 ]]; then
      [[ -f $file ]] && return 0
      errmsg "$errMsg: [$file]."
   else
      [[ -f $file ]] && return 0
      warnmsg "$errMsg: [$file]."
   fi
   return 1
}

#------------------------------------------------------------------------------
# Function: check_configurable ($instance, $configurable, $scope, $expectedVal, $errMsg1, $errMsg2)
#
# Check that a configurable is set, and optionally check that it is set to
# an expected value.
#
# Inputs:
# $1 - SDP Instance. Required.
# $2 - Configurable name. Required.
# $3 - Configurable scope/ServerID, as per 'p4 help configure'.  The default
#      is "any", meaning what it means with 'p4 configure set', i.e. that the
#      configurable is a global default.  The special value 'ALL' can
#      also be supplied parameter, which is has the special meaning of checking
#      if the configurable is defined for any ServerID, including the 'any'
#      value.  The value returned is that of the first setting encountered.
# $4 - Expected value of configurable. Optional. If defined, an additional check is
#      done, checking the current value against the expected value.  Optionally,
#      the special value UNDEF can be used, which reverses the exit code, such
#      that a happy zero is returned only if the value is not set.
# $5 - Optional error message to display if no value is defined.  See code
#      below for the default message.
# $6 - Optional error message to display if a value is defined but does not
#      match the expected value.  See code below for the default message.
#
# Return Codes:
# 1 - Verifications failed.
# 0 - Verifications passed.
# 
# Sample Usage: 
# check_configurable "$SDP_INSTANCE" journalPrefix
#
# check_configurable "$SDP_INSTANCE" journalPrefix any "$CHECKPOINTS/$P4SERVER"
#
# check_configurable "$SDP_INSTANCE" journalPrefix any "$CHECKPOINTS/$P4SERVER" ||\
#   bail "Yikes, journalPrefix is not set, all bets are off. Aborting."
#------------------------------------------------------------------------------
function check_configurable () {
   local instance="$1"
   local configurable="$2"
   local scope="${3:-any}"
   local expectedValue="${4:-NoExpectedValue}"
   local errMsgMissing="${5:-No value defined}"
   local errMsgBadValue="${6:-Value does not match what is expected}"
   local detectedScope=
   local value=
   CheckCount+=1

   if [[ "$scope" != "ALL" ]]; then
      value=$($P4DBIN -r "$P4ROOT" -cshow | grep "^${scope}: ${configurable} = ")
   else
      value=$($P4DBIN -r "$P4ROOT" -cshow | grep ": ${configurable} = " | head -1)
      detectedScope="$value"
      value=${value##* = }
      detectedScope="${detectedScope%%:*}"
   fi

   if [[ "$expectedValue" != "UNDEF" ]]; then
      if [[ -n "$value" ]]; then
         value=${value##* = }
         if [[ "$scope" != "ALL" ]]; then
            msg "Verified: Configurable ${scope}:${configurable} is defined."
         else
            msg "Verified: Configurable ${configurable} is defined for at least once."
         fi
      else
         errmsg "$errMsgMissing for configurable [${scope}:${configurable}]."
         return 1
      fi
   else
      if [[ -n "$value" ]]; then
         if [[ "$scope" != "ALL" ]]; then
            errmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${scope} of: ${value}"
            return 1
         else
            errmsg "Configurable ${configurable} should not be set with 'p4 configure set' but has a value for ServerID ${detectedScope} of: ${value} (and possibly for other ServerIDs)."
            return 1
         fi
      else
         if [[ "$scope" != "ALL" ]]; then
            msg "Verified: Configurable ${scope}:${configurable} is undefined."
         else
            msg "Verified: Configurable ${configurable} is undefined."
         fi
      fi
   fi

   [[ "$expectedValue" == "NoExpectedValue" ]] && return 0

   CheckCount+=1

   if [[ "$expectedValue" != "UNDEF" ]]; then
      if [[ "$value" == "$expectedValue" ]]; then
         msg "Verified: Configurable ${scope}:${configurable} has expected value [$value]."
      else
         errmsg "$errMsgBadValue for variable [${scope}:${configurable}]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
         return 1
      fi
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: check_env_var ($instance, $var, $expectedval, $msg1, $msg2)
#
# Check that a shell environment variable is set when sourcing the SDP
# environment. Optionally checks that variables are set to expected values.
#
# Inputs:
# $1 - SDP Instance. Required.
# $2 - Variable name. Required.
# $3 - Expected value of variable. Optional. If defined, an additional check is
#      done, checking the current value against the expected value.
# $4 - Optional error message to display if no value is defined.  See code
#      below for the default message.
# $5 - Optional error message to display if a value is defined but does not match
#      the expected value.  See code below for the default message.
# 
# Return Codes:
# 1 - Verifications failed.
# 0 - Verifications passed.
# Sample Usage: 
# check_env_var $SDP_INSTANCE P4JOURNAL "/p4/$SDP_INSTANCE/logs/journal"
#
# check_env_var $SDP_INSTANCE P4JOURNAL "/p4/$SDP_INSTANCE/logs/journal" ||\
#   bail "Yikes, P4JOURNAL is not set, all bets are off. Aborting."
#------------------------------------------------------------------------------
function check_env_var () {
   local instance="$1"
   local var="$2"
   local expectedValue="${3:-NoExpectedValue}"
   local errMsgMissing="${4:-No value defined}"
   local errMsgBadValue="${5:-Value does not match what is expected}"
   local value=
   CheckCount+=1

   eval unset "${var}"
   # shellcheck disable=SC1090
   source "$VS_SDP_ENV" "$instance"

   set +u
   if [[ -n "$(eval echo \$"${var}")" ]]; then
      msg "Verified: Variable ${var} is defined."
      set -u
   else
      errmsg "$errMsgMissing for variable [$var]."
      set -u
      return 1
   fi

   [[ "$expectedValue" == "NoExpectedValue" ]] && return 0

   CheckCount+=1
   value="$(eval echo \$"${var}")"

   if [[ "$value" == "$expectedValue" ]]; then
      msg "Verified: Variable ${var} has expected value [$value]."
   else
      errmsg "$errMsgBadValue for variable [$var]\\n\\tExpected value: [$expectedValue]\\n\\tActual value:   [$value]"
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: check_local_instance_home_dir ($instance)
#
# Check that the '/p4' directory and the instance home directory '/p4/N' are
# local directories on the root volume, per SDP structural intent.
#
# Inputs:
# $1 - SDP Instance. Required.
#
# Return Codes:
# 0 - Verifications were able to at least run; ErrorCount is incremented
#     if tests fail.
# 1 - Verifications could not even complete. This is a pre-flight failure.
#
# This increments globals CheckCount and possibly ErrorCount.
#
# Sample Usage: 
# check_local_instance_home_dir "$SDP_INSTANCE" ||\
#    bail "Error checking p4dir and/dor local instance home dir."
#------------------------------------------------------------------------------
function check_local_instance_home_dir () {
   local instance="$1"
   local p4Dir="/p4"
   local p4HomeDir="$p4Dir/$instance"

   CheckCount+=1
   if [[ "$P4HOME" == "$p4HomeDir" ]]; then
      msg "Verified: P4HOME has expected value: $p4HomeDir"
   else
      errmsg "P4HOME has unexpected value: $p4HomeDir"
   fi

   CheckCount+=1
   if [[ -L "$p4HomeDir" ]]; then
      errmsg "This is a symlink; it should be a local directory: $p4HomeDir"
   else
      msg "Verified: This P4HOME path is not a symlink: $p4HomeDir"
   fi

   CheckCount+=1
   if cd "$p4Dir"; then
      msg "Verified: cd to $p4Dir OK."
      CheckCount+=1
      if [[ "$(pwd -P)" == "$p4Dir" ]]; then
         msg "Verified: Dir $p4Dir is a local dir."
      else
         errmsg "Dir $p4Dir is NOT a local dir."
      fi
      cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."

      CheckCount+=1
      if cd "$p4HomeDir"; then
         msg "Verified: cd to $p4HomeDir OK."
         CheckCount+=1
         if [[ "$(pwd -P)" == "$p4HomeDir" ]]; then
            msg "Verified: P4HOME dir $p4HomeDir is a local dir."
         else
            errmsg "P4HOME dir $p4HomeDir is NOT a local dir."
         fi
         cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."
      else
         errmsg "Could not cd to $p4HomeDir."
         return 1
      fi
   else
      errmsg "Could not cd to $p4Dir."
      return 1
   fi

   return 0
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-online) ServerOnline=1;;
      (-si) ShowLog=0;;
      (-L) Log="$2"; shiftArgs=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown command line option ($1).";;
      (*) export SDP_INSTANCE=$1;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SDP_INSTANCE" == UnsetSDPInstance ]] && \
   usage man "Missing <instance> parameter. The <instance> must be given as a parameter to this script, or else the \$SDP_INSTANCE environment variable defined.  It can be set by doing:\\n\\n\\tsource $VS_SDP_ENV <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n"

#==============================================================================
# Main Program

# shellcheck disable=SC1090
source "$VS_SDP_ENV" "$SDP_INSTANCE" ||\
   bail "Failed to load SDP environment for instance $SDP_INSTANCE."

# shellcheck disable=SC1090
source "$P4CBIN/backup_functions.sh" ||\
   bail "Failed to load backup_functions.sh."

[[ "${OSUSER:-Unset}" == "Unset" ]] &&\
   bail "The critical OSUSER setting is not defined in $VS_SDP_ENV. Aborting."

# If this verify_sdp.sh script is called by root, change to the root user.
if [[ $(id -u) -eq 0 ]]; then
   exec su - "$OSUSER" -c "$VS_SDP_P4CBIN/${0##*/} $CmdArgs"
elif [[ $(id -u -n) != "${OSUSER:-UnknownOSUSER}" ]]; then
   bail "${0##*/} can only be run by root or $OSUSER"
fi

# Logs should be defined to /p4/N/logs after sourcing the environment
# file above; default to /tmp for cases of incomplete environment where
# LOGS is not defined.
export LOGS="${LOGS:-/tmp}"

[[ "$Log" == Unset ]] && Log="${LOGS}/verify_sdp.log"

if [[ -f "$Log" ]]; then
   if [[ ! -w "$Log" ]]; then
      BadLog="$Log"
      Log=Unset
      bail "Existing log file [$BadLog] is not writable. Aborting."
   fi
else
   if [[ ! -d "${LOGS}" ]]; then
      Log=Unset
      bail "Logs directory [$LOGS] is not writable. Aborting."
   fi
fi

[[ "$P4TMP" != Unset && -d "$P4TMP" && -w "$P4TMP" ]] ||\
   bail "SDP environment must define required P4TMP variable. Value must be a directory that is writable; value is: $P4TMP"

if [[ "$Log" != "off" ]]; then
   rotate_log_file "$Log" ".gz"
fi

msg "$ThisScript v$Version Starting SDP verification on host ${HOSTNAME%%.*} at $(date +'%a %Y-%m-%d %H:%M:%S %Z') with this command line:\\n$CmdLine"

msg "\\nIf you have any questions about the output from this script, contact support@perforce.com."

do_preflight_checks "$SDP_INSTANCE" ||\
   bail "Preflight checks failed. Aborting further checks."

msg "${H2}\\nChecking environment variables."
check_env_var "$SDP_INSTANCE" SDP_INSTANCE
check_env_var "$SDP_INSTANCE" P4ROOT "/p4/$SDP_INSTANCE/root"
check_env_var "$SDP_INSTANCE" P4JOURNAL "/p4/$SDP_INSTANCE/logs/journal"
check_env_var "$SDP_INSTANCE" P4MASTER_ID
check_env_var "$SDP_INSTANCE" P4MASTER
check_env_var "$SDP_INSTANCE" P4MASTERPORT

if ! check_env_var "$SDP_INSTANCE" SDP_ADMIN_PASSWORD_FILE "$P4CCFG/.p4passwd.$P4SERVER.admin"; then
   check_file "$SDP_ADMIN_PASSWORD_FILE" "SDP admin password file doesn't exist"
fi

msg "${H2}\\nRunning standard checks typically called within SDP scripts."
check_vars
CheckCount+=1
set_vars
check_dirs
CheckCount+=1

msg "${H2}\\nChecking for a few database files."
# Check db files exist
for file in db.counters db.domain db.config; do
   for dir in $P4ROOT $OFFLINE_DB; do
      check_file "$dir/$file" "Expected database file doesn't exist."
   done
done

msg "${H2}\\nChecking for existance of key files."
check_file "$OFFLINE_DB/offline_db_usable.txt" "Offline database not in a usable state."
check_file "$P4BIN" "The p4 binary (or symlink) doesn't exist."
# shellcheck disable=SC2154
check_file "$P4DInitScript" "The p4d init script doesn't exist."
check_file "$P4TICKETS" "The P4TICKETS file doesn't exist."

msg "${H2}\\nLicense Checks."

#------------------------------------------------------------------------------
# A sample license line looks like this:
# License: Perforce Battle School 28 users (support ends 2020/06/01) (expires 2020/06/01)
# Existence of 'support ends' indicates the license valid.
# Existence of 'expires' indicates a temp or subscription license.
#------------------------------------------------------------------------------
if check_file "$P4ROOT/license" "The \$P4ROOT/license file doesn't exist" 1; then
   LicenseInfo=$($P4DBIN -V | grep '^License:')
   CheckCount+=1
   if [[ -n "$LicenseInfo" ]]; then
      if [[ "$LicenseInfo" == *" expired "* ]]; then
         errmsg "The license is expired."
      elif [[ "$LicenseInfo" == *"expires"* ]]; then
         LicenseExpiration=${LicenseInfo##*(expires }
         LicenseExpiration=${LicenseExpiration%%)*}

         # Check License
         CheckCount+=1
         CurrentTime=$(date +%s)
         ExpirationTime=$(date +%s --date "$LicenseExpiration")
         TimeDiff=$((ExpirationTime-CurrentTime))
         DaysDiff=$((TimeDiff/(3600*24)))

         msg "Info: License expires on $LicenseExpiration (in $DaysDiff days)."
         if [[ "$DaysDiff" -le "$LicenseDaysExpireWarning" ]]; then
            errmsg "License will expire within $LicenseDaysExpireWarning days."
         fi
      elif [[ "$LicenseInfo" == *"support ends"* ]]; then
         msg "Info: License is perpetual."
      fi
   else
      errmsg "Could not determine license info from license file."
   fi
fi

msg "${H2}\\nChecking configurables values."

check_configurable "$SDP_INSTANCE" P4JOURNAL ALL UNDEF
msg "Note: P4JOURNAL must be defined as a shell environment variable and must not set in db.config.\\n"
check_configurable "$SDP_INSTANCE" journalPrefix any "$CHECKPOINTS/$P4SERVER"
check_configurable "$SDP_INSTANCE" server.depot.root any "$DEPOTS"

msg "${H2}\\nChecking P4ROOT and offline_db syminks."
CheckCount+=1
LinkP4ROOT="$(readlink "$P4ROOT")"

if [[ "$LinkP4ROOT" == *"/db1" || "$LinkP4ROOT" == *"/db2" ]]; then
   msg "Verified: Symlink for P4ROOT points to a db1 or db2 path."
else
   errmsg "Symlink for P4ROOT does not point to a db1 or db2 path."
fi

CheckCount+=1
LinkOfflineDB="$(readlink "$OFFLINE_DB")"
if [[ "$LinkOfflineDB" == *"/db1" || "$LinkOfflineDB" == *"/db2" ]]; then
   msg "Verified: Symlink for offline_db points to a db1 or db2 path."
else
   errmsg "Symlink for offline_db does not point to a db1 or db2 path."
fi

CheckCount+=1
if [[ "$LinkP4ROOT" != "$LinkOfflineDB" ]]; then
   msg "Verified: Symlinks for P4ROOT and offline_db do not point to the same target."
else
   errmsg "Symlinks for P4ROOT and offline_db point to the same target."
fi

msg "${H2}\\nChecking for standard symlink/dir structure."

CheckCount+=1
if cd "$P4HOME"; then
   CheckCount+=1
   if [[ -d "$PWD/bin" ]]; then
      msg "Verified: $PWD/bin is a regular directory."
   else
      warnmsg "$PWD/bin is not a regular directory."
   fi

   for link in checkpoints depots root offline_db logs tmp; do
      CheckCount+=1
      if [[ -L "$PWD/$link" ]]; then
         msg "Verified: $PWD/$link is a symlink."
      else
         errmsg "$PWD/$link is not a symlink directory."
      fi
   done

   CheckCount+=1
   if [[ -L "/p4/common" ]]; then
      msg "Verified: /p4/common is a symlink."
   else
      errmsg "Path /p4/common is not a symlink."
   fi

   CheckCount+=1
   if [[ -L "/p4/sdp" ]]; then
      msg "Verified: /p4/sdp is a symlink."
   else
      errmsg "Path /p4/sdp is not a symlink."
   fi

   cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."
else
   errmsg "Could not cd to $P4HOME." 
fi

if [[ "$ServerOnline" -eq 1 ]]; then
   msg "${H2}\\nDoing online checks."
   CheckCount+=1
   if run "$P4CBIN/p4login" "Login check" 0; then
      msg "Verified: Login OK."
   else
      errmsg "Login as P4USER $P4USER to P4PORT $P4PORT could not be verified."
   fi
fi

### P4="$P4BIN -p $P4PORT -u $P4USER"

# TODO:
# - Ensure links are all present and correct for instance
# - Ensure crontab includes at least daily_checkpoint.sh [make this skippable]
# - Ensure checkpoints dir contains a checkpoint or two
# - service password verified as set for a replica (value doesn't matter, it just needs to be set).
# - Add flag to check less-critical SDP configurables, and generate
#   warnings (rather than errors) if they are not set as expected,
#   using SDP configure_new_servers.sh script as a guide.

if [[ "$ErrorCount" -eq 0 && "$WarningCount" -eq 0 ]]; then
   msg "\\n${H1}\\n\\nALL CLEAN: $CheckCount verifications completed OK."
elif [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\n${H1}\\n\\nNO ERRORS: $CheckCount verifications completed, with $WarningCount warnings detected."
else
   msg "\\n${H1}\\n\\nVerifications completed, with $ErrorCount errors and $WarningCount warninigs detected in $CheckCount checks."
   ExitCode=1
fi

[[ "$ShowLog" -eq 1 && -s "$Log" ]] && cat "$Log"
 
exit $ExitCode