p4verify.sh #28

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

declare StatusMessage="OK: All scanned depots verified OK."
declare ThisScript=${0##*}
declare ThisUser=
declare Version=5.8.8
declare CmdLine="$0 $*"
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i NoOp=0
declare -i Debug=0
declare -i ShowLog=0
declare -i ExitCode=0
declare -i VerifyArchived=0
declare -i VerifyChunks=0
declare -i VerifyShelved=1
declare -i VerifySpecDepot=1
declare -i VerifySubmitted=1
declare -i VerifyUnload=1
declare -i GzipOldLog=1
declare -i ScheduleTransfers=1
declare -i ShelvedCLCount=0
declare -i ShelvedCLErrorCount=0
declare -i DepotCount=0
declare -a DepotList

# VerifyErrors is an associative array of errors indexed as follows:
# archive
# shelved
# spec
# submitted
# unload
# Values are the number of errors (BAD or MISSING) for each type of depot.
declare -A VerifyErrors

declare SDPInstance=
declare SDPEnv="/p4/common/bin/p4_vars"
declare DepotListFile=
declare RevRange=
declare RecentChangesToVerify=200
declare VerifyCmd=
declare ChangesCmd=
declare ShelvedChangesToVerify=
declare ShelvedCL=
declare VerifyOnlyOption=
declare VerifyOptions=
declare ChunkFile=
declare ChunkPath=
declare ChunkSize=
declare ChunkScript="/p4/common/bin/depot_verify_chunks.py"
declare Log=Unset
declare SummaryLog=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare TmpFile=
declare TmpLog=

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
   fi

   echo "USAGE for $ThisScript v$Version:

p4verify.sh [<instance>] [-nu] [-nr] [-ns] [-nS] [-a] [-nt] [-nz] [-o BAD|MISSING] [-chunks <ChunkSize>] [-recent] [-dlf <depot_list_file>] [-n] [-L <log>] [-v] [-d] [-D]

   or

p4verify.sh -h|-man
"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:

	This script performs a 'p4 verify' of all submitted and shelved versioned
	files in depots of all types except 'remote' and 'archive' type depots.

	If run on a replica, it schedules archive failures for transfer to the
	replica.

OPTIONS:
<instance>
	Specify the SDP instance.  If not specified, the SDP_INSTANCE
	environment variable is used instead.  If the instance is not
	defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
	exists immediately with an error message.

 -nu	Specify '-nu' (No Unload) to skip verification of the singleton depot
	of type 'unload' (if created).  The 'unload' depot is verified
	by default.

 -nr	Specify '-nr' (No Regular) to skip verification of regular submitted
	archive files.  The '-nr' option is not compatible with '-recent'.
	Regular submitted archive files are verified by default.

 -ns	Specify '-ns' (No Spec Depot) to skip verification of singleton depot
	of type 'spec' (if created). The 'spec' depot is verified by default.

 -nS	Specify '-nS' (No Shelves) to skip verification of shelved archive
	files, i.e. to skip the 'p4 verify -qS'.

 -a	Specify '-a' (Archive Depots) to do verification of depots of type
	'archive'.  Depots of type 'archive' are not verified by default, as
	archive depots are often physically removed from the server's
	storage subsystem for long-term cold storage.
 
-nt	Specify the '-nt' option to avoid passing the '-t' flag to 'p4 verify' 
	on a replica. By default, p4verify.sh detects if it is running on a
	replica, and if so automatically applies the '-t' flag to 'p4 verify'.
	That causes the replica to attempt to self-heal, as files that fail
	verification are scheduled for transfer from the P4TARGET server. This
	default behavior results in 'Transfer scheduled' messages in the log,
	and MISSING/BAD files are listed as 'info:' rather than 'error:'. There
	is no clear indication of whether or which of the scheduled transfers
	complete successfully, and so there may be a mix of transient/correctable
	and \"real\"/persistent transfer errors for files that are also BAD/MISSING
	on the master server.  Specify '-nt' to ensure the log contains a list
	of files that currently fail a 'p4 verify' without attempting to transfer
	them from the master.

 -nz	Specify '-nt' to skip the gzip of the old log file.  By default, if a
	log with the default name or the name specified with '-L' exists at the
	start of processing, the old log is rotated and gzipped.  With this option
	the old log is not zipped when rotated.

 -o BAD|MISSING
	Specify '-o MISSING' to check only whether expected archive files exist
	or not, skipping the checksum calculation of existing files.  This results
	in dramatically faster, if less comprehensive, verification.  This
	is particularly well suited when verification is being used to schedule
	archive file transfers of missing files on replicas.  This translates into
	passing the '--only MISSING' option to 'p4 verify'.

	Specify '-o BAD' to check only for BAD revisions.  This translates into
	passing the '--only BAD' option to 'p4 verify'.

	This option requires p4d to be 2021.1 or newer. For older p4d versions,
	this option is silently ignored.

 -chunks <ChunkSize>
	Specify the maximum amount of content by size to verify at once.  If
	this is specified, the depot_verify_chunks.py script is used to
	break up depots into chunks of a given size, e.g. 100M or 4G.

	The <ChunkSize> parameter must be a size value valid to pass to the
	depot_verify_chunks.py script with the '-m' option.

	This requires the perforce-p4python3 module to be installed and the
	python3 in the PATH must be the correct one that uses the P4 module.

	The -recent and -chunks options are mutually exclusive.

 -recent
	Specify that only recent changelists should be verified.
	The \$SDP_RECENT_CHANGES_TO_VERIFY variable defines how many
	changelists are considered recent; the default is $RecentChangesToVerify.

	If the default is not appropriate for your site, add
	\"export SDP_RECENT_CHANGES_TO_VERIFY\" to /p4/common/config/p4_N.vars to
	change the default for an instance, or to /p4/common/bin/p4_vars to
	change it globally.  If \$SDP_RECENT_CHANGES_TO_VERIFY is unset, the
	default is $RecentChangesToVerify.

	When -recent is used, neither shelves nor files in the unload depot
	are verified.

 -dlf <depot_list_file>
	Specify a file containing a list of depots to process in the desired
	order.  By default, all depots reported by 'p4 depots', which
	effectively results in depots processed in alphabetical order.

	This can be useful in time-sensitive situations where the order
	of processing can be prioritized, and/or to prevent processing
       	certain depots.

	The format fo the depot list file is straighforward, one line per
	depot, without the '//' nor trailling /..., so a list might look
	like this sample:

	ProjA
	ProjB
	spec
	.swarm
	unload
	archive
	ProjC

	Blank lines and lines starting with a '#' are treated as comments and
	ignored.

	WARNING: This is not intended to be the primary method of verification,
	because it would be easy to forget to add new depots to the list file.

	If the depot list file is not readable, processing aborts.

 -v     Verbose.  Show output of verify attempts, which is suppressed by default.
	Setting SDP_SHOW_LOG=1 in the shell environment has the same effect as -v.

	The default behavior of this script is to generate no terminal output,
	but instead to write output into a log file -- see LOGGING below.  If
	'-v' is specified, the generated log is sent to stdout at the end of
	processing.  This flag is not recommended for routine cron operation or
	for large data sets.

	The -chunks and -recent options are mutually exclusive.

 -L <log>
	Specify the log file to use.  The default is /p4/N/logs/p4verify.log

	Log rotation and old log cleanup logic does not apply to log files
	specified with -L.  Thus, using -L is not recommended for routine
	scheduled operation, e.g. via crontab.

DEBUGGING OPTIONS:
 -n	No-Operation (NO_OP) mode, for debugging.

 	Display certain commands that would be executed without executing
	them. When '-n' is used, commands that might take a long time to
	run or affect data are only displayed.

	Even in '-n' mode, some information-gathering commands such as
	listing shelved CLs are executed, which may cause the script to take
	a bit of time to run on a large data set even in dry run mode.

 -d	Specify that debug messages should be displayed.

 -D     Use bash 'set -x' extreme debugging verbosity, and imply '-d'.

 -L off
	The special value '-L off' disables logging. This can only be used
	with '-n' for debugging.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	Example 1: Full Verify

	This script is typically called via cron with only the instance
	parameter as an argument, e.g.:

	p4verify.sh 1

	Example 2: Fast Verify

	A \"fast\" verify is one in which only the check for MISSING archives
	is done, while the resource-intensive checksum calculation of
	potentially BAD existing archives is skipped.  This is especially
	useful when used on a replica.

	p4verify.sh 1 -o MISSING

	Example 3: Fast and Recent Verify

	The '-o MISSING' and '-recent' flags can be combined for a very
	fast check.  This check might be incorporated into a failover
	procedure.

	p4verify.sh 1 -o MISSING -recent

	Example 4: Submitted Files Only

	This will verify only use submitted files, ignoring shelves and the
	spec and unload depots, putting the results in a specified log:

	p4verify.sh 1 -ns -nS -nu -L  -L /p4/1/logs/p4verify.submitted.log

	Example 5: Shelved Files Only

	This will verify only use submitted files, ignoring shelves and the
	spec and unload depots, putting them in a specified log:

	p4verify.sh 1 -nr -ns -nu -L /p4/1/logs/p4verify.shelved.log

	Example 6: A Dry Run

	The '-n' option can be used for a dry run.  Output may also be
	displayed to the screen ('-v') for a dry run and the log file optionally
	discarded:
	
	p4verify.sh 1 -n -nS -L off -v
	
NOHUP USAGE:
	Because archive verification is typically a long running task,
	it is advisable to use 'nohup' to call each command, and combine
	that by running the command as a background process.  Alternately,
	use 'screen' or similar.

	Any of the examples above can be used with 'nohup', without output
	redirected to /dev/null (i.e. to \"the void\", as this script handles
	logging and output redirection).

	To use 'nohup', start the command line with 'nohup', and then after
	the command, add this text exactly:

	< /dev/null > /dev/null 2>&1 &

	As a example, Example 2 above, called with nohup, would look like:

	nohup /p4/common/bin/p4verify.sh 1 -o MISSING < /dev/null > /dev/null 2>&1 &

	With the ampersand '&' at the end, the command will appear to return
	immediately as the process continues to run in the background.

	Then optionally monitor the log:

	tail -f /p4/1/logs/p4verify.log

LOGGING:
	This script generates no output by default.  All (stdout and stderr) is
	logged to /p4/N/logs/p4verify.log.

	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.

NOTIFICATIONS:
	In addition to logging, a short summary of the verify is sent as a
	notification.  The summary is reliably short even if the output of the
	verifications done by this script results in a large log file.

	There are two notification schemes with this script:

	* Email notification is always attempted.

	* AWS SNS notification is attempted if the SNS_ALERT_TOPIC_ARN custom
	setting is defined. This is typically set in:

	/p4/common/site/config/p4_N.vars.local

EXIT CODES:
	An exit code of 0 indicates no errors were encountered attempting to
	perform verifications, AND that all verifications attempted
	reported no problems.

	A exit status of 1 indicates that verifications could not be
	attempted for some reason.

	A exit status of 2 indicates that verifications were successfully
	performed, but that problems such as BAD or MISSING files
	were detected, or else system limits prevented verification.
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   [[ "$Log" == "off" ]] || msg "Log is: $Log\\n${H1}"

   # Be tidy.
   [[ -n "$TmpFile" && -e "$TmpFile" ]] && rm -f "$TmpFile"

   # With the trap removed, exit.
   exit "$ExitCode"
}

function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit; }

#------------------------------------------------------------------------------
# Function: get_verify_rev_range ($depot, $recentChanges)
#------------------------------------------------------------------------------
function get_verify_rev_range () {
   declare depot=${1:-}
   declare recentChangesToVerify=${2:-}
   declare nowChange=
   [[ -z "$depot" || -z "$recentChangesToVerify" ]] && return 0

   nowChange=$("$P4BIN" -ztag -F %change% changes -m 1 "//$depot/...")
   thenChange=$("$P4BIN" -ztag -F %change% changes -m "$recentChangesToVerify" "//$depot/..." | tail -1)
   [[ -z "$nowChange" ]] && return 1
   [[ -z "$thenChange" ]] && return 1

   echo "@$thenChange,@$nowChange"

   return 0
}

#------------------------------------------------------------------------------
# Function: call_p4verify($verifyErrorType, $verifyCommand)
#
# This calls the 'p4 verify' command with the command given, and increments
# count of errors indexted by verify, updating the VerifiyErrors array
# displayed in the summary.
#
# VerifyErrors is an associative array of errors indexed as follows:
# archive
# shelved
# spec
# submitted
# unload
#
# Values are the number of errors (including BAD/MISSING/other) for each type.
#
function call_p4verify () { 
   local -i status
   local verifyErrorType="${1:-}"
   local verifyCommand="${2:-}"
   local tmpf=
   local oldErrors=
   local newErrors=
   local totalErrors=

   [[ -n "$verifyErrorType" || -n "$verifyCommand" ]] || return

   msg "${H2}\\nRunning: $verifyCommand"
   tmpf="${P4TMP}/t_verify.$$.$RANDOM"

   if [[ "$NoOp" -eq 0 ]]; then
      eval "$verifyCommand" > "$tmpf" 2>&1
      status=$?

      # Suppress 'no such file' errors from older p4d versions to avoid
      # confusion, as these are not actual errors. In newer p4d versions,
      # this is a mere warning, which we don't suppress.
      grep -Ev "^error: //.*/... - no such file(s)." "$tmpf"

      # See if there are any new errors:
      newErrors=$(grep -Ec '(BAD!|MISSING!|p4 help max)' "$tmpf")

      if [[ "$newErrors" =~ [0-9]+ ]]; then
         if [[ "$newErrors" != "0" ]]; then
            oldErrors=${VerifyErrors[$verifyErrorType]:-0}
            totalErrors=$((oldErrors+newErrors))
            VerifyErrors[$verifyErrorType]="$totalErrors"
         fi
      else
         errmsg "Unable to count verify errors in [$tmpf]."
      fi

      rm -f "$tmpf"
   else
      msg "NO_OP: Would run: $verifyCommand"
      status=0
   fi

   return "$status"
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case "$1" in
      (-h) usage -h;;
      (-man) usage -man;;
      (-ns) VerifySpecDepot=0;;
      (-nS) VerifyShelved=0;;
      (-nr) VerifySubmitted=0;;
      (-nu) VerifyUnload=0;;
      (-a) VerifyArchived=1;;
      (-nt) ScheduleTransfers=0;;
      (-nz) GzipOldLog=0;;
      (-o) VerifyOnlyOption="--only ${2^^}"; shiftArgs=1;;
      (-chunks) VerifyChunks=1; ChunkSize="$2"; shiftArgs=1;;
      (-recent) VerifyOnlyRecentChanges=1;;
      (-dlf) DepotListFile="${2:-}"; shiftArgs=1;;
      (-n) NoOp=1;;
      (-v) ShowLog=1;;
      (-L) Log="$2"; shiftArgs=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Debug; use bash 'set -x' extreme debugging mode.
      (-*) usage -h "Unknown command line option ($1).";;
      (*)
         if [[ -z "$SDPInstance" ]]; then
            SDPInstance="$1"
         else
            usage -h "Unknown extra parameter [$1]."
         fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ -z "$SDPInstance" ]] && SDPInstance="${SDP_INSTANCE:-UnsetSDPInstance}"

[[ "$SDPInstance" == "UnsetSDPInstance" ]] && \
   usage -h "The \\$SDP_INSTANCE setting is not defined. It must be defined by doing:\\n\\n\\tsource /p4/common/bin/p4_vars <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n"

# Load SDP Environment. In case this generates output on stdout, suppress it to
# avoid triggering crontab emails, but do not suppress stderr (output related
# to errors).
# shellcheck disable=SC1090
source "$SDPEnv" "$SDPInstance" > /dev/null ||\
   bail "Failed to load SDP environment with: source $SDPEnv $SDPInstance"

[[ "$VerifySubmitted" -eq 0 && "$VerifyOnlyRecentChanges" -eq 1 ]] && \
   usage -h "The '-recent' and '-nr' options are mutually exclusive."

if [[ -n "$VerifyOnlyOption" ]]; then
   # shellcheck disable=SC2072
   if [[ "$P4D_VERSION" > "2021.1" ]]; then
      if [[ ! "$VerifyOnlyOption" =~ ^--only\ (MISSING|BAD)$ ]]; then
         usage -h "The option specified with '-o' must be BAD or MISSING. Specified value was: $VerifyOnlyOption"
      fi
   else
      VerifyOnlyOption=
   fi
fi

[[ "$VerifyOnlyRecentChanges" -eq 1 && "$VerifyChunks" -eq 1 ]] && \
   usage -h "The -recent and -chunks options are mutually exclusive."

if [[ -n "$ChunkSize" && ! -x "$ChunkScript" ]]; then
   msg "Warning: -chunks ignored because chunking script is not executable by user $ThisUser: $ChunkScript"
   VerifyChunks=0
fi

if [[ -n "$DepotListFile" && ! -r "$DepotListFile" ]]; then
   usage -h "The file specified with '-dlf <depot_list-file>' does not exist: $DepotListFile"
fi

[[ "$Log" == off && "$NoOp" -eq 0 ]] && \
   usage -h "The '-L off' option can only be specified with '-n' (NO_OP/Dry Run mode)."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

# If $SDP_RECENT_CHANGES_TO_VERIFY, use that, else default to the value set for
# $RecentChangesToVerify in this script.
[[ -n "${SDP_RECENT_CHANGES_TO_VERIFY:-}" ]] && \
   RecentChangesToVerify="${SDP_RECENT_CHANGES_TO_VERIFY}"

[[ "$Log" == Unset ]] && Log="$LOGS/p4verify.log"

if [[ "$Log" != off ]]; then
   SummaryLog="$(mktemp "${P4TMP}/email_summary.XXXXXXXXX.txt")"
   if [[ -e "$Log" ]]; then
      # shellcheck disable=SC2012
      OldLogTimestamp="$(ls -l --time-style +'%Y%m%d-%H%M%S' "$Log" | awk '{print $6}')"
      OldLog="$LOGS/p4verify.${OldLogTimestamp}.log"
      mv -f "$Log" "$OldLog" || bail "Could not do: mv -f \"$Log\" \"$OldLog\""

      # Gzip old log.
      if [[ "$GzipOldLog" -eq 1 && -n "$(command -v gzip)" ]]; then
         dbg "gzip --best \"$OldLog\""
         gzip --best "$OldLog"
      fi
      dbg "Rotated old log $Log to $OldLog"
   fi

   touch "$Log" || bail "Couldn't touch new log file [$Log]."?

   # Redirect stdout and stderr to a log file.
   if [[ "$ShowLog" -eq 1 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@${HOSTNAME%%.*} on $(date +'%a %Y-%m-%d %H:%M:%S %Z') as:\\n$CmdLine"

P4="$P4BIN -p $P4PORT -u $P4USER"
"$P4CBIN/p4login"

msg "If there are errors in this log, contact support-helix-core@perforce.com"

VerifyFailed=0

if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
   msg "Info: Verifying only $RecentChangesToVerify changelists per depot due to -recent.\\n"
fi

RevRange=

# Determine the list of depots to process, either from 'p4 depots' or from a
# specified depot list file.
if [[ -n "$DepotListFile" ]]; then
   TmpFile=$(mktemp)
   grep -Ev '^\s*$|^\s*\#' "$DepotListFile" > "$TmpFile"
   if [[ -s "$TmpFile" ]]; then
      while read -r d; do
         DepotList[$DepotCount]="$d"
         DepotCount+=1
      done < "$TmpFile"
   else
      bail "No depots to process from the depot list file $DepotListFile (trimmed in $TmpFile)."
   fi
else
   for d in $($P4 -ztag -F %name% depots); do
      DepotList[$DepotCount]="$d"
      DepotCount+=1
   done
fi

msg "Processing ${#DepotList[@]} depots."
for d in "${DepotList[@]}"; do
   depotType=$($P4 -ztag -F %Type% depot -o "$d")

   case "$depotType" in
      (archive)
         [[ "$VerifyArchived" -eq 1 ]] || continue

         # For archive depots, use -A and -q but not -z.  
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption -A"
         else
            VerifyOptions="-q -A"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         msg "=== Started verify of archive depot //$d/... at $(date)."
         VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
         call_p4verify archive "$VerifyCmd" || VerifyFailed=1
      ;;
      (remote)
         msg "\\nInfo: Skipping verify of remote depot $d.\\n"
         continue
      ;;
      (spec)
         [[ "$VerifySpecDepot" -eq 1 ]] || continue

         # For the spec depot, there are no lazy copies, so no need for -z.
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption"
         else
            VerifyOptions="-q"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         msg "=== Started verify of spec depot //$d/... at $(date)."
         VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
         call_p4verify spec "$VerifyCmd" || VerifyFailed=1
      ;;
      (unload)
         [[ "$VerifyUnload" -eq 1 ]] || continue

         # For the unload depot, there are no lazy copies, so no need for -z.
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption -U"
         else
            VerifyOptions="-q -U"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         msg "=== Started verify of unload depot at $(date)."
         VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
         call_p4verify unload "$VerifyCmd" || VerifyFailed=1
      ;;
      (*)
         [[ "$depotType" != "local" && "$depotType" != "stream" ]] && \
            msg "Warning: Unknown depot type [$depotType]. Attempting normal verify."

         if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
            RevRange=$(get_verify_rev_range "$d" "$RecentChangesToVerify")
         fi

         ChunkFile="$P4TMP/depot_chunks.$d.txt"
         rm -f "$ChunkFile"

         if [[ "$VerifyChunks" -eq 1 ]]; then
            msg "Running: $ChunkScript -v ERROR -m $ChunkSize //$d/..."

            if ! "$ChunkScript" -v ERROR -m "$ChunkSize" "//$d/..." > "$ChunkFile"; then
               msg "Warning: Not chunking //$d/... due to script error."
               rm -f "$ChunkFile"
            fi

            if [[ ! -s "$ChunkFile" ]]; then
               msg "Warning: Not chunking //$d/... due to zero-size chunk file."
               rm -f "$ChunkFile"
            fi
         else
            rm -f "$ChunkFile"
         fi

         # For stream/local depots, use -z unless --only is used.
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption"
         else
            VerifyOptions="-qz"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         if [[ "$VerifySubmitted" -eq 1 ]]; then
            if [[ "$VerifyChunks" -eq 1 && -s "$ChunkFile" ]]; then
               msg "=== Started verify of submitted files in //$d/... in $ChunkSize chunks at $(date)."

               while read -r ChunkPath; do
                  VerifyCmd="$P4 -s verify $VerifyOptions \"$ChunkPath\""
                  call_p4verify submitted "$VerifyCmd" || VerifyFailed=1
               done < "$ChunkFile"
            else
               msg "=== Started verify of submitted files in //$d/... at $(date)."
               VerifyCmd="$P4 -s verify $VerifyOptions //$d/...$RevRange"
               call_p4verify submitted "$VerifyCmd" || VerifyFailed=1
            fi
         fi
      ;;
   esac
done

if [[ "$VerifyShelved" -eq 1 ]]; then
   msg "=== Started verify of shelved changelists at $(date)."
   TmpFile=$(mktemp)
   TmpLog=$(mktemp)
   ChangesCmd="$P4BIN -ztag -F %change% changes -s shelved"
   [[ "$VerifyOnlyRecentChanges" -eq 1 ]] && \
      ChangesCmd+=" -m $RecentChangesToVerify"

   ShelvedCLCount=0
   ShelvedCLErrorCount=0

   # For shelves, don't use -z since  we verify change-by-change.
   if [[ -n "$VerifyOnlyOption" ]]; then
      VerifyOptions="-q $VerifyOnlyOption -S"
   else
      VerifyOptions="-q -S"
   fi

   [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

   if $ChangesCmd > "$TmpFile"; then
      ShelvedChangesToVerify=$(wc -l "$TmpFile" | cut -d ' ' -f 1)
      msg "Verifying $ShelvedChangesToVerify shelved changelists."
      while read -r ShelvedCL; do
         ShelvedCLCount+=1
         VerifyCmd="$P4 -s verify $VerifyOptions @=$ShelvedCL"

         # Capture temp log for each CL verify command. Display only logs with
         # error; discard others.
         if [[ "$NoOp" -eq 0 ]]; then
            if eval "$VerifyCmd" > "$TmpLog" 2>&1; then
               rm -f "$TmpLog"
            else
               ShelvedCLErrorCount+=1
               cat "$TmpLog"
               rm -f "$TmpLog"
            fi
         else
            # There may be a great many shelves. In NoOp we are silent here
            # unless '-d' is used.
            dbg "NO_OP: Would run: $VerifyCmd"
         fi
      done < "$TmpFile"
   else
      msg "Error: Could not get or write list of shelved changes."
      VerifyFailed=1
   fi

   if [[ "$ShelvedCLCount" -ne 0 ]]; then
      if [[ "$ShelvedCLErrorCount" -eq 0 ]]; then
         msg "\\nAll $ShelvedCLCount shelved changes verified OK."
      else
         msg "\\nOf $ShelvedCLCount shelved changes verified, $ShelvedCLErrorCount had errors."
         VerifyFailed=1
      fi
   else
      msg "\\nThere were no shelved changelists to verify."
   fi
else
   msg "Skipping verify of shelved changelists due to '-nS'."
fi

if [[ "$VerifyFailed" -ne 0 ]]; then
   StatusMessage="Error: Verify attempt failed.  Review the log [$Log]."
   ExitCode=1
fi

if [[ "$ExitCode" -eq 0 ]]; then
   # Note: Ignore SC2196 due to 'grep -e' and 'egrep' not having consistent
   # results on all platforms.
   # shellcheck disable=SC2196
   if egrep -m1 '(BAD!|MISSING!|p4 help max)' "$Log" > /dev/null 2>&1; then
      StatusMessage="Error: Verify errors detected.  Review the log [$Log]."
      ExitCode=2
   fi
fi
 
msg "Completed verifications at $(date), taking $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

if [[ -n "$SummaryLog" ]]; then
   # Set LOGFILE as an alias for $SummaryLog. LOGFILE is used in the library functions
   # in backup_functions.sh for notification handling.
   export LOGFILE="$SummaryLog"
   # shellcheck disable=SC1091
   if source /p4/common/bin/backup_functions.sh; then
      echo -e "Summary from $ThisScript v$Version operated as $ThisUser@${HOSTNAME%%.*}

Status: $StatusMessage

Completed verifications at $(date), taking $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.

Command line was:\\n\\t$CmdLine

" > "$SummaryLog"

      if [[ "$ExitCode" -ne 0 ]]; then
         echo -e "Summary of Errors by Type:" >> "$SummaryLog"

         if [[ "$VerifyArchived" -eq 1 ]]; then
            echo -e "   Archive Depot Errors:           ${VerifyErrors[archive]:-0}" >> "$SummaryLog"
         fi

         if [[ "$VerifySubmitted" -eq 1 ]]; then
            echo -e "   Submitted File Errors:          ${VerifyErrors[submitted]:-0}" >> "$SummaryLog"
         fi

         if [[ "$VerifySpecDepot" -eq 1 ]]; then
            echo -e "   Spec Depot Errors:              ${VerifyErrors[spec]:-0}" >> "$SummaryLog"
         fi

         if [[ "$VerifyUnload" -eq 1 ]]; then
            echo -e "   Unload Depot Errors:            ${VerifyErrors[unload]:-0}" >> "$SummaryLog"
         fi

         # Do some quick math.
         if [[ "$VerifyArchived" -eq 1 || "$VerifySubmitted" -eq 1 || "$VerifySpecDepot" -eq 1 || "$VerifyUnload" -eq 1 ]]; then
            declare -i total=0
            for errorType in "${!VerifyErrors[@]}"; do
               total+=${VerifyErrors[$errorType]:-0}
            done
            echo -e "   Total Non-Shelve Errors:        $total (sum of ${#VerifyErrors[@]} error types listed above)" >> "$SummaryLog"
         fi

         if [[ "$VerifyShelved" -eq 1 ]]; then
            echo -e "\\n   Shelved Changes with Errors:    $ShelvedCLErrorCount" >> "$SummaryLog"
         fi
      fi

      echo -e "\\nDetailed log is:\\n\\t${HOSTNAME%%.*}:$Log
" >> "$SummaryLog"

      msg "${H2}\\nBEGIN Summary Log Contents:"
      cat "$SummaryLog"
      msg "END Summary Log Contents\\n${H2}"

      if [[ "${SNS_ALERT_TOPIC_ARN:-Unset}" != Unset ]]; then
         msg "Using AWS SNS for log summary delivery."
         sns_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
      fi

      msg "Using email for log summary delivery."
      mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
   else
      errmsg "Failed to load backup_functions.sh."
   fi
fi

# See the terminate() function where this script actually exits.
exit "$ExitCode"