p4verify.sh #31

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

if [[ ${BASH_VERSINFO[0]} -lt 4 ]]; then
   echo -e "\\n\\nERROR: $0 requires bash version 4.x+; current bash version is $BASH_VERSION."
   exit 1
fi

declare StatusMessage="OK: All scanned depots verified OK."
declare ThisScript=${0##*}
declare ThisUser=
declare Version=5.15.3
declare CmdLine="$0 $*"
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i NoOp=0
declare -i Debug=0
declare -i ShowLog=0
declare -i ExitCode=0
# CaseSensitive is 'i' for case-insensitive, and empty for sensitive.
# This is used to make 'grep' commands case-insensitive when appropriate.
declare CaseSensitive=
declare -i WaitBetweenVerifies=0
declare -i MaxActivePullQueueSize=0
declare -i MaxTotalPullQueueSize=0
declare -i VerifyCommandCount=0
declare -i VerifyArchived=0
declare -i VerifyChunks=0
declare -i VerifyPaths=0
declare -i VerifyShelved=1
declare -i VerifySpecDepot=1
declare -i VerifySubmitted=1
declare -i VerifyUnload=1
declare -i GzipOldLog=1
declare -i ScheduleTransfers=1
declare -i ShelvedCLCount=0
declare -i ShelvedCLErrorCount=0
declare -i DepotCount=0
declare -i OverrideCacheSafety=0
declare -i TotalSeconds=0
declare -a DepotList

# VerifyErrors is an associative array of errors indexed as follows:
# archive
# shelved
# spec
# submitted
# unload
# Values are the number of errors (BAD or MISSING) for each type of depot.
declare -A VerifyErrors
declare -i IgnoredErrors=0

declare SDPInstance=
declare SDPEnv="/p4/common/bin/p4_vars"
declare DepotListFile=
declare RegexIgnoresFile=
declare RevRange=
declare RecentChangesToVerify=
declare RecentChangesToVerifyDefault=200
declare VerifyCmd=
declare ChangesCmd=
declare ShelvedChangesToVerify=
declare ShelvedCL=
declare VerifyOnlyOption=
declare VerifyOptions=
declare ChunkFile=
declare ChunkPath=
declare ChunkSize=
declare ChunkScript="/p4/common/bin/depot_verify_chunks.py"
declare UserPathsFile=
declare DepotPathsFile=
declare DepotPath=
declare Log=Unset
declare SummaryLog=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare TmpFile=
declare TmpLog=
declare LbrReplication=
declare -i NotifyOnlyOnFailure=0 # Preserve orignal behavior of always sending a report.

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
   fi

   msg "USAGE for $ThisScript v$Version:

p4verify.sh [<instance>] [-N] [-nu] [-nr] [-ns] [-nS] [-a] [-nt] [-nz] [-o BAD|MISSING] [-chunks <ChunkSize>|-paths <paths_file>] [-w <Wait>] [-q <MaxActivePullQueueSize>] [-Q MaxTotalPullQueueSize] [-recent | -recent=N] [-dlf <depot_list_file>] [-I|-ignores <regex_ignores_file>] [-Ocache] [-n] [-L <log>] [-v] [-d] [-D]

   or

p4verify.sh -h|-man
"
   if [[ $style == -man ]]; then
      msg "DESCRIPTION:

	This script performs a 'p4 verify' of all submitted and shelved versioned
	files in depots of all types except 'remote' and 'archive' type depots.

	The singular Extensions depot is also verified, if present.

	Archive depots are not verified by defefault, but can be with the '-a'
	option.

	If run on a replica, it schedules archive failures for transfer to the
	replica.

OPTIONS:
<instance>
	Specify the SDP instance.  If not specified, the SDP_INSTANCE
	environment variable is used instead.  If the instance is not
	defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
	exists immediately with an error message.

 -N	Specify '-N' (Notify Only On Failure) to disable the default behavior
	which will always send a notification which includes a report of the p4
	verify status.  Specifying '-N' which change the behavior to only send
	a notification if there is an error during the p4 verify execution. 
	Notification methods are email, AWS SNS, and PagerDuty. Details on
	configuration can be found in the SDP documentation.  Providing the
	environment variable NOTIFY_ONLY_ON_FAILURE=1 is equivalent to the '-N'
	command line argument. 

 -nu	Specify '-nu' (No Unload) to skip verification of the singleton depot
	of type 'unload' (if created).  The 'unload' depot is verified
	by default.

 -nr	Specify '-nr' (No Regular) to skip verification of regular submitted
	archive files.  The '-nr' option is not compatible with '-recent'.
	Regular submitted archive files are verified by default.

	This option also causes Extension depot (if present) not to be
	verified.

 -ns	Specify '-ns' (No Spec Depot) to skip verification of singleton depot
	of type 'spec' (if created). The 'spec' depot is verified by default.

 -nS	Specify '-nS' (No Shelves) to skip verification of shelved archive
	files, i.e. to skip the 'p4 verify -qS'.

 -a	Specify '-a' (Archive Depots) to do verification of depots of type
	'archive'.  Depots of type 'archive' are not verified by default, as
	archive depots are often physically removed from the server's
	storage subsystem for long-term cold storage.
 
-nt	Specify the '-nt' option to avoid passing the '-t' flag to 'p4 verify' 
	on a replica. By default, p4verify.sh detects if it is running on a
	replica, and if so automatically applies the '-t' flag to 'p4 verify'.
	That causes the replica to attempt to self-heal, as files that fail
	verification are scheduled for transfer from the P4TARGET server. This
	default behavior results in 'Transfer scheduled' messages in the log,
	and MISSING/BAD files are listed as 'info:' rather than 'error:'. There
	is no clear indication of whether or which of the scheduled transfers
	complete successfully, and so there may be a mix of transient/correctable
	and \"real\"/persistent transfer errors for files that are also BAD/MISSING
	on the master server.  Specify '-nt' to ensure the log contains a list
	of files that currently fail a 'p4 verify' without attempting to transfer
	them from the master.

 -nz	Specify '-nt' to skip the gzip of the old log file.  By default, if a
	log with the default name or the name specified with '-L' exists at the
	start of processing, the old log is rotated and gzipped.  With this option
	the old log is not zipped when rotated.

 -o BAD|MISSING
	Specify '-o MISSING' to check only whether expected archive files exist
	or not, skipping the checksum calculation of existing files.  This results
	in dramatically faster, if less comprehensive, verification.  This
	is particularly well suited when verification is being used to schedule
	archive file transfers of missing files on replicas.  This translates into
	passing the '--only MISSING' option to 'p4 verify'.

	Specify '-o BAD' to check only for BAD revisions.  This translates into
	passing the '--only BAD' option to 'p4 verify'.

	This option requires p4d to be 2021.1 or newer. For older p4d versions,
	this option is silently ignored.

 -chunks <ChunkSize>
	Specify the maximum amount of content by size to verify at once.  If
	this is specified, the depot_verify_chunks.py script is used to
	break up depots into chunks of a given size, e.g. 100M or 4G.

	The <ChunkSize> parameter must be a size value valid to pass to the
	depot_verify_chunks.py script with the '-m' option.  That is,
	specifying '-chunks 200M' translates to calling depot_chunks_verify.sh
	with '-m 200M'.

	This requires the perforce-p4python3 module to be installed and the
	python3 in the PATH must be the correct one that uses the P4 module.

	Using '-chunks' is likely to result in a significantlly slower overall
	verify operation, though chuking can make it less impactful when it
	runs.  Using the '-chunks' option may be necessary on very large data
	sets, e.g. if there insufficient resources to process the largest
	depots.

	The '-recent' and '-chunks' options are mutually exclusive.

	The '-chunks' and '-paths' options can be used together; see the
	description of the '-paths' option below.

	Chunking logic applies only in depots of type 'stream' or 'local'.

 -paths <paths_file>

	Specify a file containing a list of depot paths to verify, with one
	line per entry.  Valid entries in the file start with '//', e.g.

	//mydepot/main/src/...

	In this example, when //mydepot depot is processed, only specified
	paths will be verified.  All other depots will be processed in full.
	To verify only specified paths, combine '-paths <paths_file>' with
	'-dlf <depot_list_file>' where the depot list file contains only
	'mydepot' (per the example above).

	The '-chunks' and '-paths' options can be used together for combined
	effects. If both options are specified, depots that contain specified
	paths are chunked based on the specified paths rather than the entire
	depot, and other paths in that depot are not processed.  Depots that
	do not have any specified paths listed in the <paths_file> are
	chunked at the top/depot level directory.

	The '-paths' option can be combined with '-recent' to verify only
	recent changelists in the specified paths.

	This option disables processing of the Extensions depot, though paths
	specified in the paths file can include Extensions.

	Paths specified must be in depots of type 'stream' or 'local', or
	the singular Extensionsd depot.

 -w <Wait>
	Specify the '-w' option, where <Wait> is a positive integer
	indicating the number of seconds to sleep between individual calls
	to 'p4 verify' commands.  For example, specifying '-w 300' results
	in a delay of 5 minutes between verify commands.

	This can be used with '-chunks' to inject a delay between chunked
	depot paths. Otherwise, the delay is injected between each depot
	processed. This can significantly lengthen the overall duration
	of 'p4verify.sh' operation, but can also spread out the resource
	consumption load on a server machine.

	If shelves are procossed (regardless of whether '-chunks' is used),
	the delay is injected between each individual shelved changelist, as
	shelved changes are verified one changelist at a time.  For data sets
	with a large number of shelves, it may be be wise to process shelves
	separately from submitted files if '-w' is used, a delay value to
	apply between depots may be different from that applied to
	individual changelists.

	See the '-q' option for a description of how '-q' and '-w' can be
	used together.

-q <MaxActivePullQueueSize>
	Specify the '-q' option, where <MaxActivePullQueueSize> is a positive
	integer indicating the maximum number of active pulls allowed before
	a 'p4 verify' command will be executed to transfer archives.

	The absolute maximum number of possible active pulls is affected by
	the number of 'startup.N' threads configured to pull archives files,
	and whether those threads indicate batching.

	The threads that pull archive files are those that configured to use
	the 'pull' command the '-u' option. Typically, a small number of pull
	threads are configured, between 2 and 10 or perhaps 20.

	If '-q 1' is specified, new 'p4 verify' commands will only be run
	when the active pull queue is quiet.  Specifying a too-high value,
	e.g. '-q 50' if only 3 'pull -u' archive pull threads are configured,
	will be ineffective, as the active pull threads will never exceed
	3 (let alone 50).

	The current active pull queue on a replica is reported by:

	p4 -ztag -F %replicaTransfersActive% pull -ls

	This option can be useful if using this p4verify.sh script to pull
	many or even all archives on a new replica server machine from its
	target server.  The injected delays can give the server time to transfer
	archives scheduled in one call to 'p4 verify' before calling it again
	with the goal of avoidng overloading the pull queue.

	If '-w' and '-q' options are both used, the delay specified by '-w'
	is ignored unless the active pull queue size is greater than or equal
	to the specified maximum active pull queue size.  The '-w' then
	essentially determines how frequently the 'p4 pull -ls' is run to
	check the active pull queue size.  A reasonable set of values might
	be '-q 1 -w 10'.

	The '-q' option in mutually exclusive with '-nt'.

	The '-q' option in mutually exclusive with '-Q'.

-Q <MaxTotalPullQueueSize>
	Specify the '-Q' option, where <MaxTotalPullQueueSize> is a positive
	integer indicating the maximum number of total pulls allowed before
	a 'p4 verify' command will be executed to transfer archives.

	In certain scenarios, the pull queue can become quite massive. For
	example, if a fresh standby replica is seeded from a checkpoint
	but has no archive files, and then a 'p4verify.sh' is run, the
	verify will schedule all files to be transferred, perhaps millions.

	If the pull queue gets too large, it can impact metadata replication.
	Setting this value may help mitigate issues related to scheduling
	too many archives pulls at once, by delaying scheduling new archive
	pulls until enough previously scheduled pulls are completed.

	This option can be useful in such scenarios, if this p4verify.sh script
	is used to pull many or even all archives on a new replica server machine
	from its target server.  The injected delays can give the server time to
	transfer archives scheduled in one call to 'p4 verify' before calling it
	again with the goal of avoidng overloading the pull queue.

	If individual depots contain large numbers of files, such that
	a verify on a single depot will schedule too many files to be
	transferred at once, it may be necessary to combine this option with
	the '-chunks' option to avoid overloading the transfer queue.

	**WARNING**: If there are files that cannot be tranferred from the
	replica's target server, the value of '-Q' must be set to higher than
	that number, or an infinite loop may occur.  For example, if there are
	500 permanent \"legacy\" verify errors on the commit server from 10
	years ago that have long since been abandoned, those files can never
	be transferred to any replica. Running p4verify.sh on the replica will
	cause those files to be scheduled, but as they cannot be pulled, they
	will land in the total pull queue.  In this scenario, the value set
	with '-Q' must be greater than 500, or an infinite loop is possible.

	Specify '-Q 0' to disable checking the total pull queue.

	The current total pull queue on a replica is reported by:

	p4 -ztag -F %replicaTransfersTotal% pull -ls

	This option can be useful if using this p4verify.sh script to pull
	many or even all archives on a new replica server machine from its
	target server.  The injected delays can give the server time to transfer
	archives scheduled in one call to 'p4 verify' before calling it again
	with the goal of avoidng overloading the pull queue.

	If '-w' and '-Q' options are both used, the delay specified by '-w'
	is ignored unless the total pull queue size is greater than or equal
	to the specified maximum total pull queue size.  The '-w' then
	essentially determines how frequently the 'p4 pull -ls' is run to
	check the total pull queue size.  A reasonable set of values might
	be '-q 50000 -w 10'.

	The '-Q' option in mutually exclusive with '-nt'.

	The '-Q' option in mutually exclusive with '-q'.

 -recent[=N]
	Specify that only recent changelists should be verified.

	This can be specified as '-recent' or '-recent=N', where N is an integer
	indicating the number of recent changelists to verify.

	If '-recent' is used without the optional '=N' syntax, the
	\$SDP_RECENT_CHANGES_TO_VERIFY variable defines how many changelists
	are considered recent; the default is $RecentChangesToVerifyDefault.

	If the default is not appropriate for your site, add
	\"export SDP_RECENT_CHANGES_TO_VERIFY\" to /p4/common/site/config/p4_N.vars.local to
	change the default for an instance, or to /p4/common/site/config/p4_vars.local to
	change it globally.  If \$SDP_RECENT_CHANGES_TO_VERIFY is unset, the
	default is $RecentChangesToVerifyDefault.

	When -recent is used, files in the unload depot are not verified.

 -dlf <depot_list_file>
	Specify a file containing a list of depots to process in the desired
	order.  By default, all depots in the order reported by reported by
	'p4 depots' are processed, which effectively results in depots being
	processed in alphabetical order, with the single Extensions depot being
	processed after other depots (if it exists).

	This '-dlf' option can be useful in time-sensitive situations where the
	order of processing can be prioritized, and/or to prevent processing
       	certain depots.

	The format fo the depot list file is straighforward, one line per
	depot, without the '//' nor trailling /..., so a list might look
	like this sample:

	ProjA
	ProjB
	spec
	.swarm
	unload
	archive
	ProjC

	Blank lines and lines starting with a '#' are treated as comments and
	ignored.

	WARNING: This is not intended to be the primary method of verification,
	because it would be easy to forget to add new depots to the list file.

	If the depot list file is not readable, processing aborts.

	This option disables processing of the Extensions depot unless the
	Extensions depot is list in the depot list file provided.

 -ignores <regex_ignores_file>
 	Specify the 'verify ignores' file, a file containing a series of
	regular expression patterns representing files or file revisions
	to ignore when scanning for verify errors.  Errors matching the
	pattern will be suppressed from the output captured in the log,
	and will not be considered a verification error.

	If the '-ignores' is not specified, the default verify ignores
	file is:

	/p4/common/config/p4verify.N.ignores

	where 'N' is the SDP instance name. If this file exists, it is
	considered the 'verify ignores' file.

	Specify '-ignores none' to avoid processing the standard ignores
	file.

	The patterns can be specific files, specific file paths, or broader
	patterns (e.g. in the case of entirely abandoned depots).  The file
	provided is passed as the '-f <file>' option to the 'grep' utility,
	and is expected to contain a series of one-line entries, each
	containing an expression to exclude from being considered as verify
	errors reported by this script.

	You can test your expression by first using it with grep to
	ensure it suppresses errors by using a command like this sample,
	providing an older log from this script that contains errors to
	be suppressed:

	grep -Ev -f /path/to/regex_file /path/to/old/p4verify.log

	If your server is case-sensitive, change that command to use '-i':

	grep -Evi -f /path/to/regex_file /path/to/old/p4verify.log

	This sample entry ignores a single file revision:

	//Alpha/main/docs/Expenses from February 1999.xls#3

	This sample entry ignores all revisions of a single file:

	//Alpha/main/docs/Expenses from February 1999.xls

	This sample entry ignores all entries in the spec depot
	related to client specs:

	//spec/client

	This sample uses the MD5 checksum from the verify error, just
	to illustrate that this can be used as an alternative to
	specifying file paths:

	D34989BFB8D9B0FB9866C4A604A05410

	This sample ignores BAD! (but not MISSING!) errors under the
	//Beta/main/src directory tree:

	//Beta/main/src/.* BAD!

	WARNING: Ensure that the regex file provided does NOT contain
	any blank lines or comments. The file should contain only tested
	regex patterns.

	This option is intended to provide a way to ignore unrecoverably lost
	file revisions from things like past infrastructure failures, for
	which search and recovery efforts have been abandoned. This option
	subtly changes the question answered by this script from \"Are there any
	verify errors?\" to \"Are there any new verify errors, errors we don't
	already know about?\"

	WARNING: This option is not intended to be incorporated into the primary
	method of verification, because ignoring archive errors in this script
	does not solve the problem at its source.  Ideally, the root cause of
	the verify errors should be addressed by recovering lost archives,
	injecting replacement content, or other means. So long as verify errors
	remain, even if ignored by this option, users attempting to access the
	revisions will still see Librarian errors, and replicas will encounter
	errors trying to pull the missing archives. This option could increase
	the risk that such revisions are never dealt with.

 -Ocache
	Specify '-Ocache' to attempt a verification on a replica configured
	with a 'lbr.replication' replication configuration setting value
	of 'cache'.  By default, if the 'lbr.replication' configurable is
	set to 'cache', this script aborts, as replication of such a depot
	will schedule transfers that are likely unintended.  This is a
	safety feature.

	The 'cache' mode is generally used on replicas or edge servers with
	limited disk space.  Because running a verify will cause transfers
	of any missing files, this could result in filling up the disk.

	Use of '-Ocache' is strongly discouraged unless combined with
	other options to ensure that only targeted paths are scheduled
	for transfer.

 -v     Verbose.  Show output of verify attempts, which is suppressed by default.
	Setting SDP_SHOW_LOG=1 in the shell environment has the same effect as -v.

	The default behavior of this script is to generate no terminal output,
	but instead to write output into a log file -- see LOGGING below.  If
	'-v' is specified, the generated log is sent to stdout at the end of
	processing.  This flag is not recommended for routine cron operation or
	for large data sets.

	The -chunks and -recent options are mutually exclusive.

 -L <log>
	Specify the log file to use.  The default is /p4/N/logs/p4verify.log

	Log rotation and old log cleanup logic does not apply to log files
	specified with -L.  Thus, using -L is not recommended for routine
	scheduled operation, e.g. via crontab.

DEBUGGING OPTIONS:
 -n	No-Operation (NO_OP) mode, for debugging.

 	Display certain commands that would be executed without executing
	them. When '-n' is used, commands that might take a long time to
	run or affect data are only displayed.

	Even in '-n' mode, some information-gathering commands such as
	listing shelved CLs are executed, which may cause the script to take
	a bit of time to run on a large data set even in dry run mode.

 -d	Specify that debug messages should be displayed.

 -D	Use bash 'set -x' extreme debugging verbosity, and imply '-d'.

 -L off
	The special value '-L off' disables logging. This can only be used
	with '-n' for debugging.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	Example 1: Full Verify

	This script is typically called via cron with only the instance
	parameter as an argument, e.g.:

	p4verify.sh 1

	Example 2: Fast Verify

	A \"fast\" verify is one in which only the check for MISSING archives
	is done, while the resource-intensive checksum calculation of
	potentially BAD existing archives is skipped.  This is especially
	useful when used on a replica.

	p4verify.sh 1 -o MISSING

	Example 3: Fast and Recent Verify

	The '-o MISSING' and '-recent' flags can be combined for a very
	fast check.  This check might be incorporated into a failover
	procedure.

	p4verify.sh 1 -o MISSING -recent

	Example 4: Submitted Files Only

	This will verify only use submitted files, ignoring shelves and the
	spec and unload depots, putting the results in a specified log:

	p4verify.sh 1 -ns -nS -nu -L  -L /p4/1/logs/p4verify.submitted.log

	Example 5: Shelved Files Only

	This will verify only use submitted files, ignoring shelves and the
	spec and unload depots, putting them in a specified log:

	p4verify.sh 1 -nr -ns -nu -L /p4/1/logs/p4verify.shelved.log

	Example 6: A Dry Run

	The '-n' option can be used for a dry run.  Output may also be
	displayed to the screen ('-v') for a dry run and the log file optionally
	discarded:
	
	p4verify.sh 1 -n -nS -L off -v

	Example 7: Archive File Load for New Replica

	The p4verify.sh script can be used to schedule transfers of a large
	number of files from a replica.  When doing so, however, overloading
	the new replicas pull queue with too many files may impact metadata
	replication.  This can be addressed by combining a variety of
	options, such as '-chunks' and '-Q'.  For example:

	p4verify.sh 1 -chunks 200M -Q 10000 -w 20 -o MISSING

NOHUP USAGE:
	Because archive verification is typically a long running task,
	it is advisable to use 'nohup' to call each command, and combine
	that by running the command as a background process.  Alternately,
	use 'screen' or similar.

	Any of the examples above can be used with 'nohup', without output
	redirected to /dev/null (i.e. to \"the void\", as this script handles
	logging and output redirection).

	To use 'nohup', start the command line with 'nohup', and then after
	the command, add this text exactly:

	< /dev/null > /dev/null 2>&1 &

	As a example, Example 2 above, called with nohup, would look like:

	nohup /p4/common/bin/p4verify.sh 1 -o MISSING < /dev/null > /dev/null 2>&1 &

	With the ampersand '&' at the end, the command will appear to return
	immediately as the process continues to run in the background.

	Then optionally monitor the log:

	tail -f /p4/1/logs/p4verify.log

LOGGING:
	This script generates no output by default.  All (stdout and stderr) is
	logged to /p4/N/logs/p4verify.log.

	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.

NOTIFICATIONS:
	In addition to logging, a short summary of the verify is sent as a
	notification.  The summary is reliably short even if the output of the
	verifications done by this script results in a large log file.

	There are two notification schemes with this script:

	* Email notification is always attempted.

	* AWS SNS notification is attempted if the SNS_ALERT_TOPIC_ARN custom
	setting is defined. This is typically set in:

	/p4/common/site/config/p4_N.vars.local

TIMING:
	The log file captures various timing information, including the
	time required to verify each depot, or each chunk or path if
	'-paths' or '-chunks' are used. The time to verify shelves
	in all depots is reported separately from submitted files.
	
	Timing indications all start with the text 'Time: ' on the beginning
	of a line of output in the log file, and can be extrated with a
	command like this example (adjusting the log file name as needed):

	grep ^Time: /p4/1/logs/p4verify.log

EXIT CODES:
	An exit code of 0 indicates no errors were encountered attempting to
	perform verifications, AND that all verifications attempted
	reported no problems.

	A exit status of 1 indicates that verifications could not be
	attempted for some reason.

	A exit status of 2 indicates that verifications were successfully
	performed, but that problems such as BAD or MISSING files
	were detected, or else system limits prevented verification.
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   [[ "$Log" == "off" ]] || msg "Log is: $Log\\n${H1}"

   # Be tidy.
   [[ -n "$TmpFile" && -e "$TmpFile" ]] && rm -f "$TmpFile"

   # With the trap removed, exit.
   exit "$ExitCode"
}

function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || echo -e "DEBUG: $*" >&2; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit; }

#------------------------------------------------------------------------------
# Function: get_verify_rev_range ($depotOrPath, $recentChanges)
#------------------------------------------------------------------------------
function get_verify_rev_range () {
   declare depotOrPath=${1:-}
   declare recentChangesToVerify=${2:-}
   declare nowChange=
   [[ -z "$depotOrPath" || -z "$recentChangesToVerify" ]] && return 0

   # If we are given a depot, prepend '//' and append '/...'. Otherwise assume we
   # have a raw path to use with 'p4 changes'.
   if [[ "$depotOrPath" =~ ^[a-zA-Z0-9._]+$ ]]; then
      nowChange=$("$P4BIN" -ztag -F %change% changes -m 1 "//$depotOrPath/...")
      thenChange=$("$P4BIN" -ztag -F %change% changes -m "$recentChangesToVerify" "//$depotOrPath/..." | tail -1)
   else
      nowChange=$("$P4BIN" -ztag -F %change% changes -m 1 "$depotOrPath")
      thenChange=$("$P4BIN" -ztag -F %change% changes -m "$recentChangesToVerify" "$depotOrPath" | tail -1)
   fi
   [[ -z "$nowChange" ]] && return 1
   [[ -z "$thenChange" ]] && return 1

   echo "@$thenChange,@$nowChange"

   return 0
}

#------------------------------------------------------------------------------
# Function: delay_between_verifies ($WaitBetweenVerifies, $MaxActivePullQueueSize, $MaxTotalPullQueueSize)
#
# References global(s): $VerifyCommandCount
#------------------------------------------------------------------------------
function delay_between_verifies () {
   local -i waitBetweenVerifies=${1:-0}
   local -i maxActivePullQueueSize=${2:-0}
   local -i maxTotalPullQueueSize=${3:-0}
   local -i activePullQueueSize=0
   local -i totalPullQueueSize=0
   local -i pullQueueCheckDelay=10

   # Short-circuit: Return if no verifies have yet been processed.
   [[ "$VerifyCommandCount" -eq 0 ]] && return

   # Short-circuit: Return if no delay-inducing options are set: '-q', '-Q', or '-w'.
   [[ "$maxActivePullQueueSize" -eq 0 && "$maxTotalPullQueueSize" -eq 0 && "$waitBetweenVerifies" -eq 0 ]] && return

   # If only the '-w <Wait>' was specified, just wait.
   if [[ "$maxActivePullQueueSize" -eq 0 && "$maxTotalPullQueueSize" -eq 0 ]]; then
      dbg "Sleeping $waitBetweenVerifies seconds."
      sleep "$waitBetweenVerifies"

   # If only the '-q <MaxActivePullQueueSize>' was specified, wait until the
   # max active pulls is below the specified limit.  If both '-q' and '-w' are
   # specified, apply the delay of '-w' iff the pull queue is low enough per
   # '-q'.
   elif [[ "$maxActivePullQueueSize" -ne 0 ]]; then
      [[ "$waitBetweenVerifies" -ne 0 ]] && 
         pullQueueCheckDelay=$waitBetweenVerifies

      activePullQueueSize=$($P4 -ztag -F %replicaTransfersActive% pull -ls 2>/dev/null)
      dbg "delay_between_verifies(): Active Pull Queue Size: $activePullQueueSize"
      if [[ -n "$activePullQueueSize" ]]; then
         while [[ "$activePullQueueSize" -gt "$maxActivePullQueueSize" ]]; do
            dbg "Sleeping $pullQueueCheckDelay seconds because active pull queue of $activePullQueueSize is greater than max $maxActivePullQueueSize."
            sleep "$pullQueueCheckDelay"
            activePullQueueSize=$($P4 -ztag -F %replicaTransfersActive% pull -ls 2>/dev/null)
            dbg "delay_between_verifies(): Active Pull Queue Size: $activePullQueueSize"
            if [[ -z "$activePullQueueSize" ]]; then
               errmsg "delay_between_verifies(): Could not determine active pull queue size in delay loop."
               break
            fi
         done

         dbg "Delay due to -q completed."
      else
         # We could not determine the active pull queue size.
         # Display an error and move on.
         errmsg "delay_between_verifies(): Could not determine initial active pull queue size."
      fi
   else
      [[ "$waitBetweenVerifies" -ne 0 ]] && 
         pullQueueCheckDelay=$waitBetweenVerifies

      totalPullQueueSize=$($P4 -ztag -F %replicaTransfersTotal% pull -ls 2>/dev/null)
      dbg "delay_between_verifies(): Total Pull Queue Size: $totalPullQueueSize"
      if [[ -n "$totalPullQueueSize" ]]; then
         while [[ "$totalPullQueueSize" -gt "$maxTotalPullQueueSize" ]]; do
            dbg "Sleeping $pullQueueCheckDelay seconds because total pull queue of $totalPullQueueSize is greater than max $maxTotalPullQueueSize."
            sleep "$pullQueueCheckDelay"
            totalPullQueueSize=$($P4 -ztag -F %replicaTransfersTotal% pull -ls 2>/dev/null)
            dbg "delay_between_verifies(): Active Pull Queue Size: $totalPullQueueSize"
            if [[ -z "$totalPullQueueSize" ]]; then
               errmsg "delay_between_verifies(): Could not determine total pull queue size in delay loop."
               break
            fi
         done

         dbg "Delay due to -Q completed."
      else
         # We could not determine the total pull queue size.
         # Display an error and move on.
         errmsg "delay_between_verifies(): Could not determine initial total pull queue size."
      fi
   fi
}

#------------------------------------------------------------------------------
# Function: call_p4_verify($verifyErrorType, $verifyCommand)
#
# This calls the 'p4 verify' command with the command given, and increments
# count of errors indexted by verify, updating the VerifiyErrors array
# displayed in the summary.
#
# VerifyErrors is an associative array of errors indexed as follows:
# archive
# shelved
# spec
# submitted
# unload
#
# Values are the number of errors (including BAD/MISSING/other) for each type.
#
function call_p4_verify () { 
   local -i status
   local verifyErrorType="${1:-}"
   local verifyCommand="${2:-}"
   local tmpf1=
   local tmpf2=
   local oldErrors=
   local newErrors=
   local postGrepErrors=
   local totalErrors=

   [[ -n "$verifyErrorType" || -n "$verifyCommand" ]] || return

   msg "${H2}\\nRunning: $verifyCommand"
   tmpf1="${P4TMP}/t_verify.1.$$.$RANDOM"
   tmpf2="${P4TMP}/t_verify.2.$$.$RANDOM"

   if [[ "$NoOp" -eq 0 ]]; then
      eval "$verifyCommand" > "$tmpf1" 2>&1
      status=$?

      # Suppress 'no such file' errors from older p4d versions to avoid
      # confusion, as these are not actual errors. In newer p4d versions,
      # this is a mere warning, which we don't suppress.
      grep -Ev "^error: //.*/... - no such file(s)." "$tmpf1" > "$tmpf2"
      mv -f "$tmpf2" "$tmpf1" || errmsg "Failure moving temp files."

      newErrors=$(grep -Ec '(BAD!|MISSING!|p4 help max)' "$tmpf1")

      if [[ "$newErrors" =~ [0-9]+ ]]; then
         if [[ "$newErrors" != "0" ]]; then
            # If '-ignores <regex_ignores_file>' was specified, apply it.
            if [[ -n "$RegexIgnoresFile" ]]; then
               dbg "Running: grep -E${CaseSensitive} -v -f $RegexIgnoresFile $tmpf1 .GT. $tmpf2"
               grep "-E${CaseSensitive}" -v -f "$RegexIgnoresFile" "$tmpf1" > "$tmpf2"
               postGrepErrors=$(grep -Ec '(BAD!|MISSING!|p4 help max)' "$tmpf2")
               if [[ "$postGrepErrors" =~ [0-9]+ ]]; then
                  # Determine if any verify errors were ignored here by comparing
                  # count of errors before and after the grep.
                  if [[ "$postGrepErrors" == 0 ]]; then
                     IgnoredErrors=1
                     msg "Warning: One or more verify errors were ignored here. Suppressing 'exit:' line because all errors match known ignore expressions defined in $RegexIgnoresFile."
                     grep -v ^exit: "$tmpf2" > "$tmpf1"
                  elif [[ "$postGrepErrors" != "$newErrors" ]]; then
                     IgnoredErrors=1
                     msg "Warning: One or more verify errors were ignored here. Not suppressing 'exit:' line due to non-ignored errors."
                     mv -f "$tmpf2" "$tmpf1" || errmsg "Failure moving temp files."
                  else
                     dbg "Ignore regex processed; no files matched."
                     mv -f "$tmpf2" "$tmpf1" || errmsg "Failure moving temp files."
                  fi
                  newErrors="$postGrepErrors"
               else
                  errmsg "Unable to count post-grep verify errors. Numeric value expected, got: [$postGrepErrors]."
               fi
            fi
         fi
      else
         errmsg "Unable to count verify errors. Numeric value expected, got: [$newErrors]."
      fi

      # Display errors that have not been suppressed or ignored.
      cat "$tmpf1"

      if [[ "$newErrors" =~ [0-9]+ ]]; then
         if [[ "$newErrors" != "0" ]]; then
            oldErrors=${VerifyErrors[$verifyErrorType]:-0}
            totalErrors=$((oldErrors+newErrors))
            VerifyErrors[$verifyErrorType]="$totalErrors"
         fi
      else
         errmsg "Unable to count verify errors. Numeric value expected, got: [$newErrors]."
      fi

      rm -f "$tmpf1" "$tmpf2"
   else
      msg "NO_OP: Would run: $verifyCommand"
      status=0
   fi

   VerifyCommandCount+=1

   return "$status"
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case "$1" in
      (-h) usage -h;;
      (-man) usage -man;;
      (-N) NotifyOnlyOnFailure=1;;
      (-ns) VerifySpecDepot=0;;
      (-nS) VerifyShelved=0;;
      (-nr) VerifySubmitted=0;;
      (-nu) VerifyUnload=0;;
      (-a) VerifyArchived=1;;
      (-w)
         shiftArgs=1
         if [[ "$2" =~ ^[1-9]+[0-9]*$ ]]; then
            WaitBetweenVerifies=$2
         else
            usage -h "Invalid parameter '-w $2'; the value with '-w' must be a positive integer."
         fi
      ;;
      (-q)
         shiftArgs=1
         if [[ "$2" =~ ^[1-9]+[0-9]*$ ]]; then
            MaxActivePullQueueSize=$2
         else
            usage -h "Invalid parameter '-q $2'; the value with '-q' must be a positive integer."
         fi
      ;;
      (-Q)
         shiftArgs=1
         if [[ "$2" =~ ^[1-9]+[0-9]*$ ]]; then
            MaxTotalPullQueueSize=$2
         else
            usage -h "Invalid parameter '-Q $2'; the value with '-Q' must be a positive integer."
         fi
      ;;
      (-nt) ScheduleTransfers=0;;
      (-nz) GzipOldLog=0;;
      (-o) VerifyOnlyOption="--only ${2^^}"; shiftArgs=1;;
      (-chunks) VerifyChunks=1; ChunkSize="$2"; shiftArgs=1;;
      (-paths) VerifyPaths=1; UserPathsFile="$2"; shiftArgs=1;;
      (-recent) VerifyOnlyRecentChanges=1;;
      (-recent=*) VerifyOnlyRecentChanges=1; RecentChangesToVerify=${1##*=};;
      (-dlf) DepotListFile="${2:-}"; shiftArgs=1;;
      (-ignores) RegexIgnoresFile="${2:-}"; shiftArgs=1;;
      (-Ocache) OverrideCacheSafety=1;;
      (-n) NoOp=1;;
      (-v) ShowLog=1;;
      (-L) Log="$2"; shiftArgs=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Debug; use bash 'set -x' extreme debugging mode.
      (-*) usage -h "Unknown command line option ($1).";;
      (*)
         if [[ -z "$SDPInstance" ]]; then
            SDPInstance="$1"
         else
            usage -h "Unknown extra parameter [$1]."
         fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ -z "$SDPInstance" ]] && SDPInstance="${SDP_INSTANCE:-UnsetSDPInstance}"

[[ "$SDPInstance" == "UnsetSDPInstance" ]] && \
   usage -h "The \\$SDP_INSTANCE setting is not defined. It must be defined by doing:\\n\\n\\tsource /p4/common/bin/p4_vars <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n"

# Load SDP Environment. In case this generates output on stdout, suppress it to
# avoid triggering crontab emails, but do not suppress stderr (output related
# to errors).
# shellcheck disable=SC1090
source "$SDPEnv" "$SDPInstance" > /dev/null ||\
   bail "Failed to load SDP environment with:\\n\\tsource $SDPEnv $SDPInstance"
# shellcheck disable=SC1091
source "$P4CBIN/backup_functions.sh" ||\
   bail "Failed to load backup_functions.sh with:\\n\\tsource \"$P4CBIN/backup_functions.sh\"\\n"

[[ "$VerifySubmitted" -eq 0 && "$VerifyOnlyRecentChanges" -eq 1 ]] && \
   usage -h "The '-recent' and '-nr' options are mutually exclusive."

if [[ -n "$VerifyOnlyOption" ]]; then
   # shellcheck disable=SC2072
   if [[ "$P4D_VERSION" > "2021.1" ]]; then
      if [[ ! "$VerifyOnlyOption" =~ ^--only\ (MISSING|BAD)$ ]]; then
         usage -h "The option specified with '-o' must be BAD or MISSING. Specified value was: $VerifyOnlyOption"
      fi
   else
      VerifyOnlyOption=
   fi
fi

[[ "$VerifyOnlyRecentChanges" -eq 1 && "$VerifyChunks" -eq 1 ]] && \
   usage -h "The -recent and -chunks options are mutually exclusive."

if [[ -n "$ChunkSize" && ! -x "$ChunkScript" ]]; then
   msg "Warning: -chunks ignored because chunking script is not executable by user $ThisUser: $ChunkScript"
   VerifyChunks=0
fi

[[ -n "$UserPathsFile" && ! -r "$UserPathsFile" ]] && \
   usage -h "The file specified with '-paths <paths_file>' does not exist: $UserPathsFile"

[[ -n "$DepotListFile" && ! -r "$DepotListFile" ]] && \
   usage -h "The file specified with '-dlf <depot_list_file>' does not exist: $DepotListFile"

# After this block, either RegexIgnoresFile will be set to an existing file, or the
# value will be cleared.  If set, ignores will be processed.  The special value
# 'none' means ignores are not processed, i.e. no verify errors are ignored/suppressed.
if [[ -n "$RegexIgnoresFile" ]]; then
   if [[ "${RegexIgnoresFile,,}" == none ]]; then
      RegexIgnoresFile=
   elif [[ ! -r "$RegexIgnoresFile" ]]; then
      usage -h "The file specified with '-ignores <regex_ignores_file>' does not exist: $RegexIgnoresFile"
   fi
# If '-ignores' was not used (and thus '-ignores none' was not specified), but
# the standard verify ignores file exists, use it.
elif [[ -r "${P4CCFG:-}/p4verify.${SDPInstance}.ignores" ]]; then
   RegexIgnoresFile="${P4CCFG:-}/p4verify.${SDPInstance}.ignores"
fi

[[ "$ScheduleTransfers" -eq 0 && "$MaxActivePullQueueSize" -ne 0 ]] && \
   usage -h "The file '-nt' and '-q <MaxActivePullQueueSize>' options are mutually exclusive."

[[ "$ScheduleTransfers" -eq 0 && "$MaxTotalPullQueueSize" -ne 0 ]] && \
   usage -h "The file '-nt' and '-Q <MaxTotalPullQueueSize>' options are mutually exclusive."

[[ "$MaxActivePullQueueSize" -ne 0 && "$MaxTotalPullQueueSize" -ne 0 ]] && \
   usage -h "The file '-q <MaxActivePullQueueSize>' and '-Q <MaxTotalPullQueueSize>' options are mutually exclusive."

[[ "${P4REPLICA}" == "FALSE" && "$MaxActivePullQueueSize" -ne 0 ]] && \
   usage -h "The '-q <MaxActivePullQueueSize>' option can only be used on a replica; the ServerID [$SERVERID] does not appear to be a replica."

[[ "$Log" == off && "$NoOp" -eq 0 ]] && \
   usage -h "The '-L off' option can only be specified with '-n' (NO_OP/Dry Run mode)."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

# '-N' command line argument is available to disable the default notification behavior
# check if the other option, environment variable, was used
[[ -n "${NOTIFY_ONLY_ON_FAILURE:-}" ]] && \
   NotifyOnlyOnFailure="${NOTIFY_ONLY_ON_FAILURE}"

# If the number of recent changes to verify was specified on the command
# line with '-recent=N' syntax, use that number. Otherwise, if
# $SDP_RECENT_CHANGES_TO_VERIFY is defined, use that. Finally, fall-back
# to the default to the value set in this script.
if [[ -z "$RecentChangesToVerify" ]]; then
   if [[ -n "${SDP_RECENT_CHANGES_TO_VERIFY:-}" ]]; then
      RecentChangesToVerify="${SDP_RECENT_CHANGES_TO_VERIFY}"
   else
      RecentChangesToVerify="$RecentChangesToVerifyDefault"
   fi
fi

[[ "$Log" == Unset ]] && Log="$LOGS/p4verify.log"

if [[ "$Log" != off ]]; then
   SummaryLog="$(mktemp "${P4TMP}/email_summary.XXXXXXXXX.txt")"
   if [[ -e "$Log" ]]; then
      OldLogTimestamp=$(get_old_log_timestamp "$Log")
      OldLog="$LOGS/p4verify.${OldLogTimestamp}.log"
      mv -f "$Log" "$OldLog" || bail "Could not do: mv -f \"$Log\" \"$OldLog\""

      # Gzip old log.
      if [[ "$GzipOldLog" -eq 1 && -n "$(command -v gzip)" ]]; then
         dbg "gzip --best \"$OldLog\""
         gzip --best "$OldLog"
      fi
      dbg "Rotated old log $Log to $OldLog"
   fi

   touch "$Log" || bail "Couldn't touch new log file [$Log]."?

   # Redirect stdout and stderr to a log file.
   if [[ "$ShowLog" -eq 1 ]]; then
      exec > >(tee "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi

   msg "${H1}\\nLog is: $Log\\n"
fi

ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@${HOSTNAME%%.*} on $(date +'%a %Y-%m-%d %H:%M:%S %Z') as:\\n$CmdLine"

P4="$P4BIN -p $P4PORT -u $P4USER"
"$P4CBIN/p4login"

msg "If there are errors in this log, contact support-helix-core@perforce.com"

VerifyFailed=0

if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
   msg "Info: Verifying only $RecentChangesToVerify changelists per depot due to -recent.\\n"
fi

# If we are on a replica or edge server, the lbr.replication value will be defined.
# If it is set to 'cache' (or the unsupported alias 'ondemand'), abort unless the
# '-Ocache' option was specified.
LbrReplication=$($P4 -ztag -F %Value% configure show lbr.replication 2>/dev/null)
if [[ -n "$LbrReplication" ]]; then
   if [[ "$LbrReplication" =~ ^(cache|ondemand)$ ]]; then
      if [[ "$OverrideCacheSafety" -eq 1 ]]; then
         msg "The lbr.replication value is '$LbrReplication'. Proceeding due to '-Ocache'."
      else
         bail "Aborting because the lbr.replication value is '$LbrReplication'.  See description of '-Ocache' opion for more info."
      fi
   fi
fi

if [[ "$($P4 -ztag -F %caseHandling% info -s)" == insensitive ]]; then
   CaseSensitive=i
fi

RevRange=

# Determine the list of depots to process, either from 'p4 depots' or from a
# specified depot list file.
if [[ -n "$DepotListFile" ]]; then
   TmpFile=$(mktemp)
   grep -Ev '^\s*$|^\s*\#' "$DepotListFile" > "$TmpFile"
   if [[ -s "$TmpFile" ]]; then
      while read -r d; do
         DepotList[DepotCount]="$d"
         DepotCount+=1
      done < "$TmpFile"
   else
      bail "No depots to process from the depot list file $DepotListFile (trimmed in $TmpFile)."
   fi
else
   for d in $($P4 -ztag -F %name% depots); do
      DepotList[DepotCount]="$d"
      DepotCount+=1
   done

   # Add the Extensions depot to the list if there is one.
   d=$($P4 -ztag -F %name% depots -t extension)
   if [[ -n "$d" ]]; then
      DepotList[DepotCount]="$d"
      DepotCount+=1
   fi
fi

msg "Time: Setup processing took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
TotalSeconds+=$SECONDS; SECONDS=0

msg "Processing ${#DepotList[@]} depots."
for d in "${DepotList[@]}"; do
   depotType=$($P4 -ztag -F %Type% depot -o "$d")

   delay_between_verifies "$WaitBetweenVerifies" "$MaxActivePullQueueSize" "$MaxTotalPullQueueSize"

   case "$depotType" in
      (archive)
         [[ "$VerifyArchived" -eq 1 ]] || continue

         # For archive depots, use -A and -q but not -z.  
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption -A"
         else
            VerifyOptions="-q -A"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         msg "=== Started verify of archive depot //$d/... at $(date)."
         VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
         call_p4_verify archive "$VerifyCmd" || VerifyFailed=1

         msg "Time: Verification of archive depot //$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
         TotalSeconds+=$SECONDS; SECONDS=0
      ;;
      (remote)
         msg "\\nInfo: Skipping verify of remote depot $d.\\n"
         continue
      ;;
      (spec)
         [[ "$VerifySpecDepot" -eq 1 ]] || continue

         # For the spec depot, there are no lazy copies, so no need for -z.
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption"
         else
            VerifyOptions="-q"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         msg "=== Started verify of spec depot //$d/... at $(date)."
         VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
         call_p4_verify spec "$VerifyCmd" || VerifyFailed=1

         msg "Time: Verification of spec depot //$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
         TotalSeconds+=$SECONDS; SECONDS=0
      ;;
      (unload)
         [[ "$VerifyUnload" -eq 1 ]] || continue

         # For the unload depot, there are no lazy copies, so no need for -z.
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption -U"
         else
            VerifyOptions="-q -U"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         msg "=== Started verify of unload depot at $(date)."
         VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
         call_p4_verify unload "$VerifyCmd" || VerifyFailed=1

         msg "Time: Verification of unload depot //$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
         TotalSeconds+=$SECONDS; SECONDS=0
      ;;
      (*)
         [[ "$depotType" != "local" && "$depotType" != "stream" && "$depotType" != "extension" ]] && \
            msg "Warning: Unknown depot type [$depotType]. Attempting normal verify."

         if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
            RevRange=$(get_verify_rev_range "$d" "$RecentChangesToVerify")
         fi

         DepotPathsFile="$(mktemp "$P4TMP"/verify_paths."$d".XXXXX.txt)"
         rm -f "$DepotPathsFile"

         # If '-paths' was specified providing a file containing a list of
         # user-specified, generate a depot paths file containing the subset
         # of paths in the current depot being processed.
         # This block of code will result in a DepotPathsFile existing if
         # user-specified paths are specified in the current depot being
         # processed.
         if [[ "$VerifyPaths" -eq 1 ]]; then
            grep "^//$d/" "$UserPathsFile" > "$DepotPathsFile" ||\
               errmsg "Could not generate depot paths file [$DepotPathsFile] from user paths file [$UserPathsFile] with: grep \"^//$d/\" \"$UserPathsFile\""

            [[ -s "$DepotPathsFile" ]] || rm -f "$DepotPathsFile"
         fi

         ChunkFile="$(mktemp "$P4TMP"/depot_chunks."$d".XXXXXX.txt)"
         rm -f "$ChunkFile"

         # This block of code will result in a ChunkFile existing if chunking
         # is to be done. If the ChunkFile does not exist, chunking will not
         # be applied.
         if [[ "$VerifyChunks" -eq 1 ]]; then
            # Combined '-paths' and '-chunks' behaviors.
            if [[ -e "$DepotPathsFile" ]]; then
               while read -r DepotPath; do
                  msg "Running: $ChunkScript -v ERROR -m $ChunkSize //$DepotPath"
                  if "$ChunkScript" -v ERROR -m "$ChunkSize" "$DepotPath" >> "$ChunkFile"; then
                     msg "Time: Chunking \"$DepotPath\" took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
                     TotalSeconds+=$SECONDS; SECONDS=0
                  else
                     errmsg "Chunking of path \"$DepotPath\" failed."
                  fi
               done < "$DepotPathsFile"
            # Handle '-chunks' behaviour without '-paths'.
            else
               msg "Running: $ChunkScript -v ERROR -m $ChunkSize //$d/..."

               if "$ChunkScript" -v ERROR -m "$ChunkSize" "//$d/..." > "$ChunkFile"; then
                  msg "Time: Chunking //$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
                  TotalSeconds+=$SECONDS; SECONDS=0
               else
                  msg "Warning: Not chunking //$d/... due to script error."
                  rm -f "$ChunkFile"
               fi

               if [[ ! -s "$ChunkFile" ]]; then
                  msg "Warning: Not chunking //$d/... due to zero-size chunk file."
                  rm -f "$ChunkFile"
               fi
            fi
         fi

         # For stream/local depots, use -z unless --only is used.
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption"
         else
            VerifyOptions="-qz"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         if [[ "$VerifySubmitted" -eq 1 ]]; then
            # If the ChunkFile exists, verify the specified chunks.  This may be
            # chunks of entire depots or chunks of specified paths (per '-paths').
            if [[ "$VerifyChunks" -eq 1 && -s "$ChunkFile" ]]; then
               msg "=== Started verify of submitted files in //$d/... in $ChunkSize chunks at $(date)."

               while read -r ChunkPath; do
                  VerifyCmd="$P4 -s verify $VerifyOptions \"$ChunkPath\""
                  call_p4_verify submitted "$VerifyCmd" || VerifyFailed=1

                  msg "Time: Verification of submitted files in \"$ChunkPath\" took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
                  TotalSeconds+=$SECONDS; SECONDS=0
               done < "$ChunkFile"
               rm -f "$ChunkFile"
            # If the ChunkFile does not exist, but paths were specified, handle
            # the targeted paths without chunking.
            elif [[ "$VerifyPaths" -eq 1 && -e "$DepotPathsFile" ]]; then
               if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
                  msg "=== Started verify of recently submitted files in //$d/... in user-specified paths file at $(date)."
               else
                  msg "=== Started verify of submitted files in //$d/... in user-specified paths file at $(date)."
               fi

               while read -r DepotPath; do
                  if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
                     RevRange=$(get_verify_rev_range "$DepotPath" "$RecentChangesToVerify")
                     VerifyCmd="$P4 -s verify $VerifyOptions \"$DepotPath$RevRange\""
                  else
                     VerifyCmd="$P4 -s verify $VerifyOptions \"$DepotPath\""
                  fi
                  call_p4_verify submitted "$VerifyCmd" || VerifyFailed=1

                  msg "Time: Verification of submitted files in \"$DepotPath\" took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
                  TotalSeconds+=$SECONDS; SECONDS=0
               done < "$DepotPathsFile"
               rm -f "$DepotPathsFile"
            else
               if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
                  msg "=== Started verify of recently submitted files in //$d/... at $(date)."
                  RevRange=$(get_verify_rev_range "$d" "$RecentChangesToVerify")
                  VerifyCmd="$P4 -s verify $VerifyOptions //$d/...$RevRange"
               else
                  msg "=== Started verify of submitted files in //$d/... at $(date)."
                  VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
               fi
               call_p4_verify submitted "$VerifyCmd" || VerifyFailed=1

               msg "Time: Verification of submitted files in //$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
               TotalSeconds+=$SECONDS; SECONDS=0
            fi
         fi
      ;;
   esac
done

if [[ "$VerifyShelved" -eq 1 ]]; then
   msg "=== Started verify of shelved changelists at $(date)."
   TmpFile=$(mktemp)
   TmpLog=$(mktemp)
   ChangesCmd="$P4BIN -ztag -F %change% changes -s shelved"
   [[ "$VerifyOnlyRecentChanges" -eq 1 ]] && \
      ChangesCmd+=" -m $RecentChangesToVerify"

   ShelvedCLCount=0
   ShelvedCLErrorCount=0

   # For shelves, don't use -z since  we verify change-by-change.
   if [[ -n "$VerifyOnlyOption" ]]; then
      VerifyOptions="-q $VerifyOnlyOption -S"
   else
      VerifyOptions="-q -S"
   fi

   [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

   if $ChangesCmd > "$TmpFile"; then
      ShelvedChangesToVerify=$(wc -l "$TmpFile" | cut -d ' ' -f 1)
      msg "Verifying $ShelvedChangesToVerify shelved changelists."
      while read -r ShelvedCL; do
         delay_between_verifies "$WaitBetweenVerifies" "$MaxActivePullQueueSize" "$MaxTotalPullQueueSize"
         VerifyCommandCount+=1
         ShelvedCLCount+=1
         VerifyCmd="$P4 -s verify $VerifyOptions @=$ShelvedCL"

         # Capture temp log for each CL verify command. Display only logs with
         # error; discard others.
         if [[ "$NoOp" -eq 0 ]]; then
            if eval "$VerifyCmd" > "$TmpLog" 2>&1; then
               rm -f "$TmpLog"
            else
               # If '-ignores <regex_ignores_file>' was specified, apply it.
               if [[ -n "$RegexIgnoresFile" ]]; then
                  dbg "Running: grep -E${CaseSensitive} -v -f $RegexIgnoresFile $TmpLog .GT. ${TmpLog}.2"
                  grep -E${CaseSensitive} -v -f "$RegexIgnoresFile" "$TmpLog" > "${TmpLog}.2"
                  EC1=$(grep -Ec '(BAD!|MISSING!|p4 help max)' "$TmpLog")
                  EC2=$(grep -Ec '(BAD!|MISSING!|p4 help max)' "${TmpLog}.2")

                  if [[ "$EC1" =~ [0-9]+ && "$EC2" =~ [0-9]+ ]]; then
                     if [[ "$EC2" == 0 ]]; then
                        IgnoredErrors=1
                        msg "Warning: Some shelve verify errors were ignored here. Suppressing 'exit:' line because all errors match known ignore expressions defined in $RegexIgnoresFile."
                        grep -v ^exit: "${TmpLog}.2" > "$TmpLog"
                     elif [[ "$EC1" != "$EC2" ]]; then
                        IgnoredErrors=1
                        ShelvedCLErrorCount+=1
                        msg "Warning: Some shelve verify errors were ignored here. Not suppressing 'exit:' line due to non-ignored errors."
                        mv -f "${TmpLog}.2" "$TmpLog" || errmsg "Failure moving temp files."
                     else
                        ShelvedCLErrorCount+=1
                        dbg "Ignore regex processed; no files matched."
                        mv -f "${TmpLog}.2" "$TmpLog" || errmsg "Failure moving temp files."
                     fi
                  else
                     errmsg "Unable to count shelve verify errors. Numeric value expected, got: [$EC1] and [$EC2]."
                  fi
               else
                  ShelvedCLErrorCount+=1
               fi

               cat "$TmpLog"
               rm -f "$TmpLog"
            fi
         else
            # There may be a great many shelves. In NoOp we are silent here
            # unless '-d' is used.
            dbg "NO_OP: Would run: $VerifyCmd"
         fi
      done < "$TmpFile"

      msg "Time: Verification of shelved files took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
      TotalSeconds+=$SECONDS; SECONDS=0
   else
      msg "Error: Could not get or write list of shelved changes."
      VerifyFailed=1
   fi

   if [[ "$ShelvedCLCount" -ne 0 ]]; then
      if [[ "$ShelvedCLErrorCount" -eq 0 ]]; then
         msg "\\nAll $ShelvedCLCount shelved changes verified OK."
      else
         msg "\\nOf $ShelvedCLCount shelved changes verified, $ShelvedCLErrorCount had errors."
         VerifyFailed=1
      fi
   else
      msg "\\nThere were no shelved changelists to verify."
   fi
else
   msg "Skipping verify of shelved changelists due to '-nS'."
fi

if [[ "$VerifyFailed" -ne 0 ]]; then
   StatusMessage="Error: Verify attempt failed.  Review the log [$Log]."
   ExitCode=1
fi

if [[ "$ExitCode" -eq 0 ]]; then
   # Note: Ignore SC2196 due to 'grep -e' and 'egrep' not having consistent
   # results on all platforms.
   # shellcheck disable=SC2196
   if egrep -m1 '(BAD!|MISSING!|p4 help max)' "$Log" > /dev/null 2>&1; then
      StatusMessage="Error: Verify errors detected.  Review the log [$Log]."
      ExitCode=2
   fi
fi

msg "Time: Summarizing results took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
TotalSeconds+=$SECONDS; SECONDS=0
 
msg "Time: Completed verifications at $(date), taking $((TotalSeconds/3600)) hours $((TotalSeconds%3600/60)) minutes $((TotalSeconds%60)) seconds.\\n"

if [[ -n "$SummaryLog" ]]; then
   # Set LOGFILE as an alias for $SummaryLog. LOGFILE is used in the library functions
   # in backup_functions.sh for notification handling.
   export LOGFILE="$SummaryLog"
   msg "Summary from $ThisScript v$Version operated as $ThisUser@${HOSTNAME%%.*}

Status: $StatusMessage

Time: Completed verifications at $(date), taking $((TotalSeconds/3600)) hours $((TotalSeconds%3600/60)) minutes $((TotalSeconds%60)) seconds.

Command line was:\\n\\t$CmdLine

" > "$SummaryLog"

   if [[ "$ExitCode" -ne 0 ]]; then
      msg "Summary of Errors by Type:" >> "$SummaryLog"

      if [[ "$VerifyArchived" -eq 1 ]]; then
         msg "   Archive Depot Errors:           ${VerifyErrors[archive]:-0}" >> "$SummaryLog"
      fi

      if [[ "$VerifySubmitted" -eq 1 ]]; then
         msg "   Submitted File Errors:          ${VerifyErrors[submitted]:-0}" >> "$SummaryLog"
      fi

      if [[ "$VerifySpecDepot" -eq 1 ]]; then
         msg "   Spec Depot Errors:              ${VerifyErrors[spec]:-0}" >> "$SummaryLog"
      fi

      if [[ "$VerifyUnload" -eq 1 ]]; then
         msg "   Unload Depot Errors:            ${VerifyErrors[unload]:-0}" >> "$SummaryLog"
      fi

      # Do some quick math.
      if [[ "$VerifyArchived" -eq 1 || "$VerifySubmitted" -eq 1 || "$VerifySpecDepot" -eq 1 || "$VerifyUnload" -eq 1 ]]; then
         declare -i total=0
         for errorType in "${!VerifyErrors[@]}"; do
            total+=${VerifyErrors[$errorType]:-0}
         done
         msg "   Total Non-Shelve Errors:        $total (sum of error types listed above)" >> "$SummaryLog"
      fi

      if [[ "$VerifyShelved" -eq 1 ]]; then
         msg "\\n   Shelved Changes with Errors:    $ShelvedCLErrorCount" >> "$SummaryLog"
      fi
   fi

   if [[ -n "$RegexIgnoresFile" ]]; then
      if [[ "$IgnoredErrors" -ne 0 ]]; then
         msg "\\n   Note: One or more verify errors were ignored due to '-ignores $RegexIgnoresFile'." >> "$SummaryLog"
      else
         msg "\\n   Note: The '-ignores $RegexIgnoresFile' option was specified, but no ignore expressions matched." >> "$SummaryLog"
      fi
   fi

   msg "\\nA total of $VerifyCommandCount 'p4 verify' commands were executed." >> "$SummaryLog"

   msg "\\nDetailed log is:\\n\\t${HOSTNAME%%.*}:$Log
" >> "$SummaryLog"

   msg "${H2}\\nBEGIN Summary Log Contents:"
   cat "$SummaryLog"
   msg "END Summary Log Contents\\n${H2}"

   if [[ "$NotifyOnlyOnFailure" -eq 0 ]]; then
      mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
   else
      if [[ "$ExitCode" -ne 0 ]]; then
         die "p4 verify had error: $HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
      else
         msg "NotifyOnlyOnFailure is set to 1 and the Exitcode is 0. Notifications disabled."
      fi
   fi

   msg "Using email for log summary delivery."
   mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
fi

# See the terminate() function where this script actually exits.
exit "$ExitCode"