#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ #============================================================================== # Declarations and Environment if [[ ${BASH_VERSINFO[0]} -lt 4 ]]; then echo -e "\\n\\nERROR: $0 requires bash version 4.x+; current bash version is $BASH_VERSION." exit 1 fi declare StatusMessage="OK: All scanned depots verified OK." declare ThisScript=${0##*} declare ThisUser= declare Version=5.13.2 declare CmdLine="$0 $*" declare -i VerifyOnlyRecentChanges=0 declare -i VerifyFailed= declare -i NoOp=0 declare -i Debug=0 declare -i ShowLog=0 declare -i ExitCode=0 declare -i WaitBetweenVerifies=0 declare -i MaxActivePullQueueSize=0 declare -i MaxTotalPullQueueSize=0 declare -i VerifyCommandCount=0 declare -i VerifyArchived=0 declare -i VerifyChunks=0 declare -i VerifyPaths=0 declare -i VerifyShelved=1 declare -i VerifySpecDepot=1 declare -i VerifySubmitted=1 declare -i VerifyUnload=1 declare -i GzipOldLog=1 declare -i ScheduleTransfers=1 declare -i ShelvedCLCount=0 declare -i ShelvedCLErrorCount=0 declare -i DepotCount=0 declare -i OverrideCacheSafety=0 declare -i TotalSeconds=0 declare -a DepotList # VerifyErrors is an associative array of errors indexed as follows: # archive # shelved # spec # submitted # unload # Values are the number of errors (BAD or MISSING) for each type of depot. declare -A VerifyErrors declare -i IgnoredErrors=0 declare SDPInstance= declare SDPEnv="/p4/common/bin/p4_vars" declare DepotListFile= declare RegexIgnoresFile= declare RevRange= declare RecentChangesToVerify=200 declare VerifyCmd= declare ChangesCmd= declare ShelvedChangesToVerify= declare ShelvedCL= declare VerifyOnlyOption= declare VerifyOptions= declare ChunkFile= declare ChunkPath= declare ChunkSize= declare ChunkScript="/p4/common/bin/depot_verify_chunks.py" declare UserPathsFile= declare DepotPathsFile= declare DepotPath= declare Log=Unset declare SummaryLog= declare H1="==============================================================================" declare H2="------------------------------------------------------------------------------" declare TmpFile= declare TmpLog= declare LbrReplication= declare -i NotifyOnlyOnFailure=0 # Preserve orignal behavior of always sending a report. #============================================================================== # Local Functions #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). # The default is -h. # # $2 - error message (optional). Specify this if usage() is called due to # user error, in which case the given message displayed first, followed by the # standard usage message (short or long depending on $1). If displaying an # error, usually $1 should be -h so that the longer usage message doesn't # obscure the error message. # # Sample Usage: # usage # usage -man # usage -h "Incorrect command line usage." # # This last example generates a usage error message followed by the short # '-h' usage summary. #------------------------------------------------------------------------------ function usage { declare style=${1:--h} declare errorMessage=${2:-Unset} if [[ $errorMessage != Unset ]]; then msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2 fi msg "USAGE for $ThisScript v$Version: p4verify.sh [<instance>] [-N] [-nu] [-nr] [-ns] [-nS] [-a] [-nt] [-nz] [-o BAD|MISSING] [-chunks <ChunkSize>|-paths <paths_file>] [-w <Wait>] [-q <MaxActivePullQueueSize>] [-Q MaxTotalPullQueueSize] [-recent] [-dlf <depot_list_file>] [-ignores <regex_ignores_file>] [-Ocache] [-n] [-L <log>] [-v] [-d] [-D] or p4verify.sh -h|-man " if [[ $style == -man ]]; then msg "DESCRIPTION: This script performs a 'p4 verify' of all submitted and shelved versioned files in depots of all types except 'remote' and 'archive' type depots. If run on a replica, it schedules archive failures for transfer to the replica. OPTIONS: <instance> Specify the SDP instance. If not specified, the SDP_INSTANCE environment variable is used instead. If the instance is not defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh exists immediately with an error message. -N Specify '-N' (Notify Only On Failure) to disable the default behavior which will always send a notification which includes a report of the p4 verify status. Specifying '-N' which change the behavior to only send a notification if there is an error during the p4 verify execution. Notification methods are email, AWS SNS, and PagerDuty. Details on configuration can be found in the SDP documentation. Providing the environment variable NOTIFY_ONLY_ON_FAILURE=1 is equivalent to the '-N' command line argument. -nu Specify '-nu' (No Unload) to skip verification of the singleton depot of type 'unload' (if created). The 'unload' depot is verified by default. -nr Specify '-nr' (No Regular) to skip verification of regular submitted archive files. The '-nr' option is not compatible with '-recent'. Regular submitted archive files are verified by default. -ns Specify '-ns' (No Spec Depot) to skip verification of singleton depot of type 'spec' (if created). The 'spec' depot is verified by default. -nS Specify '-nS' (No Shelves) to skip verification of shelved archive files, i.e. to skip the 'p4 verify -qS'. -a Specify '-a' (Archive Depots) to do verification of depots of type 'archive'. Depots of type 'archive' are not verified by default, as archive depots are often physically removed from the server's storage subsystem for long-term cold storage. -nt Specify the '-nt' option to avoid passing the '-t' flag to 'p4 verify' on a replica. By default, p4verify.sh detects if it is running on a replica, and if so automatically applies the '-t' flag to 'p4 verify'. That causes the replica to attempt to self-heal, as files that fail verification are scheduled for transfer from the P4TARGET server. This default behavior results in 'Transfer scheduled' messages in the log, and MISSING/BAD files are listed as 'info:' rather than 'error:'. There is no clear indication of whether or which of the scheduled transfers complete successfully, and so there may be a mix of transient/correctable and \"real\"/persistent transfer errors for files that are also BAD/MISSING on the master server. Specify '-nt' to ensure the log contains a list of files that currently fail a 'p4 verify' without attempting to transfer them from the master. -nz Specify '-nt' to skip the gzip of the old log file. By default, if a log with the default name or the name specified with '-L' exists at the start of processing, the old log is rotated and gzipped. With this option the old log is not zipped when rotated. -o BAD|MISSING Specify '-o MISSING' to check only whether expected archive files exist or not, skipping the checksum calculation of existing files. This results in dramatically faster, if less comprehensive, verification. This is particularly well suited when verification is being used to schedule archive file transfers of missing files on replicas. This translates into passing the '--only MISSING' option to 'p4 verify'. Specify '-o BAD' to check only for BAD revisions. This translates into passing the '--only BAD' option to 'p4 verify'. This option requires p4d to be 2021.1 or newer. For older p4d versions, this option is silently ignored. -chunks <ChunkSize> Specify the maximum amount of content by size to verify at once. If this is specified, the depot_verify_chunks.py script is used to break up depots into chunks of a given size, e.g. 100M or 4G. The <ChunkSize> parameter must be a size value valid to pass to the depot_verify_chunks.py script with the '-m' option. That is, specifying '-chunks 200M' translates to calling depot_chunks_verify.sh with '-m 200M'. This requires the perforce-p4python3 module to be installed and the python3 in the PATH must be the correct one that uses the P4 module. Using '-chunks' is likely to result in a significantlly slower overall verify operation, though chuking can make it less impactful when it runs. Using the '-chunks' option may be necessary on very large data sets, e.g. if there insufficient resources to process the largest depots. The '-recent' and '-chunks' options are mutually exclusive. The '-chunks' and '-paths' options can be used together; see the description of the '-paths' option below. Chunking logic applies only in depots of type 'stream' or 'local'. -paths <paths_file> Specify a file containing a list of depot paths to verify, with one line per entry. Valid entries in the file start with '//', e.g. //mydepot/main/src/... In this example, when //mydepot depot is processed, only specified paths will be verified. All other depots will be processed in full. To verify only specified paths, combine '-paths <paths_file>' with '-dlf <depot_list_file>' where the depot list file contains only 'mydepot' (per the example above). The '-recent' and '-paths' options are mutually exclusive. The '-chunks' and '-paths' options can be used together for combined effects. If both options are specified, depots that contain specified paths are chunked based on the specified paths rather than the entire depot, and other paths in that depot are not processed. Depots that do not have any specified paths listed in the <paths_file> are chunked at the top/depot level directory. Paths specified must be in depots of type 'stream' or 'local'. -w <Wait> Specify the '-w' option, where <Wait> is a positive integer indicating the number of seconds to sleep between individual calls to 'p4 verify' commands. For example, specifying '-w 300' results in a delay of 5 minutes between verify commands. This can be used with '-chunks' to inject a delay between chunked depot paths. Otherwise, the delay is injected between each depot processed. This can significantly lengthen the overall duration of 'p4verify.sh' operation, but can also spread out the resource consumption load on a server machine. If shelves are procossed (regardless of whether '-chunks' is used), the delay is injected between each individual shelved changelist, as shelved changes are verified one changelist at a time. For data sets with a large number of shelves, it may be be wise to process shelves separately from submitted files if '-w' is used, a delay value to apply between depots may be different from that applied to individual changelists. See the '-q' option for a description of how '-q' and '-w' can be used together. -q <MaxActivePullQueueSize> Specify the '-q' option, where <MaxActivePullQueueSize> is a positive integer indicating the maximum number of active pulls allowed before a 'p4 verify' command will be executed to transfer archives. The absolute maximum number of possible active pulls is affected by the number of 'startup.N' threads configured to pull archives files, and whether those threads indicate batching. The threads that pull archive files are those that configured to use the 'pull' command the '-u' option. Typically, a small number of pull threads are configured, between 2 and 10 or perhaps 20. If '-q 1' is specified, new 'p4 verify' commands will only be run when the active pull queue is quiet. Specifying a too-high value, e.g. '-q 50' if only 3 'pull -u' archive pull threads are configured, will be ineffective, as the active pull threads will never exceed 3 (let alone 50). The current active pull queue on a replica is reported by: p4 -ztag -F %replicaTransfersActive% pull -ls This option can be useful if using this p4verify.sh script to pull many or even all archives on a new replica server machine from its target server. The injected delays can give the server time to transfer archives scheduled in one call to 'p4 verify' before calling it again with the goal of avoidng overloading the pull queue. If '-w' and '-q' options are both used, the delay specified by '-w' is ignored unless the active pull queue size is greater than or equal to the specified maximum active pull queue size. The '-w' then essentially determines how frequently the 'p4 pull -ls' is run to check the active pull queue size. A reasonable set of values might be '-q 1 -w 10'. The '-q' option in mutually exclusive with '-nt'. The '-q' option in mutually exclusive with '-Q'. -Q <MaxTotalPullQueueSize> Specify the '-Q' option, where <MaxTotalPullQueueSize> is a positive integer indicating the maximum number of total pulls allowed before a 'p4 verify' command will be executed to transfer archives. In certain scenarios, the pull queue can become quite massive. For example, if a fresh standby replica is seeded from a checkpoint but has no archive files, and then a 'p4verify.sh' is run, the verify will schedule all files to be transferred, perhaps millions. If the pull queue gets too large, it can impact metadata replication. Setting this value may help mitigate issues related to scheduling too many archives pulls at once, by delaying scheduling new archive pulls until enough previously scheduled pulls are completed. This option can be useful in such scenarios, if this p4verify.sh script is used to pull many or even all archives on a new replica server machine from its target server. The injected delays can give the server time to transfer archives scheduled in one call to 'p4 verify' before calling it again with the goal of avoidng overloading the pull queue. If individual depots contain large numbers of files, such that a verify on a single depot will schedule too many files to be transferred at once, it may be necessary to combine this option with the '-chunks' option to avoid overloading the transfer queue. **WARNING**: If there are files that cannot be tranferred from the replica's target server, the value of '-Q' must be set to higher than that number, or an infinite loop may occur. For example, if there are 500 permanent \"legacy\" verify errors on the commit server from 10 years ago that have long since been abandoned, those files can never be transferred to any replica. Running p4verify.sh on the replica will cause those files to be scheduled, but as they cannot be pulled, they will land in the total pull queue. In this scenario, the value set with '-Q' must be greater than 500, or an infinite loop is possible. Specify '-Q 0' to disable checking the total pull queue. The current total pull queue on a replica is reported by: p4 -ztag -F %replicaTransfersTotal% pull -ls This option can be useful if using this p4verify.sh script to pull many or even all archives on a new replica server machine from its target server. The injected delays can give the server time to transfer archives scheduled in one call to 'p4 verify' before calling it again with the goal of avoidng overloading the pull queue. If '-w' and '-Q' options are both used, the delay specified by '-w' is ignored unless the total pull queue size is greater than or equal to the specified maximum total pull queue size. The '-w' then essentially determines how frequently the 'p4 pull -ls' is run to check the total pull queue size. A reasonable set of values might be '-q 50000 -w 10'. The '-Q' option in mutually exclusive with '-nt'. The '-Q' option in mutually exclusive with '-q'. -recent Specify that only recent changelists should be verified. The \$SDP_RECENT_CHANGES_TO_VERIFY variable defines how many changelists are considered recent; the default is $RecentChangesToVerify. If the default is not appropriate for your site, add \"export SDP_RECENT_CHANGES_TO_VERIFY\" to /p4/common/config/p4_N.vars to change the default for an instance, or to /p4/common/bin/p4_vars to change it globally. If \$SDP_RECENT_CHANGES_TO_VERIFY is unset, the default is $RecentChangesToVerify. When -recent is used, neither shelves nor files in the unload depot are verified. -dlf <depot_list_file> Specify a file containing a list of depots to process in the desired order. By default, all depots reported by 'p4 depots', which effectively results in depots processed in alphabetical order. This can be useful in time-sensitive situations where the order of processing can be prioritized, and/or to prevent processing certain depots. The format fo the depot list file is straighforward, one line per depot, without the '//' nor trailling /..., so a list might look like this sample: ProjA ProjB spec .swarm unload archive ProjC Blank lines and lines starting with a '#' are treated as comments and ignored. WARNING: This is not intended to be the primary method of verification, because it would be easy to forget to add new depots to the list file. If the depot list file is not readable, processing aborts. -ignores <regex_ignores_file> Specify a file containing a series of regular expression patterns representing file revisions to ignore when scanning for verify errors. The patterns can be specific files, specific file paths, or broader patterns (e.g. in the case of entirely abandoned depots). The file provided is passed as the '-f <file>' option to the 'grep' utility, and is expected to contain a series of one-line entries, each containing an expression to exclude. You can test your expression by first using it with grep to ensure it suppresses errors by using a command like this sample, providing an older log from this script that contains errors to be suppressed: grep -E -v -f /path/to/regex_file /path/to/old/p4verify.log This sample entry ignores a single file revision: //Alpha/main/docs/Expenses from February 1999.xls#3 This sample entry ignores all revisions of a single file: //Alpha/main/docs/Expenses from February 1999.xls This sample entry ignores all entries in the spec depot related to client specs: //spec/client This sample uses the MD5 checksum from the verify error, just to illustrate that this can be used as an alternative to specifying file paths: D34989BFB8D9B0FB9866C4A604A05410 This sample ignores BAD! (but not MISSING!) errors under the //Beta/main/src directory tree: //Beta/main/src/.* BAD! This option is intended to provide a way to ignore unrecoverably lost file revisions from things like past infrastructure failures, for which search and recovery efforts have been abandoned. This option subtly changes the question answered by this script from \"Are there any verify errors?\" to \"Are there any new verify errors, errors we don't already know about?\" WARNING: This option is not intended to be incorporated into the primary method of verification, because ignoring archive errors in this script does not solve the problem at its source. Ideally, the root cause of the verify errors should be addressed by recovering lost archives, injecting replacement content, or other means. So long as verify errors remain, even if ignored by this option, users attempting to access the revisions will still see Librarian errors. This option could increase the risk that such revisions are never dealt with. -Ocache Specify '-Ocache' to attempt a verification on a replica configured with a 'lbr.replication' replication configuration setting value of 'cache'. By default, if the 'lbr.replication' configurable is set to 'cache', this script aborts, as replication of such a depot will schedule transfers that are likely unintended. This is a safety feature. The 'cache' mode is generally used on replicas or edge servers with limited disk space. Because running a verify will cause transfers of any missing files, this could result in filling up the disk. Use of '-Ocache' is strongly discouraged unless combined with other options to ensure that only targeted paths are scheduled for transfer. -v Verbose. Show output of verify attempts, which is suppressed by default. Setting SDP_SHOW_LOG=1 in the shell environment has the same effect as -v. The default behavior of this script is to generate no terminal output, but instead to write output into a log file -- see LOGGING below. If '-v' is specified, the generated log is sent to stdout at the end of processing. This flag is not recommended for routine cron operation or for large data sets. The -chunks and -recent options are mutually exclusive. -L <log> Specify the log file to use. The default is /p4/N/logs/p4verify.log Log rotation and old log cleanup logic does not apply to log files specified with -L. Thus, using -L is not recommended for routine scheduled operation, e.g. via crontab. DEBUGGING OPTIONS: -n No-Operation (NO_OP) mode, for debugging. Display certain commands that would be executed without executing them. When '-n' is used, commands that might take a long time to run or affect data are only displayed. Even in '-n' mode, some information-gathering commands such as listing shelved CLs are executed, which may cause the script to take a bit of time to run on a large data set even in dry run mode. -d Specify that debug messages should be displayed. -D Use bash 'set -x' extreme debugging verbosity, and imply '-d'. -L off The special value '-L off' disables logging. This can only be used with '-n' for debugging. HELP OPTIONS: -h Display short help message -man Display man-style help message EXAMPLES: Example 1: Full Verify This script is typically called via cron with only the instance parameter as an argument, e.g.: p4verify.sh 1 Example 2: Fast Verify A \"fast\" verify is one in which only the check for MISSING archives is done, while the resource-intensive checksum calculation of potentially BAD existing archives is skipped. This is especially useful when used on a replica. p4verify.sh 1 -o MISSING Example 3: Fast and Recent Verify The '-o MISSING' and '-recent' flags can be combined for a very fast check. This check might be incorporated into a failover procedure. p4verify.sh 1 -o MISSING -recent Example 4: Submitted Files Only This will verify only use submitted files, ignoring shelves and the spec and unload depots, putting the results in a specified log: p4verify.sh 1 -ns -nS -nu -L -L /p4/1/logs/p4verify.submitted.log Example 5: Shelved Files Only This will verify only use submitted files, ignoring shelves and the spec and unload depots, putting them in a specified log: p4verify.sh 1 -nr -ns -nu -L /p4/1/logs/p4verify.shelved.log Example 6: A Dry Run The '-n' option can be used for a dry run. Output may also be displayed to the screen ('-v') for a dry run and the log file optionally discarded: p4verify.sh 1 -n -nS -L off -v Example 7: Archive File Load for New Replica The p4verify.sh script can be used to schedule transfers of a large number of files from a replica. When doing so, however, overloading the new replicas pull queue with too many files may impact metadata replication. This can be addressed by combining a variety of options, such as '-chunks' and '-Q'. For example: p4verify.sh 1 -chunks 200M -Q 10000 -w 20 -o MISSING NOHUP USAGE: Because archive verification is typically a long running task, it is advisable to use 'nohup' to call each command, and combine that by running the command as a background process. Alternately, use 'screen' or similar. Any of the examples above can be used with 'nohup', without output redirected to /dev/null (i.e. to \"the void\", as this script handles logging and output redirection). To use 'nohup', start the command line with 'nohup', and then after the command, add this text exactly: < /dev/null > /dev/null 2>&1 & As a example, Example 2 above, called with nohup, would look like: nohup /p4/common/bin/p4verify.sh 1 -o MISSING < /dev/null > /dev/null 2>&1 & With the ampersand '&' at the end, the command will appear to return immediately as the process continues to run in the background. Then optionally monitor the log: tail -f /p4/1/logs/p4verify.log LOGGING: This script generates no output by default. All (stdout and stderr) is logged to /p4/N/logs/p4verify.log. The exception is usage errors, which result an error being sent to stderr followed usage info on stdout, followed by an immediate exit. NOTIFICATIONS: In addition to logging, a short summary of the verify is sent as a notification. The summary is reliably short even if the output of the verifications done by this script results in a large log file. There are two notification schemes with this script: * Email notification is always attempted. * AWS SNS notification is attempted if the SNS_ALERT_TOPIC_ARN custom setting is defined. This is typically set in: /p4/common/site/config/p4_N.vars.local TIMING: The log file captures various timing information, including the time required to verify each depot, or each chunk or path if '-paths' or '-chunks' are used. The time to verify shelves in all depots is reported separately from submitted files. Timing indications all start with the text 'Time: ' on the beginning of a line of output in the log file, and can be extrated with a command like this example (adjusting the log file name as needed): grep ^Time: /p4/1/logs/p4verify.log EXIT CODES: An exit code of 0 indicates no errors were encountered attempting to perform verifications, AND that all verifications attempted reported no problems. A exit status of 1 indicates that verifications could not be attempted for some reason. A exit status of 2 indicates that verifications were successfully performed, but that problems such as BAD or MISSING files were detected, or else system limits prevented verification. " fi exit 1 } #------------------------------------------------------------------------------ # Function: terminate function terminate { # Disable signal trapping. trap - EXIT SIGINT SIGTERM [[ "$Log" == "off" ]] || msg "Log is: $Log\\n${H1}" # Be tidy. [[ -n "$TmpFile" && -e "$TmpFile" ]] && rm -f "$TmpFile" # With the trap removed, exit. exit "$ExitCode" } function msg () { echo -e "$*"; } function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; } function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; } function bail () { errmsg "${1:-Unknown Error}"; exit; } #------------------------------------------------------------------------------ # Function: get_verify_rev_range ($depot, $recentChanges) #------------------------------------------------------------------------------ function get_verify_rev_range () { declare depot=${1:-} declare recentChangesToVerify=${2:-} declare nowChange= [[ -z "$depot" || -z "$recentChangesToVerify" ]] && return 0 nowChange=$("$P4BIN" -ztag -F %change% changes -m 1 "//$depot/...") thenChange=$("$P4BIN" -ztag -F %change% changes -m "$recentChangesToVerify" "//$depot/..." | tail -1) [[ -z "$nowChange" ]] && return 1 [[ -z "$thenChange" ]] && return 1 echo "@$thenChange,@$nowChange" return 0 } #------------------------------------------------------------------------------ # Function: delay_between_verifies ($WaitBetweenVerifies, $MaxActivePullQueueSize, $MaxTotalPullQueueSize) # # References global(s): $VerifyCommandCount #------------------------------------------------------------------------------ function delay_between_verifies () { local -i waitBetweenVerifies=${1:-0} local -i maxActivePullQueueSize=${2:-0} local -i maxTotalPullQueueSize=${3:-0} local -i activePullQueueSize=0 local -i totalPullQueueSize=0 local -i pullQueueCheckDelay=10 # Short-circuit: Return if no verifies have yet been processed. [[ "$VerifyCommandCount" -eq 0 ]] && return # Short-circuit: Return if no delay-inducing options are set: '-q', '-Q', or '-w'. [[ "$maxActivePullQueueSize" -eq 0 && "$maxTotalPullQueueSize" -eq 0 && "$waitBetweenVerifies" -eq 0 ]] && return # If only the '-w <Wait>' was specified, just wait. if [[ "$maxActivePullQueueSize" -eq 0 && "$maxTotalPullQueueSize" -eq 0 ]]; then dbg "Sleeping $waitBetweenVerifies seconds." sleep "$waitBetweenVerifies" # If only the '-q <MaxActivePullQueueSize>' was specified, wait until the # max active pulls is below the specified limit. If both '-q' and '-w' are # specified, apply the delay of '-w' iff the pull queue is low enough per # '-q'. elif [[ "$maxActivePullQueueSize" -ne 0 ]]; then [[ "$waitBetweenVerifies" -ne 0 ]] && pullQueueCheckDelay=$waitBetweenVerifies activePullQueueSize=$($P4 -ztag -F %replicaTransfersActive% pull -ls 2>/dev/null) dbg "delay_between_verifies(): Active Pull Queue Size: $activePullQueueSize" if [[ -n "$activePullQueueSize" ]]; then while [[ "$activePullQueueSize" -gt "$maxActivePullQueueSize" ]]; do dbg "Sleeping $pullQueueCheckDelay seconds because active pull queue of $activePullQueueSize is greater than max $maxActivePullQueueSize." sleep "$pullQueueCheckDelay" activePullQueueSize=$($P4 -ztag -F %replicaTransfersActive% pull -ls 2>/dev/null) dbg "delay_between_verifies(): Active Pull Queue Size: $activePullQueueSize" if [[ -z "$activePullQueueSize" ]]; then errmsg "delay_between_verifies(): Could not determine active pull queue size in delay loop." break fi done dbg "Delay due to -q completed." else # We could not determine the active pull queue size. # Display an error and move on. errmsg "delay_between_verifies(): Could not determine initial active pull queue size." fi else [[ "$waitBetweenVerifies" -ne 0 ]] && pullQueueCheckDelay=$waitBetweenVerifies totalPullQueueSize=$($P4 -ztag -F %replicaTransfersTotal% pull -ls 2>/dev/null) dbg "delay_between_verifies(): Total Pull Queue Size: $totalPullQueueSize" if [[ -n "$totalPullQueueSize" ]]; then while [[ "$totalPullQueueSize" -gt "$maxTotalPullQueueSize" ]]; do dbg "Sleeping $pullQueueCheckDelay seconds because total pull queue of $totalPullQueueSize is greater than max $maxTotalPullQueueSize." sleep "$pullQueueCheckDelay" totalPullQueueSize=$($P4 -ztag -F %replicaTransfersTotal% pull -ls 2>/dev/null) dbg "delay_between_verifies(): Active Pull Queue Size: $totalPullQueueSize" if [[ -z "$totalPullQueueSize" ]]; then errmsg "delay_between_verifies(): Could not determine total pull queue size in delay loop." break fi done dbg "Delay due to -Q completed." else # We could not determine the total pull queue size. # Display an error and move on. errmsg "delay_between_verifies(): Could not determine initial total pull queue size." fi fi } #------------------------------------------------------------------------------ # Function: call_p4_verify($verifyErrorType, $verifyCommand) # # This calls the 'p4 verify' command with the command given, and increments # count of errors indexted by verify, updating the VerifiyErrors array # displayed in the summary. # # VerifyErrors is an associative array of errors indexed as follows: # archive # shelved # spec # submitted # unload # # Values are the number of errors (including BAD/MISSING/other) for each type. # function call_p4_verify () { local -i status local verifyErrorType="${1:-}" local verifyCommand="${2:-}" local tmpf1= local tmpf2= local lines= local headlines= local oldErrors= local newErrors= local totalErrors= [[ -n "$verifyErrorType" || -n "$verifyCommand" ]] || return msg "${H2}\\nRunning: $verifyCommand" tmpf1="${P4TMP}/t_verify.1.$$.$RANDOM" tmpf2="${P4TMP}/t_verify.2.$$.$RANDOM" if [[ "$NoOp" -eq 0 ]]; then eval "$verifyCommand" > "$tmpf1" 2>&1 status=$? # Suppress 'no such file' errors from older p4d versions to avoid # confusion, as these are not actual errors. In newer p4d versions, # this is a mere warning, which we don't suppress. grep -Ev "^error: //.*/... - no such file(s)." "$tmpf1" > "$tmpf2" # If '-ignores <regex_ignores_file>' was specified, apply it. if [[ -n "$RegexIgnoresFile" ]]; then mv -f "$tmpf2" "$tmpf1" || errmsg "Failure moving temp files." grep -E -v -f "$RegexIgnoresFile" "$tmpf1" > "$tmpf2" if ! diff -q "$tmpf1" "$tmpf2" > /dev/null; then IgnoredErrors=1 # If there are no errors after processing the ignores file, # suppress the 'exit: 1' from the output. newErrors=$(grep -Ec '(BAD!|MISSING!|p4 help max)' "$tmpf2") if [[ "$newErrors" =~ [0-9]+ ]]; then if [[ "$newErrors" == "0" ]]; then msg "Warning: Some verify errors were ignored here. Suppressing 'exit:' line because all errors match expressions." mv -f "$tmpf2" "$tmpf1" || errmsg "Failure moving temp files." # The 'exit: ' line will be the last in the file; exclude it # by doing 'head -N' for all ones but the last. lines=$(wc -l "$tmpf1" | cut -d ' ' -f 1) headlines=$((lines-1)) head -$headlines "$tmpf1" > "$tmpf2" else msg "Warning: Some verify errors were ignored here. Not suppressing 'exit:' line due to other errors." fi else msg "Warning: Some verify errors were ignored here; processing errors." errmsg "Unable to count verify errors in [$tmpf2]." fi fi fi # Display any errors that have not been ignored. cat "$tmpf2" # See if there are any new errors: newErrors=$(grep -Ec '(BAD!|MISSING!|p4 help max)' "$tmpf2") if [[ "$newErrors" =~ [0-9]+ ]]; then if [[ "$newErrors" != "0" ]]; then oldErrors=${VerifyErrors[$verifyErrorType]:-0} totalErrors=$((oldErrors+newErrors)) VerifyErrors[$verifyErrorType]="$totalErrors" fi else errmsg "Unable to count verify errors in [$tmpf2]." fi rm -f "$tmpf1" "$tmpf2" else msg "NO_OP: Would run: $verifyCommand" status=0 fi VerifyCommandCount+=1 return "$status" } #============================================================================== # Command Line Processing declare -i shiftArgs=0 set +u while [[ $# -gt 0 ]]; do case "$1" in (-h) usage -h;; (-man) usage -man;; (-N) NotifyOnlyOnFailure=1;; (-ns) VerifySpecDepot=0;; (-nS) VerifyShelved=0;; (-nr) VerifySubmitted=0;; (-nu) VerifyUnload=0;; (-a) VerifyArchived=1;; (-w) shiftArgs=1 if [[ "$2" =~ ^[1-9]+[0-9]*$ ]]; then WaitBetweenVerifies=$2 else usage -h "Invalid parameter '-w $2'; the value with '-w' must be a positive integer." fi ;; (-q) shiftArgs=1 if [[ "$2" =~ ^[1-9]+[0-9]*$ ]]; then MaxActivePullQueueSize=$2 else usage -h "Invalid parameter '-q $2'; the value with '-q' must be a positive integer." fi ;; (-Q) shiftArgs=1 if [[ "$2" =~ ^[1-9]+[0-9]*$ ]]; then MaxTotalPullQueueSize=$2 else usage -h "Invalid parameter '-Q $2'; the value with '-Q' must be a positive integer." fi ;; (-nt) ScheduleTransfers=0;; (-nz) GzipOldLog=0;; (-o) VerifyOnlyOption="--only ${2^^}"; shiftArgs=1;; (-chunks) VerifyChunks=1; ChunkSize="$2"; shiftArgs=1;; (-paths) VerifyPaths=1; UserPathsFile="$2"; shiftArgs=1;; (-recent) VerifyOnlyRecentChanges=1;; (-dlf) DepotListFile="${2:-}"; shiftArgs=1;; (-ignores) RegexIgnoresFile="${2:-}"; shiftArgs=1;; (-Ocache) OverrideCacheSafety=1;; (-n) NoOp=1;; (-v) ShowLog=1;; (-L) Log="$2"; shiftArgs=1;; (-d) Debug=1;; (-D) Debug=1; set -x;; # Debug; use bash 'set -x' extreme debugging mode. (-*) usage -h "Unknown command line option ($1).";; (*) if [[ -z "$SDPInstance" ]]; then SDPInstance="$1" else usage -h "Unknown extra parameter [$1]." fi ;; esac # Shift (modify $#) the appropriate number of times. shift; while [[ $shiftArgs -gt 0 ]]; do [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments." shiftArgs=$shiftArgs-1 shift done done set -u #============================================================================== # Command Line Verification [[ -z "$SDPInstance" ]] && SDPInstance="${SDP_INSTANCE:-UnsetSDPInstance}" [[ "$SDPInstance" == "UnsetSDPInstance" ]] && \ usage -h "The \\$SDP_INSTANCE setting is not defined. It must be defined by doing:\\n\\n\\tsource /p4/common/bin/p4_vars <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n" # Load SDP Environment. In case this generates output on stdout, suppress it to # avoid triggering crontab emails, but do not suppress stderr (output related # to errors). # shellcheck disable=SC1090 source "$SDPEnv" "$SDPInstance" > /dev/null ||\ bail "Failed to load SDP environment with:\\n\\tsource $SDPEnv $SDPInstance" # shellcheck disable=SC1090 source "$P4CBIN/backup_functions.sh" ||\ bail "Failed to load backup_functions.sh with:\\n\\tsource \"$P4CBIN/backup_functions.sh\"\\n" [[ "$VerifySubmitted" -eq 0 && "$VerifyOnlyRecentChanges" -eq 1 ]] && \ usage -h "The '-recent' and '-nr' options are mutually exclusive." if [[ -n "$VerifyOnlyOption" ]]; then # shellcheck disable=SC2072 if [[ "$P4D_VERSION" > "2021.1" ]]; then if [[ ! "$VerifyOnlyOption" =~ ^--only\ (MISSING|BAD)$ ]]; then usage -h "The option specified with '-o' must be BAD or MISSING. Specified value was: $VerifyOnlyOption" fi else VerifyOnlyOption= fi fi [[ "$VerifyOnlyRecentChanges" -eq 1 && "$VerifyChunks" -eq 1 ]] && \ usage -h "The -recent and -chunks options are mutually exclusive." [[ "$VerifyOnlyRecentChanges" -eq 1 && "$VerifyPaths" -eq 1 ]] && \ usage -h "The -recent and -paths options are mutually exclusive." if [[ -n "$ChunkSize" && ! -x "$ChunkScript" ]]; then msg "Warning: -chunks ignored because chunking script is not executable by user $ThisUser: $ChunkScript" VerifyChunks=0 fi [[ -n "$UserPathsFile" && ! -r "$UserPathsFile" ]] && \ usage -h "The file specified with '-paths <paths_file>' does not exist: $UserPathsFile" [[ -n "$DepotListFile" && ! -r "$DepotListFile" ]] && \ usage -h "The file specified with '-dlf <depot_list_file>' does not exist: $DepotListFile" [[ -n "$RegexIgnoresFile" && ! -r "$RegexIgnoresFile" ]] && \ usage -h "The file specified with '-ignores <regex_ignores_file>' does not exist: $RegexIgnoresFile" [[ "$ScheduleTransfers" -eq 0 && "$MaxActivePullQueueSize" -ne 0 ]] && \ usage -h "The file '-nt' and '-q <MaxActivePullQueueSize>' options are mutually exclusive." [[ "$ScheduleTransfers" -eq 0 && "$MaxTotalPullQueueSize" -ne 0 ]] && \ usage -h "The file '-nt' and '-Q <MaxTotalPullQueueSize>' options are mutually exclusive." [[ "$MaxActivePullQueueSize" -ne 0 && "$MaxTotalPullQueueSize" -ne 0 ]] && \ usage -h "The file '-q <MaxActivePullQueueSize>' and '-Q <MaxTotalPullQueueSize>' options are mutually exclusive." [[ "${P4REPLICA}" == "FALSE" && "$MaxActivePullQueueSize" -ne 0 ]] && \ usage -h "The '-q <MaxActivePullQueueSize>' option can only be used on a replica; the ServerID [$SERVERID] does not appear to be a replica." [[ "$Log" == off && "$NoOp" -eq 0 ]] && \ usage -h "The '-L off' option can only be specified with '-n' (NO_OP/Dry Run mode)." #============================================================================== # Main Program trap terminate EXIT SIGINT SIGTERM # '-N' command line argument is available to disable the default notification behavior # check if the other option, environment variable, was used [[ -n "${NOTIFY_ONLY_ON_FAILURE:-}" ]] && \ NotifyOnlyOnFailure="${NOTIFY_ONLY_ON_FAILURE}" # If $SDP_RECENT_CHANGES_TO_VERIFY, use that, else default to the value set for # $RecentChangesToVerify in this script. [[ -n "${SDP_RECENT_CHANGES_TO_VERIFY:-}" ]] && \ RecentChangesToVerify="${SDP_RECENT_CHANGES_TO_VERIFY}" [[ "$Log" == Unset ]] && Log="$LOGS/p4verify.log" if [[ "$Log" != off ]]; then SummaryLog="$(mktemp "${P4TMP}/email_summary.XXXXXXXXX.txt")" if [[ -e "$Log" ]]; then OldLogTimestamp=$(get_old_log_timestamp "$Log") OldLog="$LOGS/p4verify.${OldLogTimestamp}.log" mv -f "$Log" "$OldLog" || bail "Could not do: mv -f \"$Log\" \"$OldLog\"" # Gzip old log. if [[ "$GzipOldLog" -eq 1 && -n "$(command -v gzip)" ]]; then dbg "gzip --best \"$OldLog\"" gzip --best "$OldLog" fi dbg "Rotated old log $Log to $OldLog" fi touch "$Log" || bail "Couldn't touch new log file [$Log]."? # Redirect stdout and stderr to a log file. if [[ "$ShowLog" -eq 1 ]]; then exec > >(tee "$Log") exec 2>&1 else exec >"$Log" exec 2>&1 fi msg "${H1}\\nLog is: $Log\\n" fi ThisUser=$(id -n -u) msg "Started $ThisScript v$Version as $ThisUser@${HOSTNAME%%.*} on $(date +'%a %Y-%m-%d %H:%M:%S %Z') as:\\n$CmdLine" P4="$P4BIN -p $P4PORT -u $P4USER" "$P4CBIN/p4login" msg "If there are errors in this log, contact support-helix-core@perforce.com" VerifyFailed=0 if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then msg "Info: Verifying only $RecentChangesToVerify changelists per depot due to -recent.\\n" fi # If we are on a replica or edge server, the lbr.replication value will be defined. # If it is set to 'cache' (or the unsupported alias 'ondemand'), abort unless the # '-Ocache' option was specified. LbrReplication=$($P4 -ztag -F %Value% configure show lbr.replication 2>/dev/null) if [[ -n "$LbrReplication" ]]; then if [[ "$LbrReplication" =~ ^(cache|ondemand)$ ]]; then if [[ "$OverrideCacheSafety" -eq 1 ]]; then msg "The lbr.replication value is '$LbrReplication'. Proceeding due to '-Ocache'." else bail "Aborting because the lbr.replication value is '$LbrReplication'. See description of '-Ocache' opion for more info." fi fi fi RevRange= # Determine the list of depots to process, either from 'p4 depots' or from a # specified depot list file. if [[ -n "$DepotListFile" ]]; then TmpFile=$(mktemp) grep -Ev '^\s*$|^\s*\#' "$DepotListFile" > "$TmpFile" if [[ -s "$TmpFile" ]]; then while read -r d; do DepotList[$DepotCount]="$d" DepotCount+=1 done < "$TmpFile" else bail "No depots to process from the depot list file $DepotListFile (trimmed in $TmpFile)." fi else for d in $($P4 -ztag -F %name% depots); do DepotList[$DepotCount]="$d" DepotCount+=1 done fi msg "Time: Setup processing took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 msg "Processing ${#DepotList[@]} depots." for d in "${DepotList[@]}"; do depotType=$($P4 -ztag -F %Type% depot -o "$d") delay_between_verifies "$WaitBetweenVerifies" "$MaxActivePullQueueSize" "$MaxTotalPullQueueSize" case "$depotType" in (archive) [[ "$VerifyArchived" -eq 1 ]] || continue # For archive depots, use -A and -q but not -z. if [[ -n "$VerifyOnlyOption" ]]; then VerifyOptions="-q $VerifyOnlyOption -A" else VerifyOptions="-q -A" fi [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t" msg "=== Started verify of archive depot //$d/... at $(date)." VerifyCmd="$P4 -s verify $VerifyOptions //$d/..." call_p4_verify archive "$VerifyCmd" || VerifyFailed=1 msg "Time: Verification of archive depot //$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 ;; (remote) msg "\\nInfo: Skipping verify of remote depot $d.\\n" continue ;; (spec) [[ "$VerifySpecDepot" -eq 1 ]] || continue # For the spec depot, there are no lazy copies, so no need for -z. if [[ -n "$VerifyOnlyOption" ]]; then VerifyOptions="-q $VerifyOnlyOption" else VerifyOptions="-q" fi [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t" msg "=== Started verify of spec depot //$d/... at $(date)." VerifyCmd="$P4 -s verify $VerifyOptions //$d/..." call_p4_verify spec "$VerifyCmd" || VerifyFailed=1 msg "Time: Verification of spec depot //$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 ;; (unload) [[ "$VerifyUnload" -eq 1 ]] || continue # For the unload depot, there are no lazy copies, so no need for -z. if [[ -n "$VerifyOnlyOption" ]]; then VerifyOptions="-q $VerifyOnlyOption -U" else VerifyOptions="-q -U" fi [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t" msg "=== Started verify of unload depot at $(date)." VerifyCmd="$P4 -s verify $VerifyOptions //$d/..." call_p4_verify unload "$VerifyCmd" || VerifyFailed=1 msg "Time: Verification of unload depot //$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 ;; (*) [[ "$depotType" != "local" && "$depotType" != "stream" ]] && \ msg "Warning: Unknown depot type [$depotType]. Attempting normal verify." if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then RevRange=$(get_verify_rev_range "$d" "$RecentChangesToVerify") fi DepotPathsFile="$(mktemp "$P4TMP"/verify_paths."$d".XXXXX.txt)" rm -f "$DepotPathsFile" # If '-paths' was specified providing a file containing a list of # user-specified, generate a depot paths file containing the subset # of paths in the current depot being processed. # This block of code will result in a DepotPathsFile existing if # user-specified paths are specified in the current depot being # processed. if [[ "$VerifyPaths" -eq 1 ]]; then grep "^//$d/" "$UserPathsFile" > "$DepotPathsFile" ||\ errmsg "Could not generate depot paths file [$DepotPathsFile] from user paths file [$UserPathsFile] with: grep \"^//$d/\" \"$UserPathsFile\"" [[ -s "$DepotPathsFile" ]] || rm -f "$DepotPathsFile" fi ChunkFile="$(mktemp "$P4TMP"/depot_chunks."$d".XXXXXX.txt)" rm -f "$ChunkFile" # This block of code will result in a ChunkFile existing if chunking # is to be done. If the ChunkFile does not exist, chunking will not # be applied. if [[ "$VerifyChunks" -eq 1 ]]; then # Combined '-paths' and '-chunks' behaviors. if [[ -e "$DepotPathsFile" ]]; then while read -r DepotPath; do msg "Running: $ChunkScript -v ERROR -m $ChunkSize //$DepotPath" if "$ChunkScript" -v ERROR -m "$ChunkSize" "$DepotPath" >> "$ChunkFile"; then msg "Time: Chunking \"$DepotPath\" took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 else errmsg "Chunking of path \"$DepotPath\" failed." fi done < "$DepotPathsFile" # Handle '-chunks' behaviour without '-paths'. else msg "Running: $ChunkScript -v ERROR -m $ChunkSize //$d/..." if "$ChunkScript" -v ERROR -m "$ChunkSize" "//$d/..." > "$ChunkFile"; then msg "Time: Chunking //$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 else msg "Warning: Not chunking //$d/... due to script error." rm -f "$ChunkFile" fi if [[ ! -s "$ChunkFile" ]]; then msg "Warning: Not chunking //$d/... due to zero-size chunk file." rm -f "$ChunkFile" fi fi fi # For stream/local depots, use -z unless --only is used. if [[ -n "$VerifyOnlyOption" ]]; then VerifyOptions="-q $VerifyOnlyOption" else VerifyOptions="-qz" fi [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t" if [[ "$VerifySubmitted" -eq 1 ]]; then # If the ChunkFile exists, verify the specified chunks. This may be # chunks of entire depots or chunks of specified paths (per '-paths'). if [[ "$VerifyChunks" -eq 1 && -s "$ChunkFile" ]]; then msg "=== Started verify of submitted files in //$d/... in $ChunkSize chunks at $(date)." while read -r ChunkPath; do VerifyCmd="$P4 -s verify $VerifyOptions \"$ChunkPath\"" call_p4_verify submitted "$VerifyCmd" || VerifyFailed=1 msg "Time: Verification of submitted files in \"$ChunkPath\" took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 done < "$ChunkFile" rm -f "$ChunkFile" # If the ChunkFile does not exist, but paths were specified, handle # the targeted paths without chunking. elif [[ "$VerifyPaths" -eq 1 && -e "$DepotPathsFile" ]]; then msg "=== Started verify of submitted files in //$d/... in user-specified paths file at $(date)." while read -r DepotPath; do VerifyCmd="$P4 -s verify $VerifyOptions \"$DepotPath\"" call_p4_verify submitted "$VerifyCmd" || VerifyFailed=1 msg "Time: Verification of submitted files in \"$DepotPath\" took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 done < "$DepotPathsFile" rm -f "$DepotPathsFile" else msg "=== Started verify of submitted files in //$d/... at $(date)." VerifyCmd="$P4 -s verify $VerifyOptions //$d/...$RevRange" call_p4_verify submitted "$VerifyCmd" || VerifyFailed=1 msg "Time: Verification of submitted files in /$d/... took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 fi fi ;; esac done if [[ "$VerifyShelved" -eq 1 ]]; then msg "=== Started verify of shelved changelists at $(date)." TmpFile=$(mktemp) TmpLog=$(mktemp) ChangesCmd="$P4BIN -ztag -F %change% changes -s shelved" [[ "$VerifyOnlyRecentChanges" -eq 1 ]] && \ ChangesCmd+=" -m $RecentChangesToVerify" ShelvedCLCount=0 ShelvedCLErrorCount=0 # For shelves, don't use -z since we verify change-by-change. if [[ -n "$VerifyOnlyOption" ]]; then VerifyOptions="-q $VerifyOnlyOption -S" else VerifyOptions="-q -S" fi [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t" if $ChangesCmd > "$TmpFile"; then ShelvedChangesToVerify=$(wc -l "$TmpFile" | cut -d ' ' -f 1) msg "Verifying $ShelvedChangesToVerify shelved changelists." while read -r ShelvedCL; do delay_between_verifies "$WaitBetweenVerifies" "$MaxActivePullQueueSize" "$MaxTotalPullQueueSize" VerifyCommandCount+=1 ShelvedCLCount+=1 VerifyCmd="$P4 -s verify $VerifyOptions @=$ShelvedCL" # Capture temp log for each CL verify command. Display only logs with # error; discard others. if [[ "$NoOp" -eq 0 ]]; then if eval "$VerifyCmd" > "$TmpLog" 2>&1; then rm -f "$TmpLog" else ShelvedCLErrorCount+=1 cat "$TmpLog" rm -f "$TmpLog" fi else # There may be a great many shelves. In NoOp we are silent here # unless '-d' is used. dbg "NO_OP: Would run: $VerifyCmd" fi done < "$TmpFile" msg "Time: Verification of shelved files took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 else msg "Error: Could not get or write list of shelved changes." VerifyFailed=1 fi if [[ "$ShelvedCLCount" -ne 0 ]]; then if [[ "$ShelvedCLErrorCount" -eq 0 ]]; then msg "\\nAll $ShelvedCLCount shelved changes verified OK." else msg "\\nOf $ShelvedCLCount shelved changes verified, $ShelvedCLErrorCount had errors." VerifyFailed=1 fi else msg "\\nThere were no shelved changelists to verify." fi else msg "Skipping verify of shelved changelists due to '-nS'." fi if [[ "$VerifyFailed" -ne 0 ]]; then StatusMessage="Error: Verify attempt failed. Review the log [$Log]." ExitCode=1 fi if [[ "$ExitCode" -eq 0 ]]; then # Note: Ignore SC2196 due to 'grep -e' and 'egrep' not having consistent # results on all platforms. # shellcheck disable=SC2196 if egrep -m1 '(BAD!|MISSING!|p4 help max)' "$Log" > /dev/null 2>&1; then StatusMessage="Error: Verify errors detected. Review the log [$Log]." ExitCode=2 fi fi msg "Time: Summarizing results took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" TotalSeconds+=$SECONDS; SECONDS=0 msg "Time: Completed verifications at $(date), taking $((TotalSeconds/3600)) hours $((TotalSeconds%3600/60)) minutes $((TotalSeconds%60)) seconds.\\n" if [[ -n "$SummaryLog" ]]; then # Set LOGFILE as an alias for $SummaryLog. LOGFILE is used in the library functions # in backup_functions.sh for notification handling. export LOGFILE="$SummaryLog" msg "Summary from $ThisScript v$Version operated as $ThisUser@${HOSTNAME%%.*} Status: $StatusMessage Time: Completed verifications at $(date), taking $((TotalSeconds/3600)) hours $((TotalSeconds%3600/60)) minutes $((TotalSeconds%60)) seconds. Command line was:\\n\\t$CmdLine " > "$SummaryLog" if [[ "$ExitCode" -ne 0 ]]; then msg "Summary of Errors by Type:" >> "$SummaryLog" if [[ "$VerifyArchived" -eq 1 ]]; then msg " Archive Depot Errors: ${VerifyErrors[archive]:-0}" >> "$SummaryLog" fi if [[ "$VerifySubmitted" -eq 1 ]]; then msg " Submitted File Errors: ${VerifyErrors[submitted]:-0}" >> "$SummaryLog" fi if [[ "$VerifySpecDepot" -eq 1 ]]; then msg " Spec Depot Errors: ${VerifyErrors[spec]:-0}" >> "$SummaryLog" fi if [[ "$VerifyUnload" -eq 1 ]]; then msg " Unload Depot Errors: ${VerifyErrors[unload]:-0}" >> "$SummaryLog" fi # Do some quick math. if [[ "$VerifyArchived" -eq 1 || "$VerifySubmitted" -eq 1 || "$VerifySpecDepot" -eq 1 || "$VerifyUnload" -eq 1 ]]; then declare -i total=0 for errorType in "${!VerifyErrors[@]}"; do total+=${VerifyErrors[$errorType]:-0} done msg " Total Non-Shelve Errors: $total (sum of error types listed above)" >> "$SummaryLog" fi if [[ "$VerifyShelved" -eq 1 ]]; then msg "\\n Shelved Changes with Errors: $ShelvedCLErrorCount" >> "$SummaryLog" fi fi if [[ -n "$RegexIgnoresFile" ]]; then if [[ "$IgnoredErrors" -ne 0 ]]; then msg "\\n Note: Some verify errors were ignored due to '-ignores $RegexIgnoresFile'." >> "$SummaryLog" else msg "\\n Note: The '-ignores $RegexIgnoresFile' option was specified, but no ignore expressions matched." >> "$SummaryLog" fi fi msg "\\nA total of $VerifyCommandCount 'p4 verify' commands were executed." >> "$SummaryLog" msg "\\nDetailed log is:\\n\\t${HOSTNAME%%.*}:$Log " >> "$SummaryLog" msg "${H2}\\nBEGIN Summary Log Contents:" cat "$SummaryLog" msg "END Summary Log Contents\\n${H2}" if [[ "$NotifyOnlyOnFailure" -eq 0 ]]; then mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)" else if [[ "$ExitCode" -ne 0 ]]; then die "p4 verify had error: $HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)" else msg "NotifyOnlyOnFailure is set to 1 and the Exitcode is 0. Notifications disabled." fi fi msg "Using email for log summary delivery." mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)" fi # See the terminate() function where this script actually exits. exit "$ExitCode"
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#64 | 30344 | C. Thomas Tyler | Fixed typo in usage synopsis. | ||
#63 | 30272 | C. Thomas Tyler |
Added '-p4config' option to verify an arbitrary p4d server, maybe non-SDP and non-Linux. Changed mechansim for checking P4D_VERSION to use 'p4 info' rather than 'p4d -V' output, so as to work for local and remote servers. Added warnings for short-duration tickets. Added early abort if user does not have admin or better access (as is required for running 'p4 verify'). |
||
#62 | 30261 | C. Thomas Tyler |
p4verify.sh: Fixed bug that caused duplicate emails to be sent. #review-30262 |
||
#61 | 30255 | C. Thomas Tyler |
Added '-a' to all grep commands. #review-30256 |
||
#60 | 30251 | C. Thomas Tyler |
p4verify.sh: Added '-no_z' option to verify without '-z'. Removed restriction on using '-recent' with '-nr'. It is useful to verify recent shelves. #review-30252 |
||
#59 | 30156 | C. Thomas Tyler |
p4verify.sh v5.16.2: Doc fixes, no functional change. Thanks, Andrei! |
||
#58 | 30141 | C. Thomas Tyler | p4verify.sh: For shelved files that fail verify, show the command line that generated the error. | ||
#57 | 29992 | C. Thomas Tyler |
p4verify.sh: Added support for handling the 'trait' depot (new in 2023.2). Refactored get_old_log_timestamp() function, moving it from backup_functions.sh into new log_functions.sh file. #review-29993 |
||
#56 | 29861 | C. Thomas Tyler |
p4verify.sh v5.15: * The '-paths' and '-recent' options can now be used together, so recent files in specified paths can be verified. Particularly useful for edge servers and failover scenarios where earlier and more thorough vetting has been done. * The '-recent' now sports an optional '-recent=N' syntax to specify # of revs. * Doc updates for above. * Fixed minor ShellCheck compliance issues. #review-29862 |
||
#55 | 29827 | C. Thomas Tyler |
Added support for verifying the Extensions depot. #review-29828 @robert_cowham |
||
#54 | 29717 | C. Thomas Tyler | p4verify.sh: Cosmetic issue in 'Time:' output in p4verify.log. | ||
#53 | 29689 | C. Thomas Tyler |
Slight tweak to new '-ignores' option introduced in @29655. Added concept of a standard ignores file; if it exists in expected path, '-ignores' option is not needed. Can use '-ignores none' to avoid processing any ignores. Ignores output clearly indicates whether errors were ignored. Improved docs for ignores as well. #review-29690 |
||
#52 | 29655 | C. Thomas Tyler |
p4verify.sh v5.13.0: * Added '-ignores <regex_file_pattern>' option to ignore known errors. * Corrected issues with summary of messages in summary log file. * Changed call_p4verify() function name to call_p4_verify() to more clearly indicate that the underlying 'p4' command line is called. * Fixed cosmet tab/space errors. #review-29656 |
||
#51 | 29633 | C. Thomas Tyler |
Added '-paths' option to specify a file containing a list of paths to verify. Can be combined with '-chunks' or used independently. Enhanced timing detail, and made it easier to extract timing detail with soemthing like: grep ^Time: /p4/1/logs/p4verify.log Added docs for above, and also did some general cleanup of docs. #review-29634 |
||
#50 | 29586 | C. Thomas Tyler |
Added safety feature to avoid processing cache mode replica. Added '-Ocache' override option to bypass the safety feature. Added relevant docs. |
||
#49 | 29580 | C. Thomas Tyler | Added check for bash v4+. | ||
#48 | 29578 | C. Thomas Tyler |
Fixed issue with log rotation login on some platforms. #review-29579 @Domenic |
||
#47 | 29577 | C. Thomas Tyler | Added usage exmple for archive loading with delay. | ||
#46 | 29570 | Andy Boutte |
Enhance p4verify to utilize die() function which will allow user to leverage AWS SNS or PagerDuty. Added '-N' option to notify only in event of failure. |
||
#45 | 29559 | C. Thomas Tyler |
Added '-Q <MaxTotalPullQueueSize>' option. Fixed bugs from earlier review, 29555. #review-29560 @robert_cowham |
||
#44 | 29554 | C. Thomas Tyler |
Added '-w <Wait>' and '-q <MaxActivePullQueueSize>' options to p4verify.sh. #review-29555 |
||
#43 | 29106 | C. Thomas Tyler |
Split p4verify.log into full and summary logs, with summary sent to AWS SNS if configured. Summary is always sent via email, referencing log on host. Added rough counts of verify errors of various types to summary. Docs: Added more examples with more complete information. Enhanced general log file handling. Fixed a minor bug in suppression of 'no such file' errors. Enhanced internal code comments. Bypassing pre-commit review to engage test suite. #review-29025 @andy_boutte |
||
#42 | 28977 | C. Thomas Tyler |
Fixed bug found in review. #review-28975 |
||
#41 | 28974 | C. Thomas Tyler |
Fixed bug with P4D_VERSION unbound variable if '-o MISSING' is used and SDP shell environment has not been sourced. The script sourced p4_vars to set P4D_VERSION, but too late, after P4D_VERSION was referenced. This has been fixed. Moved cmd() function to be in the 'Local Functions' section of the file. #review-28975 |
||
#40 | 28835 | C. Thomas Tyler |
p4verify.sh v5.6.2: * Added '-dlf <depot_list_file>' option to specify prioritized list of depots to process. * Added '-chunks', useful for avoiding RAM exhaustion. Uses the depot_verify_chunks.py option to verify depots in chunks. Testing indicates this may be much slower than a standard verify, but can be a useful option if RAM is scarce. If used, this introduces a dependency on the perforce-p4python3 package. * Optimized options to 'p4 verify' on a per-depot basis, e.g. not applying '-z' in depot types that can't have lazy copies. * Eliminated the "on replica" portion of a message displayed on some verify commands, as it had become potentially inaccurate with introduction of the '-nt' option. * Added 'Processing N depots' message. To Do: The verify_depot_chunks.py sometimes lists empty paths, e.g. //MyDepot/Binaries/*, which could be eliminated if empty. This is mostly harmless but results in some cruft in the log. |
||
#39 | 28771 | C. Thomas Tyler |
Changed email address for Perforce Support. #review-28772 @amo @robert_cowham |
||
#38 | 28211 | C. Thomas Tyler |
p4verify.sh v5.4.0: * Added '-o BAD|MISSING' to expose the p4d '--only BAD|MISSING' option. * Fix: Completed implementation of documented '-a' option. * Added '-nt' option to avoid passing '-t' to 'p4 verify' on replicas. * Reduced excess noise from change-by-change verification of shelves. Bypassing pre-commit to engage test suite. #review-28212 |
||
#37 | 28143 | C. Thomas Tyler |
Fixed issue operating in environments where the shell USER variable is not defined. The current USER is now determined with 'id -n -u'. Also silenced a harmless ShellCheck warning. #review-28144 |
||
#36 | 27722 | C. Thomas Tyler |
Refinements to @27712: * Resolved one out-of-date file (verify_sdp.sh). * Added missing adoc file for which HTML file had a change (WorkflowEnforcementTriggers.adoc). * Updated revdate/revnumber in *.adoc files. * Additional content updates in Server/Unix/p4/common/etc/cron.d/ReadMe.md. * Bumped version numbers on scripts with Version= def'n. * Generated HTML, PDF, and doc/gen files: - Most HTML and all PDF are generated using Makefiles that call an AsciiDoc utility. - HTML for Perl scripts is generated with pod2html. - doc/gen/*.man.txt files are generated with .../tools/gen_script_man_pages.sh. #review-27712 |
||
#35 | 27616 | C. Thomas Tyler |
Corrected p4verify.sh doc error implying multiple instances can be processed in a single invocation. Also fixed a shellcheck warning. Non-functional change. |
||
#34 | 27216 | C. Thomas Tyler |
Non-functional change: Fixed a common doc type: paramter -> parameter |
||
#33 | 27107 | C. Thomas Tyler |
Reworked how shelved files are verified. If shelved files are to be verified, shelved changelists are processed rather than verifying by association with depots. This now complies with documented restriction per 'p4 help verify' that only the @= revision specifier is valid when the '-S' flag is used. If '-recent' is specifed, only the most recent N shelved changelists are verified. #review-27108 |
||
#32 | 26847 | Robert Cowham | Put back the include tags for AsciiDoc | ||
#31 | 26815 | C. Thomas Tyler | Patch: Fixed bug in '-recent' flag to p4verify.sh. | ||
#30 | 26755 | Robert Cowham | Include p4verify.sh man page in SDP Guide automatically for usage section. | ||
#29 | 25547 | C. Thomas Tyler |
p4verify.sh v5.2.0: Added targeting flags: -nu - No Unload -nr - No Regular archived files. -ns - No Spec Depot -nS - No Shelved -a - Archive Depot verify. Verification of archive depots with '-A' is now supported, though not enabled by default as archive depots are typically in cold storage. Also made doc tweaks, enhanced description of '-L' flag and correcting '-h' usage. Added warning message if an unknown depot type is encountered. Unknown depot types are ignored, as new depot types typically require new logic to handle. More updates for shellcheck compliance. |
||
#28 | 25452 | C. Thomas Tyler | Bumped Version ID for change @25451. | ||
#27 | 25451 | Robert Cowham | Fix failing test - local cmd function was being overwritten by sourcing backup_functions.sh | ||
#26 | 25238 | C. Thomas Tyler |
Replaced references to /tmp with P4TMP (e.g. /p4/N/tmp) to avoid issues with SELinux incompatibility. For p4verify.sh, enhanced: * Minor performance enhancement: Added '-m1' flag to egrep that determines whether to exit code with non-zero, as we only need to detect one occurence. * Changed 'Warning:' to 'Error:' if verify errors are detected. * Removed redundant check that SDP_INSTANCE is set. * Added total time info. * Adjusted to pass 'shellcheck.sh' check. #review-25239 |
||
#25 | 24967 | C. Thomas Tyler | Merged patch fix from Karl in main to dev. | ||
#24 | 24218 | C. Thomas Tyler |
Enhanced performance of "p4 verify.sh -recent" by lowering default changelists to scan per depot, and skipping shelves. Also fixed indentation/style issues. |
||
#23 | 23750 | C. Thomas Tyler |
In p4verify.sh, fixed handling of unload and spec depots. Unload depots were not being identified correctly, and so special flags needed for unload-type depots were not applied. For spec-type depots, 'p4 verify -qS' was run, which isn't necessary or helpful since it is not possible to shelve files in the spec depot. The fix included replacing parsing logic with optimized p4 commands that don't require parsing. |
||
#22 | 22074 | Russell C. Jackson (Rusty) |
Corrected the name of the variable. Already reviewed with Tom. |
||
#21 | 22066 | Russell C. Jackson (Rusty) | Added rotate for p4verify.log instead of just deleting the prior one. | ||
#20 | 21233 | Russell C. Jackson (Rusty) | Review 21010 from Adrian commit. | ||
#19 | 21007 | adrian_waters | Fix specification of depot path in get_verify_rev_range when -recent used | ||
#18 | 20995 | C. Thomas Tyler |
Style tweak; using double-square brakets rather than single-square brackets. |
||
#17 | 20994 | adrian_waters |
Refactor around changes to support -recent arg - had broken code even where -recent not used because of unbounded variable ($d); use of the -recent option also fails as 'verify -S' not valid for shelved files (only @= is supported) ; stripping out archive,unload+remote depots also not working + get_verify_rev_range used depot name incorrectly in changes (eg, changed -m 1 depot/... rather than //depot/...) Raised job000634 for changes. Further aim is to modify script for depots with no shelved files does not result in an 'error' line being written to p4verify.log - it confuses clients - will do that as a separate change. |
||
#16 | 20909 | Russell C. Jackson (Rusty) | Fixed missing LOGFILE variable issue. | ||
#15 | 20879 | Robert Cowham | Fix typo in variable name. | ||
#14 | 20868 | C. Thomas Tyler |
Add '-recent' flag to p4verify.sh to verify only recent changes. Usage info is added with '-h' and '-man' flags. This change involved changing the command line parsing to accept the <instance> positional parameter and '-recent' flag, as well as standard '-v' and '-D' flags. This change also should reduce memory requirement in cases where there are a large number of verify errors. This was done by combining greps and tossing 'grep' output to /dev/null, using only the grep exit code to determine if BAD/MISSING (etc.) appear in the log. Added a Version id. (Need to avoid +k due to the way SDP files move around from server to server). This must be updated manually. |
||
#13 | 20086 | Russell C. Jackson (Rusty) | Added -t to verify of shelves on a replica, and added verify of shelves on the master. | ||
#12 | 20083 | Russell C. Jackson (Rusty) | Added missing echo of command into log file. | ||
#11 | 19314 | Russell C. Jackson (Rusty) |
Change p4verify.sh to use -S to verify shelves on a replica instead of printing the files on the shelf. Removed the HOST_IP settings from mkdirs and instance_vars since it causes problems in a shared depotdata environment, and it was a temporary fix to work around a bug with auth.id that is being fixed. |
||
#10 | 19102 | Russell C. Jackson (Rusty) |
Added check for SHAREDDATA to avoid using -t on a shared data volume replica. Allows running verify on the replica instead of the master. |
||
#9 | 16029 | C. Thomas Tyler |
Routine merge to dev from main using: p4 merge -b perforce_software-sdp-dev |
||
#8 | 15778 | C. Thomas Tyler | Routine Merge Down to dev from main. | ||
#7 | 13906 | C. Thomas Tyler |
Normalized P4INSTANCE to SDP_INSTANCE to get Unix/Windows implementations in sync. Reasons: 1. Things that interact with SDP in both Unix and Windows environments shoudn't have to account for this obscure SDP difference between Unix and Windows. (I came across this doing CBD work). 2. The Windows and Unix scripts have different variable names for defining the same concept, the SDP instance. Unix uses P4INSTANCE, while Windows uses SDP_INSTANCE. 3. This instance tag, a data set identifier, is an SDP concept. I prefer the SDP_INSTANCE name over P4INSTANCE, so I prpose to normalize to SDP_INSTANCE. 4. The P4INSTANCE name makes it look like a setting that might be recognized by the p4d itself, which it is not. (There are other such things such as P4SERVER that could perhaps be renamed as a separate task; but I'm not sure we want to totally disallow the P4 prefix for variable names. It looks too right to be wrong in same cases, like P4BIN and P4DBIN. That's a discussion for another day, outside the scope of this task). Meanwhile: * Fixed a bug in the Windows 2013.3 upgrade script that was referencing undefined P4INSTANCE, as the Windows environment defined only SDP_INSTANCE. * Had P4INSTANCE been removed completely, this change would likely cause trouble for users doing updates for existing SDP installations. So, though it involves slight technical debt, I opted to keep a redundant definition of P4INSTANCE in p4_vars.template, with comments indicating SDP_INSTANCE should be used in favor of P4INSTANCE, with a warning that P4INSTANCE may go away in a future release. This should avoid unnecessary upgrade pain. * In mkdirs.sh, the varialbe name was INSTANCE rather than SDP_INSTANCE. I changed that as well. That required manual change rather than sub/replace to avoid corrupting other similar varialbe names (e.g. MASTERINSTANCE). This is a trivial change technically (a substitute/replace, plus tweaks in p4_vars.template), but impacts many files. |
||
#6 | 13586 | C. Thomas Tyler |
Routine merge down from main -> dev. Trivial merges, all resolved with 'p4 resolve -as.' |
||
#5 | 12169 | Russell C. Jackson (Rusty) |
Updated copyright date to 2015 Updated shell scripts to require an instance parameter to eliminate the need for calling p4master_run. Python and Perl still need it since you have to set the environment for them to run in. Incorporated comments from reviewers. Left the . instead of source as that seems more common in the field and has the same functionality. |
||
#4 | 12028 | C. Thomas Tyler | Refreshed SDP dev branch, merging down from main. | ||
#3 | 11485 | Russell C. Jackson (Rusty) |
Brought over changes from RCJ sdp to properly handle Edge servers and to properly replicate shelves when replicating from Windows to Linux |
||
#2 | 11476 | Russell C. Jackson (Rusty) | Fixed echo command to match the actual command run. | ||
#1 | 10638 | C. Thomas Tyler | Populate perforce_software-sdp-dev. | ||
//guest/perforce_software/sdp/main/Server/Unix/p4/common/bin/p4verify.sh | |||||
#1 | 10148 | C. Thomas Tyler | Promoted the Perforce Server Deployment Package to The Workshop. |