#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
#==============================================================================
# Declarations and Environment
export SDP_ENV=/p4/common/bin/p4_vars
export SDP_INSTANCE=${SDP_INSTANCE:-Unset}
declare StatusMessage="OK: All scanned depots verified OK."
declare ThisScript=${0##*}
declare Version=5.5.5
declare CmdLine="$0 $*"
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i ShowLog=0
declare -i ExitCode=0
declare -i VerifyArchived=0
declare -i VerifyShelved=1
declare -i VerifySpecDepot=1
declare -i VerifySubmitted=1
declare -i VerifyUnload=1
declare -i ShelvedCLCount=0
declare -i ShelvedCLErrorCount=0
declare -i SubmittedCLCount=0
declare -i SubmittedCLErrorCount=0
declare RevRange=
declare RecentChangesToVerify=200
declare VerifyCmd=
declare ChangesCmd=
declare ShelvedChangesToVerify=
declare SubmittedChangesToVerify=
declare SubmittedCL=
declare ShelvedCL=
declare Log=Unset
declare TmpFile=
#==============================================================================
# Local Functions
# Micro-functions, one-liners used to avoid external dependencies.
function msg () { if [[ $Log != Unset ]]; then echo -e "$*" >> $Log; else echo -e "$*"; fi; }
function bail () { msg "\\nError: ${1:-Unknown Error}"; exit "${2:-1}"; }
#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional). Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1). If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
declare style=${1:--h}
declare errorMessage=${2:-Unset}
if [[ $errorMessage != Unset ]]; then
echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
fi
# tag::includeManual[] - required for AsciiDoc
echo "USAGE for $ThisScript v$Version:
p4verify.sh [<instance>] [-nu] [-nr] [-ns] [-nS] [-a] [-recent] [-L <log>] [-v] [-D]
or
p4verify.sh -h|-man
"
if [[ $style == -man ]]; then
echo -e "DESCRIPTION:
This script performs a 'p4 verify' of all submitted and shelved versioned
files in depots of all types except 'remote' and 'archive' type depots.
If run on a replica, it schedules archive failures for transfer to the
replica.
OPTIONS:
<instance>
Specify the SDP instance. If not specified, the SDP_INSTANCE
environment variable is used instead. If the instance is not
defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
exists immediately with an error message.
-nu Specify '-nu' (No Unload) to skip verification of the singleton depot
of type 'unload' (if created). The 'unload' depot is verified
by default.
-nr Specify '-nr' (No Regular) to skip verification of regular submitted
archive files. The '-nr' option is not compatible with '-recent'.
Regular submitted archive files are verified by default.
-ns Specify '-ns' (No Spec Depot) to skip verification of singleton depot
of type 'spec' (if created). The 'spec' depot is verified by default.
-nS Specify '-nS' (No Shelves) to skip verification of shelved archive
files, i.e. to skip the 'p4 verify -qS'.
-a Specify '-a' (Archive Depots) to do verification of depots of type
'archive'. Depots of type 'archive' are not verified by default, as
archive depots are often physically removed from the server's
storage subsystem for long-term cold storage.
-recent
Specify that only recent changelists should be verified.
The \$SDP_RECENT_CHANGES_TO_VERIFY variable defines how many
changelists are considered recent; the default is $RecentChangesToVerify.
If the default is not appropriate for your site, add
\"export SDP_RECENT_CHANGES_TO_VERIFY\" to /p4/common/config/p4_N.vars to
change the default for an instance, or to /p4/common/bin/p4_vars to
change it globally. If \$SDP_RECENT_CHANGES_TO_VERIFY is unset, the
default is $RecentChangesToVerify.
When -recent is used, neither shelves nor files in the unload depot
are verified.
-v Verbose. Show output of verify attempts, which is suppressed by default.
Setting SDP_SHOW_LOG=1 in the shell environment has the same effect as -v.
The default behavior of this script is to generate no terminal output,
but instead to write output into a log file -- see LOGGING below. If
'-v' is specified, the generated log is sent to stdout at the end of
processing. This flag is not recommended for routine cron operation or
for large data sets.
-L <log>
Specify the log file to use. The default is /p4/N/logs/p4verify.log
Log rotation and old log cleanup logic does not apply to log files
specified with -L. Thus, using -L is not recommended for routine scheduled
operation, e.g. via crontab.
-D Set extreme debugging verbosity.
HELP OPTIONS:
-h Display short help message
-man Display man-style help message
EXAMPLES:
This script is typically called via cron with only the instance
parameter as an argument, e.g.:
p4verify.sh N
LOGGING:
This script generates no output by default. All (stdout and stderr) is
logged to /p4/N/logs/p4verify.log.
The exception is usage errors, which result an error being sent to
stderr followed usage info on stdout, followed by an immediate exit.
If the '-v' flag is used, the contents of the log are displayed to
stdout at the end of processing.
EXIT CODES:
An exit code of 0 indicates no errors were encountered attempting to
perform verifications, AND that all verifications attempted
reported no problems.
A exit status of 1 indicates that verifications could not be
attempted for some reason.
A exit status of 2 indicates that verifications were successfully
performed, but that problems such as BAD or MISSING files
were detected, or else system limits prevented verification.
"
# end::includeManual[] - required for AsciiDoc
fi
exit 1
}
#------------------------------------------------------------------------------
# Function: get_verify_rev_range ($depot, $recentChanges)
#------------------------------------------------------------------------------
function get_verify_rev_range () {
declare depot=${1:-}
declare recentChangesToVerify=${2:-}
declare nowChange=
[[ -z "$depot" || -z "$recentChangesToVerify" ]] && return 0
nowChange=$("$P4BIN" -ztag -F %change% changes -m 1 "//$depot/...")
thenChange=$("$P4BIN" -ztag -F %change% changes -m "$recentChangesToVerify" "//$depot/..." | tail -1)
[[ -z "$nowChange" ]] && return 1
[[ -z "$thenChange" ]] && return 1
echo "@$thenChange,@$nowChange"
return 0
}
#==============================================================================
# Command Line Processing
declare -i shiftArgs=0
set +u
while [[ $# -gt 0 ]]; do
case "$1" in
(-h) usage -h;;
(-man) usage -man;;
(-ns) VerifySpecDepot=0;;
(-nS) VerifyShelved=0;;
(-nr) VerifySubmitted=0;;
(-nu) VerifyUnload=0;;
(-recent) VerifyOnlyRecentChanges=1;;
(-v) ShowLog=1;;
(-L) Log=$2; shiftArgs=1;;
(-D) set -x;; # Debug; use 'set -x' mode.
(-*) usage -h "Unknown command line option ($1).";;
(*) export SDP_INSTANCE=$1;;
esac
# Shift (modify $#) the appropriate number of times.
shift; while [[ $shiftArgs -gt 0 ]]; do
[[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
shiftArgs=$shiftArgs-1
shift
done
done
set -u
#==============================================================================
# Command Line Verification
[[ $SDP_INSTANCE == Unset ]] && \
bail "The \\$SDP_INSTANCE setting is not defined. It must be defined by doing:\\n\\n\\tsource /p4/common/bin/p4_vars <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n"
[[ "$VerifySubmitted" -eq 0 && "$VerifyOnlyRecentChanges" -eq 1 ]] && \
bail "The '-recent' and '-nr' options are mutually exclusive."
#==============================================================================
# Main Program
# shellcheck disable=SC1090
source "$SDP_ENV" "$SDP_INSTANCE" ||\
bail "Failed to load SDP environment for instance $SDP_INSTANCE."
# shellcheck disable=SC1090
source "$P4CBIN/backup_functions.sh" ||\
bail "Failed to load backup_functions.sh."
#------------------------------------------------------------------------------
# Function: cmd()
# We need to define cmd() after we have sourced backup_functions.sh.
function cmd ()
{
tmpf="${P4TMP}/t_verify.$$.$RANDOM"
msg "$*" >> "$Log"
# Disable SC2048 for $* because we want any spaces to be expanded here.
# shellcheck disable=SC2048
$* > "$tmpf" 2>&1
status=$?
# Suppress 'no such file' errors from older p4d versions. In newer p4d
# versions, this is a mere warning, which we don't suppress.
grep -v "^error: //$d/... - no such file(s)." "$tmpf" >> "$Log"
rm -f "$tmpf"
return $status
}
# If $SDP_RECENT_CHANGES_TO_VERIFY, use that, else default to the value set for
# $RecentChangesToVerify in this script.
[[ -n "${SDP_RECENT_CHANGES_TO_VERIFY:-}" ]] && \
RecentChangesToVerify="${SDP_RECENT_CHANGES_TO_VERIFY}"
[[ "$Log" == Unset ]] && Log=$LOGS/p4verify.log
# This is needed because everything in backup_functions.sh uses LOGFILE including the mail_log_file function.
LOGFILE=$Log
rotate_log_file "$LOGFILE" ".gz"
msg "Started $ThisScript v$Version as $USER@${HOSTNAME%%.*} on $(date +'%a %Y-%m-%d %H:%M:%S %Z') as:\\n$CmdLine"
P4="$P4BIN -p $P4PORT -u $P4USER"
"$P4CBIN/p4login"
msg "If there are errors in this log, contact support@perforce.com"
VerifyFailed=0
if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
msg "Info: Verifying only $RecentChangesToVerify changelists per depot due to -recent.\\n"
fi
RevRange=
for d in $($P4 -ztag -F %name% depots); do
depotType=$($P4 -ztag -F %Type% depot -o "$d")
case "$depotType" in
(archive)
[[ "$VerifyArchived" -eq 1 ]] || continue
if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
msg "=== Started verify of archive depot //$d/... at $(date)."
VerifyCmd="$P4 -s verify -A -q //$d/..."
cmd "$VerifyCmd" || VerifyFailed=1
else
msg "=== Started verify of archive depot on replica at $(date)."
VerifyCmd="$P4 -s verify -A -q -t //$d/..."
cmd "$VerifyCmd" || VerifyFailed=1
fi
;;
(remote)
msg "\\nInfo: Skipping verify of remote depot $d.\\n"
continue
;;
(spec)
[[ "$VerifySpecDepot" -eq 1 ]] || continue
if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
msg "=== Started verify of spec depot //$d/... at $(date)."
VerifyCmd="$P4 -s verify -q //$d/..."
cmd "$VerifyCmd" || VerifyFailed=1
else
msg "=== Started verify of spec depot //$d/... on replica at $(date)."
VerifyCmd="$P4 -s verify -q -t //$d/..."
cmd "$VerifyCmd" || VerifyFailed=1
fi
;;
(unload)
[[ "$VerifyUnload" -eq 1 ]] || continue
if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
msg "=== Started verify of unload depot at $(date)."
VerifyCmd="$P4 -s verify -U -q //$d/..."
cmd "$VerifyCmd" || VerifyFailed=1
else
msg "=== Started verify of unload depot on replica at $(date)."
VerifyCmd="$P4 -s verify -U -q -t //$d/..."
cmd "$VerifyCmd" || VerifyFailed=1
fi
;;
(*)
[[ "$depotType" != "local" && "$depotType" != "stream" ]] && \
msg "Warning: Unknown depot type [$depotType]. Attempting normal verify."
if [[ "$VerifySubmitted" -eq 1 ]]; then
if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
RevRange=$(get_verify_rev_range "$d" "$RecentChangesToVerify")
ChangesCmd="$P4BIN -ztag -F %change% changes -s submitted //$d/...$RevRange"
else
ChangesCmd="$P4BIN -ztag -F %change% changes -s submitted //$d/..."
fi
TmpFile=$(mktemp)
SubmittedCLCount=0
SubmittedCLErrorCount=0
if $ChangesCmd > "$TmpFile"; then
SubmittedChangesToVerify=$(wc -l "$TmpFile" | cut -d ' ' -f 1)
msg "Starting verify of $SubmittedChangesToVerify submmited changelists in //$d/... at $(date)."
while read -r SubmittedCL; do
SubmittedCLCount+=1
if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
VerifyCmd="$P4 -s verify -q @=$SubmittedCL"
else
VerifyCmd="$P4 -s verify -q -t @=$SubmittedCL"
fi
cmd "$VerifyCmd" || SubmittedCLErrorCount+=1
done < "$TmpFile"
else
msg "Error: Could not get or write list of submitted changes in //$d/..."
VerifyFailed=1
fi
if [[ "$SubmittedCLCount" -ne 0 ]]; then
if [[ "$SubmittedCLErrorCount" -eq 0 ]]; then
msg "\\nAll $SubmittedCLCount submitted changes in //$d/... verified OK."
else
msg "\\nOf $SubmittedCLCount submitted changes verified in //$d/..., $SubmittedCLErrorCount had errors."
VerifyFailed=1
fi
else
msg "\\nThere were no submitted changelists to verify in //$d/..."
fi
else
msg "Skipping verify of submitted changelists in depot $d due to '-nr'."
fi
;;
esac
done
if [[ "$VerifyShelved" -eq 1 ]]; then
msg "=== Started verify of shelved changelists at $(date)."
TmpFile=$(mktemp)
ChangesCmd="$P4BIN -ztag -F %change% changes -s shelved"
[[ "$VerifyOnlyRecentChanges" -eq 1 ]] && \
ChangesCmd+=" -m $RecentChangesToVerify"
ShelvedCLCount=0
ShelvedCLErrorCount=0
if $ChangesCmd > "$TmpFile"; then
ShelvedChangesToVerify=$(wc -l "$TmpFile" | cut -d ' ' -f 1)
msg "Verifying $ShelvedChangesToVerify shelved changelists."
while read -r ShelvedCL; do
ShelvedCLCount+=1
if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
VerifyCmd="$P4 -s verify -qS @=$ShelvedCL"
else
VerifyCmd="$P4 -s verify -qS -t @=$ShelvedCL"
fi
cmd "$VerifyCmd" || ShelvedCLErrorCount+=1
done < "$TmpFile"
else
msg "Error: Could not get or write list of shelved changes."
VerifyFailed=1
fi
if [[ "$ShelvedCLCount" -ne 0 ]]; then
if [[ "$ShelvedCLErrorCount" -eq 0 ]]; then
msg "\\nAll $ShelvedCLCount shelved changes in //$d/... verified OK."
else
msg "\\nOf $ShelvedCLCount shelved changes verified in //$d/..., $ShelvedCLErrorCount had errors."
VerifyFailed=1
fi
else
msg "\\nThere were no shelved changelists to verify."
fi
else
msg "Skipping verify of shelved changelists due to '-nS'."
fi
if [[ "$VerifyFailed" -ne 0 ]]; then
StatusMessage="Error: Verify attempt failed. Review the log [$Log]."
ExitCode=1
fi
if [[ $ExitCode -eq 0 ]]; then
# Note: Ignore SC2196 due to 'grep -e' and 'egrep' not having consistent
# results on all platforms.
# shellcheck disable=SC2196
if egrep -m1 '(BAD!|MISSING!|p4 help max)' "$Log" > /dev/null 2>&1; then
StatusMessage="Error: Verify errors detected. Review the log [$Log]."
ExitCode=2
fi
fi
msg "Completed verifications at $(date), taking $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"
mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
[[ "$ShowLog" -eq 1 && -s "$Log" ]] && cat "$Log"
exit $ExitCode