p4verify.sh #26

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

export SDP_ENV=/p4/common/bin/p4_vars
export SDP_INSTANCE=${SDP_INSTANCE:-Unset}

declare StatusMessage="OK: All scanned depots verified OK."
declare ThisScript=${0##*}
declare ThisUser=
declare Version=5.6.2
declare CmdLine="$0 $*"
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i ShowLog=0
declare -i ExitCode=0
declare -i VerifyArchived=0
declare -i VerifyChunks=0
declare -i VerifyShelved=1
declare -i VerifySpecDepot=1
declare -i VerifySubmitted=1
declare -i VerifyUnload=1
declare -i ScheduleTransfers=1
declare -i ShelvedCLCount=0
declare -i ShelvedCLErrorCount=0
declare -i DepotCount=0
declare -a DepotList
declare DepotListFile=
declare RevRange=
declare RecentChangesToVerify=200
declare VerifyCmd=
declare ChangesCmd=
declare ShelvedChangesToVerify=
declare ShelvedCL=
declare VerifyOnlyOption=
declare VerifyOptions=
declare ChunkFile=
declare ChunkPath=
declare ChunkSize=
declare ChunkScript="/p4/common/bin/depot_verify_chunks.py"
declare Log=Unset
declare TmpFile=
declare TmpLog=

#==============================================================================
# Local Functions

# Micro-functions, one-liners used to avoid external dependencies.
function msg () { if [[ $Log != Unset ]]; then echo -e "$*" >> $Log; else echo -e "$*"; fi; }
function bail () { msg "\\nError: ${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
   fi

   echo "USAGE for $ThisScript v$Version:

p4verify.sh [<instance>] [-nu] [-nr] [-ns] [-nS] [-a] [-nt] [-o BAD|MISSING] [-chunks <ChunkSize>] [-recent] [-dlf <depot_list_file>] [-L <log>] [-v] [-D]

   or

p4verify.sh -h|-man
"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:

	This script performs a 'p4 verify' of all submitted and shelved versioned
	files in depots of all types except 'remote' and 'archive' type depots.

	If run on a replica, it schedules archive failures for transfer to the
	replica.

OPTIONS:
<instance>
	Specify the SDP instance.  If not specified, the SDP_INSTANCE
	environment variable is used instead.  If the instance is not
	defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
	exists immediately with an error message.

 -nu	Specify '-nu' (No Unload) to skip verification of the singleton depot
	of type 'unload' (if created).  The 'unload' depot is verified
	by default.

 -nr	Specify '-nr' (No Regular) to skip verification of regular submitted
	archive files.  The '-nr' option is not compatible with '-recent'.
	Regular submitted archive files are verified by default.

 -ns	Specify '-ns' (No Spec Depot) to skip verification of singleton depot
	of type 'spec' (if created). The 'spec' depot is verified by default.

 -nS	Specify '-nS' (No Shelves) to skip verification of shelved archive
	files, i.e. to skip the 'p4 verify -qS'.

 -a	Specify '-a' (Archive Depots) to do verification of depots of type
	'archive'.  Depots of type 'archive' are not verified by default, as
	archive depots are often physically removed from the server's
	storage subsystem for long-term cold storage.
 
-nt	Specify the '-nt' option to avoid passing the '-t' flag to 'p4 verify' 
	on a replica. By default, p4verify.sh detects if it is running on a
	replica, and if so automatically applies the '-t' flag to 'p4 verify'.
	That causes the replica to attempt to self-heal, as files that fail
	verification are scheduled for transfer from the P4TARGET server. This
	default behavior results in 'Transfer scheduled' messages in the log,
	and MISSING/BAD files are listed as 'info:' rather than 'error:'. There
	is no clear indication of whether or which of the scheduled transfers
	complete successfully, and so there may be a mix of transient/correctable
	and \"real\"/persistent transfer errors for files that are also BAD/MISSING
	on the master server.  Specify '-nt' to ensure the log contains a list
	of files that currently fail a 'p4 verify' without attempting to transfer
	them from the master.

 -o BAD|MISSING
	Specify '-o MISSING' to check only whether expected archive files exist
	or not, skipping the checksum calculation of existing files.  This results
	in dramatically faster, if less comprehensive, verification.  This
	is particularly well suited when verification is being used to schedule
	archive file transfers of missing files on replicas.  This translates into
	passing the '--only MISSING' option to 'p4 verify'.

	Specify '-o BAD' to check only for BAD revisions.  This translates into
	passing the '--only BAD' option to 'p4 verify'.

	This option requires p4d to be 2021.1 or newer. For older p4d versions,
	this option is silently ignored.

 -chunks <ChunkSize>
	Specify the maximum amount of content by size to verify at once.  If
	this is specified, the depot_verify_chunks.py script is used to
	break up depots into chunks of a given size, e.g. 100M or 4G.

	The <ChunkSize> parameter must be a size value valid to pass to the
	depot_verify_chunks.py script with the '-m' option.

	This requires the perforce-p4python3 module to be installed and the
	python3 in the PATH must be the correct one that uses the P4 module.

	The -recent and -chunks options are mutually exclusive.

 -recent
	Specify that only recent changelists should be verified.
	The \$SDP_RECENT_CHANGES_TO_VERIFY variable defines how many
	changelists are considered recent; the default is $RecentChangesToVerify.

	If the default is not appropriate for your site, add
	\"export SDP_RECENT_CHANGES_TO_VERIFY\" to /p4/common/config/p4_N.vars to
	change the default for an instance, or to /p4/common/bin/p4_vars to
	change it globally.  If \$SDP_RECENT_CHANGES_TO_VERIFY is unset, the
	default is $RecentChangesToVerify.

	When -recent is used, neither shelves nor files in the unload depot
	are verified.

 -dlf <depot_list_file>
	Specify a file containing a list of depots to process in the desired
	order.  By default, all depots reported by 'p4 depots', which
	effectively results in depots processed in alphabetical order.

	This can be useful in time-sensitive situations where the order
	of processing can be prioritized, and/or to prevent processing
       	certain depots.

	The format fo the depot list file is straighforward, one line per
	depot, without the '//' nor trailling /..., so a list might look
	like this sample:

	ProjA
	ProjB
	spec
	.swarm
	unload
	archive
	ProjC

	Blank lines and lines starting with a '#' are treated as comments and
	ignored.

	WARNING: This is not intended to be the primary method of verification,
	because it would be easy to forget to add new depots to the list file.

	If the depot list file is not readable, processing aborts.

 -v     Verbose.  Show output of verify attempts, which is suppressed by default.
	Setting SDP_SHOW_LOG=1 in the shell environment has the same effect as -v.

	The default behavior of this script is to generate no terminal output,
	but instead to write output into a log file -- see LOGGING below.  If
	'-v' is specified, the generated log is sent to stdout at the end of
	processing.  This flag is not recommended for routine cron operation or
	for large data sets.

	The -chunks and -recent options are mutually exclusive.

 -L <log>
	Specify the log file to use.  The default is /p4/N/logs/p4verify.log

	Log rotation and old log cleanup logic does not apply to log files
	specified with -L.  Thus, using -L is not recommended for routine scheduled
	operation, e.g. via crontab.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	This script is typically called via cron with only the instance
	parameter as an argument, e.g.:
	p4verify.sh N

LOGGING:
	This script generates no output by default.  All (stdout and stderr) is
	logged to /p4/N/logs/p4verify.log.

	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.

	If the '-v' flag is used, the contents of the log are displayed to
	stdout at the end of processing.

EXIT CODES:
	An exit code of 0 indicates no errors were encountered attempting to
	perform verifications, AND that all verifications attempted
	reported no problems.

	A exit status of 1 indicates that verifications could not be
	attempted for some reason.

	A exit status of 2 indicates that verifications were successfully
	performed, but that problems such as BAD or MISSING files
	were detected, or else system limits prevented verification.
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function: get_verify_rev_range ($depot, $recentChanges)
#------------------------------------------------------------------------------
function get_verify_rev_range () {
   declare depot=${1:-}
   declare recentChangesToVerify=${2:-}
   declare nowChange=
   [[ -z "$depot" || -z "$recentChangesToVerify" ]] && return 0

   nowChange=$("$P4BIN" -ztag -F %change% changes -m 1 "//$depot/...")
   thenChange=$("$P4BIN" -ztag -F %change% changes -m "$recentChangesToVerify" "//$depot/..." | tail -1)
   [[ -z "$nowChange" ]] && return 1
   [[ -z "$thenChange" ]] && return 1

   echo "@$thenChange,@$nowChange"

   return 0
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case "$1" in
      (-h) usage -h;;
      (-man) usage -man;;
      (-ns) VerifySpecDepot=0;;
      (-nS) VerifyShelved=0;;
      (-nr) VerifySubmitted=0;;
      (-nu) VerifyUnload=0;;
      (-a) VerifyArchived=1;;
      (-nt) ScheduleTransfers=0;;
      (-o) VerifyOnlyOption="--only ${2^^}"; shiftArgs=1;;
      (-chunks) VerifyChunks=1; ChunkSize="$2"; shiftArgs=1;;
      (-recent) VerifyOnlyRecentChanges=1;;
      (-dlf) DepotListFile="${2:-}"; shiftArgs=1;;
      (-v) ShowLog=1;;
      (-L) Log=$2; shiftArgs=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown command line option ($1).";;
      (*) export SDP_INSTANCE=$1;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification
[[ $SDP_INSTANCE == Unset ]] && \
   usage -h "The \\$SDP_INSTANCE setting is not defined. It must be defined by doing:\\n\\n\\tsource /p4/common/bin/p4_vars <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n"

[[ "$VerifySubmitted" -eq 0 && "$VerifyOnlyRecentChanges" -eq 1 ]] && \
   usage -h "The '-recent' and '-nr' options are mutually exclusive."

if [[ -n "$VerifyOnlyOption" ]]; then
   # shellcheck disable=SC2072
   if [[ "$P4D_VERSION" > "2021.1" ]]; then
      if [[ ! "$VerifyOnlyOption" =~ ^--only\ (MISSING|BAD)$ ]]; then
         usage -h "The option specified with '-o' must be BAD or MISSING. Specified value was: $VerifyOnlyOption"
      fi
   else
      VerifyOnlyOption=
   fi
fi

[[ "$VerifyOnlyRecentChanges" -eq 1 && "$VerifyChunks" -eq 1 ]] && \
   usage -h "The -recent and -chunks options are mutually exclusive."

if [[ -n "$ChunkSize" && ! -x "$ChunkScript" ]]; then
   msg "Warning: -chunks ignored because chunking script is not executable by user $ThisUser: $ChunkScript"
   VerifyChunks=0
fi

if [[ -n "$DepotListFile" && ! -r "$DepotListFile" ]]; then
   usage -h "The file specified with '-dlf <depot_list-file>' does not exist: $DepotListFile"
fi

#==============================================================================
# Main Program

# shellcheck disable=SC1090
source "$SDP_ENV" "$SDP_INSTANCE" ||\
   bail "Failed to load SDP environment for instance $SDP_INSTANCE."

# shellcheck disable=SC1090 disable=SC1091
source "$P4CBIN/backup_functions.sh" ||\
   bail "Failed to load backup_functions.sh."

#------------------------------------------------------------------------------
# Function: cmd()
# We need to define cmd() after we have sourced backup_functions.sh.
function cmd ()
{ 
   tmpf="${P4TMP}/t_verify.$$.$RANDOM"
   msg "$*" >> "$Log"
   # Disable SC2048 for $* because we want any spaces to be expanded here.
   # shellcheck disable=SC2048
   eval "$*" > "$tmpf" 2>&1
   status=$?

   # Suppress 'no such file' errors from older p4d versions. In newer p4d
   # versions, this is a mere warning, which we don't suppress.
   grep -v "^error: //$d/... - no such file(s)." "$tmpf" >> "$Log"

   rm -f "$tmpf"
   return $status
}

# If $SDP_RECENT_CHANGES_TO_VERIFY, use that, else default to the value set for
# $RecentChangesToVerify in this script.
[[ -n "${SDP_RECENT_CHANGES_TO_VERIFY:-}" ]] && \
   RecentChangesToVerify="${SDP_RECENT_CHANGES_TO_VERIFY}"

[[ "$Log" == Unset ]] && Log=$LOGS/p4verify.log

# This is needed because everything in backup_functions.sh uses LOGFILE including the mail_log_file function.
LOGFILE=$Log

rotate_log_file "$LOGFILE" ".gz"

ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@${HOSTNAME%%.*} on $(date +'%a %Y-%m-%d %H:%M:%S %Z') as:\\n$CmdLine"

P4="$P4BIN -p $P4PORT -u $P4USER"
"$P4CBIN/p4login"

msg "If there are errors in this log, contact support-helix-core@perforce.com"

VerifyFailed=0

if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
   msg "Info: Verifying only $RecentChangesToVerify changelists per depot due to -recent.\\n"
fi

RevRange=

# Determine the list of depots to process, either from 'p4 depots' or from a
# specified depot list file.
if [[ -n "$DepotListFile" ]]; then
   TmpFile=$(mktemp)
   grep -Ev '^\s*$|^\s*\#' "$DepotListFile" > "$TmpFile"
   if [[ -s "$TmpFile" ]]; then
      while read -r d; do
         DepotList[$DepotCount]="$d"
         DepotCount+=1
      done < "$TmpFile"
   else
      bail "No depots to process from the depot list file $DepotListFile (trimmed in $TmpFile)."
   fi
else
   for d in $($P4 -ztag -F %name% depots); do
      DepotList[$DepotCount]="$d"
      DepotCount+=1
   done
fi

msg "Processing ${#DepotList[@]} depots."
for d in "${DepotList[@]}"; do
   depotType=$($P4 -ztag -F %Type% depot -o "$d")

   case "$depotType" in
      (archive)
         [[ "$VerifyArchived" -eq 1 ]] || continue

         # For archive depots, use -A and -q but not -z.  
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption -A"
         else
            VerifyOptions="-q -A"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         msg "=== Started verify of archive depot //$d/... at $(date)."
         VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
         cmd "$VerifyCmd" || VerifyFailed=1
      ;;
      (remote)
         msg "\\nInfo: Skipping verify of remote depot $d.\\n"
         continue
      ;;
      (spec)
         [[ "$VerifySpecDepot" -eq 1 ]] || continue

         # For the spec depot, there are no lazy copies, so no need for -z.
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption"
         else
            VerifyOptions="-q"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         msg "=== Started verify of spec depot //$d/... at $(date)."
         VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
         cmd "$VerifyCmd" || VerifyFailed=1
      ;;
      (unload)
         [[ "$VerifyUnload" -eq 1 ]] || continue

         # For the unload depot, there are no lazy copies, so no need for -z.
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption -U"
         else
            VerifyOptions="-q -U"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         msg "=== Started verify of unload depot at $(date)."
         VerifyCmd="$P4 -s verify $VerifyOptions //$d/..."
         cmd "$VerifyCmd" || VerifyFailed=1
      ;;
      (*)
         [[ "$depotType" != "local" && "$depotType" != "stream" ]] && \
            msg "Warning: Unknown depot type [$depotType]. Attempting normal verify."

         if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
            RevRange=$(get_verify_rev_range "$d" "$RecentChangesToVerify")
         fi

         ChunkFile="$P4TMP/depot_chunks.$d.txt"
         rm -f "$ChunkFile"

         if [[ "$VerifyChunks" -eq 1 ]]; then
            msg "Running: $ChunkScript -v ERROR -m $ChunkSize //$d/..."

            if ! "$ChunkScript" -v ERROR -m "$ChunkSize" "//$d/..." > "$ChunkFile"; then
               msg "Warning: Not chunking //$d/... due to script error."
               rm -f "$ChunkFile"
            fi

            if [[ ! -s "$ChunkFile" ]]; then
               msg "Warning: Not chunking //$d/... due to zero-size chunk file."
               rm -f "$ChunkFile"
            fi
         else
            rm -f "$ChunkFile"
         fi

         # For stream/local depots, use -z unless --only is used.
         if [[ -n "$VerifyOnlyOption" ]]; then
            VerifyOptions="-q $VerifyOnlyOption"
         else
            VerifyOptions="-qz"
         fi

         [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

         if [[ "$VerifySubmitted" -eq 1 ]]; then
            if [[ "$VerifyChunks" -eq 1 && -s "$ChunkFile" ]]; then
               msg "=== Started verify of submitted files in //$d/... in $ChunkSize chunks at $(date)."

               while read -r ChunkPath; do
                  VerifyCmd="$P4 -s verify $VerifyOptions \"$ChunkPath\""
                  cmd "$VerifyCmd" || VerifyFailed=1
               done < "$ChunkFile"
            else
               msg "=== Started verify of submitted files in //$d/... at $(date)."
               VerifyCmd="$P4 -s verify $VerifyOptions //$d/...$RevRange"
               cmd "$VerifyCmd" || VerifyFailed=1
            fi
         fi
      ;;
   esac
done

if [[ "$VerifyShelved" -eq 1 ]]; then
   msg "=== Started verify of shelved changelists at $(date)."
   TmpFile=$(mktemp)
   TmpLog=$(mktemp)
   ChangesCmd="$P4BIN -ztag -F %change% changes -s shelved"
   [[ "$VerifyOnlyRecentChanges" -eq 1 ]] && \
      ChangesCmd+=" -m $RecentChangesToVerify"

   ShelvedCLCount=0
   ShelvedCLErrorCount=0

   # For shelves, don't use -z since  we verify change-by-change.
   if [[ -n "$VerifyOnlyOption" ]]; then
      VerifyOptions="-q $VerifyOnlyOption -S"
   else
      VerifyOptions="-q -S"
   fi

   [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" || "$ScheduleTransfers" -eq 0 ]] || VerifyOptions+=" -t"

   if $ChangesCmd > "$TmpFile"; then
      ShelvedChangesToVerify=$(wc -l "$TmpFile" | cut -d ' ' -f 1)
      msg "Verifying $ShelvedChangesToVerify shelved changelists."
      while read -r ShelvedCL; do
         ShelvedCLCount+=1
         VerifyCmd="$P4 -s verify $VerifyOptions @=$ShelvedCL"

         # Capture temp log for each CL verify command. Display logs with error; discard others.
         if eval "$VerifyCmd" > "$TmpLog" 2>&1; then
            rm -f "$TmpLog"
         else
            ShelvedCLErrorCount+=1
            cat "$TmpLog" >> "$Log"
            rm -f "$TmpLog"
         fi
      done < "$TmpFile"
   else
      msg "Error: Could not get or write list of shelved changes."
      VerifyFailed=1
   fi

   if [[ "$ShelvedCLCount" -ne 0 ]]; then
      if [[ "$ShelvedCLErrorCount" -eq 0 ]]; then
         msg "\\nAll $ShelvedCLCount shelved changes verified OK."
      else
         msg "\\nOf $ShelvedCLCount shelved changes verified, $ShelvedCLErrorCount had errors."
         VerifyFailed=1
      fi
   else
      msg "\\nThere were no shelved changelists to verify."
   fi
else
   msg "Skipping verify of shelved changelists due to '-nS'."
fi

if [[ "$VerifyFailed" -ne 0 ]]; then
   StatusMessage="Error: Verify attempt failed.  Review the log [$Log]."
   ExitCode=1
fi

if [[ $ExitCode -eq 0 ]]; then
   # Note: Ignore SC2196 due to 'grep -e' and 'egrep' not having consistent
   # results on all platforms.
   # shellcheck disable=SC2196
   if egrep -m1 '(BAD!|MISSING!|p4 help max)' "$Log" > /dev/null 2>&1; then
      StatusMessage="Error: Verify errors detected.  Review the log [$Log]."
      ExitCode=2
   fi
fi
 
msg "Completed verifications at $(date), taking $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"

[[ "$ShowLog" -eq 1 && -s "$Log" ]] && cat "$Log"
 
exit $ExitCode