p4verify.sh #24

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

export SDP_ENV=/p4/common/bin/p4_vars
export SDP_INSTANCE=${SDP_INSTANCE:-Unset}

declare StatusMessage="OK: All scanned depots verified OK."
declare ThisScript=${0##*}
declare Version=5.3.3
declare CmdLine="$0 $*"
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i ShowLog=0
declare -i ExitCode=0
declare -i VerifyArchived=0
declare -i VerifyShelved=1
declare -i VerifySpecDepot=1
declare -i VerifySubmitted=1
declare -i VerifyUnload=1
declare -i ShelvedCLCount=0
declare -i ShelvedCLErrorCount=0
declare RevRange=
declare RecentChangesToVerify=200
declare VerifyCmd=
declare ChangesCmd=
declare ShelvedChangesToVerify=
declare ShelvedCL=
declare Log=Unset
declare TmpFile=

#==============================================================================
# Local Functions

# Micro-functions, one-liners used to avoid external dependencies.
function msg () { if [[ $Log != Unset ]]; then echo -e "$*" >> $Log; else echo -e "$*"; fi; }
function bail () { msg "\\nError: ${1:-Unknown Error}"; exit "${2:-1}"; }


#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
   fi

   # tag::includeManual[] - required for AsciiDoc
   echo "USAGE for $ThisScript v$Version:

p4verify.sh [<instance>] [-nu] [-nr] [-ns] [-nS] [-a] [-recent] [-L <log>] [-v] [-D]

   or

p4verify.sh -h|-man
"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:

	This script performs a 'p4 verify' of all submitted and shelved versioned
	files in depots of all types except 'remote' and 'archive' type depots.

	If run on a replica, it schedules archive failures for transfer to the
	replica.

OPTIONS:
<instance>
	Specify the SDP instance.  If not specified, the SDP_INSTANCE
	environment variable is used instead.  If the instance is not
	defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
	exists immediately with an error message.

 -nu	Specify '-nu' (No Unload) to skip verification of the singleton depot
	of type 'unload' (if created).  The 'unload' depot is verified
	by default.

 -nr	Specify '-nr' (No Regular) to skip verification of regular submitted
	archive files.  The '-nr' option is not compatible with '-recent'.
	Regular submitted archive files are verified by default.

 -ns	Specify '-ns' (No Spec Depot) to skip verification of singleton depot
	of type 'spec' (if created). The 'spec' depot is verified by default.

 -nS	Specify '-nS' (No Shelves) to skip verification of shelved archive
	files, i.e. to skip the 'p4 verify -qS'.

 -a	Specify '-a' (Archive Depots) to do verification of depots of type
	'archive'.  Depots of type 'archive' are not verified by default, as
	archive depots are often physically removed from the server's
	storage subsystem for long-term cold storage.

 -recent
	Specify that only recent changelists should be verified.
	The \$SDP_RECENT_CHANGES_TO_VERIFY variable defines how many
	changelists are considered recent; the default is $RecentChangesToVerify.

	If the default is not appropriate for your site, add
	\"export SDP_RECENT_CHANGES_TO_VERIFY\" to /p4/common/config/p4_N.vars to
	change the default for an instance, or to /p4/common/bin/p4_vars to
	change it globally.  If \$SDP_RECENT_CHANGES_TO_VERIFY is unset, the
	default is $RecentChangesToVerify.

	When -recent is used, neither shelves nor files in the unload depot
	are verified.

 -v     Verbose.  Show output of verify attempts, which is suppressed by default.
	Setting SDP_SHOW_LOG=1 in the shell environment has the same effect as -v.

	The default behavior of this script is to generate no terminal output,
	but instead to write output into a log file -- see LOGGING below.  If
	'-v' is specified, the generated log is sent to stdout at the end of
	processing.  This flag is not recommended for routine cron operation or
	for large data sets.

 -L <log>
	Specify the log file to use.  The default is /p4/N/logs/p4verify.log

	Log rotation and old log cleanup logic does not apply to log files
	specified with -L.  Thus, using -L is not recommended for routine scheduled
	operation, e.g. via crontab.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	This script is typically called via cron with only the instance
	parameter as an argument, e.g.:
	p4verify.sh N

LOGGING:
	This script generates no output by default.  All (stdout and stderr) is
	logged to /p4/N/logs/p4verify.log.

	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.

	If the '-v' flag is used, the contents of the log are displayed to
	stdout at the end of processing.

EXIT CODES:
	An exit code of 0 indicates no errors were encountered attempting to
	perform verifications, AND that all verifications attempted
	reported no problems.

	A exit status of 1 indicates that verifications could not be
	attempted for some reason.

	A exit status of 2 indicates that verifications were successfully
	performed, but that problems such as BAD or MISSING files
	were detected, or else system limits prevented verification.
"
   # end::includeManual[] - required for AsciiDoc
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function: get_verify_rev_range ($depot, $recentChanges)
#------------------------------------------------------------------------------
function get_verify_rev_range () {
   declare depot=${1:-}
   declare recentChangesToVerify=${2:-}
   declare nowChange=
   [[ -z "$depot" || -z "$recentChangesToVerify" ]] && return 0

   nowChange=$("$P4BIN" -ztag -F %change% changes -m 1 "//$depot/...")
   thenChange=$("$P4BIN" -ztag -F %change% changes -m "$recentChangesToVerify" "//$depot/..." | tail -1)
   [[ -z "$nowChange" ]] && return 1
   [[ -z "$thenChange" ]] && return 1

   echo "@$thenChange,@$nowChange"

   return 0
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case "$1" in
      (-h) usage -h;;
      (-man) usage -man;;
      (-ns) VerifySpecDepot=0;;
      (-nS) VerifyShelved=0;;
      (-nr) VerifySubmitted=0;;
      (-nu) VerifyUnload=0;;
      (-recent) VerifyOnlyRecentChanges=1;;
      (-v) ShowLog=1;;
      (-L) Log=$2; shiftArgs=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown command line option ($1).";;
      (*) export SDP_INSTANCE=$1;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification
[[ $SDP_INSTANCE == Unset ]] && \
   bail "The \\$SDP_INSTANCE setting is not defined. It must be defined by doing:\\n\\n\\tsource /p4/common/bin/p4_vars <instance>\\n\\nor by passing in the instance name as a parameter to this script.\\n"

[[ "$VerifySubmitted" -eq 0 && "$VerifyOnlyRecentChanges" -eq 1 ]] && \
   bail "The '-recent' and '-nr' options are mutually exclusive."

#==============================================================================
# Main Program

# shellcheck disable=SC1090
source "$SDP_ENV" "$SDP_INSTANCE" ||\
   bail "Failed to load SDP environment for instance $SDP_INSTANCE."

# shellcheck disable=SC1090
source "$P4CBIN/backup_functions.sh" ||\
   bail "Failed to load backup_functions.sh."

#------------------------------------------------------------------------------
# Function: cmd()
# We need to define cmd() after we have sourced backup_functions.sh.
function cmd ()
{ 
   tmpf="${P4TMP}/t_verify.$$.$RANDOM"
   msg "$*" >> "$Log"
   # Disable SC2048 for $* because we want any spaces to be expanded here.
   # shellcheck disable=SC2048
   $* > "$tmpf" 2>&1
   status=$?

   # Suppress 'no such file' errors from older p4d versions. In newer p4d
   # versions, this is a mere warning, which we don't suppress.
   grep -v "^error: //$d/... - no such file(s)." "$tmpf" >> "$Log"

   rm -f "$tmpf"
   return $status
}

# If $SDP_RECENT_CHANGES_TO_VERIFY, use that, else default to the value set for
# $RecentChangesToVerify in this script.
[[ -n "${SDP_RECENT_CHANGES_TO_VERIFY:-}" ]] && \
   RecentChangesToVerify="${SDP_RECENT_CHANGES_TO_VERIFY}"

[[ "$Log" == Unset ]] && Log=$LOGS/p4verify.log

# This is needed because everything in backup_functions.sh uses LOGFILE including the mail_log_file function.
LOGFILE=$Log

rotate_log_file "$LOGFILE" ".gz"

msg "Started $ThisScript v$Version as $USER@${HOSTNAME%%.*} on $(date +'%a %Y-%m-%d %H:%M:%S %Z') as:\\n$CmdLine"

P4="$P4BIN -p $P4PORT -u $P4USER"
"$P4CBIN/p4login"

msg "If there are errors in this log, contact support@perforce.com"

VerifyFailed=0

if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
   msg "Info: Verifying only $RecentChangesToVerify changelists per depot due to -recent.\\n"
fi

RevRange=
for d in $($P4 -ztag -F %name% depots); do
   depotType=$($P4 -ztag -F %Type% depot -o "$d")

   case "$depotType" in
      (archive)
         [[ "$VerifyArchived" -eq 1 ]] || continue

         if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
            msg "=== Started verify of archive depot //$d/... at $(date)."
            VerifyCmd="$P4 -s verify -A -q //$d/..."
            cmd "$VerifyCmd" || VerifyFailed=1
         else
            msg "=== Started verify of archive depot on replica at $(date)."
            VerifyCmd="$P4 -s verify -A -q -t //$d/..."
            cmd "$VerifyCmd" || VerifyFailed=1
         fi
      ;;
      (remote)
         msg "\\nInfo: Skipping verify of remote depot $d.\\n"
         continue
      ;;
      (spec)
         [[ "$VerifySpecDepot" -eq 1 ]] || continue

         if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
            msg "=== Started verify of spec depot //$d/... at $(date)."
            VerifyCmd="$P4 -s verify -qz //$d/..."
            cmd "$VerifyCmd" || VerifyFailed=1

         else
            msg "=== Started verify of spec depot //$d/... on replica at $(date)."
            VerifyCmd="$P4 -s verify -qz -t //$d/..."
            cmd "$VerifyCmd" || VerifyFailed=1
         fi
      ;;
      (unload)
         [[ "$VerifyUnload" -eq 1 ]] || continue

         if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
            msg "=== Started verify of unload depot at $(date)."
            VerifyCmd="$P4 -s verify -U -q //$d/..."
            cmd "$VerifyCmd" || VerifyFailed=1
         else
            msg "=== Started verify of unload depot on replica at $(date)."
            VerifyCmd="$P4 -s verify -U -q -t //$d/..."
            cmd "$VerifyCmd" || VerifyFailed=1
         fi
      ;;
      (*)
         [[ "$depotType" != "local" && "$depotType" != "stream" ]] && \
            msg "Warning: Unknown depot type [$depotType]. Attempting normal verify."

         if [[ "$VerifyOnlyRecentChanges" -eq 1 ]]; then
            RevRange=$(get_verify_rev_range "$d" "$RecentChangesToVerify")
         fi

         if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
            if [[ "$VerifySubmitted" -eq 1 ]]; then
               msg "=== Started verify of submitted files in //$d/... at $(date)."
               VerifyCmd="$P4 -s verify -qz //$d/...$RevRange"
               cmd "$VerifyCmd" || VerifyFailed=1
            fi

         else
            if [[ "$VerifySubmitted" -eq 1 ]]; then
               msg "=== Started verify of submitted files on replica in //$d/... at $(date)."
               VerifyCmd="$P4 -s verify -qz -t //$d/...$RevRange"
               cmd "$VerifyCmd" || VerifyFailed=1
            fi
         fi
      ;;
   esac
done

if [[ "$VerifyShelved" -eq 1 ]]; then
   msg "=== Started verify of shelved changelists at $(date)."
   TmpFile=$(mktemp)
   ChangesCmd="$P4BIN -ztag -F %change% changes -s shelved"
   [[ "$VerifyOnlyRecentChanges" -eq 1 ]] && \
      ChangesCmd+=" -m $RecentChangesToVerify"

   ShelvedCLCount=0
   ShelvedCLErrorCount=0

   if $ChangesCmd > "$TmpFile"; then
      ShelvedChangesToVerify=$(wc -l "$TmpFile" | cut -d ' ' -f 1)
      msg "Verifying $ShelvedChangesToVerify shelved changelists."
      while read -r ShelvedCL; do
         ShelvedCLCount+=1
         if [[ "${P4REPLICA}" == "FALSE" || "${SHAREDDATA}" == "TRUE" ]]; then
            VerifyCmd="$P4 -s verify -qS @=$ShelvedCL"
         else
            VerifyCmd="$P4 -s verify -qS -t @=$ShelvedCL"
         fi
         cmd "$VerifyCmd" || ShelvedCLErrorCount+=1
      done < "$TmpFile"
   else
      msg "Error: Could not get or write list of shelved changes."
      VerifyFailed=1
   fi

   if [[ "$ShelvedCLCount" -ne 0 ]]; then
      if [[ "$ShelvedCLErrorCount" -eq 0 ]]; then
         msg "\\nAll $ShelvedCLCount shelved changes verified OK."
      else
         msg "\\nOf $ShelvedCLCount shelved changes verified, $ShelvedCLErrorCount had errors."
         VerifyFailed=1
      fi
   else
      msg "\\nThere were no shelved changelists to verify."
   fi
else
   msg "Skipping verify of shelved changelists due to '-nS'."
fi

if [[ "$VerifyFailed" -ne 0 ]]; then
   StatusMessage="Error: Verify attempt failed.  Review the log [$Log]."
   ExitCode=1
fi

if [[ $ExitCode -eq 0 ]]; then
   # Note: Ignore SC2196 due to 'grep -e' and 'egrep' not having consistent
   # results on all platforms.
   # shellcheck disable=SC2196
   if egrep -m1 '(BAD!|MISSING!|p4 help max)' "$Log" > /dev/null 2>&1; then
      StatusMessage="Error: Verify errors detected.  Review the log [$Log]."
      ExitCode=2
   fi
fi
 
msg "Completed verifications at $(date), taking $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"

[[ "$ShowLog" -eq 1 && -s "$Log" ]] && cat "$Log"
 
exit $ExitCode