p4verify.sh #5

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

export SDP_ENV=/p4/common/bin/p4_vars
export SDP_INSTANCE=${SDP_INSTANCE:-Unset}

export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ $SDP_INSTANCE == Undefined ]]; then
   echo "Instance parameter not supplied."
   echo "You must supply the Perforce instance as a parameter to this script."
   exit 1
fi

declare RecentChangesToVerify=${SDP_RECENT_CHANGES_TO_VERIFY:-2500}

declare StatusMessage="OK: All scanned depots verified OK."
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i ShowLog=0
declare -i ExitCode=0
declare RevRange=
declare VerifyCmd=
declare Log=Unset
declare Version=6.0.0
declare Threads=1

#==============================================================================
# Local Functions

# Micro-functions, one-liners used to avoid external dependencies.
function msg () { if [[ $Log != Unset ]]; then echo -e "$*" >> $Log; else echo -e "$*"; fi; }
function bail () { msg "\nError: ${1:-Unknown Error}"; exit ${2:-1}; }

function cmd ()
{ 
  echo === Running $* on $(date). >> $cmd_log 
  $* >> $cmd_log 2>&1
  status=$?
  if [ $status -ne 0 ]; then
    VerifyFailed = 1
  fi
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\n\nUsage Error:\n\n$errorMessage\n\n" >&2
   fi

   echo "USAGE for p4verify.sh v$Version:

p4verify.sh [<instance>] [-recent]

   or

p4verify.sh -h|-man
"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:

	This script performs a 'p4 verify' of all submitted and shelved versioned
	files in depots of all types except 'remote' and 'archive' type depots.

	If run on a replica, it schedules archive failures for transfer to the
	replica.

OPTIONS:
<instance>
	Specify the SDP instances.  If not specified, the SDP_INSTANCE
	environment variable is used instead.  If the instance is not
	defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
	exists immediately with an error message.

 -recent
	Specify that only recent changelists should be verified.
	The \$SDP_RECENT_CHANGES_TO_VERIFY variable defines how many
	changelists are considered recent; the default is $RecentChangesToVerify.

	If the default is not appropriate for your site, add
	\"export SDP_RECENT_CHANGES_TO_VERIFY\" to /p4/common/config/p4_N.vars to
	change the default for an instance, or to /p4/common/bin/p4_vars to
	change it globally.  If unset, the default is $RecentChangesToVerify.

 -v     Show ouptput of verify attempts, which is suppressed by default.
	Setting SDP_SHOW_LOG=1 in the shell environment has the same
	effect as -v.

 -L <log>
	Specify the log file to use.  The default is /p4/N/logs/p4verify.log

 -D     Set extreme debugging verbosity.

-P #
	Specify the number of depots to verify in parallel.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	This script is typically called via cron with only the instance
	paramter as an argument, e.g.:
	p4verify.sh N

LOGGING:
	This script generates no output by default.  All (stdout and stderr) is
	logged to /p4/N/logs/p4verify.log.

	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.

	If the '-v' flag is used, the contents of the log are displayed to
	stdout at the end of processing.

EXIT CODES:
	An exit code of 0 indicates no errors were encounted attempting to
	perform verifications, AND that all verifications attempted
	reported no problems.

	A exit status of 1 indicates that verifications could not be
	attempted for some reason.

	A exit status of 2 indicates that verifications were successfully
	performed, but that problems such as BAD or MISSING files
	were detected, or else system limits prevented verification.
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function: get_verify_rev_range ($depot, $recentChanges)
#------------------------------------------------------------------------------
function get_verify_rev_range () {
   declare depot=${1:-}
   declare recentChangesToVerify=${2:-}
   declare nowChange=
   [[ -z "$depot" || -z "$recentChangesToVerify" ]] && return 0

   nowChange=$($P4 -ztag -F %change% changes -m 1 //$depot/...)
   thenChange=$($P4 -ztag -F %change% changes -m $recentChangesToVerify //$depot/... | tail -1)
   [[ -z "$nowChange" ]] && return 1
   [[ -z "$thenChange" ]] && return 1

   echo "@$thenChange,@$nowChange"

   return 0
}

#==============================================================================
# Command Line Processing
declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-recent) VerifyOnlyRecentChanges=1;;
      (-v) ShowLog=1;;
      (-L) Log=$2; shiftArgs=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-P) Threads=$2; shiftArgs=1;;
      (-*) usage -h "Unknown command line option ($1).";;
      (*) export SDP_INSTANCE=$1;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SDP_INSTANCE == Unset ]] && \
   bail "The \$SDP_INSTANCE setting is not defined. It must be defined by doing:\n\n\tsource /p4/common/bin/p4_vars <instance>\n\nor by passing in the instance name as a parameter to this script.\n"

#==============================================================================
# Main Program

source $SDP_ENV $SDP_INSTANCE ||\
   bail "Failed to load SDP environment for instance $SDP_INSTANCE."

source $P4CBIN/backup_functions.sh ||\
   bail "Failed to load backup_functions.sh."

check_vars
set_vars
get_journalnum

[[ $Log == Unset ]] && Log=$LOGS/p4verify.log

# This is needed because everything in backup_functions.sh uses LOGFILE including the mail_log_file function.
LOGFILE=$Log
LOG=$LOGS/p4verify

rotate_log_file $LOGFILE ".gz"

msg "${0##*/} v$Version Starting verify at $(date +'%a %Y-%m-%d %H:%M:%S %Z')."

P4="$P4BIN -p $P4PORT -u $P4USER"
$P4CBIN/p4login

msg "If there are errors in this log, contact your local support team."

VerifyFailed=0

RevRange=

depots=() # Clear array
thread=0  # Set current threads to 0


# Build array depots of all of the depot names
for d in `$P4 -s depots|grep "^info: Depot " | cut -d" " -f3,5 | egrep -v 'archive$|remote$' | cut -d" " -f1 ` ; do
   depots+=( "$d" ) # Append depot to the array
done

# loop depots running the number of them in parraell that is specified by the command line
for d in "${depots[@]}"; do
  # Loop to see if we are over our thread count.  If so wait until we drop below it again
  while [ $thread -ge $Threads ]; do
    sleep 5
    #loop thread process id's and see if any have finished.
    spot=0
    for p in "${ids[@]}"; do
      running=$(ps cax | grep $p)
      if [ -n "$running" ]; then
        run[$spot]=$p
        spot=$((spot+1))
        ids=("${run[@]}")
      else
        thread=$((thread-1))
      fi
    done
  done

  # check the depot type as behaviour / flags differ
  depot_type=`$P4 depot -o $d | sed -n "s/^Type:[     ]*\(.*\)/\1/p"`

  if [[ $depot_type != "unload" ]]; then
    if [[ $VerifyOnlyRecentChanges -eq 1 ]]; then
      RevRange=$(get_verify_rev_range "$d" "$RecentChangesToVerify")
    fi

    if [[ "${P4REPLICA}" == "FALSE" || ${SHAREDDATA} == "TRUE" ]]; then
      VerifyCmd="$P4 -s verify -qz //$d/...$RevRange"
      cmd_log=$LOG-$d.log
      cmd $VerifyCmd & 
      ids[$thread]=$! # add the process ID into the array of running processes
      thread=$((thread+1))  #  add one to the thread count and start a new verify

      VerifyCmd="$P4 -s verify -qS //$d/..."        # cannot specify range with -S
      cmd_log=$LOG-$d-S.log
      cmd $VerifyCmd &
      ids[$thread]=$! # add the process ID into the array of running processes
      thread=$((thread+1))  #  add one to the thread count and start a new verify
    else
      VerifyCmd="$P4 -s verify -qz -t //$d/...$RevRange"
      cmd_log=$LOG-$d.log
      cmd $VerifyCmd &
      ids[$thread]=$! # add the process ID into the array of running processes
      thread=$((thread+1))  #  add one to the thread count and start a new verify

      VerifyCmd="$P4 -s verify -qS -t //$d/..."    # cannot specify range with -S
      cmd_log=$LOG-$d-S.log
      cmd $VerifyCmd &
      ids[$thread]=$! # add the process ID into the array of running processes
      thread=$((thread+1))  #  add one to the thread count and start a new verify
    fi
  else     # unload depot;  revision range not applicable
    if [[ "${P4REPLICA}" == "FALSE" || ${SHAREDDATA} == "TRUE" ]]; then
      VerifyCmd="$P4 -s verify -U -q //$d/..."
      cmd_log=$LOG-$d.log
      cmd $VerifyCmd &
      ids[$thread]=$! # add the process ID into the array of running processes
      thread=$((thread+1))  #  add one to the thread count and start a new verify
    else
      VerifyCmd="$P4 -s verify -U -q -t //$d/..."
      cmd_log=$LOG-$d.log
      cmd $VerifyCmd &
      ids[$thread]=$! # add the process ID into the array of running processes
      thread=$((thread+1))  #  add one to the thread count and start a new verify
    fi
  fi
done

# now that we have started all of them wait until all of our processes have finished before continuing.
while [ $thread -gt 0 ]; do
  sleep 5
  #loop thread process id's and see if any have finished.
  spot=0
  for p in "${ids[@]}"; do
  running=$(ps cax | grep $p)
  if [ -n "$running" ]; then
    run[$spot]=$p
    spot=$((spot+1))
  else
    thread=$((thread-1))
  fi
  ids=("${run[@]}")
  done
done

# now that the processes have finished combine all of the log file together
for d in "${depots[@]}"; do
   cat $LOG-$d.log >> "$Log"
   if [ -f $LOG-$d-S.log ]; then
     cat $LOG-$d-S.log >> "$Log"
     rm -f $LOG-$d-S.log
   fi
   rm -f $LOG-$d.log
done

if [[ $VerifyFailed -ne 0 ]]; then
   StatusMessage="Error: Verify attempt failed.  Review the log [$Log]."
   ExitCode=1
fi

if [[ $ExitCode -eq 0 ]]; then
   egrep '(BAD!|MISSING!|p4 help max)' $Log > /dev/null 2>&1
   if [[ $? -eq 0 ]]; then
      StatusMessage="Warning: Verify errors detected.  Review the log [$Log]."
      ExitCode=2
   fi
fi
 
msg "Completed verifications at $(date)."

if [[ $ExitCode -ne 0 ]]; then
   mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
fi

[[ $ShowLog -eq 1 && -s $Log ]] && cat $Log

exit $ExitCode