p4verify.sh #26

#!/bin/bash
# Declarations and Environment

export SDP_ENV=/p4/common/bin/p4_vars
export SDP_INSTANCE=${SDP_INSTANCE:-Unset}

export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ $SDP_INSTANCE == Undefined ]]; then
  echo "Instance parameter not supplied."
  echo "You must supply the Perforce instance as a parameter to this script."
  exit 1
fi

declare StatusMessage="OK: All scanned depots verified OK."
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i ExitCode=0
declare VerifyCmd=
declare Log=Unset
declare Version=6.0.0
declare Threads=1

#==============================================================================
# Local Functions

# Micro-functions, one-liners used to avoid external dependencies.
function msg () { if [[ $Log != Unset ]]; then echo -e "$*" >> $Log; else echo -e "$*"; fi; }
function bail () { msg "\nError: ${1:-Unknown Error}"; exit ${2:-1}; }

function cmd ()
{
  echo === Running $* on $(date). >> $cmd_log
  $* >> $cmd_log 2>&1
  status=$?
  if [ $status -ne 0 ]; then
    VerifyFailed=1
  fi
  echo === $* completed on $(date). >> $cmd_log
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
  declare style=${1:--h}
  declare errorMessage=${2:-Unset}

  if [[ $errorMessage != Unset ]]; then
    echo -e "\n\nUsage Error:\n\n$errorMessage\n\n" >&2
  fi

  echo "USAGE for p4verify.sh v$Version:

p4verify.sh [<instance>] [-P num_threads]

  or

p4verify.sh -h|-man
"
  if [[ $style == -man ]]; then
    echo -e "DESCRIPTION:

        This script performs a 'p4 verify' of all submitted and shelved versioned
        files in depots of all types except 'remote' and 'archive' type depots.

        If run on a replica, it schedules archive failures for transfer to the
        replica.

OPTIONS:
<instance>
        Specify the SDP instances.  If not specified, the SDP_INSTANCE
        environment variable is used instead.  If the instance is not
        defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
        exists immediately with an error message.

 -D     Set extreme debugging verbosity.

-P #
        Specify the number of depots to verify in parallel.

HELP OPTIONS:
 -h     Display short help message
 -man   Display man-style help message

EXAMPLES:
        This script is typically called via cron with only the instance
        paramter as an argument, e.g.:
        p4verify.sh N

LOGGING:
        This script generates no output by default.  All (stdout and stderr) is
        logged to /p4/N/logs/p4verify.log.

        The exception is usage errors, which result an error being sent to
        stderr followed usage info on stdout, followed by an immediate exit.

        If the '-v' flag is used, the contents of the log are displayed to
        stdout at the end of processing.

EXIT CODES:
        An exit code of 0 indicates no errors were encounted attempting to
        perform verifications, AND that all verifications attempted
        reported no problems.

        A exit status of 1 indicates that verifications could not be
        attempted for some reason.

        A exit status of 2 indicates that verifications were successfully
        performed, but that problems such as BAD or MISSING files
        were detected, or else system limits prevented verification.
"
  fi

  exit 1
}

#==============================================================================
# Command Line Processing
declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
  case $1 in
    (-h) usage -h;;
    (-man) usage -man;;
    (-D) set -x;; # Debug; use 'set -x' mode.
    (-P) Threads=$2; shiftArgs=1;;
    (-*) usage -h "Unknown command line option ($1).";;
    (*) export SDP_INSTANCE=$1;;
  esac

  # Shift (modify $#) the appropriate number of times.
  shift; while [[ $shiftArgs -gt 0 ]]; do
    [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
    shiftArgs=$shiftArgs-1
    shift
  done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SDP_INSTANCE == Unset ]] && \
  bail "The \$SDP_INSTANCE setting is not defined. It must be defined by doing:\n\n\tsource /p4/common/bin/p4_vars <instance>\n\nor by passing in the instance name as a parameter to this script.\n"

#==============================================================================
# Main Program

source $SDP_ENV $SDP_INSTANCE ||\
  bail "Failed to load SDP environment for instance $SDP_INSTANCE."

source $P4CBIN/backup_functions.sh ||\
  bail "Failed to load backup_functions.sh."

[[ $Log == Unset ]] && Log=$LOGS/p4verify.log

# This is needed because everything in backup_functions.sh uses LOGFILE including the mail_log_file function.
LOGFILE=$Log
LOG=$LOGS/p4verify

check_vars
set_vars
get_journalnum

# Clean up old logs
rm -f ${LOG}-*
rotate_log_file $LOGFILE ".gz"

msg "${0##*/} v$Version Starting verify at $(date +'%a %Y-%m-%d %H:%M:%S %Z')."

P4="$P4BIN -p $P4PORT -u $P4USER"
$P4CBIN/p4login

msg "If there are errors in this log, contact your local support team."

VerifyFailed=0

depots=() # Clear array
depotdirs=() # Clear array

if [[ "${P4REPLICA}" == "FALSE" || ${SHAREDDATA} == "TRUE" ]]; then
  msg "Starting verify of shelves."
  $P4 -s verify -qS //... >> $Log 2>&1 &
  msg "Starting verify of unload depot."
  $P4 -s verify -qU //$($P4 -ztag -F %name%,%type% depots | grep unload | cut -d "," -f 1)/... >> $Log 2>&1 &
else
  msg "Starting verify of shelves."
  $P4 -s verify -qSt //... >> $Log 2>&1 &
  msg "Starting verify of unload depot."
  $P4 -s verify -qUt //$($P4 -ztag -F %name%,%type% depots | grep unload | cut -d "," -f 1)/... >> $Log 2>&1 &
fi

# Build array depots of all of the depot names
for d in $($P4 -ztag -F %name% depots); do
  depot_type=$($P4 -ztag -F %Type% depot -o $d)
  [[ "$depot_type" == remote || "$depot_type" == achive || "$depot_type" == unload ]] && continue
  depots+=( "$d" ) # Append depot to the array
  for dir in $($P4 -ztag -F %dir% dirs //$d/*);do
    depotdirs+=( "$dir" )
  done
done

thread=$($P4 monitor show | grep -c verify)

# We skip verify on edges and edgereps because they should be running in cache mode.
#if [[ $SERVER_TYPE != "p4d_edge" && $SERVER_TYPE != "p4d_edgerep" ]]; then
  for dir in "${depotdirs[@]}"; do
    depotname=$(echo $dir | cut -d "/" -f 3)
    cd $P4TMP

    $P4 -ztag -F %depotFile% files $dir/... > files.txt
    split -l100000 files.txt files.txt_

    for f in $(ls files.txt_*); do
      # Loop to see if we are over our thread count.  If so wait until we drop below it again
      while [ $thread -ge $Threads ]; do
        echo "Waiting thread count=$thread"
        sleep 60
        thread=$($P4 monitor show | grep -c verify)
      done

      if [[ "${P4REPLICA}" == "FALSE" || ${SHAREDDATA} == "TRUE" ]]; then
        VerifyCmd="verify -q"
      else
        VerifyCmd="verify -qt"
      fi

      cmd_log=${LOG}-${depotname}.log
      echo "Verifying files in $f" >> $cmd_log
      cmd $P4 -s -x $f $VerifyCmd & 
      thread=$((thread+1))  #  add one to the thread count and start a new verify
      echo "Thread count=$thread"
      sleep 1
    done
    while [ $thread -gt 0 ]; do
      sleep 60    
      thread=$($P4 monitor show | grep -c verify)
      echo "Waiting for $f file verify to finish. Thread count=$thread"
    done
    rm -f files.*
  done
#fi

# now that we have started all of them wait until all of our processes have finished before continuing.
while [ $thread -gt 0 ]; do
  sleep 60    
  thread=$($P4 monitor show | grep -c verify)
  echo "Wait for finish thread count=$thread"
done

# now that the processes have finished combine all of the log file together
for d in "${depots[@]}"; do
  if [ -f $LOG-$d.log ]; then
    cat $LOG-$d.log >> "$Log"
    rm -f $LOG-$d.log
  fi
done

if [[ $VerifyFailed -ne 0 ]]; then
  StatusMessage="Error: Verify attempt failed.  Review the log [$Log]."
  ExitCode=1
fi

if [[ $ExitCode -eq 0 ]]; then
  egrep '(BAD!|MISSING!|p4 help max)' $Log > /dev/null 2>&1
  if [[ $? -eq 0 ]]; then
    StatusMessage="Warning: Verify errors detected.  Review the log [$Log]."
    ExitCode=2
  fi
fi

msg "Completed verifications at $(date)."

if [[ $ExitCode -ne 0 ]]; then
  mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
fi

/p4/common/bin/rsync_missing.sh $SDP_INSTANCE

exit $ExitCode