#!/bin/bash
# Declarations and Environment

export SDP_ENV=/p4/common/bin/p4_vars
export SDP_INSTANCE=${SDP_INSTANCE:-Unset}

export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ $SDP_INSTANCE == Undefined ]]; then
  echo "Instance parameter not supplied."
  echo "You must supply the Perforce instance as a parameter to this script."
  exit 1
fi

declare RecentChangesToVerify=${SDP_RECENT_CHANGES_TO_VERIFY:-2500}

declare StatusMessage="OK: All scanned depots verified OK."
declare -i VerifyOnlyRecentChanges=0
declare -i VerifyFailed=
declare -i ExitCode=0
declare RevRange=
declare VerifyCmd=
declare Log=Unset
declare Version=6.0.0
declare Threads=1
declare running=0

#==============================================================================
# Local Functions

# Micro-functions, one-liners used to avoid external dependencies.
function msg () { if [[ $Log != Unset ]]; then echo -e "$*" >> $Log; else echo -e "$*"; fi; }
function bail () { msg "\nError: ${1:-Unknown Error}"; exit ${2:-1}; }

function cmd ()
{ 
  echo === Running $* on $(date). >> $cmd_log 
  $* >> $cmd_log 2>&1
  status=$?
  if [ $status -ne 0 ]; then
    VerifyFailed=1
  fi
  echo === $* completed on $(date). >> $cmd_log 
}

function check_running ()
{
  sleep 30
  #loop thread process id's and see if any have finished.
  spot=0
  run=()
  for p in "${ids[@]}"; do
    if [ -n "$p" ]; then
      running=$(ps cax | grep "$p")
    fi
    if [ -n "$running" ]; then
      run[$spot]=$p
      spot=$((spot+1))
    else
      thread=$((thread-1))
    fi
  done
  if [[ $spot -ne 0 ]]; then
    ids=("${run[@]}")
  else
    ids=()
  fi
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
  declare style=${1:--h}
  declare errorMessage=${2:-Unset}

  if [[ $errorMessage != Unset ]]; then
    echo -e "\n\nUsage Error:\n\n$errorMessage\n\n" >&2
  fi

  echo "USAGE for p4verify.sh v$Version:

p4verify.sh [<instance>] [-P num_threads]

  or

p4verify.sh -h|-man
"
  if [[ $style == -man ]]; then
    echo -e "DESCRIPTION:

	This script performs a 'p4 verify' of all submitted and shelved versioned
	files in depots of all types except 'remote' and 'archive' type depots.

	If run on a replica, it schedules archive failures for transfer to the
	replica.

OPTIONS:
<instance>
	Specify the SDP instances.  If not specified, the SDP_INSTANCE
	environment variable is used instead.  If the instance is not
	defined by a parameter and SDP_INSTANCE is not defined, p4verify.sh
	exists immediately with an error message.

 -D     Set extreme debugging verbosity.

-P #
	Specify the number of depots to verify in parallel.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	This script is typically called via cron with only the instance
	paramter as an argument, e.g.:
	p4verify.sh N

LOGGING:
	This script generates no output by default.  All (stdout and stderr) is
	logged to /p4/N/logs/p4verify.log.

	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.

	If the '-v' flag is used, the contents of the log are displayed to
	stdout at the end of processing.

EXIT CODES:
	An exit code of 0 indicates no errors were encounted attempting to
	perform verifications, AND that all verifications attempted
	reported no problems.

	A exit status of 1 indicates that verifications could not be
	attempted for some reason.

	A exit status of 2 indicates that verifications were successfully
	performed, but that problems such as BAD or MISSING files
	were detected, or else system limits prevented verification.
"
  fi

  exit 1
}

#==============================================================================
# Command Line Processing
declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
  case $1 in
    (-h) usage -h;;
    (-man) usage -man;;
    (-D) set -x;; # Debug; use 'set -x' mode.
    (-P) Threads=$2; shiftArgs=1;;
    (-*) usage -h "Unknown command line option ($1).";;
    (*) export SDP_INSTANCE=$1;;
  esac

  # Shift (modify $#) the appropriate number of times.
  shift; while [[ $shiftArgs -gt 0 ]]; do
    [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
    shiftArgs=$shiftArgs-1
    shift
  done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SDP_INSTANCE == Unset ]] && \
  bail "The \$SDP_INSTANCE setting is not defined. It must be defined by doing:\n\n\tsource /p4/common/bin/p4_vars <instance>\n\nor by passing in the instance name as a parameter to this script.\n"

#==============================================================================
# Main Program

source $SDP_ENV $SDP_INSTANCE ||\
  bail "Failed to load SDP environment for instance $SDP_INSTANCE."

source $P4CBIN/backup_functions.sh ||\
  bail "Failed to load backup_functions.sh."

[[ $Log == Unset ]] && Log=$LOGS/p4verify.log

# This is needed because everything in backup_functions.sh uses LOGFILE including the mail_log_file function.
LOGFILE=$Log
LOG=$LOGS/p4verify

check_vars
set_vars
get_journalnum

# Clean up old logs
rm -f ${LOG}-*
rotate_log_file $LOGFILE ".gz"

msg "${0##*/} v$Version Starting verify at $(date +'%a %Y-%m-%d %H:%M:%S %Z')."

P4="$P4BIN -p $P4PORT -u $P4USER"
$P4CBIN/p4login

msg "If there are errors in this log, contact your local support team."

VerifyFailed=0

depots=() # Clear array
depotdirs=() # Clear array
thread=0  # Set current threads to 0


if [[ "${P4REPLICA}" == "FALSE" || ${SHAREDDATA} == "TRUE" ]]; then
  $P4 -s verify -qS //... >> $Log 2>&1 &
  $P4 -s verify -qU //$($P4 -ztag -F %name%,%type% depots | grep unload | cut -d "," -f 1)/... >> $Log 2>&1 &
else
  $P4 -s verify -qSt //... >> $Log 2>&1 &
  $P4 -s verify -qUt //$($P4 -ztag -F %name%,%type% depots | grep unload | cut -d "," -f 1)/... >> $Log 2>&1 &
fi

sleep 1
pid=$($P4 monitor show -al | grep -F "verify -qU" | awk '{print $1;}')
if [ -n "$pid" ]; then
  ids[$thread]=$pid # add the process ID into the array of running processes
  thread=$((thread+1))  #  add one to the thread count and start a new verify
fi
 
pid=$($P4 monitor show -al | grep -F "verify -qS" | awk '{print $1;}')
if [ -n "$pid" ]; then
  ids[$thread]=$pid # add the process ID into the array of running processes
  thread=$((thread+1))  #  add one to the thread count and start a new verify
fi

# Build array depots of all of the depot names
for d in $($P4 -ztag -F %name% depots); do 
  depot_type=$($P4 -ztag -F %Type% depot -o $d)
  [[ "$depot_type" == remote || "$depot_type" == achive || "$depot_type" == unload ]] && continue
  depots+=( "$d" ) # Append depot to the array
  for dir in $($P4 -ztag -F %dir% dirs //$d/*);do
    depotdirs+=( "$dir" )
  done
done
 
# We skip verify on edges and edgereps because they should be running in cache mode.
if [[ $SERVER_TYPE != "p4d_edge" && $SERVER_TYPE != "p4d_edgerep" ]]; then
  # loop depots running the number of them in parraell that is specified by the command line
  for dir in "${depotdirs[@]}"; do
    depotname=$(echo $dir | cut -d "/" -f 3)
    # Loop to see if we are over our thread count.  If so wait until we drop below it again
    while [ $thread -ge $Threads ]; do
      check_running
    done

    if [[ "${P4REPLICA}" == "FALSE" || ${SHAREDDATA} == "TRUE" ]]; then
      VerifyCmd="verify -qz $dir/..."
    else
      VerifyCmd="verify -qzt $dir/..."
    fi

    echo $VerifyCmd > greppattern.txt
    cmd_log=${LOG}-${depotname}.log
    cmd $P4 -s $VerifyCmd & 
    sleep 1
    pid=$($P4 monitor show -ale | grep -F -f greppattern.txt | awk '{print $1;}')
    if [ -n "$pid" ]; then
      ids[$thread]=$pid # add the process ID into the array of running processes
      thread=$((thread+1))  #  add one to the thread count and start a new verify
    fi
  done
fi

rm greppattern.txt > /dev/null 2>&1

# now that we have started all of them wait until all of our processes have finished before continuing.
while [ $thread -gt 0 ]; do
  check_running
done

# now that the processes have finished combine all of the log file together
for d in "${depots[@]}"; do
  if [ -f $LOG-$d.log ]; then
    cat $LOG-$d.log >> "$Log"
    rm -f $LOG-$d.log
  fi
done

if [[ $VerifyFailed -ne 0 ]]; then
  StatusMessage="Error: Verify attempt failed.  Review the log [$Log]."
  ExitCode=1
fi

if [[ $ExitCode -eq 0 ]]; then
  egrep '(BAD!|MISSING!|p4 help max)' $Log > /dev/null 2>&1
  if [[ $? -eq 0 ]]; then
    StatusMessage="Warning: Verify errors detected.  Review the log [$Log]."
    ExitCode=2
  fi
fi
 
msg "Completed verifications at $(date)."

if [[ $ExitCode -ne 0 ]]; then
  mail_log_file "$HOSTNAME $P4SERVER P4Verify Log ($StatusMessage)"
fi

/p4/common/bin/rsync_missing.sh $SDP_INSTANCE

exit $ExitCode