sdp_health_check.sh #5

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#------------------------------------------------------------------------------
# sdp_health_check.sh
#
#==============================================================================
# Declarations and Environment

# Prefix global vars with HC_ to avoid name collisions.
export HC_SDP_ENV=/p4/common/bin/p4_vars
export HC_SDP_MRUN=/p4/common/bin/p4master_run
export HC_SDP_P4CBIN=/p4/common/bin
export HC_SDP_INSTANCE_LIST=
export HC_SDP_INSTALLED_VERSION=Unknown
export HC_SDP_341_URL="https://swarm.workshop.perforce.com/jobs/SDP-341"
export HC_THISSCRIPT=${0##*/}

declare -i ExitCode=0
declare -i ErrorCount=0
declare -i WarningCount=0
declare -i OverallExitCode=0
declare Version=1.0.5

declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"

if [[ -n "$(which date 2>/dev/null)" ]]; then
   export LOGFILE=/tmp/sdp_health_check.$(date +'%Y%m%d-%H%M%s').log
else
   export LOGFILE=/tmp/sdp_health_check.log
fi

#==============================================================================
# Local Functions

# Note: This script does not use SDP library files, as its purpose is to
# verify the integrity of an SDP installation.  Thus, it has its own
# self-contained versions of some functions that would normally be
# sourced in from files like /p4/common/lib/libcore.sh.

# Micro-functions, one-liners used to avoid external dependencies.
function msg () { echo -e "$*" >> $LOGFILE; }
function errmsg () { msg "\nError: ${1:-Unknown Error}\n"; ErrorCount+=1; }
function warnmsg () { msg "\nWarning: ${1:-Unknown Warning}\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; OverallExitCode=1; exit ${2:-1}; }

function run () { 
   local cmd="${1:-echo}"
   local desc="${2:-}"
   local -i showOutput="${3:-1}"
   local log=$(mktemp)
   local -i exitCode=

   [[ -n "$desc" ]] && msg "$desc"
   msg "Executing: $cmd"
   $cmd > "$log" 2>&1
   exitCode="$?"

   if [[ "$showOutput" ]]; then
      echo "EXIT_CODE: $exitCode" >> "$log"
      cat "$log" >> $LOGFILE
   fi

   /bin/rm -f "$log"
   return $exitCode
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\n\nUsage Error:\n\n$errorMessage\n\n" >&2
   fi

   echo "USAGE for sdp_health_check.sh v$Version:

sdp_health_check.sh

   or

sdp_health_check.sh -h|-man
"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:

	This script does a health check of the SDP, and generates a
	short repot.  It identifies SDP instances and reports on
	whether the installation is at risk due to a bug, SDP-341.

	For details, see:  $HC_SDP_341_URL

	It must be run as the OS user who owns the /p4/common/bin
	directory.

	Characteristics of this script:
	* It is always safe to run.  It does only analysis and reporting.
	* It does only fast checks, and has no interactive prompts.
	* It does only checks that can be done regardless of whether p4d is
	  running.
	* It requires no command line arguments.
	* It works for any SDP version since 2007.

	Assumptions:
	* The SDP has always used /p4/common/bin/p4_vars as the shell
	  environment file.  This is consistent across all SDP versions.

OPTIONS:
 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	This script is typically called with no arguments.

LOGGING:
	This script generates a log file and also displays it to stdout at the
	end of processing.  By default, the log is:
	
	/tmp/sdp_health_check.<datestamp>.log

	or

	/tmp/sdp_health_check.log

	The exception is usage errors, which result an error being sent to
	stderr followed usage info on stdout, followed by an immediate exit.

EXIT CODES:
	An exit code of 0 indicates no errors or warnings were encountered.
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function: do_341_check ()
function do_341_check () {

   declare returnCode=0

   msg "$H2\nChecking for susceptibility to SDP-341.\n"
   Script1="/p4/common/bin/recreate_db_checkpoint.sh"
   Script2="/p4/common/bin/recreate_db_sync_replica.sh"

   if [[ -r "$Script1" ]]; then
      if [[ -n "$(grep 'SAVEDIR since we know' $Script1)" ]]; then
         msg "Verified: Known-safe version exists: $Script1"
      elif [[ -n "$(grep 'save directory since we know' $Script1)" ]]; then 
         errmsg "At risk due to existing known-broken version of script $Script1\nSuggested actions and background information are provided here: $HC_SDP_341_URL"
         returnCode=1
      else
         warnmsg "Unknown version of this script exists: $Script1"
      fi
   else
      msg "Verified: This does not exist: $Script1"
   fi

   if [[ -r "$Script2" ]]; then
      if [[ -n "$(grep 'rm -f rdb.lbr' $Script2)" ]]; then
         msg "Verified: Known-safe version exists: $Script2"
      elif [[ -n "$(grep 'RsyncCmd=' $Script2)" ]]; then 
         errmsg "At risk due to existing known-broken version of script $Script2\nSuggested actions and background information are provided here: $HC_SDP_341_URL"
         returnCode=1
      else
         warnmsg "Unknown version of this script exists: $Script2"
      fi
   else
      msg "Verified: This does not exist: $Script2"
   fi

   if [[ "$returnCode" -eq 0 ]]; then
      msg "Verified: There is no susceptibilty to SDP-341."
   else
      errmsg "Known-broken version of one or more key scripts detected."
   fi

   return $returnCode
}

#------------------------------------------------------------------------------
# Function: do_preflight_checks ()
function do_preflight_checks () {

   ExitCode=0
   declare ToolsList="date ls grep awk id"

   msg "$H2\nDoing preflight sanity checks."
   msg "Preflight Check 1: Ensuring basic tools are in the PATH."

   for tool in $ToolsList; do
      if [[ -z "$(which $tool 2>/dev/null)" ]]; then
         errmsg "Required tool '$tool' not found in PATH."
         ExitCode=1
      fi
   done

   [[ $ExitCode -ne 0 ]] && return 1

   msg "Verified: Essential tools are in the PATH."

   msg "Preflight Check 2: cd $HC_SDP_P4CBIN"

   cd "$HC_SDP_P4CBIN" >> $LOGFILE 2>&1
   if [[ "$?" -ne 0 ]]; then
      errmsg "Could not cd to: $HC_SDP_P4CBIN"
      OverallExitCode=1
      return 1
   fi

   msg "Verified: cd works to: $HC_SDP_P4CBIN"

   msg "Preflight Check 3: Checking current user owns $HC_SDP_P4CBIN"
   export HC_SDP_OWNER=$(ls -ld . | awk '{print $3}')
   export ThisUser="$(id -n -u)"

   if [[ "$ThisUser" == "$HC_SDP_OWNER" ]]; then
      msg "Verified: Current user [$ThisUser] owns $HC_SDP_P4CBIN"
   else
      errmsg "Current user [$ThisUser] does not own $HC_SDP_P4CBIN."
      OverallExitCode=1
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: get_sdp_instances ()
#
# Get the list of SDP instances after doing some preliminary sanity
# checks.
function get_sdp_instances () {
   export HC_SDP_INSTANCE_LIST=
   cd /p4
   for e in $(ls); do
      if [[ -r "/p4/$e/root/db.counters" ]]; then
         export HC_SDP_INSTANCE_LIST+=" $e"
      fi
   done

   # Trim leading space.
   export HC_SDP_INSTANCE_LIST=$(echo $HC_SDP_INSTANCE_LIST)
}

#------------------------------------------------------------------------------
# Function: check_sdp_instance ()
function check_sdp_instance () {
   export SDP_INSTANCE=${1:-UnsetSDPInstance}

   if [[ "$SDP_INSTANCE" == UnsetSDPInstance ]]; then
      errmsg "Invalid call to check_sdp_instance(), no instance parameter. Skipping further checks for this instance."
      return 1
   fi

   msg "$H1\nChecking SDP instance $SDP_INSTANCE."

   source "$HC_SDP_ENV" "$SDP_INSTANCE"
   if [[ $? -ne 0 ]]; then
      errmsg "Failed to load SDP environment for instance $SDP_INSTANCE. Skipping fuirther checks for this isntance."
      return 1
   fi

   if [[ -L "/p4/$SDP_INSTANCE" ]]; then
      warnmsg "Instance $SDP_INSTANCE uses old-style symlink structure and should be upgraded."
      return 1
   fi

   msg "Verified: SDP instance $SDP_INSTANCE is OK."

   return $ExitCode
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   msg "$H1\nErrors detected: $ErrorCount"
   msg "Warnings detected: $WarningCount"
   msg "$HC_THISSCRIPT: EXITCODE: $OverallExitCode"
   msg "$H2\nThis log file: $LOGFILE"

   cat "$LOGFILE"

   exit $OverallExitCode
}
#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown command line flag ($1).";;
      (*) usage -h "Unknown command line fragment ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

msg "$HC_THISSCRIPT v$Version Starting validation at $(date +'%a %Y-%m-%d %H:%M:%S %Z')."

msg "This log file is: $LOGFILE"

do_preflight_checks || bail "Preflight checks failed. Aborting."

do_341_check || OverallExitCode=1

get_sdp_instances
if [[ -n "$HC_SDP_INSTANCE_LIST" ]]; then
   msg "$H2\nList of SDP Instances to verify: $HC_SDP_INSTANCE_LIST"
else
   bail "No SDP instances detected."
fi

for i in $HC_SDP_INSTANCE_LIST; do
   check_sdp_instance "$i" || OverallExitCode=1
done

if [[ "$OverallExitCode" -eq 0 ]]; then
   msg "The SDP installation has passed basic health checks."
fi

msg "If you have any questions about the output from this script, contact support@perforce.com"

exit $OverallExitCode