#!/bin/bash
set -u

#==============================================================================
# Declarations and Environment
declare ThisScript=${0##*/}
declare CmdLine="$0 $*"
declare Version=1.0.4
declare ThisUser=
declare ThisHost=
declare HostList=
declare HostList2=
declare ExcludeHostList=
declare -i Excluded=0
declare -i HostCount=0
declare -i HostProcessedCount=0
declare -i OKCount=0
declare -i ErrorCount=0
declare -A HostStatus
declare CmdAndArgs=
declare SDPEnvFile="/p4/common/bin/p4_vars"
declare SDPInstance="${SDP_INSTANCE:-1}"
export NO_OP=0
export P4U_LOG=

if ! source "$SDPEnvFile" "$SDPInstance"; then
   echo -e "\\nError: Failed to load SDP shell environment for instancce $SDPInstance."
   exit 1
fi

if ! source "$P4CLIB/p4u_env.sh"; then
   echo -e "\\nError: Failed to load p4u_env.sh."
   exit 1
fi

if ! source "$P4CLIB/libcore.sh"; then
   echo -e "\\nError: Failed to load libcore.sh."
   exit 1
fi

if ! source "$P4CCFG/sdp_hosts.cfg"; then
   echo -e "\\nError: Failed to load sdp_hosts.cfg file."
   exit 1
fi

export VERBOSITY=4
P4U_LOG="${LOGS}/hrun.$(date +'%Y%m%d-%H%M%S').log"

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Stop logging.
   [[ "$P4U_LOG" != off && -f "$P4U_LOG" ]] && stoplog

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ "$errorMessage" != Unset ]]; then
      msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n"
   fi

   msg "USAGE for $ThisScript v$Version:

$ThisScript {-a [-e <host1>[,<host2>...]] | -r [-e <host1>[,<host2>...]] | -H <host1>[,<host2>...]} [-n] [-L <log>] [-si] [-v<n>] 
[-n] [-c cmd and args ...]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script executes a command on multiple SDP hosts.

OPTIONS:
 -c	Specify the command with options/arguments/flags to run on remote hosts.
	After the '-c' flag occurs on the command line, the remainder of the
	command line is interpreted as part of the command, not as options to
	this script. Therefore, '-c' must always be specified after all other
	options to this script.

	If '-c' is not specified, the 'hostname' command is executed. This can be
	useful for exercising and verifying SSH access to all machines.

	Quoting what comes after '-c' is necessary if the command to run on
	others hosts uses '&&' or ';' to separate multi-part commands.
	
 -H <host1>[,<host2>...]
	Specify an explict, comma-delimted list of hosts to execute
	the command on.

 -a	Specify the command is to be run on all SDP hosts, per
	the ALL_SDP_HOSTS setting in $P4CCFG/sdp_hosts.cfg.

 -r	Specify the command is to be run on all replica hosts. per
	the SDP_REPLICA_HOSTS setting in $P4CCFG/sdp_hosts.cfg.

 -e <host1>[,<host2>...]
	Specify an explict, comma-delimted list of hosts to exclude from
	the list of hosts to command on; use this with '-a' or '-r'.

 -v<N>	Specify the verbosity level, from most quiet (1) to loudest (6):
	-v1 quiet (errors, warnings, vital info)
	-v3 normal
	-v6 debugging verbosity

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to
	${LOGS}/log

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

 -si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -n	No-Op.  Prints commands instead of running them.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Display version info for this script and its libraries.

FILES:
	This script uses $P4CCFG/sdp_hosts.cfg to provide a list of all hosts.

EXAMPLES:
	EXAMPLE 1: Check the p4d version running on all hosts:

	hrun -a -c p4 -ztag -F %serverVersion% info -s

	EXAMPLE 2: Check the p4d version running on a list of hosts:

	hrun -H perforce1,fmt-p4edge,perforceindev1,perforce1-tx,perforce2-tx,perforce3-tx,perforce1-ot,perforce2-ot -c p4 -ztag -F %serverVersion% info

	EXAMPLE 3: Check the p4d version staged on all hosts. Note usage of quoting:

	hrun -a -c \"/p4/sdp/helix_binaries/p4d -V | grep ^Rev\"

	EXAMPLE 4: Check replica health (replicas only). Note usage of quoting:

	hrun -r -c \"p4 pull -lj; p4 pull -ls\"

	EXAMPLE 5: Check replica health (replicas only), skipping some. Note usage of
	quoting:

	hrun -r -e perforce2-ot,perforce2 -c \"p4 pull -lj; p4 pull -ls\"

	EXAMPLE 6: Check server.id on all hosts:
	hrun -a -c cat /p4/1/root/server.id
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
function host_cmd_failed () {
   msg "FAILED on ${1:-UnknownHost}"
   ErrorCount+=1
}


#==============================================================================
# Command Line Processing

declare -i shiftArgs=0
declare -i SilentMode=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-H) HostList="$2"; shiftArgs=1;;
      (-e) ExcludeHostList="$2"; shiftArgs=1;;
      (-a) HostList="${ALL_SDP_HOSTS:-}";;
      (-r) HostList="${SDP_REPLICA_HOSTS:-}";;
      (-v1) export VERBOSITIY=1;;
      (-v2) export VERBOSITIY=2;;
      (-v3) export VERBOSITIY=3;;
      (-v4) export VERBOSITIY=4;;
      (-v5) export VERBOSITIY=5;;
      (-v6) export VERBOSITIY=6;;
      (-L) P4U_LOG="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-c) shift; CmdAndArgs="$*"; shiftArgs=$#; shiftArgs=$((shiftArgs-1));;
      (*) usageError "Unknown argument/option [$1].";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

ThisUser=$(whoami)
ThisHost=${HOSTNAME%%.*}
[[ -z "$CmdAndArgs" ]] && CmdAndArgs="hostname"

if [[ -n "$HostList" ]]; then
   HostList=$(echo "$HostList"| tr ',' ' ')
else
   bail "No target hosts specified. Specify '-a', '-r', or '-H <host1>[,<host2>...]'."
fi

if [[ -n "$ExcludeHostList" ]]; then
   ExcludeHostList=$(echo "$ExcludeHostList" | tr ',' ' ')

   HostList2=
   for host in $HostList; do
      Excluded=0
      for ehost in $ExcludeHostList; do
         [[ "$host" == "$ehost" ]] && Excluded=1
      done
      [[ "$Excluded" -eq 0 ]] && HostList2+=" $host"
   done
   HostList=$(echo "$HostList2")
fi

if [[ "$P4U_LOG" != off ]]; then
   touch "$P4U_LOG" || bail "Couldn't touch log file: $P4U_LOG"

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "$P4U_LOG")
      exec 2>&1
   else
      exec >"$P4U_LOG"
      exec 2>&1
   fi

   initlog
fi

msg "User: $ThisUser\\nLauch Host: $ThisHost\\nCommand and Args:\\n\\t$CmdAndArgs\\n"
msg "Host List: $HostList"

if [[ "$NO_OP" -eq 1 ]]; then
   msg "\\nNO_OP (No Operation/preview) mode enabled."
fi

for host in $HostList; do
   HostCount+=1
   HostStatus[$host]="Not Attempted."
done

for host in $HostList; do
   HostProcessedCount+=1
   if rrun "$host" "$CmdAndArgs"; then
      msg "Command OK on host $host."
      OKCount+=1
      HostStatus[$host]="OK"
   else
      host_cmd_failed "$host"
      HostStatus[$host]="FAIL"
   fi
done

msg "\\nSummary:"

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "Command executed successfully on all $HostProcessedCount hosts."
else
   msg "Command executed on $HostProcessedCount of $HostCount hosts, OK on $OKCount, $ErrorCount with errors."

fi

msg "\\nStatus on each host:"
for host in $HostList; do
   printf " %-16s %-s\n" "$host" "${HostStatus[$host]}"
done

msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"