hms #5

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce_software-hms/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment
set -u

if [[ ${BASH_VERSINFO[0]} -lt 4 ]]; then
   echo -e "\n\nERROR: $0 requires bash version 4.x+; current bash version is $BASH_VERSION."
   exit 1
fi

EnvFile=${SDP_ENV:-/p4/common/bin/p4_vars}

if [[ -r "$EnvFile" ]]; then
   source "$EnvFile" hms
else
   echo -e "\n\nERROR: The environment file $EnvFile is missing.\n\n"
   exit 1
fi

if [[ $(id -u) -eq 0 ]]; then
   exec su - $OSUSER -c "$0 $*"
elif [[ $(id -u -n) != $OSUSER ]]; then
   echo "$0 can only be run by root or $OSUSER"
   exit 1
fi

# Allow override of P4U_HOME, which is set only when testing P4U utility scripts.
export P4U_HOME=${P4U_HOME:-$P4CBIN}
export P4U_LIB=${P4U_LIB:-$P4CLIB}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=Unset
export HMS_HOME=${HMS_HOME:-/p4/common/hms}
export HMS_SCRIPTS=$HMS_HOME/scripts

# Indicate whether we are called by the broker wrapper.
export HMS_CALLED_BY_WRAPPER=${HMS_CALLED_BY_WRAPPER:-0}

# Load bash libs.
declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"
BASH_LIBS+=" $P4U_LIB/hms_actions.sh"
BASH_LIBS+=" $P4U_LIB/hms_load_and_verify.sh"
BASH_LIBS+=" $P4U_LIB/hms_upgrade.sh"

for bash_lib in $BASH_LIBS; do
   source $bash_lib
done

# After sourcing bash libs, set default VERBOSITY for this program, which can
# be overridden on the command line.
export VERBOSITY=3

# SDP Instance data, with associative arrays (i.e. arrays that can be indexed
# by a string rather than a purely numeric name).  These Instance* vars are
# indexed by the SDP instance name.
declare -A InstanceUserPorts
declare -A InstanceMasterHost
declare -A InstanceServerPort
declare -A InstanceBrokerPort
declare -A InstanceManaged
declare -A InstanceDesc
declare -A InstanceComponents
declare -A InstanceFailoverOptions

# Helix Topology Components data, indexed by the fully qualified component name
# of the form "SDPInstance:ComponentName", e.g. "1:p4d-r01".
declare -A ComponentType
declare -A ComponentMasterHost
declare -A ComponentURL
declare -A ComponentBackupHost
declare -A ComponentManaged
declare -A ComponentDesc
declare -A ComponentStatusCode
declare -A ComponentStatusMsg
declare -A ComponentMajorVersion
declare -A ComponentMinorVersion
declare -A ComponentVersion

# Failover Options, indexed by the fully qualified failover option name of the
# form "SDPInstance:FailoverOptionName", e.g. "1:dr".
declare -A FailoverType
declare -A FailoverMasterHost
declare -A FailoverBackupHost
declare -A FailoverInstanceList
declare -A FailoverActive
declare -A FailoverDesc

declare -i SilentMode=0

# Increment the Version manually on each submit.  (We avoid +k filetype of
# Perforce due to various complications moving across p4d instances).
declare Version=1.1.0

# The RequiredCfgVersion check is done with a lexiographic compare vs.
# the value defined in the Helix Topology file.  Helix Topology files define
# their file format version with 3 digits, e.g. 3.0.0 or 3.0.1.  The
# RequiredCfgVersion here must be defined with two digits, e.g. "3.0" if
#"3.0.0" is good enough, since 3.0.0 is lexiographically greater than 3.0.
declare RequiredCfgVersion=1.0

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit $OverallReturnStatus
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\n\nUsage Error:\n\n$errorMessage\n\n"
   fi

   echo -e "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT show

$THISSCRIPT status {all|<instance>[:<component>]} [-o]

$THISSCRIPT {start|stop} <instance>[:<component>]

$THISSCRIPT dfm {on|off|status} <instance>

$THISSCRIPT {update|upgrade} [-n|-N]

$THISSCRIPT failover <plan> {i|h}:<instance_or_host> {u|s}

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This is the Perforce Helix Management System,
	$THISSCRIPT v$Version.

	This script is self-logging.  That is, all output displayed on the
	screen (stdout and stderr) is simultaneously captured in a log file.
	You do not need to run this script with redirection operators like
	'> log' or '2>&1', and do not need to use 'tee.'  The default log
	file is:

	$LOGS/hms.<NAME>.<DATESTAMP>.log


HMS CONFIG FILES:
	The HMS config file defines a Helix global topology.

HMS COMMANDS:

status {all|<instance>[:<component>]} [-o]
	Get status of components defined in the Helix Topology
	file.  Specify all for a site-wide global status, an
	instance name for all components related to a given instance
	or a specific component within an instance.

start|stop <instance>[:<component>]
	Start or stop a specific component in the topology, or start/stop
	all components associated with a given instance.

dfm {on|off|status} <instance>
	The 'dfm on' command puts all brokers for the specified SDP
	instance into \"Down For Maintenance\" mode.

	This involves stopping the broker running with the default config
	(p4_N.broker.cfg) and starting the broker using the DFM config
	file, p4_N.broker.dfm.cfg.  The dfm broker config file is expected
	to be configured to reject all user commands and display a friendly
	message for users telling them that the Helix Server is offline
	for maintenance.

	The 'dfm off' command brings all brokers for the specified SDP
	instance (or all instances) back online with the default config,
	after shutting down brokers running with the dfm config.

	The 'dfm status' command checks the status of brokers running with
	the dfm config.

update <instance> [-n|-N]
	Update to the latest versions of software available for the current
	major release.  An update will not upgrade to a new major version.

	Specify an instance name to update.  The global topology for the
	specified instance will be updated.

	Update processing aborts in event of a failure to update a component.

	The update processing varies depending on the component to be updated.
	It essentially consists of:
	 1. Stop the service.
	 2. Replace the executable (e.g p4d, p4broker, p4p, p4) to the latest
	    binary patch.
	 3. Update SDP symlinks as needed.
	 4. Restart the service.

	For p4d, a database schema upgrade is not done for a patch, and no
	checkpoint processing occurs.

	Updates occur in the proper 'outer-to-inner' order, with the master/commit
	server being updated last.  In case of daisy chain of replicas and edges,
	those farthest removed from the commit server are updated first.

	Use with the '-n' flag to do a dry run.

	Use with '-N' to do a more serious dry run that performs actual staging of
	binaries on all machines so they can be compared against local versions to
	see what would be upgraded, but without performing the actual upgrade.

upgrade <instance> [-n|-N]
	Upgrade to the latest versions of software available for the latest GA
	release, upgrading to new major versions if available.

	Specify an instance name to upgrade.  The global topology for the
	specified instance will be upgraded.

	The upgrade processing varies depending on the component to be upgraded.
	It essentially consists of:
	 1. Stop the service.
	 2. Replace the executable (e.g p4d, p4broker, p4p, p4) to the latest
	    binary patch.
	 3. Update SDP symlinks as needed.
	 4. Restart the service.

	For p4d, a database schema upgrade is done.

	Upgrades occur in the proper 'outer-to-inner' order, with the master/commit
	server being upgraded last.  In case of daisy chain of replicas and edges,
	those farthest removed from the commit server are upgraded first.

	Use with '-N' to do a more serious dry run that performs actual staging of
	binaries on all machines so they can be compared against local versions to
	see what would be upgraded, but without performing the actual upgrade.

failover <plan> {h|i}:<scope> {u|s} [-n]
	Execute a failover using a pre-defined plan name defined in the
	Helix Topology file.  Multiple failover plans can be defined
	for a given SDP instance, defining (for example) a \"local\" failover
	plan (using local offline databases on the same host), an HA plan, and
	one or more DR plans.

	The <scope> must be prefixed with i: or h:, as in i:<SDPInstance> or
	h:<Hostname>.  If an SDP instance is specified, that instance fails
	over to the designated backup host for the plan selected.  If a
	machine is specified, all instances currently mastered on that machine
	failover.  In event of hardware failure of a host, the host option
	should be considered.

	When a machine is specified for the scope, the plan should be valid
	for all instances on that machine.  Any instances for which the plan
	is not valid will be skipped in the failover processing (but will not
	prevent it from handling other instances).

	The last paramter {u|s} indicates whether the failover is unscheduled
	('u') or scheduled ('s').  This impacts how the failover process
	occurs.  In a scheduled failover, all systems are expected to be
	operating normally at the start of the failover process, and the
	failover only proceeds if all systems are operating normally at the
	start of the process.

	Scheduled failover can be used to take a perfectly working machine
	offline for a time, e.g. to add memory.  An 'unscheduled' failover is
	executed in reaction to a problem of some kind, such as hardware
	failure.

	An unscheduled failover should be used when it is known that master
	server is offline or otherwise not usable.  If it can be reached, the
	master server is shutdown as part of an unscheduled failover.

	The 'failover' command can be abbreviated as 'fo'.

	HOST ALIAS UPDATE: External to processing of this script, the host
	alias used by users to taget the master server must be updated.
	This may involve a DNS change, virtual IP change, anycast update, etc.
	This must be done before the failover is complete and service can be
	restored for users.

HMS GENERAL COMMAND OPTIONS (valid for all commands):
 -c <helix_topologoy_config>
	Specify the path to your hand-crafted Helix Topology configuration
	file.  See documentation in the default config file regarding the
	required format for topology configuration.  The default
	file is $P4CCFG/HelixTopology.cfg.

	Specify something like '-c /full/path/HelixTopology.cfg' or
	'-c HelixTopology.test.cfg'.  With the latter example, the
	file is presumed to be in the $P4CCFG directory.

	The toplogy configuration file defines SDP Instances, Helix
	Topology Components, and Failover Options.

	This option is intended for development of changes to
	Helix Topology configuration files.

HMS COMMAND OPTIONS (valid for certain commands as indicated):
 -o	Specify '-o' to be optimistic regarding status of components
	for which status logic is not yet implememented.  By default,
	components that don't yet have status logic coded report as
	failed.  With '-o', components of known types report as OK.
	Components of unknown type always report as failed.

	This can be helpful in verifying whether the Helix Topology
	config file contains only valid/known component types.
	Components of unknown types should be commented out or
	marked as unmanaged.

 -n	No-Op preview mode.  This shows operations that would be
	performed, but takes no actions that affect the live system
	or data.  This is valid only for command for which it is
	indicated.

GENERAL OPTIONS:
 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) is captured
	in a log file named:

	$LOGS/hms.<NAME>.<DATESTAMP>.log

 -C	Check the HMS config file and then stop.  This can be used to
	verify the syntax of the HMS config file without starting an actual
	failover.

-si	Operate silently.  All output (stdout and stderr) is redirected to
	the log only; no output appears on the terminal.  This cannot be used
	with '-L off'.
      
 -D     Set extreme debugging verbosity.

 -S     Step mode - if running interactively prompt user to enter return
	before progressing to the next step.  The '-S' option is not supported
	when called via p4broker (i.e. called as 'p4 hms' rather than calling
	the $THISSCRIPT script directly).

	THIS FLAG IS NOT IMPLEMENTED AND WILL BE DEFERRED TO hms 2.0 or beyond.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

DEPENDENCIES:
	The bash shell version must be 4.0 or higher. This precludes
	operation on Mac OSX, at least up to and including OSX 10.14.5
	(Mojave), which ships with bash 3.x.

	This script operates in an SDP environment, and the 'hms' instance
	(/p4/hms) must be configured.

EXAMPLES:
	Get a status in the master p4d for Instance 1:
	$THISSCRIPT status 1:master

	Get a status in the second replica p4d for Instance 2:
	$THISSCRIPT status 2:p4d-r02

	Get a status for all known components, optimisitically assuming
	those we haven't coded status logic for yet are configured are happy:
	$THISSCRIPT status all -o

	Update all components in Instance 1 in the correct order.
	$THISSCRIPT update 1

	Upgrade all components in Instance 1 in the correct order.
	$THISSCRIPT upgrade 1

	Perform an unscheduled failover for all instances mastered on host
	bos-helix-01:
	$THISSCRIPT failover ha h:bos-helix-01 u

	Perform a scheduled failover for all instances mastered on host
	syd-helix-04:
	$THISSCRIPT failover ha h:syd-helix-04 s

	Perform an unscheduled local failover for Instance 1 on its currently
	configured master host:
	$THISSCRIPT failover local i:1 u
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare HelixTopologyCfg=$P4CCFG/HelixTopology.cfg
declare GlobalOptions=
declare Command=
declare SubCommand=
declare Target=
declare FailoverPath=
declare FailoverScope=
declare FailoverStyle=
declare -i PreflightCheck=0
declare -i RunTestSuite=0
declare -i OptimisticStatus=0
declare -i Interactive=1
declare -i StepMode=0
declare -i OverallReturnStatus=0
declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (show) Command=show;;
      (df)
         Command=df
         Target=$2
         shiftArgs=1
      ;;
      (dfm)
         Command=dfm
         usageMsg="Invalid Usage.  Usage for 'hms $Command' command is:\n\nhms $Command {on|off|status} <instance>"
         [[ $# -lt 3 ]] && bail "$usageMsg"
         SubCommand=$2
         [[ $SubCommand =~ ^(on|off|status)$ ]] || bail "$usageMsg"
         Target=$3
         shiftArgs=2
      ;;
      (start|stop|status)
         Command=$1
         usageMsg="Invalid Usage.  Usage for 'hms $Command' command is:\n\nhms $Command <instance>[:<component>]"
         [[ $# -lt 2 ]] && bail "$usageMsg"
         Target=$2
         shiftArgs=1
      ;;
      (failover|fo)
         Command=failover
         usageMsg="Invalid Usage.  Usage for 'hms failover' command is:\n\nhms failover <plan> {i|h}:<scope> <style>"
         [[ $# -lt 4 ]] && bail "$usageMsg"
         FailoverPath=$2
         FailoverScope=$3
         FailoverStyle=$4
         [[ $FailoverPath == "-"* || $FailoverScope == "-"* || $FailoverStyle == "-"* ]] && \
            bail "$usageMsg"

         [[ $FailoverScope =~ ^(i|h): ]] || bail "$usageMsg"

         # Convert short form of 'style' argument, 'u' or 's', to long form for code
         # readability.
         [[ $FailoverStyle =~ ^(s|scheduled|u|unscheduled)$ ]] || bail "$usageMsg"
         [[ $FailoverStyle =~ ^(s|scheduled)$ ]] && FailoverStyle=Scheduled
         [[ $FailoverStyle =~ ^(u|unscheduled)$ ]] && FailoverStyle=Unscheduled
            
         shiftArgs=3
      ;;
      (update|upgrade)
         Command="$1"
         [[ $# -lt 2 ]] && bail "Invalid Usage.  Usage for 'hms $Command' command is:\\n\\nhms $Command <instance> [-n|-N]"
         Target="$2"
         shiftArgs=1
      ;;
      (-c) HelixTopologyCfg=$2; shiftArgs=1;;
      (-C) PreflightCheck=1;;
      (-S) StepMode=1;;
      (-y) Interactive=0;;
      (-o) OptimisticStatus=1;;
      (-T) RunTestSuite=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 0;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-v) export VERBOSITY=${2:-5}; shiftArgs=1;;
      (-L) P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-N) export NO_OP=2;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usage -h "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Bad usage; incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usage -h "Cannot use '-si' with '-L off'."

if [[ $RunTestSuite -eq 1 ]]; then
   [[ -n "$HelixTopologyCfg" ]] && \
      usage -h "The '-c <helix_topology_cfg>' argument is not allowed with -T."

else
   [[ -z "$HelixTopologyCfg" ]] && \
      usage -h "The '-c <helix_topology_cfg>' argument is required."
fi

# Only allow interactive mode 
if [[ $Interactive -eq 1 && $HMS_CALLED_BY_WRAPPER -eq 1 ]]; then
   if [[ ! $Command =~ ^(show|status)$ ]]; then
      usage -h "The -y (auto-confirm) flag is required when called via the broker. Try calling the hms script directly, not with 'p4 hms'."
   fi
fi

# Only allow step mode if running interactively so prompt/response can be handled
if [[ $StepMode -eq 1 && $HMS_CALLED_BY_WRAPPER -eq 1 ]]; then
   bail "The -S (Step Mode) isn't supported when called via the broker. Try calling the hms script directly, not with 'p4 hms'."
fi

# Disable logging by default if we're just doing a status or show command
# called from the broker.
if [[ $HMS_CALLED_BY_WRAPPER -eq 1 && $Command =~ ^(show|status)$ ]]; then
   P4U_LOG=off
fi

if [[ $OptimisticStatus -eq 1 && $Command != status ]]; then
   usage -h "The '-o' option (optimistic status) is only valid with the 'status' command."
fi

#==============================================================================
# Main Program
declare component=
declare instanceComponents=
declare componentList=
declare -i statusAllOK=1
declare -i i

if [[ $RunTestSuite -eq 1 ]]; then
   if [[ -x "$HMS_HOME/test/run_test_suite.sh" ]]; then
      "$HMS_HOME/test/run_test_suite.sh"
      OverallReturnStatus=$?
      exit $OverallReturnStatus
   else
      bail "The expected test suite wrapper script is missing: $HMS_HOME/test/run_test_suite.sh."
   fi
fi

# Very first thing - Check BASH_VERSION
case $BASH_VERSION in
   (1*|2*|3*) echo -e "\n The version of bash is too old.  This script requires bash 4.0 or higher.\n This version is $BASH_VERSION.  If working on Mac, do a web search for\n 'bash 4 mac' for options on acquring a bash 4.x.  Please acquire a new bash\n and install it somewhere, perhaps /usr/local/bin/bash, and change the '#!'\n line at the top of this script ($THISSCRIPT) to reference the newer bash.\n"
   exit 1;;
esac

# If the topology file is specified as '/absolute/path/top.cfg', use it verbatim.
# If the topolgoy file is specified as 'top.cfg', look for top.cfg in the
# current dir first, and then check for top.cfg in /p4/common/config.  Otherwise
# give up.
if [[ ! -r "$HelixTopologyCfg" ]]; then
   if [[ "$HelixTopologyCfg" != "/"* && -r "$P4CCFG/$HelixTopologyCfg" ]]; then
      HelixTopologyCfg=$P4CCFG/$HelixTopologyCfg
   else
      bail "The Helix Topology file [$HelixTopologyCfg] does not exist."
   fi
fi

HelixRegion=$(grep ^NAME= "$HelixTopologyCfg" 2>/dev/null | cut -d '=' -f 2)
[[ -z "$HelixRegion" ]] && \
   bail "The Helix Topology file [$HelixTopologyCfg] is missing a 'NAME=' definition."

if [[ $P4U_LOG == Unset ]]; then
   if [[ -n "$Command" ]]; then
      P4U_LOG="$LOGS/hms.$Command.$HelixRegion.$(date +'%Y%m%d-%H%M%S').log"
   else
      P4U_LOG="$LOGS/hms.$Command.$HelixRegion.$(date +'%Y%m%d-%H%M%S').log"
   fi
fi

trap terminate EXIT SIGINT SIGTERM

if [[ "${P4U_LOG}" != off ]]; then
   # If we can't put the log where we want it, try putting it in /tmp
   # instead.  We don't want to abort failover operations for lack of
   # a log file, as long as we can write one somehwere.
   if ! touch "${P4U_LOG}"; then
      P4U_LOG="/tmp/${P4U_LOG##*/}"
      touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."
   fi

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee "${P4U_LOG}")
      exec 2>&1
   else
      exec >"${P4U_LOG}"
      exec 2>&1
   fi

   initlog
fi

msg "${H}\\nStarted at $(date).  Versions:"

[[ $VERBOSITY -gt 3 ]] && show_versions

# Environment isolation:  Clear Perforce server environment settings.
unset P4AUDIT P4AUTH P4CHANGE P4CONFIG P4DEBUG P4DESCRIPTION P4JOURNAL \
   P4LOG P4NAME P4PORT P4ROOT P4SSLDIR P4TARGET P4TICKETS P4TRUST

# Environment isolation:  Unset P4CONFIG, and point P4ENVIRO to /dev/null
# to avoid accidentally interacting with a personal DVCS repo which slickly
# applies a P4CONFIG value even after we unset it, by reading it from the
# P4ENVIRO if it finds one (defaulting to ~/.p4enviro).
export P4CONFIG=Unset
export P4ENVIRO=/dev/null/.p4enviro

[[ -z "$P4BIN" || ! -x "$P4BIN" || -z "$P4DBIN" || ! -x "$P4DBIN" ]] && \
   bail "\n\nThe p4 and p4d binaries are missing or are not executable.\n"

P4BinRev=$($P4BIN -V|grep ^Rev)
P4DBinRev=$($P4DBIN -V|grep ^Rev)

msg "Using these p4/p4d executables:\n$P4BIN:  $P4BinRev\n$P4DBIN: $P4DBinRev\n"

if [[ $PreflightCheck -eq 0 ]]; then
   load_helix_topology "$HelixTopologyCfg" "$RequiredCfgVersion" ||\
      bail "Helix Topology definition failed sanity check."

else
   load_helix_topology "$HelixTopologyCfg" "$RequiredCfgVersion" ||\
      bail "Helix Topology definition in $HelixTopologyCfg FAILED preflight sanity checks."

   msg "Verified: Helix Topology defined in $HelixTopologyCfg PASSED preflight sanity checks."
fi

msg "\n${H}\n"

case "$Command" in
   (show)
      msg "Command: $Command"
      show_helix_topology
   ;;

   (start|stop)
      msg "Command: $Command Target=[$Target]"
      change_component_status "$Command" "$Target"
   ;;

   (dfm)
      msg "Command: $Command SubCommand=[$SubCommand] Target=[$Target]"
      down_for_maintenance "$SubCommand" "$Target"
   ;;

   (df)
      msg "Command: $Command Target=[$Target]"
      get_diskspace "$Target"
   ;;

   (status)
      msg "Command: $Command Target=[$Target]"
      componentList=$(get_component_list "$Target" 0) ||\
         bail "Failed to get a valid list of components."

      for component in $componentList; do
         get_component_status "$component" $OptimisticStatus ||\
            statusAllOK=0
         statusCode=${ComponentStatusCode[$component]:-Unset}
         statusMsg=${ComponentStatusMsg[$component]:-Unset}
         if [[ $statusCode != Unset && $statusMsg != Unset ]]; then
            qmsg "Status: $component is $statusMsg (code $statusCode)."
         fi
      done

      if [[ $statusAllOK -eq 1 ]]; then
         msg "${H}\nStatus for all components checked is OK."
      else
         msg "${H}\nStatus for one or more components FAILED."
         OverallReturnStatus=1
      fi
   ;;

   (failover)
      msg "Command: $Command Path=[$FailoverPath] Scope=[$FailoverScope] Style=[$FailoverStyle]"

      [[ "$NO_OP" -ne 0 ]] && msg "Running in NO-OP Preview Mode due to '-n'."

      if failover "$FailoverPath" "$FailoverScope" "$FailoverStyle"; then
         msg "${H}\\nFailover completed successfully.\\n"
      else
         errmsg "Failover did NOT complete successfully.\\n"
         OverallReturnStatus=1
      fi
   ;;

   (update|upgrade)
      msg "Command: $Command Target=[$Target]"
      [[ "$NO_OP" -ne 0 ]] && msg "Running $Command in NO-OP Preview Mode due to '-n'."

      msg "Global Replica Status preflight check for $Command"
      if run "global_replica_status.sh -i $Target"; then
         msg "Global replica status OK."
      else
         bail "Global replica status was not OK. Aborting."
      fi

      if update_or_upgrade "$Command" "$Target"; then
         msg "\\nSUCCESS:  Upgrade completed successfully on all hosts."
      else
         bail "\\nUpgrade did not complete successfully. Review the log: $P4U_LOG"
      fi
   ;;

   (*)
      bail "Unknown command.  Bad usage."
   ;;
esac

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\\nAll processing completed successfully.\\n"
else
   msg "${H}\\nProcessing completed, but with errors.  Scan above output carefully.\\n" 
fi

msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

[[ -r "${P4U_LOG}" ]] && msg "Log file is: $P4U_LOG\n${H}\n"

exit $OverallReturnStatus