#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
#==============================================================================
# Declarations and Environment
set -u
if [[ ${BASH_VERSINFO[0]} -lt 4 ]]; then
echo -e "\n\nERROR: $0 requires bash version 4.x+; current bash version is $BASH_VERSION."
exit 1
fi
EnvFile=${SDP_ENV:-/p4/common/bin/p4_vars}
if [[ -r "$EnvFile" ]]; then
source "$EnvFile" hms
else
echo -e "\n\nERROR: The environment file $EnvFile is missing.\n\n"
exit 1
fi
if [[ $(id -u) -eq 0 ]]; then
exec su - $OSUSER -c "$0 $*"
elif [[ $(id -u -n) != $OSUSER ]]; then
echo "$0 can only be run by root or $OSUSER"
exit 1
fi
# Allow override of P4U_HOME, which is set only when testing P4U utility scripts.
export P4U_HOME=${P4U_HOME:-$P4CBIN}
export P4U_LIB=${P4U_LIB:-$P4CLIB}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=Unset
export HMS_HOME=${HMS_HOME:-/p4/common/hms}
export HMS_SCRIPTS=$HMS_HOME/scripts
# Indicate whether we are called by the broker wrapper.
export HMS_CALLED_BY_WRAPPER=${HMS_CALLED_BY_WRAPPER:-0}
# Load bash libs.
declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"
BASH_LIBS+=" $P4U_LIB/hms_actions.sh"
BASH_LIBS+=" $P4U_LIB/hms_load_and_verify.sh"
for bash_lib in $BASH_LIBS; do
source $bash_lib
done
# After sourcing bash libs, set default VERBOSITY for this program, which can
# be overridden on the command line.
export VERBOSITY=3
# SDP Instance data, with associative arrays (i.e. arrays that can be indexed
# by a string rather than a purely numeric name). These Instance* vars are
# indexed by the SDP instance name.
declare -A InstanceUserPorts
declare -A InstanceMasterHost
declare -A InstanceServerPort
declare -A InstanceBrokerPort
declare -A InstanceManaged
declare -A InstanceDesc
declare -A InstanceComponents
declare -A InstanceFailoverOptions
# Helix Topology Components data, indexed by the fully qualified component name
# of the form "SDPInstance:ComponentName", e.g. "1:p4d-r01".
declare -A ComponentType
declare -A ComponentMasterHost
declare -A ComponentURL
declare -A ComponentBackupHost
declare -A ComponentManaged
declare -A ComponentDesc
declare -A ComponentStatusCode
declare -A ComponentStatusMsg
declare -A ComponentMajorVersion
declare -A ComponentMinorVersion
declare -A ComponentVersion
# Failover Options, indexed by the fully qualified failover option name of the
# form "SDPInstance:FailoverOptionName", e.g. "1:dr".
declare -A FailoverType
declare -A FailoverMasterHost
declare -A FailoverBackupHost
declare -A FailoverInstanceList
declare -A FailoverActive
declare -A FailoverDesc
declare -i SilentMode=0
# Increment the Version manually on each submit. (We avoid +k filetype of
# Perforce due to various complications moving across p4d instances).
declare Version=1.0.34
# The RequiredCfgVersion check is done with a lexiographic compare vs.
# the value defined in the Helix Topology file. Helix Topology files define
# their file format version with 3 digits, e.g. 3.0.0 or 3.0.1. The
# RequiredCfgVersion here must be defined with two digits, e.g. "3.0" if
#"3.0.0" is good enough, since 3.0.0 is lexiographically greater than 3.0.
declare RequiredCfgVersion=1.0
#==============================================================================
# Local Functions
#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
# Disable signal trapping.
trap - EXIT SIGINT SIGTERM
# Don't litter.
cleanTrash
vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"
# Stop logging.
[[ "${P4U_LOG}" == off ]] || stoplog
# With the trap removed, exit.
exit $OverallReturnStatus
}
#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional). Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1). If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
declare style=${1:--h}
declare errorMessage=${2:-Unset}
if [[ $errorMessage != Unset ]]; then
echo -e "\n\nUsage Error:\n\n$errorMessage\n\n"
fi
echo -e "USAGE for $THISSCRIPT v$Version:
$THISSCRIPT show
$THISSCRIPT status {all|<instance>[:<component>]} [-o]
$THISSCRIPT {start|stop} <instance>[:<component>]
$THISSCRIPT dfm {on|off|status} <instance>
$THISSCRIPT {update|upgrade} <instance>[:<component>]
$THISSCRIPT failover <path> {i|h}:<instance_or_host> <style>
$THISSCRIPT [-h|-man|-V]
"
if [[ $style == -man ]]; then
echo -e "
DESCRIPTION:
This is the Perforce Helix Management System,
$THISSCRIPT v$Version.
This script is self-logging. That is, all output displayed on the
screen (stdout and stderr) is simultaneously captured in a log file.
You do not need to run this script with redirection operators like
'> log' or '2>&1', and do not need to use 'tee.' The default log
file is:
$LOGS/hms.<NAME>.<DATESTAMP>.log
HMS CONFIG FILES:
The HMS config file defines a Helix global topology.
HMS COMMANDS:
status {all|<instance>[:<component>]} [-o]
Get status of components defined in the Helix Topology
file. Specify all for a site-wide global status, an
instance name for all components related to a given instance
or a specific component within an instance.
start|stop <instance>[:<component>]
Start or stop a specific component in the topology, or start/stop
all components associated with a given instance.
dfm {on|off|status} <instance>
The 'dfm on' command puts all brokers for the specified SDP
instance into \"Down For Maintenance\" mode.
This involves stopping the broker running with the default config
(p4_N.broker.cfg) and starting the broker using the DFM config
file, p4_N.broker.dfm.cfg. The dfm broker config file is expected
to be configured to reject all user commands and display a friendly
message for users telling them that the Helix Server is offline
for maintenance.
The 'dfm off' command brings all brokers for the specified SDP
instance (or all instances) back online with the default config,
after shutting down brokers running with the dfm config.
The 'dfm status' command checks the status of brokers running with
the dfm config.
update <instance>[:<component>] [-n]
Update to the latest versions of software available for the current
major release for the given component. An update will not upgrade to a
new major version.
Specify 'all' to perform a site-wide update of all defined Helix
topology components. Updates are done one instance at a time. Update
processing aborts in event of a failure to upgrade any component.
Specify an instance name to update all components associated with a
given Helix instance. Update processing aborts in event of a failure
to update any component.
The update processing varies depending on the component to be updated.
It essentially consists of:
1. Stop the service.
2. Replace the executable (e.g .p4d, p4broker, Swarm files) to the
latest patched executable.
3. Update SDP symlinks as needed.
4. Restart the service.
For p4d, a database upgrade is not done, and no checkpoint processing
occurs.
For any given instance, updates occur in the proper order, with the
master/commit server being updated last. In case of daisy chain of
replicas, those farthest removed from the commit server are updated
first.
Use with the '-n' flag to see what versions would be updated.
upgrade <instance>[:<component>] [-n]
Upgrade to the latest versions of software available for the latest GA
release, upgrading to new major versions if needed.
Specify 'all' to perform a site-wide upgrade of all defined Helix
topology components. upgrades are done one instance at a time.
Upgrade processing aborts in event of a failure to upgrade or update
any component.
Specify an instance name to upgrade all components associated with a
given Helix instance. Upgrade processing aborts in event of a failure
to upgrade any component.
The upgrade processing varies depending on the component to be
upgraded. It essentially consists of:
1. Stop the service.
2. Replace the executable (e.g .p4d, p4broker, Swarm files) to the
latest patched executable.
3. Restart the service.
For p4d, an offline checkpoint is initiated at the start of processing.
For any given instance, upgrades occur in the proper order, with the
master/commit server being upgraded last. In case of a daisy chain of
replicas, those farthest removed from the commit server are upgraded
first.
Use with the '-n' flag to see what versions would be updated or
upgraded.
failover <path> {h|i}:<scope> <style> [-n]
Execute a failover using a pre-defined <path> name defined in the
Helix Topology file. Multiple failover paths can be defined
for a given SDP instance, defining (for example) a "local" failover
option (using offline databases on the same host), an HA option, and
one or more DR options.
The <scope> must be prefixed with i: or h:, as in i:<SDPInstance> or
h:<Hostname>. If an SDP instance is specified, that instance fails
over to the designated backup host for the path selected. If a
machine is specified, all instances currently mastered on that machine
failover. In event of hardware failure of a host, this option should
be considered.
When a machine is specified for the scope, the <path> should be valid
for all instances on that machine. Any instances for which the path
is not valid will be skipped in the failover processing (but will not
prevent it from handling other instances).
The <style> is either 'scheduled' ('s' for short) or 'unscheduled' ('u'
for short). This impacts how the failover process occurs. In a
scheduled failover, all systems are expected to be operating normally
at the start of the failover process, and the failover only occurs if
all systems are operating normally.
Scheduled failover can be used to take a perfectly working machine
offline for a time, e.g. to add memory. An 'unscheduled' failover is
executed in reaction to a problem of some kind, such as hardware
failure.
An 'unscheduled' failover should be used when it is known that master
server is offline or otherwise not usable. If it can be reached, the
master server is shutdown as part of an unscheduled failover.
The 'failover' command can be abbreviated as 'fo'.
HOST ALIAS UPDATE: External to processing of this script, the host
alias used by users to taget the master server must be updated.
This may involve a DNS change, virtual IP change, anycast update, etc.
This must be done before the failover is complete and service can be
restored for users.
HMS GENERAL COMMAND OPTIONS (valid for all commands):
-c <helix_topologoy_config>
Specify the path to your hand-crafted Helix Topology configuration
file. See documentation in the default config file regarding the
required format for topology configuration. The default
file is $P4CCFG/HelixTopology.cfg.
Specify something like '-c /full/path/HelixTopology.cfg' or
'-c HelixTopology.test.cfg'. With the latter example, the
file is presumed to be in the $P4CCFG directory.
The toplogy configuration file defines SDP Instances, Helix
Topology Components, and Failover Options.
This option is intended for development of changes to
Helix Topology configuration files.
HMS COMMAND OPTIONS (valid for certain commands as indicated):
-o Specify '-o' to be optimistic regarding status of components
for which status logic is not yet implememented. By default,
components that don't yet have status logic coded report as
failed. With '-o', components of known types report as OK.
Components of unknown type always report as failed.
This can be helpful in verifying whether the Helix Topology
config file contains only valid/known component types.
Components of unknown types should be commented out or
marked as unmanaged.
-n No-Op preview mode. This shows operations that would be
performed, but takes no actions that affect the live system
or data. This is valid only for command for which it is
indicated.
GENERAL OPTIONS:
-v<n> Set verbosity 1-5 (-v1 = quiet, -v5 = highest).
-L <log>
Specify the path to a log file, or the special value 'off' to disable
logging. By default, all output (stdout and stderr) is captured
in a log file named:
$LOGS/hms.<NAME>.<DATESTAMP>.log
-C Check the HMS config file and then stop. This can be used to
verify the syntax of the HMS config file without starting an actual
failover.
-si Operate silently. All output (stdout and stderr) is redirected to
the log only; no output appears on the terminal. This cannot be used
with '-L off'.
-D Set extreme debugging verbosity.
-S Step mode - if running interactively prompt user to enter return
before progressing to the next step. The '-S' option is not supported
when called via p4broker (i.e. called as 'p4 hms' rather than calling
the $THISSCRIPT script directly).
THIS FLAG IS NOT IMPLEMENTED AND WILL BE DEFERRED TO hms 2.0 or beyond.
HELP OPTIONS:
-h Display short help message
-man Display man-style help message
-V Dispay version info for this script and its libraries.
DEPENDENCIES:
The bash shell in /bin/bash must be 4.0 or higher.
Note: This precludes operation on Mac OSX, at least
up thru and include OSX 10.13.4 (High Sierra), which
ships with bash 3.x.
EXAMPLES:
Get a status in the master p4d for Instance 1:
$THISSCRIPT status 1:master
Get a status in the second replica p4d for Instance 2:
$THISSCRIPT status 2:p4d-r02
Get a status for all known components, optimisitically assuming
those we haven't coded status logic for yet are configured are happy:
$THISSCRIPT status all -o
Show whether the second broker for Instance 2 can be updated to a
newer patch version of p4broker, within the current major version:
$THISSCRIPT update 1:p4b02 -n
Upgrade the second broker for Instance 2 to the latest patch version
of p4broker, within the current major version:
$THISSCRIPT update 1:p4b02
Show what components in Instance 1 would be affected by site-wide
topology upgrade:
$THISSCRIPT upgrade 1 -n
Upgrade all components in Instance 1 in the correct order.
$THISSCRIPT upgrade 1 -n
Perform an unscheduled failover for all instances mastered on host
bos-helix-01:
$THISSCRIPT failover ha h:bos-helix-01 u
Perform a scheduled failover for all instances mastered on host
syd-helix-04:
$THISSCRIPT failover ha h:syd-helix-04 s
Perform an unscheduled local failover for Instance 1 on its currently
configured master host:
$THISSCRIPT failover local i:1 u
"
fi
exit 1
}
#==============================================================================
# Command Line Processing
declare HelixTopologyCfg=$P4CCFG/HelixTopology.cfg
declare GlobalOptions=
declare Command=
declare SubCommand=
declare Target=
declare FailoverPath=
declare FailoverScope=
declare FailoverStyle=
declare -i PreflightCheck=0
declare -i RunTestSuite=0
declare -i OptimisticStatus=0
declare -i Interactive=1
declare -i StepMode=0
declare -i OverallReturnStatus=0
declare -i shiftArgs=0
set +u
while [[ $# -gt 0 ]]; do
case $1 in
(show) Command=show;;
(pull)
Command=pull
Target=$2
shiftArgs=1
;;
(df)
Command=df
Target=$2
shiftArgs=1
;;
(dfm)
Command=dfm
usageMsg="Invalid Usage. Usage for 'hms $Command' command is:\n\nhms $Command {on|off|status} <instance>"
[[ $# -lt 3 ]] && bail "$usageMsg"
SubCommand=$2
[[ $SubCommand =~ ^(on|off|status)$ ]] || bail "$usageMsg"
Target=$3
shiftArgs=2
;;
(start|stop|status)
Command=$1
usageMsg="Invalid Usage. Usage for 'hms $Command' command is:\n\nhms $Command <instance>[:<component>]"
[[ $# -lt 2 ]] && bail "$usageMsg"
Target=$2
shiftArgs=1
;;
(failover|fo)
Command=failover
usageMsg="Invalid Usage. Usage for 'hms failover' command is:\n\nhms failover <path> {i|h}:<scope> <style>"
[[ $# -lt 4 ]] && bail "$usageMsg"
FailoverPath=$2
FailoverScope=$3
FailoverStyle=$4
[[ $FailoverPath == "-"* || $FailoverScope == "-"* || $FailoverStyle == "-"* ]] && \
bail "$usageMsg"
[[ $FailoverScope =~ ^(i|h): ]] || bail "$usageMsg"
# Convert short form of 'style' argument, 'u' or 's', to long form for code
# readability.
[[ $FailoverStyle =~ ^(s|scheduled|u|unscheduled)$ ]] || bail "$usageMsg"
[[ $FailoverStyle =~ ^(s|scheduled)$ ]] && FailoverStyle=Scheduled
[[ $FailoverStyle =~ ^(u|unscheduled)$ ]] && FailoverStyle=Unscheduled
shiftArgs=3
;;
(update|upgrade)
Command=$1
usageMsg="Invalid Usage. Usage for 'hms $Command' command is:\n\nhms $Command <instance>[:<component>]"
[[ $# -lt 2 ]] && bail "$usageMsg"
Target=$2
shiftArgs=1
;;
(-c) HelixTopologyCfg=$2; shiftArgs=1;;
(-C) PreflightCheck=1;;
(-S) StepMode=1;;
(-y) Interactive=0;;
(-o) OptimisticStatus=1;;
(-T) RunTestSuite=1;;
(-h) usage -h;;
(-man) usage -man;;
(-V) show_versions; exit 0;;
(-v1) export VERBOSITY=1;;
(-v2) export VERBOSITY=2;;
(-v3) export VERBOSITY=3;;
(-v4) export VERBOSITY=4;;
(-v5) export VERBOSITY=5;;
(-v) export VERBOSITY=${2:-5}; shiftArgs=1;;
(-L) export P4U_LOG=$2; shiftArgs=1;;
(-si) SilentMode=1;;
(-n) export NO_OP=1;;
(-D) set -x;; # Debug; use 'set -x' mode.
(*) usage -h "Unknown arg ($1).";;
esac
# Shift (modify $#) the appropriate number of times.
shift; while [[ $shiftArgs -gt 0 ]]; do
[[ $# -eq 0 ]] && usage -h "Bad usage; incorrect number of arguments."
shiftArgs=$shiftArgs-1
shift
done
done
set -u
#==============================================================================
# Command Line Verification
[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
usage -h "Cannot use '-si' with '-L off'."
if [[ $RunTestSuite -eq 1 ]]; then
[[ -n "$HelixTopologyCfg" ]] && \
usage -h "The '-c <helix_topology_cfg>' argument is not allowed with -T."
else
[[ -z "$HelixTopologyCfg" ]] && \
usage -h "The '-c <helix_topology_cfg>' argument is required."
fi
# Only allow interactive mode
if [[ $Interactive -eq 1 && $HMS_CALLED_BY_WRAPPER -eq 1 ]]; then
if [[ ! $Command =~ ^(show|status)$ ]]; then
usage -h "The -y (auto-confirm) flag is required when called via the broker. Try calling the hms script directly, not with 'p4 hms'."
fi
fi
# Only allow step mode if running interactively so prompt/response can be handled
if [[ $StepMode -eq 1 && $HMS_CALLED_BY_WRAPPER -eq 1 ]]; then
bail "The -S (Step Mode) isn't supported when called via the broker. Try calling the hms script directly, not with 'p4 hms'."
fi
# Disable logging by default if we're just doing a status or show command
# called from the broker.
if [[ $HMS_CALLED_BY_WRAPPER -eq 1 && $Command =~ ^(show|status)$ ]]; then
export P4U_LOG=off
fi
if [[ $OptimisticStatus -eq 1 && $Command != status ]]; then
usage -h "The '-o' option (optimistic status) is only valid with the 'status' command."
fi
#==============================================================================
# Main Program
declare component=
declare instanceComponents=
declare componentList=
declare -i statusAllOK=1
declare -i i
if [[ $RunTestSuite -eq 1 ]]; then
if [[ -x $HMS_HOME/test/run_test_suite.sh ]]; then
$HMS_HOME/test/run_test_suite.sh
OverallReturnStatus=$?
exit $OverallReturnStatus
else
bail "The expected test suite wrapper script is missing: $HMS_HOME/test/run_test_suite.sh."
fi
fi
# Very first thing - Check BASH_VERSION
case $BASH_VERSION in
(1*|2*|3*) echo -e "\n The version of bash is too old. This script requires bash 4.0 or higher.\n This version is $BASH_VERSION. If working on Mac, do a web search for\n 'bash 4 mac' for options on acquring a bash 4.x. Please acquire a new bash\n and install it somewhere, perhaps /usr/local/bin/bash, and change the '#!'\n line at the top of this script ($THISSCRIPT) to reference the newer bash.\n"
exit 1;;
esac
# If the topology file is specified as '/absolute/path/top.cfg', use it verbatim.
# If the topolgoy file is specified as 'top.cfg', look for top.cfg in the
# current dir first, and then check for top.cfg in /p4/common/config. Otherwise
# give up.
if [[ ! -r "$HelixTopologyCfg" ]]; then
if [[ "$HelixTopologyCfg" != "/"* && -r "$P4CCFG/$HelixTopologyCfg" ]]; then
HelixTopologyCfg=$P4CCFG/$HelixTopologyCfg
else
bail "The Helix Topology file [$HelixTopologyCfg] does not exist."
fi
fi
SiteTag=$(grep ^NAME= "$HelixTopologyCfg" 2>/dev/null | cut -d '=' -f 2)
[[ -z "$SiteTag" ]] && \
bail "The Helix Topology file [$HelixTopologyCfg] is missing a 'NAME=' definition."
if [[ $P4U_LOG == Unset ]]; then
export P4U_LOG="$LOGS/hms.$SiteTag.$(date +'%Y%m%d-%H%M%S').log"
fi
trap terminate EXIT SIGINT SIGTERM
if [[ "${P4U_LOG}" != off ]]; then
touch ${P4U_LOG}
# If we can't put the log where we want it, try putting it in /tmp
# instead. We don't want to abort failover operations for lack of
# a log file, as long as we can write one somehwere.
if [[ $? -ne 0 ]]; then
export P4U_LOG=/tmp/${P4U_LOG##*/}
touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]."
fi
# Redirect stdout and stderr to a log file.
if [[ $SilentMode -eq 0 ]]; then
exec > >(tee ${P4U_LOG})
exec 2>&1
else
exec >${P4U_LOG}
exec 2>&1
fi
initlog
fi
msg "${H}\nStarted at $(date). Versions:"
[[ $VERBOSITY -gt 3 ]] && show_versions
# Environment isolation: Clear Perforce server environment settings.
unset P4AUDIT P4AUTH P4CHANGE P4CONFIG P4DEBUG P4DESCRIPTION P4JOURNAL \
P4LOG P4NAME P4PORT P4ROOT P4SSLDIR P4TARGET P4TICKETS P4TRUST
# Environment isolation: Unset P4CONFIG, and point P4ENVIRO to /dev/null
# to avoid accidentally interacting with a personal DVCS repo which slickly
# applies a P4CONFIG value even after we unset it, by reading it from the
# P4ENVIRO if it finds one (defaulting to ~/.p4enviro).
export P4CONFIG=Unset
export P4ENVIRO=/dev/null/.p4enviro
[[ -z "$P4BIN" || ! -x "$P4BIN" || -z "$P4DBIN" || ! -x "$P4DBIN" ]] && \
bail "\n\nThe p4 and p4d binaries are missing or are not executable.\n"
P4BinRev=$($P4BIN -V|grep ^Rev)
P4DBinRev=$($P4DBIN -V|grep ^Rev)
msg "Using these p4/p4d executables:\n$P4BIN: $P4BinRev\n$P4DBIN: $P4DBinRev\n"
if [[ $PreflightCheck -eq 0 ]]; then
load_helix_topology "$HelixTopologyCfg" "$RequiredCfgVersion" ||\
bail "Helix Topology definition failed sanity check."
else
load_helix_topology "$HelixTopologyCfg" "$RequiredCfgVersion" ||\
bail "Helix Topology definition in $HelixTopologyCfg FAILED preflight sanity checks."
msg "Verified: Helix Topology defined in $HelixTopologyCfg PASSED preflight sanity checks."
fi
msg "\n${H}\n"
case "$Command" in
(show)
msg "Command: $Command"
show_helix_topology
;;
(start|stop)
msg "Command: $Command Target=[$Target]"
change_component_status "$Command" "$Target"
;;
(dfm)
msg "Command: $Command SubCommand=[$SubCommand] Target=[$Target]"
down_for_maintenance "$SubCommand" "$Target"
;;
(pull)
msg "Command: $Command Target=[$Target]"
get_replica_status "$Target"
;;
(df)
msg "Command: $Command Target=[$Target]"
get_diskspace "$Target"
;;
(status)
msg "Command: $Command Target=[$Target]"
componentList=$(get_component_list "$Target" 0) ||\
bail "Failed to get a valid list of components."
for component in $componentList; do
get_component_status "$component" $OptimisticStatus ||\
statusAllOK=0
statusCode=${ComponentStatusCode[$component]:-Unset}
statusMsg=${ComponentStatusMsg[$component]:-Unset}
if [[ $statusCode != Unset && $statusMsg != Unset ]]; then
qmsg "Status: $component is $statusMsg (code $statusCode)."
fi
done
if [[ $statusAllOK -eq 1 ]]; then
msg "${H}\nStatus for all components checked is OK."
else
msg "${H}\nStatus for one or more components FAILED."
OverallReturnStatus=1
fi
;;
(failover)
msg "Command: $Command Path=[$FailoverPath] Scope=[$FailoverScope] Style=[$FailoverStyle]"
[[ $NO_OP -eq 1 ]] && msg "Running in NO-OP Preview Mode due to '-n'."
failover "$FailoverPath" "$FailoverScope" "$FailoverStyle"
if [[ $? -eq 0 ]]; then
msg "${H}\nFailover completed successfully.\n"
else
errmsg "Failover did NOT complete successfully.\n"
OverallReturnStatus=1
fi
;;
(update)
msg "Command: $Command Target=[$Target]"
[[ $NO_OP -eq 1 ]] && msg "Running in NO-OP Preview Mode due to '-n'."
componentList=$(get_component_list "$Target" 1) ||\
bail "Failed to get a valid list of components."
msg "Checking component status prior to update."
for component in $componentList; do
get_component_status "$component" $OptimisticStatus ||\
statusAllOK=0
statusCode=${ComponentStatusCode[$component]:-Unset}
statusMsg=${ComponentStatusMsg[$component]:-Unset}
if [[ $statusCode != Unset && $statusMsg != Unset ]]; then
qmsg "Status: $component is $statusMsg (code $statusCode)."
fi
done
if [[ $statusAllOK -eq 1 ]]; then
msg "${H}\nStatus for all components checked is OK."
else
bail "${H}\nStatus for one or more components FAILED. Aborting $Command action."
OverallReturnStatus=1
fi
;;
(upgrade)
msg "Command: $Command Target=[$Target]"
[[ $NO_OP -eq 1 ]] && msg "Running in NO-OP Preview Mode due to '-n'."
msg "Checking component status prior to upgrade."
componentList=$(get_component_list "$Target" 1) ||\
bail "Failed to get a valid list of components."
for component in $componentList; do
get_component_status "$component" OptimisticStatus ||\
statusAllOK=0
statusCode=${ComponentStatusCode[$component]:-Unset}
statusMsg=${ComponentStatusMsg[$component]:-Unset}
if [[ $statusCode != Unset && $statusMsg != Unset ]]; then
qmsg "Status: $component is $statusMsg (code $statusCode)."
fi
done
if [[ $statusAllOK -eq 1 ]]; then
msg "${H}\nStatus for all components checked is OK."
else
bail "${H}\nStatus for one or more components FAILED. Aborting $Command action."
OverallReturnStatus=1
fi
;;
(*)
bail "Unknown command. Bad usage."
;;
esac
if [[ $OverallReturnStatus -eq 0 ]]; then
msg "${H}\nAll processing completed successfully.\n"
else
msg "${H}\nProcessing completed, but with errors. Scan above output carefully.\n"
fi
msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"
[[ -r "${P4U_LOG}" ]] && msg "Log file is: $P4U_LOG\n${H}\n"
exit $OverallReturnStatus