hms #1

#!/p4/common/bash/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment
set -u

EnvFile=/p4/common/bin/p4_vars

if [[ -r "$EnvFile" ]]; then
   source "$EnvFile" hms
else
   echo -e "\n\nERROR: The environment file $EnvFile is missing.\n\n"
   exit 1
fi

# Override P4CONFIG to enhance environment isolation.
export P4CONFIG=Undefined
export P4ENVIRO=/dev/null/.p4enviro

# Allow override of P4U_HOME, which is set only when testing P4U utility scripts.
export P4U_HOME=${P4U_HOME:-$P4CBIN}
export P4U_LIB=${P4U_LIB:-$P4CLIB}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=Unset
export HMS_HOME=${HMS_HOME:-/p4/common/hms}

# Indicate whether we are called by the brokre wrapper.
export HMS_CALLED_BY_WRAPPER=${HMS_CALLED_BY_WRAPPER:-0}

# Load bash libs.
declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"
BASH_LIBS+=" $HMS_HOME/lib/hms_load_and_verify.sh"

for bash_lib in $BASH_LIBS; do
   source $bash_lib
done

export VERBOSITY=3

declare SiteTag=Unset
declare -a InstanceName
declare -a InstanceUserPorts
declare -a InstanceMasterHost
declare -a InstanceServerPort
declare -a InstanceBrokerPort
declare -a InstanceManaged
declare -i InstanceCount=0

declare -a FailoverName
declare -a FailoverType
declare -a FailoverMasterHost
declare -a FailoverBackupHost
declare -a FailoverInstanceList
declare -a FailoverDesc
declare -i FailoverCount=0

declare -i SilentMode=0
declare Version=1.0.3

# A lexiographic compare is used to compare required vs. actual version of
# the Helix Topology file.  Helix Topology files define their version with 3
# digits, e.g. 3.0.0 or 3.0.1.  The RequiredCfgVersion here must be defined
# with two digits, e.g. "3.0" if "3.0.0" is good enough, since 3.0.0 is
# lexiographically greater than 3.0.
declare RequiredCfgVersion=1.0

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit $OverallReturnStatus
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT {status|update|failover} [-L <log>] [-si] [-v<n>] [-T] [-n] [-D] [-S]

The *status*, *update*, and *failover* commands take additional arguments:

status all | <instance>[:<component>]

update all | <instance>[:<component>]

upgrade all | <instance>[:<component>]

failover <target> <scope> <style>

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This is the Perforce Helix Management System,
	$THISSCRIPT v$Version.

	This script is self-logging.  That is, all output displayed on the
	screen (stdout and stderr) is simultaneously captured in a log file.
	You do not need to run this script with redirection operators like
	'> log' or '2>&1', and do not need to use 'tee.'  The default log
	file is:

	$LOGS/hms.<NAME>.<DATESTAMP>.log


HMS CONFIG FILES:
	The HMS config file defines a Helix global topology.

HMS COMMANDS:

status all | <instance>[:<component>]
	Get status of components defined in the Helix Topology
	file.  Specify all for a site-wide global status, an
	instance name for all components related to a given instance
	or a specific component within an instance.

update all | <instance>[:<component>]
	Update to the latest versions of software available for the current major
	release (e.g. 2016.1).  An update will not upgrade to a new major version.

	Specify 'all' to perform a site-wide update of all defined Helix topology
	components.  Updates are done one instance at a time.  Update processing aborts
	in event of a failure to upgrade any componnet.

	Specify an instance name to update all components associated with a given Helix
	instance.

	Use with the '-n' flag to see what versions would be updated.

upgrade all | <instance>[:<component>]
	Upgrade to the latest versions of software available for the latest GA
	release, upgrading to new major versions if needed.

	Specify 'all' to perform a site-wide upgrade of all defined Helix topology
	components.  upgrades are done one instance at a time.  Upgrade processing aborts
	in event of a failure to upgrade or update any componnet.

	Use with the '-n' flag to see what versions would be updated or upgraded.

failover <target> <scope> <style>
	Execute a failover to a pre-defined <target> name defined in the
	Helix Topology file.  Multiple failover targets can be defined
	for any given instance, defining (for example) a "local" failover
	option, an HA option, and one or more DR options.

	The <scope> can be either a Helix instance name, or the hostname
	of a machine.  If an instance is specified, that instance fails over
	to the designated target.  If a machine is specified, all instances
	currently mastered on that machine fail over (e.g. in event of hardware
	failure of that host).  When a machine is defined, the <target> should
	be valid for all instances on that machine; any instances for which the
	target is not valid will be skipped in the failover processing (but will
	not prevent it from starting).

	The <style> is either 'scheduled' or 'unscheduled'.  This impacts how
	the failover process occurs.  In a scheduled failover, all systems are
	expected to be operating normally at the start of the failover process,
	and the failover only occurs if all systems are operating normally.
	This can be used to take a perfectly working machine offline for a time,
	e.g. to add memory or something.  An 'unscheduled' failover is executed
	in reaction to a problem of some kind, such as hardware failure.  An
	'unscheduled' failover should be used when it is known that master server
	is offline or otherwise not usable.  If it can be reached, the master
	server is shutdown as part of an unscheduled failover.


COMMAND LINE OPTIONS:
 -c <helix_topology_cfg>
	Specify the path to your hand-crafted Helix Topology Config.
	documentation below on the format of this file.  The default
	file is $P4CCFG/helix_topology.cfg

 -T	Execute the HMS Test Suite, as documented in the EXAMPLES
	below.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) is captured
	in a log file named:

	$LOGS/bbi.<NAME>.<DATESTAMP>.log

 -C	Check the HMS config file and then stop.  This can be used to
	verify the syntax of the HMS config file without starting an actual
	failover.

-si	Operate silently.  All output (stdout and stderr) is redirected to
	the log only; no output appears on the terminal.  This cannot be used
	with '-L off'.
      
 -n	No-Op.  Doesn't run commands that affect data.  This is for
	debugging purposes.

 -D     Set extreme debugging verbosity.

 -S     Step mode - if running interactively prompt user to enter return
	before progressing to the next step.  The '-S' option is not supported
	when called via p4broker (i.e. called as 'p4 hms' rather than just 'hms').

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

DEPENDENCIES:
	The bash shell must be installed as /p4/common/bash/bin/bash, and should be
	bash 4.3 or higher, acquired from ftp.gnu.org.

EXAMPLES:
	hms status

	hms update -n

	hms upgrade -n

	hms upgrade

	hms upgrade -S

	hms failover HA helix-01 unscheduled

"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare ConfigFile=$P4CCFG/HelixTopology.cfg
declare GlobalOptions=
declare Command=
declare Target=
declare FailoverScope=
declare FailoverStyle=
declare -i PreflightCheck=0
declare -i RunTestSuite=0
declare -i StepMode=0
declare -i OverallReturnStatus=0
declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (show) Command=show;;
      (status)
         Command=status
         usageMsg="Invalid Usage.  Usage for 'hms status' command is:\n\nhms status <target>"
         [[ $# -lt 2 ]] && bail "$usageMsg"
         Target=$2
         shiftArgs=1
      ;;
      (failover|fo)
         Command=failover
         usageMsg="Invalid Usage.  Usage for 'hms failover' command is:\n\nhms failover <target> <scope> <style>"
         [[ $# -lt 4 ]] && bail "$usageMsg"
         Target=$2
         FailoverScope=$3
         FailoverStyle=$4
         [[ "$Target" == "-"* || $FailoverScope == "-"* || $FailoverStyle == "-"* ]] && \
            bail "$usageMsg"
            
         shiftArgs=3
      ;;
      (update)
         Command=update
         usageMsg="Invalid Usage.  Usage for 'hms update' command is:\n\nhms update <target>"
         [[ $# -lt 2 ]] && bail "$usageMsg"
         Target=$2
         shiftArgs=1
      ;;
      (upgrade)
         Command=upgrade
         usageMsg="Invalid Usage.  Usage for 'hms upgrade' command is:\n\nhms upgrade <target>"
         [[ $# -lt 2 ]] && bail "$usageMsg"
         Target=$2
         shiftArgs=1
      ;;
      (-c) ConfigFile=$2; shiftArgs=1;;
      (-C) PreflightCheck=1;;
      (-S) StepMode=1;;
      (-T) RunTestSuite=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 0;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-v) export VERBOSITY=$2; shiftArgs=1;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

if [[ $RunTestSuite -eq 1 ]]; then
   [[ -n "$ConfigFile" ]] && \
      usageError "The '-c <helix_topology_cfg>' argument is not allowed with -T."

else
   [[ -z "$ConfigFile" ]] && \
      usageError "The '-c <helix_topology_cfg>' argument is required."
fi

# Only allow step mode if running interactively so prompt/response can be handled
if [[ $StepMode -eq 1 && $HMS_CALLED_BY_WRAPPER -eq 1 ]]; then
   bail "The -S (Step Mode) isn't supported when called via the broker. Try calling the hms script directly, not with 'p4 hms'."
fi

#==============================================================================
# Main Program

if [[ $RunTestSuite -eq 1 ]]; then
   if [[ -x $HMS_HOME/test/run_test_suite.sh ]]; then
      $HMS_HOME/test/run_test_suite.sh
      OverallReturnStatus=$?
      exit $OverallReturnStatus
   else
      bail "The expected test suite wrapper script is missing: $HMS_HOME/test/run_test_suite.sh."
   fi
fi

# Very first thing - Check BASH_VERSION
case $BASH_VERSION in
   (1*|2*|3*) echo -e "\n The version of bash is too old.  This script requires bash 4.0 or higher.\n This version is $BASH_VERSION.  If working on Mac, do a web search for\n 'bash 4 mac' for options on acquring a bash 4.x.  Please acquire a new bash\n and install it somewhere, perhaps /usr/local/bin/bash, and change the '#!'\n line at the top of this script ($THISSCRIPT) to reference the newer bash.\n"
   exit 1;;
esac

[[ -r "$ConfigFile" ]] || bail "The Helix Topology file [$ConfigFile] does not exist."

SiteTag=$(grep ^NAME= "$ConfigFile" 2>/dev/null | cut -d '=' -f 2)
[[ -z "$SiteTag" ]] && \
   bail "The Helix Topology file [$ConfigFile] is missing a 'NAME=' definition."

if [[ $P4U_LOG == Unset ]]; then
   export P4U_LOG="$LOGS/hms.$SiteTag.$(date +'%Y%m%d-%H%M%S').log"
fi

trap terminate EXIT SIGINT SIGTERM

if [[ "${P4U_LOG}" != off ]]; then
   touch ${P4U_LOG}
   # If we can't put the log where we want it, try putting it in /tmp
   # instead.  We don't want to abort failover operations for lack of
   # a log file, as long as we can write one somehwere.
   if [[ $? -ne 0 ]]; then
      export P4U_LOG=/tmp/${P4U_LOG##*/}
      touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]."
   fi

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee ${P4U_LOG})
      exec 2>&1
   else
      exec >${P4U_LOG}
      exec 2>&1
   fi

   initlog
fi

msg "${H}\nStarted at $(date).  Versions:"
show_versions

# Environment isolation:  Clear Perforce environment settings.
# Set P4ENVIRO to a junk value so as to avoid accidentally interacting
# with a personal DVCS repo which slickly applies a P4CONFIG value
# even after we unset it, by reading it from the P4ENVIRO file
# (~/.p4enviro by default).
unset P4AUDIT P4AUTH P4CHANGE P4CONFIG P4DEBUG P4DESCRIPTION P4JOURNAL \
   P4LOG P4NAME P4PORT P4ROOT P4SSLDIR P4TARGET P4TICKETS P4TRUST
export P4ENVIRO=/dev/null/.p4enviro

[[ -z "$P4BIN" || ! -x "$P4BIN" || -z "$P4DBIN" || ! -x "$P4DBIN" ]] && \
   bail "\n\nThe p4 and p4d binaries are missing or are not executable.\n"

P4BinRev=$($P4BIN -V|grep ^Rev)
P4DBinRev=$($P4DBIN -V|grep ^Rev)

msg "Using these p4/p4d executables:\n$P4BIN:  $P4BinRev\n$P4DBIN: $P4DBinRev\n"

if [[ $PreflightCheck -eq 0 ]]; then
   load_helix_topology "$ConfigFile" "$RequiredCfgVersion" ||\
      bail "Helix Topology definition failed sanity check."

else
   load_helix_topology "$ConfigFile" "$RequiredCfgVersion" ||\
      bail "Helix Topology definition in $ConfigFile FAILED preflight sanity checks."

   msg "Verified: Helix Topology defined in $ConfigFile PASSED preflight sanity checks."
fi

msg "\n${H}\n"

case "$Command" in
   (show)
      msg "Command: $Command"
   ;;
   (status)
      msg "Command: $Command Target=[$Target]"
   ;;
   (failover)
      msg "Command: $Command Target=[$Target] Scope=[$FailoverScope] Style=[$FailoverStyle]"
   ;;
   (update)
      msg "Command: $Command Target=[$Target]"
   ;;
   (upgrade)
      msg "Command: $Command Target=[$Target]"
   ;;
   (*)
      bail "Unknown command.  Bad usage."
   ;;
esac

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\nAll processing completed successfully.\n"
else
   msg "${H}\nProcessing completed, but with errors.  Scan above output carefully.\n" 
fi

msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

[[ -r "${P4U_LOG}" ]] && msg "Log file is: $P4U_LOG\n${H}\n"

exit $OverallReturnStatus