#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ #============================================================================== # Declarations and Environment set -u if [[ ${BASH_VERSINFO[0]} -lt 4 ]]; then echo -e "\n\nERROR: $0 requires bash version 4.x+; current bash version is $BASH_VERSION." exit 1 fi EnvFile=${SDP_ENV:-/p4/common/bin/p4_vars} if [[ -r "$EnvFile" ]]; then source "$EnvFile" hms else echo -e "\n\nERROR: The environment file $EnvFile is missing.\n\n" exit 1 fi if [[ $(id -u) -eq 0 ]]; then exec su - $OSUSER -c "$0 $*" elif [[ $(id -u -n) != $OSUSER ]]; then echo "$0 can only be run by root or $OSUSER" exit 1 fi # Allow override of P4U_HOME, which is set only when testing P4U utility scripts. export P4U_HOME=${P4U_HOME:-$P4CBIN} export P4U_LIB=${P4U_LIB:-$P4CLIB} export P4U_ENV=$P4U_LIB/p4u_env.sh export P4U_LOG=Unset export HMS_HOME=${HMS_HOME:-/p4/common/hms} export HMS_SCRIPTS=$HMS_HOME/scripts # Indicate whether we are called by the broker wrapper. export HMS_CALLED_BY_WRAPPER=${HMS_CALLED_BY_WRAPPER:-0} # Load bash libs. declare BASH_LIBS=$P4U_ENV BASH_LIBS+=" $P4U_LIB/libcore.sh" BASH_LIBS+=" $P4U_LIB/libp4u.sh" BASH_LIBS+=" $P4U_LIB/hms_actions.sh" BASH_LIBS+=" $P4U_LIB/hms_load_and_verify.sh" for bash_lib in $BASH_LIBS; do source $bash_lib done # After sourcing bash libs, set default VERBOSITY for this program, which can # be overridden on the command line. export VERBOSITY=3 # SDP Instance data, with associative arrays (i.e. arrays that can be indexed # by a string rather than a purely numeric name). These Instance* vars are # indexed by the SDP instance name. declare -A InstanceUserPorts declare -A InstanceMasterHost declare -A InstanceServerPort declare -A InstanceBrokerPort declare -A InstanceManaged declare -A InstanceDesc declare -A InstanceComponents declare -A InstanceFailoverOptions # Helix Topology Components data, indexed by the fully qualified component name # of the form "SDPInstance:ComponentName", e.g. "1:p4d-r01". declare -A ComponentType declare -A ComponentMasterHost declare -A ComponentURL declare -A ComponentBackupHost declare -A ComponentManaged declare -A ComponentDesc declare -A ComponentStatusCode declare -A ComponentStatusMsg declare -A ComponentMajorVersion declare -A ComponentMinorVersion declare -A ComponentVersion # Failover Options, indexed by the fully qualified failover option name of the # form "SDPInstance:FailoverOptionName", e.g. "1:dr". declare -A FailoverType declare -A FailoverMasterHost declare -A FailoverBackupHost declare -A FailoverInstanceList declare -A FailoverActive declare -A FailoverDesc declare -i SilentMode=0 # Increment the Version manually on each submit. (We avoid +k filetype of # Perforce due to various complications moving across p4d instances). declare Version=1.0.33 # The RequiredCfgVersion check is done with a lexiographic compare vs. # the value defined in the Helix Topology file. Helix Topology files define # their file format version with 3 digits, e.g. 3.0.0 or 3.0.1. The # RequiredCfgVersion here must be defined with two digits, e.g. "3.0" if #"3.0.0" is good enough, since 3.0.0 is lexiographically greater than 3.0. declare RequiredCfgVersion=1.0 #============================================================================== # Local Functions #------------------------------------------------------------------------------ # Function: terminate function terminate { # Disable signal trapping. trap - EXIT SIGINT SIGTERM # Don't litter. cleanTrash vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus" # Stop logging. [[ "${P4U_LOG}" == off ]] || stoplog # With the trap removed, exit. exit $OverallReturnStatus } #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). # The default is -h. # # $2 - error message (optional). Specify this if usage() is called due to # user error, in which case the given message displayed first, followed by the # standard usage message (short or long depending on $1). If displaying an # errror, usually $1 should be -h so that the longer usage message doesn't # obsure the error message. # # Sample Usage: # usage # usage -h # usage -man # usage -h "Incorrect command line usage." #------------------------------------------------------------------------------ function usage { declare style=${1:--h} declare errorMessage=${2:-Unset} if [[ $errorMessage != Unset ]]; then echo -e "\n\nUsage Error:\n\n$errorMessage\n\n" fi echo -e "USAGE for $THISSCRIPT v$Version: $THISSCRIPT show $THISSCRIPT status {all|<instance>[:<component>]} [-o] $THISSCRIPT {start|stop} <instance>[:<component>] $THISSCRIPT dfm {on|off|status} <instance> $THISSCRIPT {update|upgrade} <instance>[:<component>] $THISSCRIPT failover <path> {i|h}:<instance_or_host> <style> $THISSCRIPT [-h|-man|-V] " if [[ $style == -man ]]; then echo -e " DESCRIPTION: This is the Perforce Helix Management System, $THISSCRIPT v$Version. This script is self-logging. That is, all output displayed on the screen (stdout and stderr) is simultaneously captured in a log file. You do not need to run this script with redirection operators like '> log' or '2>&1', and do not need to use 'tee.' The default log file is: $LOGS/hms.<NAME>.<DATESTAMP>.log HMS CONFIG FILES: The HMS config file defines a Helix global topology. HMS COMMANDS: status {all|<instance>[:<component>]} [-o] Get status of components defined in the Helix Topology file. Specify all for a site-wide global status, an instance name for all components related to a given instance or a specific component within an instance. start|stop <instance>[:<component>] Start or stop a specific component in the topology, or start/stop all components associated with a given instance. dfm {on|off|status} <instance> The 'dfm on' command puts all brokers for the specified SDP instance into \"Down For Maintenance\" mode. This involves stopping the broker running with the default config (p4_N.broker.cfg) and starting the broker using the DFM config file, p4_N.broker.dfm.cfg. The dfm broker config file is expected to be configured to reject all user commands and display a friendly message for users telling them that the Helix Server is offline for maintenance. The 'dfm off' command brings all brokers for the specified SDP instance (or all instances) back online with the default config, after shutting down brokers running with the dfm config. The 'dfm status' command checks the status of brokers running with the dfm config. update <instance>[:<component>] [-n] Update to the latest versions of software available for the current major release for the given component. An update will not upgrade to a new major version. Specify 'all' to perform a site-wide update of all defined Helix topology components. Updates are done one instance at a time. Update processing aborts in event of a failure to upgrade any component. Specify an instance name to update all components associated with a given Helix instance. Update processing aborts in event of a failure to update any component. The update processing varies depending on the component to be updated. It essentially consists of: 1. Stop the service. 2. Replace the executable (e.g .p4d, p4broker, Swarm files) to the latest patched executable. 3. Update SDP symlinks as needed. 4. Restart the service. For p4d, a database upgrade is not done, and no checkpoint processing occurs. For any given instance, updates occur in the proper order, with the master/commit server being updated last. In case of daisy chain of replicas, those farthest removed from the commit server are updated first. Use with the '-n' flag to see what versions would be updated. upgrade <instance>[:<component>] [-n] Upgrade to the latest versions of software available for the latest GA release, upgrading to new major versions if needed. Specify 'all' to perform a site-wide upgrade of all defined Helix topology components. upgrades are done one instance at a time. Upgrade processing aborts in event of a failure to upgrade or update any component. Specify an instance name to upgrade all components associated with a given Helix instance. Upgrade processing aborts in event of a failure to upgrade any component. The upgrade processing varies depending on the component to be upgraded. It essentially consists of: 1. Stop the service. 2. Replace the executable (e.g .p4d, p4broker, Swarm files) to the latest patched executable. 3. Restart the service. For p4d, an offline checkpoint is initiated at the start of processing. For any given instance, upgrades occur in the proper order, with the master/commit server being upgraded last. In case of a daisy chain of replicas, those farthest removed from the commit server are upgraded first. Use with the '-n' flag to see what versions would be updated or upgraded. failover <path> {h|i}:<scope> <style> [-n] Execute a failover using a pre-defined <path> name defined in the Helix Topology file. Multiple failover paths can be defined for a given SDP instance, defining (for example) a "local" failover option (using offline databases on the same host), an HA option, and one or more DR options. The <scope> must be prefixed with i: or h:, as in i:<SDPInstance> or h:<Hostname>. If an SDP instance is specified, that instance fails over to the designated backup host for the path selected. If a machine is specified, all instances currently mastered on that machine failover. In event of hardware failure of a host, this option should be considered. When a machine is specified for the scope, the <path> should be valid for all instances on that machine. Any instances for which the path is not valid will be skipped in the failover processing (but will not prevent it from handling other instances). The <style> is either 'scheduled' ('s' for short) or 'unscheduled' ('u' for short). This impacts how the failover process occurs. In a scheduled failover, all systems are expected to be operating normally at the start of the failover process, and the failover only occurs if all systems are operating normally. Scheduled failover can be used to take a perfectly working machine offline for a time, e.g. to add memory. An 'unscheduled' failover is executed in reaction to a problem of some kind, such as hardware failure. An 'unscheduled' failover should be used when it is known that master server is offline or otherwise not usable. If it can be reached, the master server is shutdown as part of an unscheduled failover. The 'failover' command can be abbreviated as 'fo'. HOST ALIAS UPDATE: External to processing of this script, the host alias used by users to taget the master server must be updated. This may involve a DNS change, virtual IP change, anycast update, etc. This must be done before the failover is complete and service can be restored for users. HMS GENERAL COMMAND OPTIONS (valid for all commands): -c <helix_topologoy_config> Specify the path to your hand-crafted Helix Topology configuration file. See documentation in the default config file regarding the required format for topology configuration. The default file is $P4CCFG/HelixTopology.cfg. Specify something like '-c /full/path/HelixTopology.cfg' or '-c HelixTopology.test.cfg'. With the latter example, the file is presumed to be in the $P4CCFG directory. The toplogy configuration file defines SDP Instances, Helix Topology Components, and Failover Options. This option is intended for development of changes to Helix Topology configuration files. HMS COMMAND OPTIONS (valid for certain commands as indicated): -o Specify '-o' to be optimistic regarding status of components for which status logic is not yet implememented. By default, components that don't yet have status logic coded report as failed. With '-o', components of known types report as OK. Components of unknown type always report as failed. This can be helpful in verifying whether the Helix Topology config file contains only valid/known component types. Components of unknown types should be commented out or marked as unmanaged. -n No-Op preview mode. This shows operations that would be performed, but takes no actions that affect the live system or data. This is valid only for command for which it is indicated. GENERAL LINE OPTIONS: -v<n> Set verbosity 1-5 (-v1 = quiet, -v5 = highest). -L <log> Specify the path to a log file, or the special value 'off' to disable logging. By default, all output (stdout and stderr) is captured in a log file named: $LOGS/hms.<NAME>.<DATESTAMP>.log -C Check the HMS config file and then stop. This can be used to verify the syntax of the HMS config file without starting an actual failover. -si Operate silently. All output (stdout and stderr) is redirected to the log only; no output appears on the terminal. This cannot be used with '-L off'. -D Set extreme debugging verbosity. -S Step mode - if running interactively prompt user to enter return before progressing to the next step. The '-S' option is not supported when called via p4broker (i.e. called as 'p4 hms' rather than calling the $THISSCRIPT script directly). THIS FLAG IS NOT IMPLEMENTED AND WILL BE DEFERRED TO hms 2.0 or beyond. HELP OPTIONS: -h Display short help message -man Display man-style help message -V Dispay version info for this script and its libraries. GENERAL LINE OPTIONS: DEPENDENCIES: The bash shell must be installed as /usr/local/bin//bash, and should be bash 4.3 or higher, acquired from ftp.gnu.org. The bash in /bin/bash need only be bash 3.0+ or greater. EXAMPLES: Get a status in the master p4d for Instance 1: $THISSCRIPT status 1:master Get a status in the second replica p4d for Instance 2: $THISSCRIPT status 2:p4d-r02 Get a status for all known components, optimisitically assuming those we haven't coded status logic for yet are configured are happy: $THISSCRIPT status all -o Show whether the second broker for Instance 2 can be updated to a newer patch version of p4broker, within the current major version: $THISSCRIPT update 1:p4b02 -n Upgrade the second broker for Instance 2 to the latest patch version of p4broker, within the current major version: $THISSCRIPT update 1:p4b02 Show what components in Instance 1 would be affected by site-wide topology upgrade: $THISSCRIPT upgrade 1 -n Upgrade all components in Instance 1 in the correct order. $THISSCRIPT upgrade 1 -n Perform an unscheduled failover for all instances mastered on host bos-helix-01: $THISSCRIPT failover ha h:bos-helix-01 u Perform a scheduled failover for all instances mastered on host syd-helix-04: $THISSCRIPT failover ha h:syd-helix-04 s Perform an unscheduled local failover for Instance 1 on its currently configured master host: $THISSCRIPT failover local i:1 u " fi exit 1 } #============================================================================== # Command Line Processing declare HelixTopologyCfg=$P4CCFG/HelixTopology.cfg declare GlobalOptions= declare Command= declare SubCommand= declare Target= declare FailoverPath= declare FailoverScope= declare FailoverStyle= declare -i PreflightCheck=0 declare -i RunTestSuite=0 declare -i OptimisticStatus=0 declare -i Interactive=1 declare -i StepMode=0 declare -i OverallReturnStatus=0 declare -i shiftArgs=0 set +u while [[ $# -gt 0 ]]; do case $1 in (show) Command=show;; (pull) Command=pull Target=$2 shiftArgs=1 ;; (df) Command=df Target=$2 shiftArgs=1 ;; (dfm) Command=dfm usageMsg="Invalid Usage. Usage for 'hms $Command' command is:\n\nhms $Command {on|off|status} <instance>" [[ $# -lt 3 ]] && bail "$usageMsg" SubCommand=$2 [[ $SubCommand =~ ^(on|off|status)$ ]] || bail "$usageMsg" Target=$3 shiftArgs=2 ;; (start|stop|status) Command=$1 usageMsg="Invalid Usage. Usage for 'hms $Command' command is:\n\nhms $Command <instance>[:<component>]" [[ $# -lt 2 ]] && bail "$usageMsg" Target=$2 shiftArgs=1 ;; (failover|fo) Command=failover usageMsg="Invalid Usage. Usage for 'hms failover' command is:\n\nhms failover <path> {i|h}:<scope> <style>" [[ $# -lt 4 ]] && bail "$usageMsg" FailoverPath=$2 FailoverScope=$3 FailoverStyle=$4 [[ $FailoverPath == "-"* || $FailoverScope == "-"* || $FailoverStyle == "-"* ]] && \ bail "$usageMsg" [[ $FailoverScope =~ ^(i|h): ]] || bail "$usageMsg" # Convert short form of 'style' argument, 'u' or 's', to long form for code # readability. [[ $FailoverStyle =~ ^(s|scheduled|u|unscheduled)$ ]] || bail "$usageMsg" [[ $FailoverStyle =~ ^(s|scheduled)$ ]] && FailoverStyle=Scheduled [[ $FailoverStyle =~ ^(u|unscheduled)$ ]] && FailoverStyle=Unscheduled shiftArgs=3 ;; (update|upgrade) Command=$1 usageMsg="Invalid Usage. Usage for 'hms $Command' command is:\n\nhms $Command <instance>[:<component>]" [[ $# -lt 2 ]] && bail "$usageMsg" Target=$2 shiftArgs=1 ;; (-c) HelixTopologyCfg=$2; shiftArgs=1;; (-C) PreflightCheck=1;; (-S) StepMode=1;; (-y) Interactive=0;; (-o) OptimisticStatus=1;; (-T) RunTestSuite=1;; (-h) usage -h;; (-man) usage -man;; (-V) show_versions; exit 0;; (-v1) export VERBOSITY=1;; (-v2) export VERBOSITY=2;; (-v3) export VERBOSITY=3;; (-v4) export VERBOSITY=4;; (-v5) export VERBOSITY=5;; (-v) export VERBOSITY=${2:-5}; shiftArgs=1;; (-L) export P4U_LOG=$2; shiftArgs=1;; (-si) SilentMode=1;; (-n) export NO_OP=1;; (-D) set -x;; # Debug; use 'set -x' mode. (*) usage -h "Unknown arg ($1).";; esac # Shift (modify $#) the appropriate number of times. shift; while [[ $shiftArgs -gt 0 ]]; do [[ $# -eq 0 ]] && usage -h "Bad usage; incorrect number of arguments." shiftArgs=$shiftArgs-1 shift done done set -u #============================================================================== # Command Line Verification [[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \ usage -h "Cannot use '-si' with '-L off'." if [[ $RunTestSuite -eq 1 ]]; then [[ -n "$HelixTopologyCfg" ]] && \ usage -h "The '-c <helix_topology_cfg>' argument is not allowed with -T." else [[ -z "$HelixTopologyCfg" ]] && \ usage -h "The '-c <helix_topology_cfg>' argument is required." fi # Only allow interactive mode if [[ $Interactive -eq 1 && $HMS_CALLED_BY_WRAPPER -eq 1 ]]; then if [[ ! $Command =~ ^(show|status)$ ]]; then usage -h "The -y (auto-confirm) flag is required when called via the broker. Try calling the hms script directly, not with 'p4 hms'." fi fi # Only allow step mode if running interactively so prompt/response can be handled if [[ $StepMode -eq 1 && $HMS_CALLED_BY_WRAPPER -eq 1 ]]; then bail "The -S (Step Mode) isn't supported when called via the broker. Try calling the hms script directly, not with 'p4 hms'." fi # Disable logging by default if we're just doing a status or show command # called from the broker. if [[ $HMS_CALLED_BY_WRAPPER -eq 1 && $Command =~ ^(show|status)$ ]]; then export P4U_LOG=off fi if [[ $OptimisticStatus -eq 1 && $Command != status ]]; then usage -h "The '-o' option (optimistic status) is only valid with the 'status' command." fi #============================================================================== # Main Program declare component= declare instanceComponents= declare componentList= declare -i statusAllOK=1 declare -i i if [[ $RunTestSuite -eq 1 ]]; then if [[ -x $HMS_HOME/test/run_test_suite.sh ]]; then $HMS_HOME/test/run_test_suite.sh OverallReturnStatus=$? exit $OverallReturnStatus else bail "The expected test suite wrapper script is missing: $HMS_HOME/test/run_test_suite.sh." fi fi # Very first thing - Check BASH_VERSION case $BASH_VERSION in (1*|2*|3*) echo -e "\n The version of bash is too old. This script requires bash 4.0 or higher.\n This version is $BASH_VERSION. If working on Mac, do a web search for\n 'bash 4 mac' for options on acquring a bash 4.x. Please acquire a new bash\n and install it somewhere, perhaps /usr/local/bin/bash, and change the '#!'\n line at the top of this script ($THISSCRIPT) to reference the newer bash.\n" exit 1;; esac # If the topology file is specified as '/absolute/path/top.cfg', use it verbatim. # If the topolgoy file is specified as 'top.cfg', look for top.cfg in the # current dir first, and then check for top.cfg in /p4/common/config. Otherwise # give up. if [[ ! -r "$HelixTopologyCfg" ]]; then if [[ "$HelixTopologyCfg" != "/"* && -r "$P4CCFG/$HelixTopologyCfg" ]]; then HelixTopologyCfg=$P4CCFG/$HelixTopologyCfg else bail "The Helix Topology file [$HelixTopologyCfg] does not exist." fi fi SiteTag=$(grep ^NAME= "$HelixTopologyCfg" 2>/dev/null | cut -d '=' -f 2) [[ -z "$SiteTag" ]] && \ bail "The Helix Topology file [$HelixTopologyCfg] is missing a 'NAME=' definition." if [[ $P4U_LOG == Unset ]]; then export P4U_LOG="$LOGS/hms.$SiteTag.$(date +'%Y%m%d-%H%M%S').log" fi trap terminate EXIT SIGINT SIGTERM if [[ "${P4U_LOG}" != off ]]; then touch ${P4U_LOG} # If we can't put the log where we want it, try putting it in /tmp # instead. We don't want to abort failover operations for lack of # a log file, as long as we can write one somehwere. if [[ $? -ne 0 ]]; then export P4U_LOG=/tmp/${P4U_LOG##*/} touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]." fi # Redirect stdout and stderr to a log file. if [[ $SilentMode -eq 0 ]]; then exec > >(tee ${P4U_LOG}) exec 2>&1 else exec >${P4U_LOG} exec 2>&1 fi initlog fi msg "${H}\nStarted at $(date). Versions:" [[ $VERBOSITY -gt 3 ]] && show_versions # Environment isolation: Clear Perforce server environment settings. unset P4AUDIT P4AUTH P4CHANGE P4CONFIG P4DEBUG P4DESCRIPTION P4JOURNAL \ P4LOG P4NAME P4PORT P4ROOT P4SSLDIR P4TARGET P4TICKETS P4TRUST # Environment isolation: Unset P4CONFIG, and point P4ENVIRO to /dev/null # to avoid accidentally interacting with a personal DVCS repo which slickly # applies a P4CONFIG value even after we unset it, by reading it from the # P4ENVIRO if it finds one (defaulting to ~/.p4enviro). export P4CONFIG=Unset export P4ENVIRO=/dev/null/.p4enviro [[ -z "$P4BIN" || ! -x "$P4BIN" || -z "$P4DBIN" || ! -x "$P4DBIN" ]] && \ bail "\n\nThe p4 and p4d binaries are missing or are not executable.\n" P4BinRev=$($P4BIN -V|grep ^Rev) P4DBinRev=$($P4DBIN -V|grep ^Rev) msg "Using these p4/p4d executables:\n$P4BIN: $P4BinRev\n$P4DBIN: $P4DBinRev\n" if [[ $PreflightCheck -eq 0 ]]; then load_helix_topology "$HelixTopologyCfg" "$RequiredCfgVersion" ||\ bail "Helix Topology definition failed sanity check." else load_helix_topology "$HelixTopologyCfg" "$RequiredCfgVersion" ||\ bail "Helix Topology definition in $HelixTopologyCfg FAILED preflight sanity checks." msg "Verified: Helix Topology defined in $HelixTopologyCfg PASSED preflight sanity checks." fi msg "\n${H}\n" case "$Command" in (show) msg "Command: $Command" show_helix_topology ;; (start|stop) msg "Command: $Command Target=[$Target]" change_component_status "$Command" "$Target" ;; (dfm) msg "Command: $Command SubCommand=[$SubCommand] Target=[$Target]" down_for_maintenance "$SubCommand" "$Target" ;; (pull) msg "Command: $Command Target=[$Target]" get_replica_status "$Target" ;; (df) msg "Command: $Command Target=[$Target]" get_diskspace "$Target" ;; (status) msg "Command: $Command Target=[$Target]" componentList=$(get_component_list "$Target" 0) ||\ bail "Failed to get a valid list of components." for component in $componentList; do get_component_status "$component" $OptimisticStatus ||\ statusAllOK=0 statusCode=${ComponentStatusCode[$component]:-Unset} statusMsg=${ComponentStatusMsg[$component]:-Unset} if [[ $statusCode != Unset && $statusMsg != Unset ]]; then qmsg "Status: $component is $statusMsg (code $statusCode)." fi done if [[ $statusAllOK -eq 1 ]]; then msg "${H}\nStatus for all components checked is OK." else msg "${H}\nStatus for one or more components FAILED." OverallReturnStatus=1 fi ;; (failover) msg "Command: $Command Path=[$FailoverPath] Scope=[$FailoverScope] Style=[$FailoverStyle]" [[ $NO_OP -eq 1 ]] && msg "Running in NO-OP Preview Mode due to '-n'." failover "$FailoverPath" "$FailoverScope" "$FailoverStyle" if [[ $? -eq 0 ]]; then msg "${H}\nFailover completed successfully.\n" else errmsg "Failover did NOT complete successfully.\n" OverallReturnStatus=1 fi ;; (update) msg "Command: $Command Target=[$Target]" [[ $NO_OP -eq 1 ]] && msg "Running in NO-OP Preview Mode due to '-n'." componentList=$(get_component_list "$Target" 1) ||\ bail "Failed to get a valid list of components." msg "Checking component status prior to update." for component in $componentList; do get_component_status "$component" $OptimisticStatus ||\ statusAllOK=0 statusCode=${ComponentStatusCode[$component]:-Unset} statusMsg=${ComponentStatusMsg[$component]:-Unset} if [[ $statusCode != Unset && $statusMsg != Unset ]]; then qmsg "Status: $component is $statusMsg (code $statusCode)." fi done if [[ $statusAllOK -eq 1 ]]; then msg "${H}\nStatus for all components checked is OK." else bail "${H}\nStatus for one or more components FAILED. Aborting $Command action." OverallReturnStatus=1 fi ;; (upgrade) msg "Command: $Command Target=[$Target]" [[ $NO_OP -eq 1 ]] && msg "Running in NO-OP Preview Mode due to '-n'." msg "Checking component status prior to upgrade." componentList=$(get_component_list "$Target" 1) ||\ bail "Failed to get a valid list of components." for component in $componentList; do get_component_status "$component" OptimisticStatus ||\ statusAllOK=0 statusCode=${ComponentStatusCode[$component]:-Unset} statusMsg=${ComponentStatusMsg[$component]:-Unset} if [[ $statusCode != Unset && $statusMsg != Unset ]]; then qmsg "Status: $component is $statusMsg (code $statusCode)." fi done if [[ $statusAllOK -eq 1 ]]; then msg "${H}\nStatus for all components checked is OK." else bail "${H}\nStatus for one or more components FAILED. Aborting $Command action." OverallReturnStatus=1 fi ;; (*) bail "Unknown command. Bad usage." ;; esac if [[ $OverallReturnStatus -eq 0 ]]; then msg "${H}\nAll processing completed successfully.\n" else msg "${H}\nProcessing completed, but with errors. Scan above output carefully.\n" fi msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n" [[ -r "${P4U_LOG}" ]] && msg "Log file is: $P4U_LOG\n${H}\n" exit $OverallReturnStatus
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#9 | 25550 | C. Thomas Tyler |
Split HMS from SDP, such that HMS will live on as a separate product, layered on the SDP. See: https://swarm.workshop.perforce.com/jobs/SDP-356 Many of the files deleted here have already been copied to HMS. See these HMS changes: https://swarm.workshop.perforce.com/changes/25531 https://swarm.workshop.perforce.com/changes/25533 And this branch spec: SDP_Split_2019.2_HMS In mkrep.sh, HMS-awareness is removed from mkrep.sh, a core SDP script, and the '-f' (Full Setup) flag has been temporarily removed. This useful functionality will be restored in an HMS script. This is a big change, so shellcheck v0.6.0 was implemented to support it. |
||
#8 | 24188 | C. Thomas Tyler |
Corrected dependencies related to bash version and tweaked comments. #review @josh |
||
#7 | 23747 | C. Thomas Tyler |
Fixed hms status issues. Fixed issue getting status of SSL-enabled servers requiring SDP environment (including P4TRUST) to be set, incorporating new irun utility. Fixed issue with illegal negative exit codes in get_component_status(). Added in-code docs for component status values. Fixed get_component_list() to remove call-stack info in high-verbosity mode, due to text output being interpreted as data. |
||
#6 | 23454 | cgeen |
Update to hms script to do two new functions: pull and df. Pull issues a pull -lj on all the edge/replica servers to get the status df issues a diskspace on all the servers. The reason for this is with a hardened edge environment the P4USER does not have permissions to run these commands on the edge servers. Therefore we need to run them centrally as a super. This replaces the replica_status commands on the edge. The updates also includes an update to the p4_vars to change the P4USER on an edge server. The idea is that P4USER only has super privs on the master on the replica edge servers the default user can only have admin privs. This is so the box adminstrator in restricted regions can't change the protection table. Hence the need for pull df centrally as these are the only commands that can't be run on the replicas/edges from the maintance scripts. It also goes without saying that ssh should only go out from the commit and never back. |
||
#5 | 21989 | C. Thomas Tyler |
Doc typo fixes only, no functional change. Bypassing pre-commit review. |
||
#4 | 21921 | C. Thomas Tyler |
HMS change, to simplify setup on compliant platforms (i.e. those using bash 4.x). The shebang line for all bash scripts has been standardized to /bin/bash, and bash checks for $BASH_VERSION added where needed. References to having a custom-built /usr/local/bin/bash have been removed. As a trade-off, non-compliant platforms (way-old Linux and Mac using bash 3.x) will have a bit more work to do to work with shebang lines. |
||
#3 | 20852 | C. Thomas Tyler | The 'hms -c <cfg>' flag now conveniently looks in /p4/common/config. | ||
#2 | 20805 | C. Thomas Tyler | Implemented global/central start stop. | ||
#1 | 20745 | C. Thomas Tyler |
Approving as is since it isn't changing core SDP functionality, and reviewing it all line by line will take some time. We can do that as we move forward with it. First addition of HMS v1.0 files. This change is a soft launch HMS for initial deployment and testing. Updates to HMS-related files are expected and will bypass pre-commit code review until stabilized. |