# hms_upgrade.sh Version=1.2.0 #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce_software-hms/view/main/LICENSE #------------------------------------------------------------------------------ #============================================================================== # HMS Library Functions. #------------------------------------------------------------------------------ # upgrade_or_upgrade($updateType, $instance) # Input: # $updateType: "update" or "upgrade"; default is "update". # $instance to update/upgrade. # #------------------------------------------------------------------------------ # This script is impacted by NO_OP mode: # NO_OP = 0: Normal/Live Operation. # NO_OP = 1: No Operation mode; set with command line '-n' flag. Displays SSH # commands # NO_OP = 2: Deeper No Op mode; executes SSH comamnds with '-n' flag to called # scripts. # #------------------------------------------------------------------------------ function update_or_upgrade () { vvmsg "CALL: update_or_upgrade ($*)" local updateType="${1:-update}" local instance="${2:-UnsetInstance}" local scriptArgs= local helixBinariesDir="/p4/sdp/helix_binaries" local desc= local host= local serverID= local p4= local -i errorCount=0 local -i hostCount=0 local upgradeArgs= #--------------------------------------------------------------------------- msg "\\nStarting $updateType processing for instance $instance." if [[ "$instance" == "UnsetInstance" ]]; then errmsg "Internal error update_or_upgrade() required 2nd parameter (instance) is missing." return 1 fi if [[ "$updateType" == "update" ]]; then msg "UPDATE: Getting latest patch of Helix binaries for the current major version." else msg "UPGRADE: Getting latest Helix binaries for the latest major version." ### DEMO HACK scriptArgs+="-r r20.2" fi #--------------------------------------------------------------------------- # shellcheck disable=SC1091 source /p4/common/bin/p4_vars "$instance" p4="p4 -u $P4USER -p $P4MASTERPORT" if ! $p4 login -s; then if $p4 login -a < "$SDP_ADMIN_PASSWORD_FILE"; then msg "Logged in using $p4." else errmsg "Could not login using: $p4 login -a .LT. $SDP_ADMIN_PASSWORD_FILE" return 1 fi fi #--------------------------------------------------------------------------- if set_outer_to_inner_upgrade_order "$instance" "$p4"; then msg "Servers will be upgraded in outer-to-inner order." else errmsg "Could not determine outer-to-inner order of ServerIDs. Aborting." return 1 fi #--------------------------------------------------------------------------- # Stage binaries. If '-n', dry run the stage. With '-N', do the actual # staging. [[ "$NO_OP" -ne 0 ]] && scriptArgs+=" -n" for host in ${ServerHostsOTI[@]}; do msg "Staging helix binaries for $updateType in $host:$helixBinariesDir" msg "Doing: ssh -q $host \"cd $helixBinariesDir; ./get_helix_binaries.sh $scriptArgs\"" if [[ "$NO_OP" -ne 0 || "$NO_OP" -eq 2 ]]; then if ! ssh -q "$host" "cd $helixBinariesDir; ./get_helix_binaries.sh $scriptArgs"; then errmsg "Could not stage helix binaries on host $host." fi else msg "NO_OP: Would have run above ssh command." fi done if [[ "$errorCount" -eq 0 ]]; then msg "Binaries staged successfully on all $hostCount hosts." else errmsg "Staging binaries failed on $errorCount of $hostCount hosts. Aborting." return 1 fi ### DEMO_HACK ### For 'update' mode, pull in latest patch of p4d 2020.1 ### For 'upgrade' mode, pull in 2020.2 binaries. if [[ "$updateType" == "update" ]]; then cp -f /hxdepots/helix_binaries/p4* /p4/sdp/helix_binaries/. cp -f /hxdepots/helix_binaries/staged_for_upgrade/2020.1/p4* /p4/sdp/helix_binaries/. else cp -f /hxdepots/helix_binaries/staged_for_upgrade/2020.2/p4* /p4/sdp/helix_binaries/. fi chmod 755 /p4/sdp/helix_binaries/p4* rsync -a /p4/sdp/helix_binaries/ helix-05:/p4/sdp/helix_binaries rsync -a /p4/sdp/helix_binaries/ helix-04:/p4/sdp/helix_binaries rsync -a /p4/sdp/helix_binaries/ helix-03:/p4/sdp/helix_binaries rsync -a /p4/sdp/helix_binaries/ helix-02:/p4/sdp/helix_binaries ### DEMO_HACK #--------------------------------------------------------------------------- # Perform upgrades. # If '-n', dry run the upgrade. If '-N', do a deeper dry run including SSH # to the host to run the 'upgrade.sh' script, but with '-n' option. upgradeArgs="$instance" if [[ "$NO_OP" -eq 2 ]]; then upgradeArgs+=" -n" fi for host in ${ServerHostsOTI[@]}; do msg "Doing: ssh -q $host \"upgrade.sh $upgradeArgs\"" if [[ "$NO_OP" -eq 1 ]]; then msg "NO_OP: Would have run above ssh command." else if ssh -q "$host" "upgrade.sh $upgradeArgs"; then msg "\\n\\nUPGRADE OK on host $host." else errmsg "\\n\\nUPGRADE FAILED on host $host." errorCount+=1 fi fi done if [[ "$errorCount" -eq 0 ]]; then msg "The $updateType completed successfully on all $hostCount hosts." else errmsg "The $updateType failed on $errorCount of $hostCount hosts." return 1 fi return 0 } #------------------------------------------------------------------------------ # Function: get_levels_removed ($s) # Return the number of levels of replication removed from the master server, # useful for determining the order of upgrade in outer-to-inner order. # Set global MaxOTILevels # This funciton is recursive. It uses bash's limited return capabilty to # return an integer in the range of 0-255, which is sufficient for our needs # as Helix Core replicas can span the globe in just a few hops of daisy # chaining. function get_levels_removed () { local s=${1:-UnsetServer} local target local -i levels=0 # Short cut: Check if we are the master. [[ "$s" == "$P4MASTER_ID" ]] && return 0 target="${ServerTargets[$s]}" # Short cut: If target the master, we're 1 level removed from it. if [[ -z "$target" && "$P4MASTER_ID" ]]; then [[ "$MaxOTILevels" -lt 1 ]] && MaxOTILevels=1 return 1 fi get_levels_removed "$target" levels=$? [[ "$MaxOTILevels" -lt $((levels+1)) ]] && MaxOTILevels=$((levels+1)) return $((levels+1)) } #------------------------------------------------------------------------------ # Function: set_outer_to_inner_upgrade_order ($instance, $p4) # # This function creates lists of ServerIDs and ServerHosts outer-to-inner order. # Ultimately, we need a simple list of hosts in order in which to run the # SDP upgrade.sh script. # # Data Requirements: # * The ReplicatingFrom field must be set in all server specs of type 'server' # meaning a p4d server (not a broker/proxy/connector), with the obvious # exception of the singular master/commit server itself. # * The ExternalAddress field must be set to a correct value for all server # specs. The hostname component of ExternalAddress must work with SSH. A # short hostname is acceptable, an FQDN may be used as well. # # We start with Helix ServerId values of 'server' type server specs, and # determine the outer-to-inner based on how many level of replication we are # removed from the master. For example, the master is 0 levels removed. Any # replica directly targeting the master is 1 level removed. The ReplicatingFrom # field values are used to determine the outer-to-inner order based on the # ServerID. The recursive get_level_removed() function follows the replication # daisy chaining. # # After that, we simply append the list of ServerIDs of types proxy and broker, # to get a complete list of ServerIDs to be upgraded. # # Next, we determine teh server host for each ServerID in order. Since the # SDP upgrade updates all ServerIDs on a given machine (p4d/p4broker/p4p), # the order of ServerIDs of all server specs is not strictly definitive. That's # OK; order only truly matters for 'p4d' servers. # # This function populate these global variables: # ServerHosts - hash of server hosts indexed by ServerID. # ServerTypes - hash of server types indexed by ServerID. # ServerTargets - hash of server targets indexed by ServerID. # ServerOTILayers - hash of server targets indexed by ServerID. # ServerIDsOTI - sorted array of ServerIDs in outer-to-inner order. # ServerHostsOTI - sorted array of ServerHosts in outer-to-inner order. # # This depends on the shell environment for the instance already having been # sourced in the calling environment. #------------------------------------------------------------------------------ function set_outer_to_inner_upgrade_order () { vvmsg "CALL: set_outer_to_inner_upgrade_order ($*)" local instance="${1:-UnsetInstance}" local p4="${2:-UnsetP4}" local serverData local -i i=0 local -i j=0 local -i k=0 local -i levelsRemoved=0 local -i preflightOK=1 local -i hostAlreadyInList=0 local server local target local address local host msg "Building Outer-to-Inner list of p4d/p4broker/p4p servers." for serverData in $($p4 -ztag -F %Type%:%ServerID%:%ReplicatingFrom%:%ExternalAddress%: servers); do type=$(echo "$serverData"|cut -d ':' -f 1) # Ignore server types we don't know about or don't handle, such as # 'connector' (for Helix4Git). [[ ! "$type" =~ ^(server|proxy|broker)$ ]] && continue server=$(echo "$serverData"|cut -d ':' -f 2) target=$(echo "$serverData"|cut -d ':' -f 3) address=$(echo "$serverData"|cut -d ':' -f 4) # Determine hostname from ExternalAddress field. host=${address%:*}; host=${host#*:} # For p4d servers, type ReplicatingFrom field must be set. if [[ "$type" == "server" ]]; then if [[ -z "$target" && "$server" != "$P4MASTER_ID" ]]; then errmsg "Could not get 'ReplicatingFrom:' field value for ServerID: $server." preflightOK=0 elif [[ -z "$target" ]]; then target=NONE fi fi # If the ServerID description's contains the word "inactive", ignore it. desc=$($p4 -ztag -F %Description% server -o $server) [[ "${desc^^}" == *"INACTIVE"* ]] && continue vvmsg "SVR=[$server] Type=[$type] Target=[$target]" ServerTypes[$server]="$type" ServerTargets[$server]="$target" ServerHosts[$server]="$host" done if [[ "$preflightOK" -eq 0 ]]; then return fi # Do p4d servers first, in outer-to-inner order. for s in ${!ServerTypes[@]}; do [[ "${ServerTypes[$s]}" == "server" ]] || continue get_levels_removed "$s" levelsRemoved=$? ServerOTILayers[$s]=$levelsRemoved done for s in ${!ServerTypes[@]}; do [[ "${ServerTypes[$s]}" == "server" ]] || continue vvmsg "Server [$s] is at level [${ServerOTILayers[$s]}]." done # Do proxies next. for s in ${!ServerTypes[@]}; do if [[ "${ServerTypes[$s]}" == "proxy" ]]; then vvmsg "Adding ServerID for proxy server $s." ServerIDsOTI[$i]="$s" i+=1 fi done # Do brokers last. for s in ${!ServerTypes[@]}; do if [[ "${ServerTypes[$s]}" == "broker" ]]; then vvmsg "Adding ServerID for broker server $s." ServerIDsOTI[$i]="$s" i+=1 fi done vvmsg "Outer to inner by ServerID ..." for ((j=$MaxOTILevels; j >= 0; j=j-1)); do vvmsg "Layer $j" for s in ${!ServerOTILayers[@]}; do vvmsg "Checking server $s at layer $j." if [[ "${ServerOTILayers[$s]}" -eq "$j" ]]; then vvmsg "Adding server $s at layer $j." ServerIDsOTI[$i]="$s" i+=1 fi done done # This is where we build the list by host. Proxies or brokers will # be upgraded in an order determined by the p4d on the machine, and will # be done earlier in the process. Lone proxies and brokers are handled # last. vvmsg "Outer to inner by Host ..." k=0 for s in ${ServerIDsOTI[@]}; do host=${ServerHosts[$s]} hostAlreadyInList=0 for h in ${ServerHostsOTI[@]}; do if [[ "$h" == "$host" ]]; then hostAlreadyInList=1 continue fi done if [[ "$hostAlreadyInList" -eq 0 ]]; then ServerHostsOTI[$k]=$host k+=1 fi done }
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#2 | 30242 | C. Thomas Tyler | Updated package reference in LICENSE to refer to HMS rather than SDP license. | ||
#1 | 29182 | C. Thomas Tyler |
Moved HMS files from /p4/common/bin -> /p4/common/site/bin. Moved HMS files from /p4/common/lib -> /p4/common/site/lib. Removed dependency on SDP libs so that HMS can be deployed with a wider variety of SDP versions. |
||
//guest/perforce_software/hms/dev/p4/common/lib/hms_upgrade.sh | |||||
#4 | 27748 | C. Thomas Tyler |
First pass at "outer to inner" implementation, adding a new test for same. Removed some DEMO HACK code; more to be removed. |
||
#3 | 27700 | C. Thomas Tyler | Refined function signature. | ||
#2 | 27699 | C. Thomas Tyler | Adjustments in preparation for demo. | ||
#1 | 27698 | C. Thomas Tyler | Implememnted '-upgrade' and '-update' options. |