hms_upgrade.sh #2

# hms_upgrade.sh
# shellcheck disable=SC2034
Version=2.0.0

#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Helm Management System (HMS), and also available online:
# https://workshop.perforce.com/projects/p4-hms/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# HMS Library Functions.

#------------------------------------------------------------------------------
# upgrade_or_upgrade($updateType, $instance)
# Input:
#   $updateType: "update" or "upgrade"; default is "update".
#   $instance to update/upgrade.
#
#------------------------------------------------------------------------------
# This script is impacted by NO_OP mode:
# NO_OP = 0: Normal/Live Operation.
# NO_OP = 1: No Operation mode; set with command line '-n' flag. Displays SSH
# commands
# NO_OP = 2: Deeper No Op mode; executes SSH comamnds with '-n' flag to called
# scripts.
# 
#------------------------------------------------------------------------------
function update_or_upgrade () {
   vvmsg "CALL: update_or_upgrade ($*)"
   local updateType="${1:-update}"
   local instance="${2:-UnsetInstance}"
   local scriptArgs=
   local helixBinariesDir="/p4/sdp/helix_binaries"
   local desc=
   local host=
   local p4=
   local -i errorCount=0
   local -i hostCount=0
   local upgradeArgs=
   local p4dMajorVersion=
   local userPrompt=

   #---------------------------------------------------------------------------
   msg "\nStarting $updateType processing for instance $instance."

   if [[ "$instance" == "UnsetInstance" ]]; then
      errmsg "Internal error update_or_upgrade() required 2nd parameter (instance) is missing."
      return 1
   fi

   if [[ "$updateType" == "update" ]]; then
      p4dMajorVersion=$("/p4/$instance/bin/p4d_${instance}" -V | grep -i Rev. | awk -F / '{print $3}')

      ### BEGIN DEMO HACK - This will break in the year 2100.
      p4dMajorVersion="r${p4dMajorVersion#20}"
      ### END DEMO HACK
      msg "UPDATE: Getting latest patch of P4 binaries for the current major version ($p4dMajorVersion)."
      scriptArgs+="-r $p4dMajorVersion"
   else
      msg "UPGRADE: Getting latest P4 binaries for the latest major version."
   fi

   # shellcheck disable=SC2154
   if [[ "$Interactive" -eq 1 ]]; then
      userPrompt="Proceed with $updateType"
      userPrompt+=" ($OpModeDesc)"
      userPrompt+=" (y/n)? "
      while true; do
         read -r -p "$userPrompt" response
         case "$response" in
            [Yy])
               msg "Confirmation received. Proceeding with $updateType ..."
               break
            ;;
            [Nn])
               msg "Aborting per user resposne [$response]."
               exit 0
            ;;
            *)
               msg "Invalid response. Please enter 'y' to proceed or 'n' to abort."
            ;;
        esac
      done
   else
      msg "Proceeding with $updateType ($OpModeDesc) due to '-y'."
   fi

   #---------------------------------------------------------------------------
   # shellcheck disable=SC1091
   source /p4/common/bin/p4_vars "$instance"
   p4="p4 -u $P4USER -p $P4MASTERPORT"
   if ! $p4 login -s; then
      if $p4 login -a < "$SDP_ADMIN_PASSWORD_FILE"; then
         msg "Logged in using $p4."
      else
         errmsg "Could not login using: $p4 login -a .LT. $SDP_ADMIN_PASSWORD_FILE"
         return 1
      fi
   fi

   #---------------------------------------------------------------------------
   if set_outer_to_inner_upgrade_order "$instance" "$p4"; then
      msg "Servers will be upgraded in outer-to-inner order."
   else
      errmsg "Could not determine outer-to-inner order of ServerIDs. Aborting."
      return 1
   fi

   #---------------------------------------------------------------------------
   # Stage binaries.  If '-n', dry run the stage.  With '-N', do the actual
   # staging.
   [[ "$NO_OP" -eq 1 ]] && scriptArgs+=" -n"

   for host in "${ServerHostsOTI[@]}"; do
      msg "Staging helix binaries for $updateType in $host:$helixBinariesDir"

      msg "Doing: ssh -q $host \"cd $helixBinariesDir; ./get_helix_binaries.sh $scriptArgs\""
      if [[ "$NO_OP" -ne 0 || "$NO_OP" -eq 2 ]]; then
         if ! ssh -q "$host" "cd $helixBinariesDir; ./get_helix_binaries.sh $scriptArgs"; then
            errmsg "Could not stage helix binaries on host $host."
         fi
      else
         msg "NO_OP: Would have run above ssh command."
      fi
   done

   if [[ "$errorCount" -eq 0 ]]; then
      msg "Binaries staged successfully on all $hostCount hosts."
   else
      errmsg "Staging binaries failed on $errorCount of $hostCount hosts. Aborting."
      return 1
   fi

   #---------------------------------------------------------------------------
   # Perform upgrades.
   # If '-n', dry run the upgrade.  If '-N', do a deeper dry run including SSH
   # to the host to run the 'upgrade.sh' script, but without the '-y' option.

   upgradeArgs="$instance"
   [[ "$NO_OP" -eq 0 ]] && upgradeArgs+=" -y"

   for host in "${ServerHostsOTI[@]}"; do
      msg "Doing: ssh -q $host \"upgrade.sh $upgradeArgs\""
      if [[ "$NO_OP" -eq 1 ]]; then
         msg "NO_OP: Would have run above ssh command."
      else
         if ssh -q "$host" "upgrade.sh $upgradeArgs"; then
            msg "\n\nUPGRADE OK on host $host."
         else
            errmsg "\n\nUPGRADE FAILED on host $host."
            errorCount+=1
         fi
      fi
   done

   if [[ "$errorCount" -eq 0 ]]; then
      msg "The $updateType ($OpModeDesc) completed successfully on all $hostCount hosts."
   else
      errmsg "The $updateType ($OpModeDesc) failed on $errorCount of $hostCount hosts."
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: get_levels_removed ($s)
# Return the number of levels of replication removed from the master server,
# useful for determining the order of upgrade in outer-to-inner order.
# Set global MaxOTILevels
# This funciton is recursive.  It uses bash's limited return capabilty to
# return an integer in the range of 0-255, which is sufficient for our needs
# as P4 replicas can span the globe in just a few hops of daisy
# chaining.
function get_levels_removed () {
   local s=${1:-UnsetServer}
   local target
   local -i levels=0

   # Short cut: Check if we are the master.
   [[ "$s" == "$P4MASTER_ID" ]] && return 0

   target="${ServerTargets[$s]}"

   # Short cut: If target the master, we're 1 level removed from it.
   if [[ -z "$target" && "$P4MASTER_ID" ]]; then
      [[ "$MaxOTILevels" -lt 1 ]] && MaxOTILevels=1
      return 1
   fi

   get_levels_removed "$target"
   levels=$?

   [[ "$MaxOTILevels" -lt $((levels+1)) ]] && MaxOTILevels=$((levels+1))

   return $((levels+1))
}

#------------------------------------------------------------------------------
# Function: set_outer_to_inner_upgrade_order ($instance, $p4)
#
# This function creates lists of ServerIDs and ServerHosts outer-to-inner order.
# Ultimately, we need a simple list of hosts in order in which to run the
# SDP upgrade.sh script.
#
# Data Requirements:
# * The ReplicatingFrom field must be set in all server specs of type 'server'
#   meaning a p4d server (not a broker/proxy/connector), with the obvious
#   exception of the singular master/commit server itself.
# * The ExternalAddress field must be set to a correct value for all server
#   specs. The hostname component of ExternalAddress must work with SSH. A
#   short hostname is acceptable, an FQDN may be used as well.
# 
# We start with P4 ServerId values of 'server' type server specs, and
# determine the outer-to-inner based on how many level of replication we are
# removed from the master. For example, the master is 0 levels removed. Any
# replica directly targeting the master is 1 level removed.  The ReplicatingFrom
# field values are used to determine the outer-to-inner order based on the
# ServerID. The recursive get_level_removed() function follows the replication
# daisy chaining.
#
# After that, we simply append the list of ServerIDs of types proxy and broker,
# to get a complete list of ServerIDs to be upgraded.
#
# Next, we determine the server host for each ServerID in order.  Since the
# SDP upgrade updates all ServerIDs on a given machine (p4d/p4broker/p4p),
# the order of ServerIDs of all server specs is not strictly definitive.  That's
# OK; order only truly matters for 'p4d' servers.
#
# This function populate these global variables:
# ServerHosts - hash of server hosts indexed by ServerID.
# ServerTypes - hash of server types indexed by ServerID.
# ServerTargets - hash of server targets indexed by ServerID.
# ServerOTILayers - hash of server targets indexed by ServerID.
# ServerIDsOTI - sorted array of ServerIDs in outer-to-inner order.
# ServerHostsOTI - sorted array of ServerHosts in outer-to-inner order.
#
# This depends on the shell environment for the instance already having been
# sourced in the calling environment.
#------------------------------------------------------------------------------
function set_outer_to_inner_upgrade_order () {
   vvmsg "CALL: set_outer_to_inner_upgrade_order ($*)"
   local instance="${1:-UnsetInstance}"
   local p4="${2:-UnsetP4}"
   local serverData
   local -i i=0
   local -i j=0
   local -i k=0
   local -i levelsRemoved=0
   local -i preflightOK=1
   local -i hostAlreadyInList=0
   local server
   local target
   local address
   local host

   msg "Building Outer-to-Inner list of p4d/p4broker/p4p servers."
   for serverData in $($p4 -ztag -F %Type%:%ServerID%:%ReplicatingFrom%:%ExternalAddress%: servers); do
      type=$(echo "$serverData"|cut -d ':' -f 1)

      # Ignore server types we don't know about or don't handle, such as Git Connector.
      [[ ! "$type" =~ ^(server|proxy|broker)$ ]] && continue
      server=$(echo "$serverData"|cut -d ':' -f 2)
      target=$(echo "$serverData"|cut -d ':' -f 3)
      address=$(echo "$serverData"|cut -d ':' -f 4)

      # Determine hostname from ExternalAddress field.
      host=${address%:*}; host=${host#*:}

      # For p4d servers, type ReplicatingFrom field must be set.
      if [[ "$type" == "server" ]]; then
         if [[ -z "$target" && "$server" != "$P4MASTER_ID" ]]; then
            errmsg "Could not get 'ReplicatingFrom:' field value for ServerID: $server."
            preflightOK=0
         elif [[ -z "$target" ]]; then
            target=NONE
         fi
      fi

      # If the ServerID description's contains the word "inactive", ignore it.
      desc=$($p4 -ztag -F %Description% server -o "$server")
      [[ "${desc^^}" == *"INACTIVE"* ]] && continue

      vvmsg "SVR=[$server] Type=[$type] Target=[$target]"
      ServerTypes[$server]="$type"
      ServerTargets[$server]="$target"
      ServerHosts[$server]="$host"
   done

   if [[ "$preflightOK" -eq 0 ]]; then
      return
   fi

   # Do p4d servers first, in outer-to-inner order.
   for s in "${!ServerTypes[@]}"; do
      [[ "${ServerTypes[$s]}" == "server" ]] || continue
      get_levels_removed "$s"
      levelsRemoved=$?
      ServerOTILayers[$s]=$levelsRemoved
   done

   for s in "${!ServerTypes[@]}"; do
      [[ "${ServerTypes[$s]}" == "server" ]] || continue
      vvmsg "Server [$s] is at level [${ServerOTILayers[$s]}]."
   done

   # Do proxies next.
   for s in "${!ServerTypes[@]}"; do
      if [[ "${ServerTypes[$s]}" == "proxy" ]]; then
         vvmsg "Adding ServerID for proxy server $s."
         ServerIDsOTI[$i]="$s"
         i+=1
      fi
   done

   # Do brokers last.
   for s in "${!ServerTypes[@]}"; do
      if [[ "${ServerTypes[$s]}" == "broker" ]]; then
         vvmsg "Adding ServerID for broker server $s."
         ServerIDsOTI[$i]="$s"
         i+=1
      fi
   done

   vvmsg "Outer to inner by ServerID ..."
   for ((j=MaxOTILevels; j >= 0; j=j-1)); do
      vvmsg "Layer $j"
      for s in "${!ServerOTILayers[@]}"; do
         vvmsg "Checking server $s at layer $j."
         if [[ "${ServerOTILayers[$s]}" -eq "$j" ]]; then
            vvmsg "Adding server $s at layer $j."
            ServerIDsOTI[$i]="$s"
            i+=1
         fi
      done
   done

   # This is where we build the list by host. Proxies or brokers will
   # be upgraded in an order determined by the p4d on the machine, and will
   # be done earlier in the process.  Lone proxies and brokers are handled
   # last.
   vvmsg "Outer to inner by Host ..."
   k=0
   for s in "${ServerIDsOTI[@]}"; do
      host=${ServerHosts[$s]}
      hostAlreadyInList=0
      for h in "${ServerHostsOTI[@]}"; do
         if [[ "$h" == "$host" ]]; then
            hostAlreadyInList=1
            continue
         fi
      done

      if [[ "$hostAlreadyInList" -eq 0 ]]; then
         ServerHostsOTI[$k]=$host
         k+=1
      fi
   done
}