hms_upgrade.sh #2

# hms_upgrade.sh
Version=1.0.2

#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce_software-hms/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# HMS Library Functions.

#------------------------------------------------------------------------------
# upgrade_or_upgrade($updateType, $instance)
# Input:
#   $updateType: "update" or "upgrade"; default is "update".
#   $instance to update/upgrade.
#
#------------------------------------------------------------------------------
# This script is impacted by NO_OP mode:
# NO_OP = 0: Normal/Live Operation.
# NO_OP = 1: No Operation mode; set with command line '-n' flag. Displays SSH
# commands
# NO_OP = 2: Deeper No Op mode; executes SSH comamnds with '-n' flag to called
# scripts.
# 
#------------------------------------------------------------------------------
function update_or_upgrade () {
   vvmsg "CALL: update_or_upgrade ($*)"
   local updateType="${1:-update}"
   local instance="${2:-UnsetInstance}"
   local scriptArgs=
   local helixBinariesDir="/p4/sdp/helix_binaries"
   local desc=
   local host=
   local serverID=
   local p4=
   local -i errorCount=0
   local -i hostCount=0
   local hostList=
   local upgradeArgs=

   #---------------------------------------------------------------------------
   msg "\\nStarting $updateType processing for instance $instance."

   if [[ "$instance" == "UnsetInstance" ]]; then
      errmsg "Internal error update_or_upgrade() required 2nd parameter (instance) is missing."
      return 1
   fi

   if [[ "$updateType" == "update" ]]; then
      msg "UPDATE: Getting latest patch of Helix binaries for the current major version."
   else
      msg "UPGRADE: Getting latest Helix binaries for the latest major version."
      ### DEMO_HACK: Hard-coding major version; need to actually get it from updates.perforce.com JSON file.
      scriptArgs+="-r r20.2"
   fi

   #---------------------------------------------------------------------------
   # shellcheck disable=SC1091
   source /p4/common/bin/p4_vars "$instance"
   p4="p4 -u $P4USER -p $P4MASTERPORT"
   if ! $p4 login -s; then
      if $p4 login -a < "$SDP_ADMIN_PASSWORD_FILE"; then
         msg "Logged in using $p4."
      else
         errmsg "Could not login using: $p4 login -a .LT. $SDP_ADMIN_PASSWORD_FILE"
         return 1
      fi
   fi

   #---------------------------------------------------------------------------
   for serverID in $(get_outer_to_inner_upgrade_order "$instance"); do
      host=$($p4 -ztag -F %ExternalAddress% server -o "$serverID")
      host=${host#ssl:}
      host=${host%:*}
      if [[ -n "$host" ]]; then
         hostList+=" $host"
         hostCount+=1
      else
         errmsg "Could not determine host for ServerID: $serverID."
         errorCount+=1
      fi
   done

   if [[ "$errorCount" -eq 0 ]]; then
      msg "Host upgrade order is: $hostList"
   else
      errmsg "Could not determine all hosts from ServerID values. Aborting."
      return 1
   fi

   #---------------------------------------------------------------------------
   # Stage binaries.  If '-n', dry run the stage.  With '-N', do the actual
   # staging.
   [[ "$NO_OP" -ne 0 ]] && scriptArgs+=" -n"

   for host in $hostList; do
      msg "Staging helix binaries for $updateType in $host:$helixBinariesDir"

      msg "Doing: ssh -q $host \"cd $helixBinariesDir; ./get_helix_binaries.sh $scriptArgs\""
      if [[ "$NO_OP" -ne 0 || "$NO_OP" -eq 2 ]]; then
         if ! ssh -q "$host" "cd $helixBinariesDir; ./get_helix_binaries.sh $scriptArgs"; then
            errmsg "Could not stage helix binaries on host $host."
         fi
      else
         msg "NO_OP: Would have run above ssh command."
      fi
   done

   if [[ "$errorCount" -eq 0 ]]; then
      msg "Binaries staged successfully on all $hostCount hosts."
   else
      errmsg "Staging binaries failed on $errorCount of $hostCount hosts. Aborting."
      return 1
   fi

   ### DEMO_HACK
   ### For 'update' mode, pull in latest patch of p4d 2020.1
   ### For 'upgrade' mode, pull in 2020.2 binaries.
   if [[ "$updateType" == "update" ]]; then
      cp -f /hxdepots/helix_binaries/p4* /p4/sdp/helix_binaries/.
      cp -f /hxdepots/helix_binaries/staged_for_upgrade/2020.1/p4* /p4/sdp/helix_binaries/.
   else
      cp -f /hxdepots/helix_binaries/staged_for_upgrade/2020.2/p4* /p4/sdp/helix_binaries/.
   fi
   chmod 755 /p4/sdp/helix_binaries/p4*
   rsync -a /p4/sdp/helix_binaries/ helix-05:/p4/sdp/helix_binaries
   rsync -a /p4/sdp/helix_binaries/ helix-04:/p4/sdp/helix_binaries
   rsync -a /p4/sdp/helix_binaries/ helix-03:/p4/sdp/helix_binaries
   rsync -a /p4/sdp/helix_binaries/ helix-02:/p4/sdp/helix_binaries
   ### DEMO_HACK

   #---------------------------------------------------------------------------
   # Perform upgrades.
   # If '-n', dry run the upgrade.  If '-N', do a deeper dry run including SSH
   # to the host to run the 'upgrade.sh' script, but with '-n' option.

   upgradeArgs="$instance"
   if [[ "$NO_OP" -eq 2 ]]; then
      upgradeArgs+=" -n"
   fi

   for host in $hostList; do
      msg "Doing: ssh -q $host \"upgrade.sh $upgradeArgs\""
      if [[ "$NO_OP" -eq 1 ]]; then
         msg "NO_OP: Would have run above ssh command."
      else
         if ssh -q "$host" "upgrade.sh $upgradeArgs"; then
            msg "\\n\\nUPGRADE OK on host $host."
         else
            errmsg "\\n\\nUPGRADE FAILED on host $host."
            errorCount+=1
         fi
      fi
   done

   if [[ "$errorCount" -eq 0 ]]; then
      msg "The $updateType completed successfully on all $hostCount hosts."
   else
      errmsg "The $updateType failed on $errorCount of $hostCount hosts."
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: get_outer_to_inner_upgrade_order ($instance)
#
# Generate a list of server specs in outer-to-inner order.
#------------------------------------------------------------------------------
function get_outer_to_inner_upgrade_order () {
   vvmsg "CALL: get_outer_to_inner_upgrade_order ($*)"

   ### DEMO_HACK: Hard-coding list of servers in "outer-to-iner" order.

   echo "p4d_ha_edge_syd p4d_edge_syd p4d_fs_nyc p4d_ha_bos master.1"

   return 0
}