sdp_upgrade.sh #28

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
set -u

# sdp_upgrade.sh
# Upgrades the Perforce Helix Server Deployment Package (SDP).

# When upgrading this script to support new SDP versions, search for the token
# TAG: to find areas of the code that should be reviewed for every release.

#==============================================================================
# Declarations and Environment

declare Version=1.7.0
declare ThisScript="${0##*/}"
declare ThisUser=
declare CmdArgs="$*"
declare CmdLine="$0 $CmdArgs"
declare -i ErrorCount=0
declare -i WarningCount=0
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare Log="Unset"
declare -i PreflightOnly=0
declare -i IgnorePreflightErrors=0
declare -i Debug=0
declare -i NoOp=1
declare -i i=0

declare SDPInstallRoot="/p4"
declare SDPCommon="$SDPInstallRoot/common"
declare SDPCommonSiteCfg="$SDPCommon/site/config"
declare SDPCommonBackup=
declare SDPBackup=
declare SDPEnvFile="$SDPCommon/bin/p4_vars"
declare SDPEnvFileTemplate=
declare SDPEnvFileNew=
declare SDPCustomEnvFile="/p4/common/site/config/p4_vars.local"
declare SDPCustomEnvFileNew=

# TAG: Update the SDPEnvFileNewFormatVersion (for p4_vars) if the file format changes.
# This setting in this file is the default. The actual format version is defined
# with the SDP_P4_VARS_FORMAT setting in the p4_vars.template file.  Keep the values
# in sync in this file and the p4_vars.template file.
declare SDPEnvFileNewFormatVersion=1.2

# TAG: Update the SDPInstanceCfgFileNewFormatVersion (for instance vars files) if the
# file format changes. This setting in this file is the default. The actual format
# version is defined with the SDP_INSTANCE_VARS_FORMAT setting in the
# instance_vars.template file. Keep the values in sync in this file and the
# instance_vars.template file.
declare SDPInstanceCfgFileNewFormatVersion=1.5

declare SDPOwner=
declare HxDepots=

# The HxDepotsDocDefault and SDPOwnerDocDefault are used when running with
# '-man' to access the documentation/manual page. This is only used in an
# environment where the site local value for HxDepots could not be determined,
# e.g. if running on a laptop with no SDP installed to generate docs with '-man'
# (which is the only useful thing that can be done with this sdp_upgrade.sh if
# SDP is not installed).
declare HxDepotsDocDefault=/hxdepots
declare SDPOwnerDocDefault=perforce

declare DownloadsDir=
declare DownloadsDirNew=
declare SDPInstanceList=
declare -a SDPInstanceCfgFiles
declare -a SDPInstanceCfgFilesNew
declare -a SDPCustomInstanceCfgFiles
declare -a SDPCustomInstanceCfgFilesNew
declare -i SDPInstanceCfgFileCount=0
declare SDPInstanceCfgFileTemplate=
declare CommonUpdateCmd=
declare SDPUpdateCmd=
declare PreSDPUpgradeScript=/p4/common/site/upgrade/pre-sdp_upgrade.sh
declare PreSDPUpgradeCmd=
declare PostSDPUpgradeScript=/p4/common/site/upgrade/post-sdp_upgrade.sh
declare PostSDPUpgradeCmd=
declare SDPTarballURL="https://swarm.workshop.perforce.com/projects/perforce-software-sdp/download/downloads/sdp.Unix.tgz"
declare SDPTarball="${SDPTarballURL##*/}"
declare DeprecatedFilesListFile=

# The SDPTargetMajorVersion is hard-coded by design. This is upgraded with every SDP major
# version.  The SDPNewVersion includes the changelist number extracted from the SDP
# 'Version' file which is updated by automation during the SDP release process.
# TAG: Update the major version for each release.
declare SDPTargetMajorVersion=2022.2
declare SDPNewVersionString=
declare SDPNewVersion=
declare SDPOldVersionString=
declare SDPOldVersion=
declare SDPDefaultUmask=

#==============================================================================
# Local Functions

# Note: This script does not use SDP library files, as its purpose is to
# upgrade the SDP installation.
function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -ne 0 ]] && msg "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# This function takes as input an SDP version string, and returns a version
# id of the form YYYY.N.CL, where YYYY is the year, N is an incrementing
# release ID with a given year, and CL is a changelist identifier. The
# YYYY.N together comprise the major version, often shortened to YY.N, e.g.
# r20.1 for the 2020.1 release.
#
# The full SDP Version string looks something like this:
# Rev. SDP/MultiArch/2019.3/26494 (2020/04/23).
#
# This function parses that full string and returns a value like: 2019.3.26494
function get_sdp_version_from_string () {
   dbg "CALL get_sdp_veresion_from_string($*)"
   local versionString="${1:-}"
   local version=
   version="20${versionString##*/20}"
   version="${version%% *}"
   version="${version/\//.}"

   [[ "$version" == "20" || "$version" == "200" ]] && version="Unknown"
   echo "$version"
}

#------------------------------------------------------------------------------
# Function: check_p4_vars_file ()
#
# Determine if the SDP p4_vars file can be safely upgraded. Generate the
# new content, but do not put it in place.
#
# Extract any custom content below the '### MAKE LOCAL CHANGES HERE' line
# and put it in a file that will be moved to
# /p4/common/site/config/p4_vars.local
#------------------------------------------------------------------------------
function check_p4_vars_file () {
   dbg "CALL check_p4_vars_file()"
   # See if SDP_P4_VARS_FORMAT is defined.
   local oldFileFormatVersion=1.0
   local -A cfgValues=
   local -i allRequiredSettingsFound=1
   local linesOfCustomCode=

   # Set the global variable referring to the generated new version.
   SDPEnvFileNew=$(mktemp)
   SDPCustomEnvFileNew=$(mktemp)

   # This logic uses the SDP_P4_VARS_FORMAT, which is updated as needed
   # if significant changes to the p4_vars file format are made.  The
   # first assigned version is 1.1; if no version is detetected, it is
   # set to 1.0 (as the first version from SDP 2020.1 did not set
   # SDP_P4_VARS_FORMAT).  This is infrastructure for future upgrades on
   # an ongoing basis.
   if grep -q ^SDP_P4_VARS_FORMAT= "$SDPEnvFile"; then
      oldFileFormatVersion=$(grep ^SDP_P4_VARS_FORMAT= "$SDPEnvFile" | tail -1 | cut -d '=' -f 2)
      msg "Old p4_vars version detected: $oldFileFormatVersion"
   else
      msg "Old p4_vars version set to: $oldFileFormatVersion"
   fi
   msg "Changing p4_vars file format from version $oldFileFormatVersion to $SDPEnvFileNewFormatVersion."

   # Check for required parameters in p4_vars file.
   for var in KEEPLOGS KEEPCKPS KEEPJNLS; do
      grepCmd="grep -qE '^\s*export $var=' $SDPEnvFile"
      if ! eval "$grepCmd"; then
         errmsg "Missing definition of variable $var in: $SDPEnvFile"
         allRequiredSettingsFound=0
      fi
   done

   if [[ "$allRequiredSettingsFound" -eq 0 ]]; then
      errmsg "The SDP Env File p4_vars was missing required settings."
      return 1
   fi

   # Extract values for KEEPLOGS, KEEPCKPS, and KEEPJNLS settings.
   for var in KEEPLOGS KEEPCKPS KEEPJNLS; do
      value=$(grep -E "^\s*export $var=" "$SDPEnvFile" | tail -1 | cut -d '=' -f 2)
      cfgValues[$var]="$value"
   done

   if sed -e "s|REPL_OSUSER|$SDPOwner|g" \
      -e "s|REPL_SDPVERSION|$SDPNewVersionString|g" \
      -e "s|export KEEPLOGS=.*|export KEEPLOGS=${cfgValues[KEEPLOGS]}|g" \
      -e "s|export KEEPCKPS=.*|export KEEPCKPS=${cfgValues[KEEPCKPS]}|g" \
      -e "s|export KEEPJNLS=.*|export KEEPJNLS=${cfgValues[KEEPJNLS]}|g" \
      "$SDPEnvFileTemplate" > "$SDPEnvFileNew"; then
      msg "Verified: $SDPEnvFile can be generated."
      msg "\\n${H2}\\nBEGIN Diffs for: p4_vars\\n"
      diff "$SDPEnvFile" "$SDPEnvFileNew"
      msg "\\n${H2}\\nEND Diffs for: p4_vars\\n"
   else
      errmsg "Failed to generate the SDP Env file p4_vars."
      return 1
   fi

   linesOfCustomCode=$(grep -a -A 9999999 '^### MAKE LOCAL CHANGES HERE' "$SDPEnvFile" | grep -c -v '^### MAKE LOCAL CHANGES HERE')

   if [[ "$linesOfCustomCode" -ne 0 ]]; then
      msg "Extracting $linesOfCustomCode lines of custom code from $SDPEnvFile, to be deposited in $SDPCustomEnvFile"
      if [[ ! -e "$SDPCustomEnvFile" ]]; then
         grep -a -A 9999999 '^### MAKE LOCAL CHANGES HERE' "$SDPEnvFile" | grep -v '^### MAKE LOCAL CHANGES HERE' > "$SDPCustomEnvFileNew"

         msg "\\n${H2}\\nBEGIN Preview of Extracted Custom Content for: $SDPCustomEnvFile"
         cat "$SDPCustomEnvFileNew"
         msg "\\n${H2}\\nEND Extracted Custom Contnet for: $SDPCustomEnvFile\\n"
      else
         errmsg "A $SDPCustomEnvFile already exists, but $linesOfCustomCode were found in $SDPEnvFile after the '### MAKE LOCAL CHANGES HERE' tag."
      fi
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: check_instance_cfg_file ($i)
#
# Determine if the SDP instance config file can be safely upgraded.
# See if SDP_INSTANCE_VARS_FORMAT is defined.
# Generate a new instance config file, but do not deploy it.
#
# Extract any custom content below the '### MAKE LOCAL CHANGES HERE' line
# and put it in a temp file that will be placed in
# the file: /p4/common/site/config/p4_vars.local
#------------------------------------------------------------------------------
function check_instance_cfg_file () {
   dbg "CALL check_instance_cfg_file($*)"
   local instanceCfgFile=
   local instanceCfgFileNew=
   local instanceCustomCfgFile=
   local instanceCustomCfgFileNew=
   local oldFileFormatVersion=1.0
   local -i allRequiredSettingsFound=1
   local -A instanceCfgValues=
   local var=
   local value=
   local linesOfCustomCode=

   instanceCfgFile=${SDPInstanceCfgFiles[$i]}
   instanceCfgFileNew=${SDPInstanceCfgFilesNew[$i]}
   instanceCustomCfgFile=${SDPCustomInstanceCfgFiles[$i]}
   instanceCustomCfgFileNew=${SDPCustomInstanceCfgFilesNew[$i]}

   # This logic uses the SDP_INSTANCE_VARS_FORMAT, which is updated as needed
   # if significant changes to the Instance Vars file format are made.  The
   # first assigned version is 1.1; if no version is detetected, it is
   # set to 1.0 (as the first version from SDP 2020.1 did not set
   # SDP_P4_VARS_FORMAT).  This is infrastructure for future upgrades on
   # an ongoing basis.
   if grep -q ^SDP_INSTANCE_VARS_FORMAT= "$instanceCfgFile"; then
      oldFileFormatVersion=$(grep ^SDP_INSTANCE_VARS_FORMAT= "$instanceCfgFile" | tail -1 | cut -d '=' -f 2)
      msg "Old Instance Vars version detected: $oldFileFormatVersion"
   else
      msg "Old Instance Vars version set to: $oldFileFormatVersion"
   fi
   msg "Changing $instanceCfgFile file format from version $oldFileFormatVersion to $SDPInstanceCfgFileNewFormatVersion."

   if [[ -r "$instanceCfgFile" ]]; then
      # Step 1: Check for required parameters in the instance config file.
      for var in MAILTO MAILFROM P4USER P4MASTER_ID SSL_PREFIX P4PORTNUM P4BROKERPORTNUM P4MASTERHOST; do
         grepCmd="grep -qE '^\s*export $var=' $instanceCfgFile"
         if ! eval "$grepCmd"; then
            errmsg "Missing definition of variable $var in: $instanceCfgFile"
            allRequiredSettingsFound=0
         fi
      done

      if [[ "$allRequiredSettingsFound" -eq 0 ]]; then
         errmsg "This instance config file cannot be upgraded: $instanceCfgFile"
         return 1
      fi

      # Step 2: Get values for all parmeters that users can modify, including
      # required and optional parameters. Some may be empty. This excludes values
      # that are calculated, such sd P4D_FLAGS, SHAREDDATA, etc.
      # Step 2A: Start with the required parmeters.
      for var in MAILTO MAILFROM P4USER P4MASTER_ID SSL_PREFIX P4PORTNUM P4BROKERPORTNUM P4MASTERHOST; do
         # For most settings, we want to preserve the *last* definition found in the Instance
         # Vars file. We make an exception for P4MASTERHOST: Due to the way it is used in the
         # instance_vars.template, we want to preserve only the *first* setting, not the last.
         if [[ "$var" == "P4MASTERHOST" ]]; then
            value=$(grep -E "^\s*export $var=" "$instanceCfgFile" | head -1 | cut -d '=' -f 2)
         else
            value=$(grep -E "^\s*export $var=" "$instanceCfgFile" | tail -1 | cut -d '=' -f 2)
         fi
         dbg "VAR 1 [$var] VALUE=[$value]"
         instanceCfgValues[$var]="$value"
      done

      # Step 2B: Get values for other parameters users can modify.
      for var in P4DTG_CFG PROXY_TARGET PROXY_PORT PROXY_V_FLAGS SNAPSHOT_SCRIPT VERIFY_SDP_SKIP_TEST_LIST SDP_ALWAYS_LOGIN SDP_MAX_START_DELAY_P4D SDP_MAX_START_DELAY_P4BROKER SDP_MAX_START_DELAY_P4P SDP_MAX_STOP_DELAY_P4BROKER SDP_MAX_STOP_DELAY_P4P SDP_AUTOMATION_USERS DO_PARALLEL_CHECKPOINTS; do
         # For most settings, we want to preserve the *last* definition found in the Instance
         # Vars file. We make an exception for SNAPSHOT_SCRIPT: Due to the way it is used in the
         # instance_vars.template, we want to preserve only the *first* setting, not the last.
         if [[ "$var" == "SNAPSHOT_SCRIPT" ]]; then
            value=$(grep -E "^\s*export $var=" "$instanceCfgFile" | head -1 | cut -d '=' -f 2)
         else
            value=$(grep -E "^\s*export $var=" "$instanceCfgFile" | tail -1 | cut -d '=' -f 2)
         fi
         dbg "VAR 2 [$var] VALUE=[$value]"
         instanceCfgValues[$var]="$value"
      done

      # Step 3: Get Umask.
      var="umask"
      value=$(grep ^umask "$instanceCfgFile" | tail -1 | awk '{print $2}')
      [[ -n "$value" ]] || value="$SDPDefaultUmask"
      dbg "VAR 3 [$var] VALUE=[$value]"
      instanceCfgValues[$var]="$value"

      # Step 4: The big substitution! Beware of legacy idiosyncracies; do not assume the
      # variable name matches the REPL value, e.g. SSL_PREFIX=REPL_SSLPREFIX and
      # P4PORTNUM=REPL_P4PORT, etc.

      dbg "Substitution Values:"
      for var in "${!instanceCfgValues[@]}"; do
         [[ "$var" == "0" ]] && continue
         msg "  [$var]=[${instanceCfgValues[$var]}]."
      done

      if sed -e "s|=REPL_MAILTO|=${instanceCfgValues[MAILTO]}|g" \
         -e "s|=REPL_MAILFROM|=${instanceCfgValues[MAILFROM]}|g" \
         -e "s|=REPL_ADMINUSER|=${instanceCfgValues[P4USER]}|g" \
         -e "s|=REPL_MASTER_ID|=${instanceCfgValues[P4MASTER_ID]}|g" \
         -e "s|=REPL_SSLPREFIX|=${instanceCfgValues[SSL_PREFIX]}|g" \
         -e "s|=REPL_P4PORT|=${instanceCfgValues[P4PORTNUM]}|g" \
         -e "s|=REPL_P4BROKERPORT|=${instanceCfgValues[P4BROKERPORTNUM]}|g" \
         -e "s|=REPL_P4MASTERHOST|=${instanceCfgValues[P4MASTERHOST]}|g" \
         -e "s|export P4DTG_CFG=.*|export P4DTG_CFG=${instanceCfgValues[P4DTG_CFG]}|g" \
         -e "s|export PROXY_TARGET=.*|export PROXY_TARGET=${instanceCfgValues[PROXY_TARGET]}|g" \
         -e "s|export PROXY_V_FLAGS=.*|export PROXY_V_FLAGS=${instanceCfgValues[PROXY_V_FLAGS]}|g" \
         -e "s|export SNAPSHOT_SCRIPT=.*|export SNAPSHOT_SCRIPT=${instanceCfgValues[SNAPSHOT_SCRIPT]}|g" \
         -e "s|export VERIFY_SDP_SKIP_TEST_LIST=.*|export VERIFY_SDP_SKIP_TEST_LIST=${instanceCfgValues[VERIFY_SDP_SKIP_TEST_LIST]}|g" \
         -e "s|export SDP_ALWAYS_LOGIN=.*|export SDP_ALWAYS_LOGIN=${instanceCfgValues[SDP_ALWAYS_LOGIN]}|g" \
         -e "s|export SDP_MAX_START_DELAY_P4D=.*|export SDP_MAX_START_DELAY_P4D=${instanceCfgValues[SDP_MAX_START_DELAY_P4D]}|g" \
         -e "s|export SDP_MAX_START_DELAY_P4BROKER=.*|export SDP_MAX_START_DELAY_P4BROKER=${instanceCfgValues[SDP_MAX_START_DELAY_P4BROKER]}|g" \
         -e "s|export SDP_MAX_START_DELAY_P4P=.*|export SDP_MAX_START_DELAY_P4P=${instanceCfgValues[SDP_MAX_START_DELAY_P4P]}|g" \
         -e "s|export SDP_MAX_STOP_DELAY_P4BROKER=.*|export SDP_MAX_STOP_DELAY_P4BROKER=${instanceCfgValues[SDP_MAX_STOP_DELAY_P4BROKER]}|g" \
         -e "s|export SDP_MAX_STOP_DELAY_P4P=.*|export SDP_MAX_STOP_DELAY_P4P=${instanceCfgValues[SDP_MAX_STOP_DELAY_P4P]}|g" \
         -e "s|export SDP_AUTOMATION_USERS=.*|export SDP_AUTOMATION_USERS=${instanceCfgValues[SDP_AUTOMATION_USERS]}|g" \
         -e "s|^umask .*|umask ${instanceCfgValues[umask]}|g" \
         "$SDPInstanceCfgFileTemplate" > "$instanceCfgFileNew"; then
         msg "\\nVerified: This instance config file can be upgraded: $instanceCfgFile"

         msg "\\n${H2}\\nBEGIN Diffs for: $instanceCfgFile"
         diff "$instanceCfgFile" "$instanceCfgFileNew"
         msg "\\n${H2}\\nEND Diffs for: $instanceCfgFile\\n"

         # Step 5: Extract any customizations.
         linesOfCustomCode=$(grep -a -A 9999999 '^### MAKE LOCAL CHANGES HERE' "$instanceCfgFile" | grep -c -v '^### MAKE LOCAL CHANGES HERE')
         if [[ "$linesOfCustomCode" -ne 0 ]]; then
            msg "Extracting $linesOfCustomCode lines of custom code from $instanceCfgFile, to be deposited in $instanceCustomCfgFile"
            if [[ ! -e "$instanceCustomCfgFile" ]]; then
               grep -a -A 9999999 '^### MAKE LOCAL CHANGES HERE' "$instanceCfgFile" | grep -v '^### MAKE LOCAL CHANGES HERE' > "$instanceCustomCfgFileNew"

               msg "\\n${H2}\\nBEGIN Preview of Extracted Custom Content for: $instanceCustomCfgFile"
               cat "$instanceCustomCfgFileNew"
               msg "\\n${H2}\\nEND Extracted Custom Contnet for: $instanceCustomCfgFile\\n"
            else
               errmsg "The file $instanceCustomCfgFile already exists, but $linesOfCustomCode were found in $instanceCfgFile after the '### MAKE LOCAL CHANGES HERE' tag."
            fi
         fi

      else
         errmsg "This instance config file cannot be upgraded: $instanceCfgFile"
      fi
   else
      errmsg "Missing SDP Instance config file: $instanceCfgFile"
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: check_sdp_instances ()
#
# Get the list of SDP instances after doing some preliminary sanity
# checks. Sets global SDPInstanceList. Verify that the SDP instance
# config file for each instance can be upgraded.
function check_sdp_instances () {
   dbg "CALL check_sdp_instance()"
   local e=
   local instanceCfg=
   local instanceCustomCfg=
   local instanceBinDir=

   SDPInstanceList=

   cd "$SDPInstallRoot" || \
      bail "Could not cd to SDP Install Root dir: $SDPInstallRoot"

   # Start with elements (files/dirs/symlinks) under /p4.
   for e in *; do
      # Silently ignore expected elements under /p4.
      [[ "$e" =~ ^(common|sdp|ssl)$ ]] && continue

      # Warn about symlinks under /p4. There should not be any symlinks
      # other than the SDP standard symlinks.
      if [[ -L "$e" ]]; then
         # Special case: The '/p4/metrics' symlink will exist if
         # P4Prometheus is installed. If that path exsits and is a
         # symlink, silently ignore it.  However, because 'metrics' is
         # a potentially valid SDP instance name, we will process it as
         # an SDP instance below if the '/p4/metrics' path is a regular
         # directory (not a symlink).
         [[ "$e" =~ ^(metrics)$ ]] && continue

         warnmsg "Ignoring unexpected symlink $SDPInstallRoot/$e"
         continue
      fi

      # Consider only at directories under /p4; silently ignore files.
      if [[ -d "$e" ]]; then
         # For a directory under /p4 to be an SDP instance, there must
         # be corresponding instance configuration file to upgrade:
         # /p4/common/config/p4_N.vars.local
         instanceCfg="$SDPInstallRoot/common/config/p4_${e}.vars"
         instanceCustomCfg="$SDPInstallRoot/common/site/config/p4_${e}.vars.local"
         instanceBinDir="$SDPInstallRoot/${e}/bin"

         if [[ -r "$instanceCfg" ]]; then
            SDPInstanceList+=" $e"
            SDPInstanceCfgFiles[$SDPInstanceCfgFileCount]="$instanceCfg"
            SDPInstanceCfgFilesNew[$SDPInstanceCfgFileCount]=$(mktemp)
            SDPCustomInstanceCfgFiles[$SDPInstanceCfgFileCount]="$instanceCustomCfg"
            SDPCustomInstanceCfgFilesNew[$SDPInstanceCfgFileCount]=$(mktemp)
            SDPInstanceCfgFileCount+=1
         elif [[ -d "$instanceBinDir" ]]; then
            warnmsg "Found instance bin dir [$instanceBinDir] but no corresponding instance config file [$instanceCfg]. Assuming $SDPInstallRoot/$e is not a valid SDP instance directory."
         fi
      fi
   done
 
   if [[ "$SDPInstanceCfgFileCount" -ne 0 ]]; then
      i=0; while [[ "$i" -lt "$SDPInstanceCfgFileCount" ]]; do
         check_instance_cfg_file "$i" ||\
            errmsg "This instance config file cannot be upgraded: ${SDPInstanceCfgFiles[$i]}"
         i+=1
      done
   else
      warnmsg "There are no SDP instance config files to update."
   fi

   # Trim leading space.
   # shellcheck disable=SC2116
   SDPInstanceList=${SDPInstanceList# }
   if [[ -n "$SDPInstanceList" ]]; then
      msg "\\nList of valid SDP Instances: $SDPInstanceList"
   else
      warnmsg "\\nNo SDP instances detected under $SDPInstallRoot"
   fi

   cd - > /dev/null || bail "Could not cd back to: $OLDPWD"
}

#------------------------------------------------------------------------------
# Function: verify_required_tools_are_available ()
#
# If these sanity checks fail, further tests are aborted. Failure of the
# very basic sanity checks is an indication that necessary tools for this
# script to operate are not available.
#
# This function is called before logging is formally started, so this
# routine operates silently unless debug mode is specified with '-d' or '-D'.
#
# These sanity checks cannot be ignored with '-I'.
#------------------------------------------------------------------------------
function verify_required_tools_are_available () {
   dbg "CALL verify_required_tools_are_available()"

   local toolsList="awk cat date grep head id ls rsync sed sort tail tee wc"

   dbg "$H2\\nDoing basic sanity checks."
   dbg "Preflight Check: Ensuring these utils are in PATH: $toolsList"

   for tool in $toolsList; do
      CheckCount+=1
      [[ -z "$(command -v "$tool")" ]] && \
         errmsg "Required OS utility '$tool' not in PATH."
   done

   # If basic tools aren't available, we don't even try to complete the
   # sanity checks, and return immediately.
   [[ "$ErrorCount" -eq 0 ]] || return 1

   dbg "Verified: Essential tools are in the PATH."

   return 0
}

#------------------------------------------------------------------------------
# Function: do_preflight_checks ()
#
# Do standard preflight checks, checking the basic SDP strucure.
#------------------------------------------------------------------------------
function do_preflight_checks () {
   dbg "CALL do_preflight_checks()"

   local oldSDPVersionFile="$SDPInstallRoot/sdp/Version"
   local newSDPVersionFile="../../../../../Version"

   msg "Preflight Check: Get Current SDP Version from: $SDPInstallRoot/sdp/Version" 

   if [[ -r "$oldSDPVersionFile" ]]; then
      SDPOldVersionString=$(cat "$oldSDPVersionFile")
      SDPOldVersion=$(get_sdp_version_from_string "$SDPOldVersionString")
      [[ "$SDPOldVersion" == "Unknown" ]] && \
         errmsg "Could not determine SDP old version from this string: $SDPOldVersionString"
   else
      # Failure of this preflight check requires a hard stop.
      bail "Could not find old SDP Version file: $oldSDPVersionFile"
   fi

   msg "Preflight Check: Get New SDP Version from: $newSDPVersionFile"
   if [[ -r "$newSDPVersionFile" ]]; then
      SDPNewVersionString=$(cat "$newSDPVersionFile")
      SDPNewVersion=$(get_sdp_version_from_string "$SDPNewVersionString")
      [[ "$SDPNewVersion" == "Unknown" ]] && \
         errmsg "Could not determine SDP target version from this string: $SDPNewVersionString"
   else
      # Failure of this preflight check requires a hard stop.
      bail "Could not find new SDP Version file: $PWD/$newSDPVersionFile"
   fi

   # shellcheck disable=SC2072
   if [[ "$SDPOldVersion" > "2020.1" ]]; then
      msg "Verified: Current SDP Version [$SDPOldVersion] is at least 2020.1."
   else
      errmsg "The old SDP Version is not 2020.1 or newer, as is required to use this script. To upgrade from older SDP versions, see the SDP Legacy Upgrades guide: https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/doc/SDP_Legacy_Upgrades.Unix.html"
   fi

   if [[ "$SDPNewVersion" > "$SDPOldVersion" ]]; then
      msg "\\nThe SDP will be upgraded from $SDPOldVersion to $SDPNewVersion.\\n"
   elif [[ "$SDPNewVersion" == "$SDPOldVersion" ]]; then
      warnmsg "The new SDP version matches the old: $SDPNewVersion."
   else
      errmsg "The target SDP version [$SDPNewVersion] is older than the current version [$SDPOldVersion]. Downgrades are not supported."
   fi

   msg "Preflight Check: cd $SDPCommon"

   if cd "$SDPCommon"; then
      cd - > /dev/null || bail "Failed to cd to $OLDPWD. Aborting."
   else
      errmsg "Could not cd to: $SDPCommon"
      return 1
   fi

   msg "Verified: cd works to: $SDPCommon"

   msg "Preflight Check: Checking current user owns $SDPCommon"

   if [[ -n "$SDPOwner" ]]; then
      msg "SDPOwner is: $SDPOwner"
   
      if [[ "$ThisUser" == "$SDPOwner" ]]; then
         msg "Verified: Current user [$ThisUser] owns $SDPCommon"
      else
         errmsg "Current user [$ThisUser] does not own $SDPCommon. This most likely means this script is running as the wrong user.  It could also mean the $SDPCommon directory is not owned by the correct owner, which should be the OS account under which the p4d process runs."
         return 1
      fi
   else
      errmsg "Could not determine value for SDPOwner. This most likely means this script is running as the wrong user. This should be run under the OS account under which the p4d process runs."
   fi

   if [[ -n "$HxDepots" ]]; then
      msg "HxDepots is: $HxDepots"
   else
      errmsg "Could not determine value for HxDepots."
   fi

   if [[ -d "$HxDepots" ]]; then
      msg "Verified: HxDepots path [$HxDepots] is a directory."
   else
      errmsg "HxDepots path [$HxDepots] is not a directory."
   fi

   [[ -r "$SDPEnvFile" ]] || errmsg "Missing SDP Environment File: $SDPEnvFile"

   DownloadsDir="$HxDepots/downloads"
   DownloadsDirNew="$DownloadsDir/new"
   SDPEnvFileTemplate="$DownloadsDirNew/sdp/Server/Unix/p4/common/config/p4_vars.template"
   SDPInstanceCfgFileTemplate="$DownloadsDirNew/sdp/Server/Unix/p4/common/config/instance_vars.template"

   [[ -r "$SDPEnvFileTemplate" ]] ||\
      errmsg "Missing SDP p4_vars tempate: $SDPEnvFileTemplate"

   [[ -r "$SDPInstanceCfgFileTemplate" ]] ||\
      errmsg "Missing SDP Instance Vars tempate: $SDPInstanceCfgFileTemplate"

   if grep -q ^SDP_P4_VARS_FORMAT= "$SDPEnvFileTemplate"; then
      SDPEnvFileNewFormatVersion=$(grep ^SDP_P4_VARS_FORMAT= "$SDPEnvFileTemplate" | tail -1 | cut -d '=' -f 2)
   else
      # The new file format version should always be defined; warn othewise.
      warnmsg "New p4_vars version not detected; defaulting to: $SDPEnvFileNewFormatVersion"
   fi

   if grep -q ^SDP_INSTANCE_VARS_FORMAT= "$SDPInstanceCfgFileTemplate"; then
      SDPInstanceCfgFileNewFormatVersion=$(grep ^SDP_INSTANCE_VARS_FORMAT= "$SDPInstanceCfgFileTemplate" | tail -1 | cut -d '=' -f 2)
   else
      # The new file format version should always be defined; warn othewise.
      warnmsg "New Instance Vars version not detected; defaulting to: $SDPInstanceCfgFileNewFormatVersion"
   fi

   msg "Detected and Assigned Settings:
   HxDepots:         $HxDepots
   DownloadsDir:     $DownloadsDir
   DownloadsDirNew:  $DownloadsDirNew

   SDPEnvFileNewFormatVersion:         $SDPEnvFileNewFormatVersion
   SDPInstanceCfgFileNewFormatVersion: $SDPInstanceCfgFileNewFormatVersion
"

   SDPDefaultUmask=$(grep umask "$SDPInstanceCfgFileTemplate" | tail -1 | awk '{print $2}')

   # Check the main p4_vars file.
   check_p4_vars_file

   # Check the main Instance Vars file for each instance.
   check_sdp_instances

   if [[ -x "$PreSDPUpgradeScript" ]]; then
      PreSDPUpgradeCmd="$PreSDPUpgradeScript"
      [[ "$NoOp" -eq 0 ]] && PreSDPUpgradeCmd+=" -y"

      if [[ "$IgnorePreflightErrors" -eq 0 ]]; then
         msg "\\nA custom pre-upgrade script exists and will be executed if preflight checks\\nare successful. The pre-upgrade command line will be:\\n\\t$PreSDPUpgradeCmd\\n"
      else
         msg "\\nA custom pre-upgrade script exists and will be executed after preflight checks\\nare attempted. The pre-upgrade command line will be:\\n\\t$PreSDPUpgradeCmd\\n"
      fi
   fi

   if [[ -x "$PostSDPUpgradeScript" ]]; then
      PostSDPUpgradeCmd="$PostSDPUpgradeScript"
      [[ "$NoOp" -eq 0 ]] && PostSDPUpgradeCmd+=" -y"

      msg "\\nA custom post-upgrade script exists and will be executed if the upgrade is\\nsuccessful. The post-upgrade command line will be:\\n\\t$PostSDPUpgradeCmd\\n"
   fi

   return "$ErrorCount"
}

#------------------------------------------------------------------------------
# Function: run ($cmd, $desc, $honorNoOp, $showOutput)
#
# Runs a command, with optional description, showing command line to execute
# and optionally also the output, and capturing and returning the exit code.
#
# Input:
# $1 - Command and arguments to execute. Defaults to 'echo'.
# $2 - Optional message to display describing what the command is doing.
# $3 - Numeric flag to determine of the global NoOp is honored.
#      Default is 1.
# $4 - Numeric flag to show output; '1' indicates to show output, 0 to
#      suppress it. Default is 1.
#------------------------------------------------------------------------------
function run () {
   local cmdAndArgs="${1:-echo}"
   local desc="${2:-}"
   local -i honorNoOp="${3:-1}"
   local -i showOutput="${4:-1}"
   local -i exitCode=
   local cmdLog=

   cmdLog="$(mktemp run.XXXXXXXXXXX.log)"

   [[ -n "$desc" ]] && msg "$desc"
   msg "Executing: $cmdAndArgs"

   # If we're not in NoOp mode, or ignoring it, then execute the command.
   if [[ "$NoOp" -eq 0 || "$honorNoOp" -eq 0 ]]; then
      eval "$cmdAndArgs" > "$cmdLog" 2>&1
      exitCode=$?
   else
      echo "NO-OP: Would run: $cmdAndArgs" > "$cmdLog" 2>&1
      exitCode=0
   fi

   if [[ "$showOutput" -eq 1 ]]; then
      echo "EXIT_CODE: $exitCode" >> "$cmdLog"
      cat "$cmdLog"
   fi

   /bin/rm -f "$cmdLog"
   return "$exitCode"
}

#------------------------------------------------------------------------------
# Function: remove_deprecated_files ($deprecatedFilesListFile)
#
# This function removes deprecated SDP files listed in a file containing the
# list of deprecated SDP files.  Review this file for more information:
#
# /p4/common/sdp_upgrade/deprecated_files.txt
#
# This function remove deprecated files, silently ignores files that are not
# there, and generates warnings when existing files fail to remove (e.g.
# due to 'root' ownership).
#
# Input:
# $1 - File containing a list of files to remove, as paths relative to the
#      /p4/common directory.
#------------------------------------------------------------------------------
function remove_deprecated_files () {
   local deprecatedFilesListFile="${1:-Unset}"
   local fileRelPath=
   local file=
   local -i removalErrors=0

   if [[ "$deprecatedFilesListFile" == Unset ]]; then
      errmsg "Bad Usage: remove_deprecated_files()."
      return 1
   fi

   msg "\\nRemoving deprecated files listed in: $deprecatedFilesListFile\\n"

   while read -r fileRelPath; do
      # Ignore comments and blank links.
      [[ -n "$fileRelPath" ]] || continue
      [[ "$fileRelPath" =~ ^[\ ]*# ]] && continue

      # Process removals.
      file="$SDPInstallRoot/$fileRelPath"
      if [[ -e "$file" ]]; then
         msg "  Removing: $file"

         if [[ "$NoOp" -eq 0 ]]; then
            if ! rm -f "$file"; then
               warnmsg "Failed to remove file: $file"
               removalErrors+=1
            fi
         else
            msg "NO-OP: Would execute: rm -f $file"
         fi
      fi
   done < "$deprecatedFilesListFile"

   if [[ "$removalErrors" -eq 0 ]]; then
      msg "\\nDeprecated file removal complete."
   else
      warnmsg "There were problems removing $removalErrors deprecated files."
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   # If a value for HxDepots was detected, use the locally detected value in
   # the man page to give site-local context to the man page. If no value 
   # for HxDepots was detected, use HxDepotsDocDefault.
   [[ -n "$HxDepots" ]] || HxDepots="$HxDepotsDocDefault"
   [[ -n "$SDPOwner" ]] || SDPOwner="$SDPOwnerDocDefault"
   DeprecatedFilesListFile="$HxDepots/downloads/new/sdp/Server/Unix/p4/common/sdp_upgrade/deprecated_files.txt"

   if [[ $errorMessage != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
   fi

   echo "USAGE for sdp_upgrade.sh v$Version:

   sdp_upgrade.sh [-y] [-p] [-L <log>|off] [-D]

   or

sdp_upgrade.sh -h|-man

This script must be executed from the 'sdp_upgrade' directory in the extracted
SDP tarball.

Typical operation starts like this:

cd $HxDepots/downloads/new/sdp/Server/Unix/p4/common/sdp_upgrade
./sdp_upgrade.sh -h

"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:

	This script upgrades Perforce Helix Server Deployment Package (SDP) from
	SDP 2020.1 to the version included in the latest SDP version, SDP
	$SDPTargetMajorVersion.

	== Pre-Upgrade Planning ==

	This script will upgrade the SDP if the pre-upgrade starting SDP version
	is SDP 2020.1 or later, including any/all patches of SDP
	2020.1.

	If the current SDP version is older than 2020.1, it must first be upgraded
	to SDP 2020.1 using the SDP Legacy Upgrade Guide.  For upgrading from
	pre-20.1 versions dating back to 2007, in-place or migration-style upgrades
	can be done.  See:

	https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/doc/SDP_Legacy_Upgrades.Unix.html

	The SDP should always be upgraded to the latest version first before
	Helix Core binaries p4d/p4broker/p4p are upgraded using the SDP
	upgrade.sh script.

	Upgrading the SDP first ensures the version of the SDP you have
	is compatible with the latest versions of p4d/p4broker/p4p/p4, and
	will always be compatible with all supported versions of these
	Helix Core binaries.

	When this script is used, i.e. when the current SDP version is 2020.1
	or newer, the SDP upgrade procedure does not require downtime for any
	running Perforce Helix services, such as p4d, p4broker, or p4p.  This
	script is safe to run in environments where live p4d instances are running,
	and does not require p4d, p4broker, p4p, or any other services to be stopped
	or upgraded.  Upgrade of the SDP is cleanly separate from the upgrade the
	Helix Core binaries. The upgrade of the SDP can be done immediately prior to
	Helix Core upgrades, or many days prior.

	There can be multiple SDP instances on a given server machine. This script
	will upgrade the SDP on the machine, and thus after the upgrade all
	instances will immediately use new SDP scripts and updated instance
	configuration files, e.g. the /p4/common/config/p4_N.vars files.  However,
	all instances will continue running the same Helix Core binaries.  Any live
	running Helix Core server process on the machine are unaffected by the
	upgrade of SDP.

	This script will upgrade the SDP on a single machine. If your Perforce
	Helix topology has multiple machines, the SDP should be upgraded on all
	machines. The upgrade of SDP on multiple machines can be done in any order,
	as there is no cross-machine dependency requiring the SDP to be the same
	version.  (The order of upgrade of Helix Core services and binaries such as
	p4d in global topologies with replicas and edge servers does matter, but is
	outside the scope of this script).

	Planning Recap:
	1. The SDP can be upgraded without downtime when this script is used, i.e.
	when the starting SDP version is 2020.1 or later.
	2. Upgrade SDP on all machines, in any order, before upgrading p4d and other
	Helix binaries.

	== NFS Sharing of HxDepots ==

	In some environments, the HxDepots volume is shared across multiple server
	machines with NFS, typically mounted as /hxdepots.  This script updates the
	/hxdepots/p4/common and /hxdepots/sdp directories, both of which are on the
	NFS mount.  Thus upgrading SDP on a single machine will effectively and
	immediately upgrade the SDP on all machines that share /hxdepots from the
	same NFS-mounted storage. This is a safe and valid configuration, as
	upgrading the SDP does not affect any live running p4d servers.

	== Acquiring the SDP Package ==

	This script is part of the SDP package (tarball). It must be run from an
	extracted tarball directory.  Acquiring the SDP tarball is a manual operation.

	The SDP tarball must be extracted such that the 'sdp' directory appears as
	<HxDepots>/downloads/new/sdp, where <HxDepots> defaults to /hxdepots. To
	determine the value for <HxDepots> at your site you can run the following:
   
     bash -c 'cd /p4/common; d=\$(pwd -P); echo \${d%/p4/common}'
   
   On this machine, that value is: $HxDepots

	Following are sample commands to acquire the latest SDP, to be executed
	as the user $SDPOwner:

	  cd $HxDepots
	  [[ -d downloads ]] || mkdir downloads
	  cd downloads
	  [[ -d new ]] && mv new old.\$(date +'%Y%m%d-%H%M')
	  curl -s -k -O $SDPTarballURL
	  mkdir new
	  cd new
	  tar -xzf ../$SDPTarball

	After extracting the SDP tarball, cd to the directory where this
	sdp_upgrade.sh script resides, and execute it from there.

	  cd $HxDepots/downloads/new/sdp/Server/Unix/p4/common/sdp_upgrade
	  ./sdp_upgrade.sh -man

	== Preflight Checks ==

	Prior to upgrading, preflight checks are performed to ensure the
	upgrade can be completed successfully. If the preflight checks
	fail, the upgrade will not start.

	Sample Preflight Checks:

	* The existing SDP version is verified to be SDP 2020.1+.
	* Various basic SDP structural checks are done.
	* The /p4/common/bin/p4_vars is checked to confirm it can be upgraded.
	* All /p4/common/config/p4_N.vars files are checked to confirm they can be upgraded.

	== Automated Upgrade Processing ==

	Step 1: Backup /p4/common.

	The existing <HxDepots>/p4/common structure is backed up to:

	<HxDepots>/p4/common.bak.<YYYYMMDD-hhmm>

	Step 2: Update /p4/common.

	The existing SDP /p4/common structure is updated with new
	versions of SDP files.

	Step 3: Generate the SDP Environment File.

	Regenerate the SDP general environment file,
	/p4/common/bin/p4_vars.

	The template is /p4/common/config/p4_vars.template.

	Step 4: Generate the SDP Instance Files.

	Regenerate the SDP instance environment files for all instances based on
	the new template.

	The template is /p4/common/config/instance_vars.template.

	For Steps 3 and 4, the re-generation logic will preserve current
	settings. If upgrading from SDP r20.1, any custom logic that
	exists below the '### MAKE LOCAL CHANGES HERE' tag will be
	split into separate files.  Custom logic in p4_vars will be moved
	to /p4/common/site/config/p4_vars.local. Custom logic in
	p4_N.vars files will be moved to /p4/common/site/config/p4_N.vars.local.

	Note: Despite these changes, the mechanism for loading the SDP shell
	environment remains unchanged since 2007, so it looks like:

	\$ source /p4/common/bin/p4_vars N

	Changes to the right-side of assignments for specific are preserved
	for all defined SDP settings.  For p4_vars, preserved settings are:
	  - OSUSER (determined by current owner of /p4/common)
	  - KEEPLOGS
	  - KEEPCKPS
	  - KEEPJNLS
	
	For instance_vars files, preserved settings are:
	  - MAILTO
	  - MAILFROM
	  - P4USER
	  - P4MASTER_ID
	  - SSL_PREFIX
	  - P4PORTNUM
	  - P4BROKERPORTNUM
	  - P4MASTERHOST
	  - PROXY_TARGET
	  - PROXY_PORT
	  - PROXY_V_FLAGS
	  - P4DTG_CFG
	  - SNAPSHOT_SCRIPT
	  - SDP_ALWAYS_LOGIN
	  - SDP_AUTOMATION_USERS
	  - SDP_MAX_START_DELAY_P4D
	  - SDP_MAX_START_DELAY_P4BROKER
	  - SDP_MAX_START_DELAY_P4P
	  - SDP_MAX_STOP_DELAY_P4D
	  - SDP_MAX_STOP_DELAY_P4BROKER
	  - SDP_MAX_STOP_DELAY_P4P
	  - VERIFY_SDP_SKIP_TEST_LIST
	  - The 'umask' setting.
	  - KEEPLOGS (if set)
	  - KEEPCKPS (if set)
	  - KEEPJNLS (if set)
	
	Note that the above list excludes any values that are calculated.
	
	Step 5: Remove Deprecated Files.

	Deprecated files will be purged from the SDP structure.  The list of
	files to be cleaned are listed in this file:

	$DeprecatedFilesListFile

	Paths listed in this file are relative to the '/p4' directory (or
	more accurately the SDP Install Root directory, which is always
	'/p4' except in SDP test production environments).

	Step 6: Update SDP crontabs.

	No crontab updates are required for this SDP upgrade.
	
	== Post-Upgrade Processing ==

	This script provides guidance on any post-processing steps. For some
	releases, this may include upgrades to crontabs.

OPTIONS:
-y	Specify the '-y' option to confirm that the SDP upgrade should be done.

	By default, this script operates in No-Op mode, meaning no actions
	that affect data or structures are taken.  Instead, commands that would
	be run are displayed.  This mode can be educational, showing various
	steps that will occur during an actual upgrade.

 -p	Specify '-p' to halt processing after preflight checks are complete,
	and before actual processing starts. By default, processing starts
	immediately upon successful completion of preflight checks.

 -L <log>
	Specify the log file to use.  The default is /tmp/sdp_upgrade.<timestamp>.log

	The special value 'off' disables logging to a file. This cannot be
	specified if '-y' is used.

 -d	Enable debugging verbosity.

 -D	Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

FILES AND DIRECTORIES:
	Name: SDPCommon
	Path: /p4/common
	Notes: This sdp_upgrade.sh script updates files in and under this folder.

	Name: HxDepots
	Default Path: /hxdepots
	Notes: The folder containing versioned files, checkpoints, and numbered
	journals, and the SDP itself. This is commonly a mount point.

	Name: DownloadsDir
	Default Path: /hxdepots/downloads

	Name: SDPInstallRoot
	Path: /p4

EXAMPLES:
	This script must be executed from 'sdp_upgrade' directory in the extracted
	SDP tarball.  Typical operation starts like this:

	cd /hxdepots/downloads/new/sdp/Server/Unix/p4/common/sdp_upgrade
	./sdp_upgrade.sh -h

	All following examples assume operation from that directory.

	Example 1: Prelight check only:

	sdp_upgrade.sh -p

	Example 2: Preview mode:

	sdp_upgrade.sh

	Example 3: Live operation:

	sdp_upgrade.sh -y

LOGGING:
	This script generates a log file, ~/sdp_upgrade.<timestamp>.log
	by default. See the '-L' option above.

CUSTOM PRE- AND POST- UPGRADE AUTOMATION HOOKS:
	This script can execute custom pre- and post- upgrade scripts. This
	can be useful to incorporate site-specific elements of an SDP upgrade.

	If the file /p4/common/site/upgrade/pre-sdp_upgrade.sh exists and is
	executable, it will be executed as a pre-upgrade script. If the file
	/p4/common/site/upgrade/post-sdp_upgrade.sh exists and is executable,
	it will be executed as a post-upgrade script.
	
	Pre- and post- upgrade scripts are passed the '-y' flag to confirm
	actual processing is to be done.  Custom scripts are expected to
	operate in preview mode by default, taking no actions that affect data
	(just as this script behaves).  If this sdp_upgrade.sh script is given
	the '-y' flag, that option is passed to the custom script as well,
	indicating active processing should occur.

	Pre- and post- upgrade scripts are expected to exit with a zero exit
	code to indicate success, and non-zero to indicate failure."

###
### Keep this block in case the undocumented '-I' flag becomes documented.
###
#	The custom pre-upgrade script is executed after standard preflight
#	checks complete successfully.  If the '-I' flag is used to ignore the
#	status of preflight checks, the custom pre-upgrade script is
#	executed regardless of the status of preflight checks. Preflight
#	checks are executed before actual upgrade processing commences. If a
#	custom pre-upgrade script indicates a failure, the overall upgrade
#	process aborts.

msg "	The custom pre-upgrade script is executed after standard preflight
	checks complete successfully.  Preflight checks are executed before
	actual upgrade processing commences. If a custom pre-upgrade script
	indicates a failure, the overall upgrade process aborts.

	The post-upgrade custom script is executed after the main SDP upgrade
	is successful.

	Success or failure of pre- and post- upgrade scripts is reported in
	the log.  These scripts do not require independent logging, as all
	standard and error output is captured in the log of this sdp_upgrade.sh
	script.

	TIP: Be sure to fully test custom scripts in a test environment
	before incorporating them into an upgrade on production systems.

EXIT CODES:
	An exit code of 0 indicates no errors were encountered during the
	upgrade. A non-zero exit code indicates the upgrade was aborted
	or failed.
"

   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log\\n${H1}\\n"

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#==============================================================================
# Command Line Processing

# Before processing command line arguments, ensure needed tools are availble
# in the PATH, and set some values that are referenced in the usage() function.

verify_required_tools_are_available || bail "Missing required tools. Aborting."

# shellcheck disable=SC2012
SDPOwner="$(ls -ld "$SDPCommon" 2>/dev/null | awk '{print $3}')"

# shellcheck disable=SC2164
HxDepots=$(cd /p4/common 2>/dev/null || exit; d=$(pwd -P); echo "${d%/p4/common}")

DeprecatedFilesListFile="$HxDepots/downloads/new/sdp/Server/Unix/p4/common/sdp_upgrade/deprecated_files.txt"

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-p) PreflightOnly=1;;
      (-y) NoOp=0;;
      (-I) IgnorePreflightErrors=1;; # Undocumented option.
      (-man) usage -man;;
      (-L) Log="$2"; shiftArgs=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown command line option ($1).";;
      (*) export SDP_INSTANCE=$1;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$Log" == "off" && "$NoOp" -eq 0 ]] && \
   usage -h "Disabling logging with '-L off' is not with '-y'."

[[ "$PreflightOnly" -eq 1 && "$IgnorePreflightErrors" -eq 1 ]] && \
   usage -h "The '-p' and '-I' options are mutually exclusive."

if [[ "$PWD" != *"/sdp_upgrade" ]]; then
   usage -h "Run this script from the sdp_upgrade directory in the extracted SDP tarball."
fi

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

[[ "$Log" == "Unset" ]] && Log="${HOME:-~}/sdp_upgrade.$(date +'%Y%m%d-%H%M%S').log"

if [[ "$Log" != "off" ]]; then
   [[ -e "$Log" ]] && bail "Log file already exists: $Log\\nAborting.\\n"

   if ! touch "$Log"; then
      BadLog="$Log"
      Log="off"
      bail "Couldn't touch log file [$BadLog]. Aborting."
   fi

   # Redirect stdout and stderr to a log file.
   exec > >(tee "$Log")
   exec 2>&1

   msg "${H1}\\nLog is: $Log"
fi

ThisUser="$(id -n -u)"
msg "$ThisScript v$Version Starting SDP upgrade as $ThisUser@${HOSTNAME%%.*} at $(date +'%a %Y-%m-%d %H:%M:%S %Z') with this command line:\\n$CmdLine"

msg "\\nIf you have any questions about the output from this script, contact support-helix-core@perforce.com."
   
if do_preflight_checks; then
   msg "\\nAll preflight checks were successful."

   if [[ "$PreflightOnly" -eq 1 ]]; then
      msg "\\nExiting early after successful preflight checks due to '-p'."
      exit 0
   fi
else
   if [[ "$IgnorePreflightErrors" -eq 1 ]]; then
      warnmsg "Preflight checks failed. Proceeding anyway due to '-I'. ErrorCount reset to 0."
      ErrorCount=0
   else
      bail "Aborting due to failed preflight checks. The current SDP remains unchanged."
   fi
fi

#------------------------------------------------------------------------------
if [[ "$ErrorCount" -eq 0 || "$IgnorePreflightErrors" -eq 1 ]]; then
   if [[ -n "$PreSDPUpgradeCmd" ]]; then
      msg "${H2}\\nSTEP 0: Custom Pre-SDP-Upgrade Processing."
      msg "\\nExecuting custom pre-upgrade command:\\n\\t$PreSDPUpgradeCmd"

      if $PreSDPUpgradeCmd; then
         msg "\\nThe custom pre-upgrade command indicated success."
      else
         bail "Although the standard preflight checks were successful, the custom pre-upgrade command indicated failure. Aborting the upgrade."
      fi
   fi
fi

#------------------------------------------------------------------------------
msg "${H2}\\nSTEP 1: Backup /p4/common."

cd "$HxDepots/p4" || bail "Could not cd to $HxDepots/p4."

SDPCommonBackup="common.bak.$(date '+%Y%m%d-%H%M%S')"
if run "rsync -a common/ $SDPCommonBackup" "Backing up $HxDepots/p4/common."; then
   msg "Backup of /p4/common completed OK."
else
   bail "The backup of the /p4/common directory failed. Aborting early: No changes to the existing SDP were made, so this is a harmless failure. Determine why the backup failed, fix the issue, and try again.  A potential cause of failure is incorrect (root?) ownership of files."
fi

#------------------------------------------------------------------------------
msg "${H2}\\nSTEP 2: Backup /p4/sdp."

cd "$HxDepots" || bail "Could not cd to $HxDepots."

SDPBackup="sdp.bak.$(date '+%Y%m%d-%H%M%S')"
if run "rsync -a sdp/ $SDPBackup" "Backing up $HxDepots/sdp."; then
   msg "Backup of /p4/sdp completed OK."
else
   bail "The backup of the /p4/sdp directory failed. Aborting early: No changes to the existing SDP were made, so this is a harmless failure. Determine why the backup failed, fix the issue, and try again.  A potential cause of failure is incorrect (root?) ownership of files."
fi

#------------------------------------------------------------------------------
msg "${H2}\\nSTEP 3: Update /p4/sdp."

SDPUpdateCmd="rsync -nav $DownloadsDirNew/sdp/ /p4/sdp"

# If we're not in NoOp mode, replace '-nav' with '-av'.
[[ "$NoOp" -eq 0 ]] && SDPUpdateCmd=${SDPUpdateCmd/-nav/-av}

run "$SDPUpdateCmd" "Updating /p4/sdp." 0 1 ||\
   bail "Update of /p4/sdp directory failed. Aborting."

#------------------------------------------------------------------------------
msg "${H2}\\nSTEP 4: Update /p4/common."

CommonUpdateCmd="rsync -nav $DownloadsDirNew/sdp/Server/Unix/p4/common/ /p4/common"

# If we're not in NoOp mode, replace '-nav' with '-av'.
[[ "$NoOp" -eq 0 ]] && CommonUpdateCmd=${CommonUpdateCmd/-nav/-av}

run "$CommonUpdateCmd" "Updating /p4/common." 0 1 ||\
   bail "Update of /p4/common directory failed. Aborting."

#------------------------------------------------------------------------------
msg "${H2}\\nSTEP 5: Replace SDP Environment File (p4_vars)."

run "mv -f $SDPEnvFileNew $SDPEnvFile" "Deploying new p4_vars file." ||\
   errmsg "Failed to replace p4_vars file. Manual corrective action required."

rm -f "$SDPEnvFileNew"

# The custom site/config/p4_vars.local is generated once, and never touched
# again.
if [[ -e "$SDPCustomEnvFileNew" && ! -e "$SDPCustomEnvFile" ]]; then
   msg "${H2}\\nSTEP 5A: Add Custom SDP Environment File (p4_vars.local)."

   if [[ ! -d "$SDPCommon/site/config" ]]; then
      run "mkdir -p $SDPCommonSiteCfg" "Initializing dir $SDPCommonSiteCfg."
   fi

   run "mv -f $SDPCustomEnvFileNew $SDPCustomEnvFile" "Adding new $SDPCustomEnvFile." ||\
      errmsg "Failed to add p4_vars.local file. Manual corrective action required."

   rm -f "$SDPCustomEnvFileNew"
fi

#------------------------------------------------------------------------------
msg "${H2}\\nSTEP 6: Replace SDP Instance Configuration Files."

if [[ "$SDPInstanceCfgFileCount" -ne 0 ]]; then
   i=0; while [[ "$i" -lt "$SDPInstanceCfgFileCount" ]]; do
      run "mv -f ${SDPInstanceCfgFilesNew[$i]} ${SDPInstanceCfgFiles[$i]}" \
         "Deploying new file: ${SDPInstanceCfgFiles[$i]}" ||\
         errmsg "Failed to replace ${SDPInstanceCfgFiles[$i]} file. Manual corrective action required."

      # The custom per-instance site/config/p4_N.vars.local files are generated
      # once, and never touched again.
      if [[ -e "${SDPCustomInstanceCfgFilesNew[$i]}" && ! -e "${SDPCustomInstanceCfgFiles[$i]}" ]]; then
         if [[ ! -d "$SDPCommon/site/config" ]]; then
            run "mkdir -p $SDPCommonSiteCfg" "Initializing dir $SDPCommonSiteCfg."
         fi

         run "mv -f ${SDPCustomInstanceCfgFilesNew[$i]} ${SDPCustomInstanceCfgFiles[$i]}" "Adding new ${SDPCustomInstanceCfgFiles[$i]}." ||\
            errmsg "Failed to add p4_N.vars.local file. Manual corrective action required."
      fi

      i+=1
   done
else
   warnmsg "Skipping SDP instance config updates as no SDP instance config files were found."
fi

#------------------------------------------------------------------------------
msg "${H2}\\nSTEP 7: Remove Deprecated Files."

if [[ -r "$DeprecatedFilesListFile" ]]; then
   remove_deprecated_files "$DeprecatedFilesListFile" ||\
      warnmsg "There were issues removing deprecated files."
else
   warnmsg "Ignoring missing file containing list of deprecated files: $DeprecatedFilesListFile"
fi

#------------------------------------------------------------------------------
msg "${H2}\\nSTEP 8: Update SDP crontabs."

msg "No SDP crontab updates are required for this upgrade."

if [[ "$ErrorCount" -eq 0 ]]; then
   if [[ "$WarningCount" -eq 0 ]]; then
      if [[ "$NoOp" -eq 0 ]]; then
         msg "\\n${H1}\\n\\nSUCCESS:  SDP Upgrade Completed OK."
      else
         msg "\\n${H1}\\n\\nSUCCESS:  SDP Upgrade Preview Completed OK (DRY RUN)."
      fi
   else
      if [[ "$NoOp" -eq 0 ]]; then
         msg "\\n${H1}\\n\\nSUCCESS: SDP Upgrade Completed with no errors. There were $WarningCount warnings."
      else
         msg "\\n${H1}\\n\\nSUCCESS: SDP Upgrade Preview Completed with no errors (DRY RUN). There were $WarningCount warnings."
      fi
      msg "\\nTo display warnings, do:\\n\\tgrep ^Warning: $Log\\n"
   fi

   if [[ -n "$PostSDPUpgradeCmd" ]]; then
      msg "${H2}\\nSTEP 9: Custom Post-SDP-Upgrade Processing."
      msg "\\nExecuting custom post-upgrade command:\\n\\t$PostSDPUpgradeCmd"

      if $PostSDPUpgradeCmd; then
         msg "\\nThe custom post-upgrade command indicated success."
      else
         errmsg "Although the primary SDP upgrade was successful, the custom post-upgrade command indicated failure."
      fi
   fi
else
   if [[ "$NoOp" -eq 0 ]]; then
      msg "\\n${H1}\\n\\nSDP Upgrade FAILED, with $ErrorCount errors and $WarningCount warnings. Review the output above."
   else
      msg "\\n${H1}\\n\\nSDP Upgrade Preview FAILED, with $ErrorCount errors and $WarningCount warnings. This was a DRY RUN, not a live upgrade. Review the output above before attempting the real upgrade."
   fi
   msg "\\nTo display errors warnings, do:\\n\\tgrep -E '^(Error|Warning):' $Log\\n"
fi

# See the terminate() function, which is really where this script exits.
exit "$ErrorCount"