sdp_upgrade.sh #2

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
set -u

# sdp_upgrade.sh
# Upgrades the Perforce Helix Server Deployment Package (SDP).

#==============================================================================
# Declarations and Environment

declare Version=1.0.1
declare ThisScript="${0##*/}"
declare ThisUser=
declare CmdArgs="$*"
declare CmdLine="$0 $CmdArgs"
declare -i ErrorCount=0
declare -i WarningCount=0
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare Log="Unset"
declare -i PreflightOnly=0
declare -i Debug=0
declare -i NoOp=0

declare SDPInstallRoot="/p4"
declare SDPCommon="$SDPInstallRoot/common"
declare SDPOwner=
declare HxDepots=
declare DownloadsDir=
declare SDPTarball=
declare SDPInstanceList=
declare -a InstanceCfgFiles
declare -i InstanceCfgFileCount
declare TarRoot=
declare TarRootCommon=
declare TarRootConfig=
declare TarRootCron=
declare TarRootBinDir=
declare TarRootInit=

# The SDPTargetMajorVersion is hard-coded by design. This is upgraded with every SDP major
# version.  The SDPTargetVersion includes the changelist number extracted from the SDP
# 'Version' file which is updated by automation during the SDP release process.
declare SDPTargetMajorVersion=2021.1
declare SDPTargetVersion=

#==============================================================================
# Local Functions

# Note: This script does not use SDP library files, as its purpose is to
# upgrade the SDP installation.
function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -ne 0 ]] && msg "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# This function takes as input an SDP version string, and returns a version
# id of the form YYYY.N.CL, where YYYY is the year, N is an incrementing
# release ID with a given year, and CL is a changelist identifier. The
# YYYY.N together comprise the major version, often shortened to YY.N, e.g.
# r20.1 for the 2020.1 release.
#
# The full SDP Version string looks something like this:
# Rev. SDP/MultiArch/2019.3/26494 (2020/04/23).
#
# This function parses that full string and returns a value like: 2019.3.26494
function get_sdp_version_from_string () {
   dbg "CALL get_sdp_veresion_from_string($*)"
   local versionString="${1:-}"
   local version=
   version="20${versionString##*/20}"
   version="${version%% *}"
   version="${version/\//.}"

   [[ "$version" == "20" || "$version" == "200" ]] && version="Unknown"
   echo "$version"
}

#------------------------------------------------------------------------------
# Function: acquire_sdp ()
#------------------------------------------------------------------------------
function acquire_sdp () {
   dbg "CALL acquire_sdp($*)"
   true; # No-op placeholder.
}

#------------------------------------------------------------------------------
# Function: check_p4_vars_file ()
#
# Determine if the SDP p4_vars file can be safely upgraded.
#------------------------------------------------------------------------------
function check_p4_vars_file () {
   dbg "CALL check_p4_vars_file($*)"
   # See if SDP_P4_VARS_FORMAT is defined.
   local fileFormatVersion=0.5
}

#------------------------------------------------------------------------------
# Function: check_instance_cfg_file ()
#
# Determine if the SDP instance config file can be safely upgraded.
# See if SDP_INSTANCE_VARS_FORMAT is defined.
#------------------------------------------------------------------------------
function check_instance_cfg_file () {
   dbg "CALL check_instance_cfg_file($*)"
   local instanceCfgFile=${1:-UnsetSDPInstanceCfgFile}
   local fileFormatVersion=0.5

   if [[ -r "$instanceCfgFile" ]]; then
      # Check for required parameters in the file.
#
#MAILTO
#MAILFROM
#P4USER
#P4MASTER_ID
#SSL_PREFIX
#P4PORTNUM
#P4MASTERHOST
#P4MASTERPORT

      msg "Verified: This instance config file can be ugpraded: $instanceCfgFile"
   else
      errmsg "Missin SDP Instance config file: $instanceCfgFile"
      return 1
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: get_sdp_instances ()
#
# Get the list of SDP instances after doing some preliminary sanity
# checks. Sets global SDPInstanceList.
function get_sdp_instances () {
   dbg "CALL get_sdp_instance($*)"
   local e=
   local instanceCfg=
   local instanceBinDir=

   SDPInstanceList=

   cd "$SDPInstallRoot" || \
      bail "Could not cd to SDP Install Root dir: $SDPInstallRoot"

   # Start with elements (files/dirs/symlinks) under /p4.
   for e in *; do
      # Silently ignore expected elements under /p4.
      [[ "$e" =~ ^(common|sdp|ssl)$ ]] && continue

      # Warn about symlinks under /p4; there should not be any, only
      # directories.
      if [[ -L "$e" ]]; then
         warnmsg "Ignoring unexpected symlink $SDPInstallRoot/$e"
         continue
      fi

      # Consider only at directories under /p4; silently ignore files.
      if [[ -d "$e" ]]; then
         # For a directory under /p4 to be an SDP instance, there must
         # be corresponding instance configuration file to upgrade,
         # /p4/common/config/p4_N.vars.
         instanceCfg="$SDPInstallRoot/common/p4_${e}.vars"
         instanceBinDir="$SDPInstallRoot/${e}/bin"

         if [[ -r "$instanceCfg" ]]; then
            SDPInstanceList+=" $e"
            SDPInstanceCfgFile[$SDPInstanceCfgFileCount]="$instanceCfg"
            SDPInstanceCfgFileCount+=1
         elif [[ -d "$instanceBinDir" ]]; then
            warnmsg "Found instance bin dir [$instanceBinDir] but no corresponding instance config file [$instanceCfgFile]. Assuming $SDPInstallRoot/$e is not a valid SDP instance directory."
         fi
      fi
   done

   # Trim leading space.
   # shellcheck disable=SC2116
   SDPInstanceList=$(echo "$SDPInstanceList")
   if [[ -n "$SDPInstanceList" ]]; then
      msg "\\nList of SDP Instances: $SDPInstanceList"
   else
      warnmsg "\\nNo SDP instances detected under $SDPInstallRoot"
   fi
}

#------------------------------------------------------------------------------
# Function: do_preflight_checks ()
#
# If preflight checks fail, further tests are aborted. Failure of the very
# basic preflight checks is an indication that the SDP structure is in
# need of repair.
#------------------------------------------------------------------------------
function do_preflight_checks () {
   dbg "CALL do_preflight_checks($*)"

   local toolsList="awk date grep head id ls rsync sort tail tee"
   local cfgFile=

   msg "$H2\\nDoing preflight sanity checks."
   msg "Preflight Check: Ensuring these utils are in PATH: $toolsList"

   for tool in $toolsList; do
      CheckCount+=1
      [[ -z "$(command -v "$tool")" ]] && \
         errmsg "Tool '$tool' not in PATH."
   done

   # If basic tools aren't available, we don't even try to complete the
   # preflight checks, and return immediately.
   [[ $ErrorCount -eq 0 ]] || return 1

   msg "Verified: Essential tools are in the PATH."

   msg "Preflight Check: cd $SDPCommon"

   if cd "$SDPCommon"; then
      cd "$OLDPWD" || bail "Failed to cd to $OLDPWD. Aborting."
   else
      errmsg "Could not cd to: $SDPCommon"
      return 1
   fi

   msg "Verified: cd works to: $SDPCommon"

   msg "Preflight Check: Checking current user owns $SDPCommon"

   # shellcheck disable=SC2012
   SDPOwner="$(ls -ld "$SDPCommon" | awk '{print $3}')"

   if [[ "$ThisUser" == "$SDPOwner" ]]; then
      msg "Verified: Current user [$ThisUser] owns $SDPCommon"
   else
      errmsg "Current user [$ThisUser] does not own $SDPCommon. This most likely means this script is running as the wrong user.  It could also mean the $SDPCommon directory is not owned by the correct owner, which should be the OS account under which the p4d process runs."
      return 1
   fi

   for cfgFile in "${SDPInstanceCfgFile[@]}"; do
      check_instance_cfg_file "$cfgFile"
   done

   return 0
}

#------------------------------------------------------------------------------
# Function: run ($cmd, $desc, $showOutput)
#
# Runs a command, with optional description, showing command line to execute
# and optionally also the output, and capturing and returning the exit code.
#
# Input:
# $1 - Command and arguments to execute. Defaults to 'echo'.
# $2 - Optional message to display describing what the command is doing.
# $3 - Numeric flag to show output; '1' indicates to show output, 0 to
#      suppress it.
#------------------------------------------------------------------------------
function run () {
   local cmd="${1:-echo}"
   local desc="${2:-}"
   local -i showOutput="${3:-1}"
   local -i exitCode=
   local cmdLog

   cmdLog="$(mktemp run.XXXXXXXXXXX.log)"

   [[ -n "$desc" ]] && msg "$desc"
   msg "Executing: $cmd"
   eval "$cmd" > "$cmdLog" 2>&1
   exitCode=$?

   if [[ "$showOutput" -eq 1 ]]; then
      echo "EXIT_CODE: $exitCode" >> "$cmdLog"
      cat "$cmdLog"
   fi

   /bin/rm -f "$cmdLog"
   return "$exitCode"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" >&2
   fi

   echo "USAGE for sdp_upgrade.sh v$Version:

sdp_upgrade.sh [-n] [-si] [-L <log>|off ] [-D]

   or

sdp_upgrade.sh -h|-man
"
   if [[ $style == -man ]]; then
      echo -e "DESCRIPTION:

	This script upgrades Perforce Helix Server Deployment Package (SDP) from
	SDP 2020.1 to the version included in the latest SDP version, SDP
	$SDPTargetMajorVersion.

	== Pre-Upgrade Planning ==

	This script will upgrade the SDP if the pre-upgrade starting SDP version
	is SDP 2020.1 or later, including any/all patches of SDP
	2020.1.

	If the current SDP version is older than 2020.1, it must first be upgraded
	to SDP 2020.1 using the SDP Legacy Upgrade Guide.  For upgrading from
	pre-20.1 versions dating back to 2007, in-place or migration-style upgrades
	can be done.  See:
	https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/doc/SDP_Legacy_Upgrades.Unix.html

	The SDP should always be upgraded to the latest version first before
	Helix Core binaries p4d/p4broker/p4p are upgraded using the SDP
	upgrade.sh script.

	Upgrading the SDP first ensures the version of the SDP you have
	is compatible with the latest versions of p4d/p4broker/p4p/p4, and
	will always be compatible with all supported versions of these
	Helix Core binaries.

	When this script is used, i.e. when the current SDP version is 2020.1
	or newer, the SDP upgrade procedure does not require downtime for any
	running Perforce Helix services, such as p4d, p4broker, or p4p.  This
	script is safe to run in environments where live p4d instances are running,
	and does not require p4d or other services to be stopped or upgraded.
	Upgrade of the SDP is cleanly separate from the upgrade the Helix Core
	binaries. The upgrade of the SDP can be done immediately prior to Helix
	Core upgrades, or many days prior.

	This script will upgrade the SDP on a single machine. If your Perforce
	Helix topology has multiple machines, the SDP can be upgraded on all
	machines in any order, as their is no cross-machine dependency requiring
	the SDP to be the same version. (The order of upgrade of Helix Core services
	and binaries such as p4d in global topologies with replicas and edge servers
	does matter, but is outside the scope of this script).

	Planning Recap:
	1. The SDP can be upgraded without downtime when this script is used,
	i.e. when the SDP version is 2020.1 or later.
	2. Upgrade SDP on all machines, in any order, before upgrading p4d and other
	Helix binaries.

	== Acquiring the SDP Package ==

	This script is part of the SDP package (tarball). It can be
	run from extracted tarball directory, implying that the SDP
	tarball has already been downloaded and extracted. However,
	this individual script can also be downloaded independently
	and use to acquire the SDP tarball if outbound internet
	access over port 443 for HTTPS is available.

	== Preflight Checks ==

	Prior to upgrading, preflight checks are performed to ensure the
	upgrade can be completed successfully. If the preflight checks
	fail, the upgrade will not start.

	Preflight Checks:

	Check 1: The existing SDP version is verified to be SDP 2020.1+.

	Check 2: The existing verify_sdp.sh script in the current, pre-
	upgrade SDP is executed as part of its preflight checks. This
	executes many detailed checks to ensure the SDP is healthy
	prior to the upgrade.

	Check 3: The /p4/common/bin/p4_vars is checked to confirm that
	it can be upgraded.

	Check 4: All /p4/common/config/p4_N.vars files are checked to
	confirm they can be upgraded.

	== Upgrade Processing ==

	Step 1: Backup the existing SDP /p4/common structure.

	Step 2: Regenerate the SDP general environment file p4_vars
	based on the new template.

	Step 3: Regenerate the SDP instance environment files for
	all instances based on the new template.

	For Steps 1 and 2, the re-generation logic will preserve current
	settings. If upgrading from SDP r20.1, any custom logic that
	exist below the '### MAKE LOCAL CHANGES HERE' tag will be
	split it separate files.  Custom logic in p4_vars will be moved
	to /p4/common/site/config/p4_vars.local. Custom logic in
	p4_N.vars files will be moved to /p4/common/site/config/p4_N.vars.local.

	Note: Despite these changes, the mechanism for loading the SDP shell
	environment remains unchanged since 2007, so it looks like:

	$ source /p4/common/bin/p4_vars N

	Changes to the right-side of assignments for specific are preserved
	for all defined SDP settings.  For p4_vars, preserved settings are:
	  - OSUSER
	  - KEEPLOGS
	  - KEEPCKPS
	  - KEEPJNLS
	
	For instance_vars files, preserved settings are:
	  - MAILTO
	  - MAILFROM
	  - P4USER
	  - P4MASTER_ID
	  - SSL_PREFIX
	  - P4PORTNUM
	  - P4BROKERPORTNUM
	  - P4MASTERHOST
	  - P4MASTERHOST
	  - PROXY_TARGET
	  - PROXY_PORT
	  - P4DTG_CFG
	  - SNAPSHOT_SCRIPT
	  - SDP_ALWAYS_LOGIN
	  - SDP_AUTOMATION_USERS
	  - SDP_MAX_START_DELAY_P4D
	  - SDP_MAX_START_DELAY_P4BROKER
	  - SDP_MAX_START_DELAY_P4P
	  - SDP_MAX_STOP_DELAY_P4D
	  - SDP_MAX_STOP_DELAY_P4BROKER
	  - SDP_MAX_STOP_DELAY_P4P
	  - VERIFY_SDP_SKIP_TEST_LIST
	  - The 'umask' setting.
	  - KEEPLOGS (if set)
	  - KEEPCKPS (if set)
	  - KEEPJNLS (if set)
	
	Note that the above list excludes any values that are calculated.
	
	Step 4: The SDP_VERSION and SDP_DATE counters will be set.
	
	Step 5: An hook will enable site-specific post-upgrade processing
	for environments with additional upgrade steps.  Sites deployed with
	the custom Helix Management System

	== Post-Upgrade Processing ==

	This script provide guidance on any post-processing steps. This may
	include upgrades to crontabs.

OPTIONS:
 -n	No-Op.  In No-Op mode, no actions that affect data or structures are
	taken.  Instead, commands that would be run are displayed.  This
	command can also be educational, showing various steps that will occur
	during an upgrade.

 -p	Specify '-p' to halt processing after preflight checks are complete,
	and before actual processing starts. By default, processing starts
	immediately upon successful completion of preflight checks.

 -L <log>
	Specify the log file to use.  The default is /tmp/sdp_upgrade.<timestamp>.log

	The special value 'off' disables logging to a file.

 -d	Enable debugging verbosity.

 -D	Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

FILES AND DIRECTORIES:
	Name: SDPCommon
	Path: /p4/common
	Notes: This sdp_upgrade.sh script updates files in and under this folder.

	Name: HxDepots
	Default Path: /hxdepots
	Notes: The folder containing versioned files, checkpoints, and numbered
	journals, and the SDP itself. This is commonly a mount point.

	Name: DownloadsDir
	Default Path: /hxdepots/sdp
	Notes: The symlink /p4/sdp points here.  The path /p4/sdp is consitent;
	the HxDepots can vary.

EXAMPLES:
	Example 1: Prelight check only:

	sdp_upgrade.sh -p

	Example 2: NoOp mode:

	sdp_upgrade.sh -n

LOGGING:
	This script generates a log file, /tmp/sdp_upgrade.<timestamp>.log
	by default. See the '-L' option above.

EXIT CODES:
	An exit code of 0 indicates no errors were encountered during the
	upgrade. A non-zero exit code indicates the upgrade was aborted
	or failed.
"

   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   [[ "$Log" == "off" ]] || msg "\\nLog is: $Log\\n${H1}\\n"

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-p) PreflightOnly=1;;
      (-n) NoOp=1;;
      (-man) usage -man;;
      (-L) Log="$2"; shiftArgs=1;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown command line option ($1).";;
      (*) export SDP_INSTANCE=$1;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

if [[ "$Log" == "off" ]]; then
   if [[ "$NoOp" -eq 0 && "$PreflightOnly" -eq 0 ]]; then
      usage -h "Disabling logging with '-L off' is only allowed with '-p' and/or '-n' flags."
   fi
fi

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

[[ "$Log" == "Unset" ]] && Log="/tmp/sdp_upgrade.$(date +'%Y%m%d-%H%M').log"

if [[ "$Log" != "off" ]]; then
   [[ -e "$Log" ]] && bail "Log file already exists: $Log\\nAborting.\\n"

   if ! touch "$Log"; then
      BadLog="$Log"
      Log="off"
      bail "Couldn't touch log file [$BadLog]. Aborting."
   fi

   # Redirect stdout and stderr to a log file.
   exec > >(tee "$Log")
   exec 2>&1

   msg "${H1}\\nLog is: $Log"
fi

ThisUser="$(id -n -u)"
msg "$ThisScript v$Version Starting SDP upgrade as $ThisUser@${HOSTNAME%%.*} at $(date +'%a %Y-%m-%d %H:%M:%S %Z') with this command line:\\n$CmdLine"

msg "\\nIf you have any questions about the output from this script, contact support@perforce.com."

if do_preflight_checks; then
   msg "\\nAll preflight checks were successful."

   if [[ "$PreflightOnly" -eq 1 ]]; then
      msg "\\nExiting early after successful preflight checks due to '-p'."
      exit 0
   fi
else
   bail "Aborting due to failed preflight checks. The current SDP remains unchanged."
fi

msg "${H2}\\nUpgrading SDP."

if [[ "$ErrorCount" -eq 0 && "$WarningCount" -eq 0 ]]; then
   msg "\\n${H1}\\n\\nSDP Upgrade Completed OK."
elif [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\n${H1}\\n\\nSDP Upgrade Completed with no errors and $WarningCount warnings."
else
   msg "\\n${H1}\\n\\nSDP Upgrade FAILED, with $ErrorCount errors and $WarningCount warnings. Review the output above."
fi

# See the terminate() function, which is really where this script exits.
exit "$ErrorCount"