hms_ts_setup.sh #4

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=Unset
export VERBOSITY=${VERBOSITY:-3}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=${P4CONFIG:-.p4config}

[[ -r "$P4U_ENV" ]] || {
   echo -e "\nError: Cannot load environment from: $P4U_ENV\n\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   source $bash_lib ||\
      { echo -e "\nFATAL: Failed to load bash lib [$bash_lib]. Aborting.\n"; exit 1; }
done

declare Version=1.0.3
declare SDPInstallDirDefault="/hxdepots/sdp"
declare SDPInstallDir=Unset
declare SDPConfigScript=
declare SDPHostsFile=
declare HMSVarsFile="/p4/common/config/p4_hms.vars"
declare -i Interactive=1
declare -i PreflightOK
declare ActiveInstances
declare ThisHost=${HOSTNAME%%.*}
declare DirList=${HOSTNAME%%.*}
declare TmpFile1=$P4U_TMPDIR/tmp.1
declare TmpFile2=$P4U_TMPDIR/tmp.2
declare TmpFile3=$P4U_TMPDIR/tmp.3
declare TmpFile4=$P4U_TMPDIR/tmp.4
declare TmpFile5=$P4U_TMPDIR/tmp.5
export VERBOSITY=3

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit $OverallReturnStatus
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      echo -e "\n\nUsage Error:\n\n$errorMessage\n\n"
   fi

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT [-y] [-d <sdp_dir>] [-L <log>] [-v<n>] [-n] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	The HMS \"tight ship\" setup script transforms a standard SDP
	installation into a tight-ship style installation on the master
	server machine.

	A tight ship installation is one in which:
	* The key SDP folders, e.g. /p4/common and /p4/N/bin (for
	each instance) are versioned locally using SDP.<hostname>
	workspaces in a local HMS instance, /p4/hms, which this script
	creates and initializes.

	* The crontab for the Perforce user is versioned.

	* Any local configuration and/or customizations of the SDP
	are versioned.

	* The HMS instance is aware of The Workshop, a public
	Helix server hosted by Perforce Software.  It fetches SDP
	updates from The Workshop into the local hms instance using
	Helix DVCS features.

	This simplifies future updates of the SDP, allowing them
	to be done with a 'p4 fetch' followed by a 'p4 merge'
	on your local server (if the server can reach The
	Workshop directly; if not an alternative
	approach using an interim personal repo can be used to
	bridge 'air gap' networks).

PRE-REQUISITES:
	The following are pre-requisites to running $THISSCRIPT:

	1. The SDP configuration script, mkdirs.sh must have been
	configured and run.

	The SDP is expected to exist in /hxdepots/sdp (or whatever
	diretory is specified with '-d <sdp_dir>').  This may have
	been populated via a tar file or workspace pulling files from
	The Workshop (P4PORT=workshop.perforce.com:1666),
	pulling files from //guest/perforce_software/sdp/main on that
	server.

	The /hxdepots/sdp/Server/Unix/setup/mkdirs.sh must already be
	configured correctly, and have been run for at least one
	instance.  The password stored in the newest of the
	/p4/common/config/.p4passwd.*.p4admin files will be used
	for the perforce user for the hms instance.

	2. The /p4/common/config/sdp_hosts.cfg file is configured to
	reference only this current master server machine.  If this
	file does not exist, a template can be copied from
	/hxdepots/sdp/Server/Unix/p4/common/config/sdp_hosts.cfg.

	3. Perforce crontab initialized.

	The crontab for the perforce user must already be configured as
	intended.

	4. No cruft symlinks in /p4.

	In the /p4 directory, and /p4/<N> symlinks which contain a
	/p4/<N>/root/server.id file are considered to be valid SDP
	instances.  This script will attempt to manage any such files.

	5. The OSUSER (typically 'perforce') has full sudo privs.
	The OSUSER must be able to execute 'sudo su' commands without
	a password.  This can be accomplished on modern Linux systems
	by creating /etc/sudoers.d/perforce, with this one line of
	contents:

	perforce	ALL=(ALL)	NOPASSWD: ALL

	If configured properly, the following command should execute
	without error:
	sudo /bin/ls /etc/sudoers.d

	6. No hms instancey

	There should not (yet) be an hms instance.

ADDITIONAL SETUP:
	The following setup is necessary before operating in a tight
	ship environment, but are not necessary before running this
	script.

	1. The SSH keys must be configured enabling the 'perforce' user
	to ssh without a password to all machines listed in sdp_hosts.cfg
	configured above, including the current host (i.e. it must be
	able to ssh to itself using the current hostname).

OPTIONS:
 -y	Continue past the preflight without an interactive prompt.

 -d <sdp_dir>
	Specify the root of the directory tree where the SDP package
	is available, the directory which contains an already-configured
	SDP configuration script, mkdirs.sh

	The default is: $SDPInstallDirDefault

	The version of the SDP installed must be recent enough to
	contain the 'hms' setup directory:
	$SDPInstallDirDefault/Server/Unix/setup/hms.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	$(dirname ${P4U_LOG}).

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

 -n	No-Op.  Prints commands instead of running them.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

FILES:

EXAMPLES:
	This is best done immediately after a standard SDP
	installation, and must be run on the same master server
	machine where the SDP was installed.

SEE ALSO:
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-d) SDPInstallDir=$2; shiftArgs=1;;
      (-y) Interactive=0;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usage -h "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $P4U_LOG == Unset ]] && \
   export P4U_LOG="/tmp/tight_ship_setup.$(date +'%Y%m%d-%H%M%S').log"

[[ $SDPInstallDir == Unset ]] && SDPInstallDir="$SDPInstallDirDefault"

[[ -d "$SDPInstallDir" ]] || \
   bail "The SDP install dir [$SDPInstallDir] does not exist. Specify the correct value with '-d <sdp_dir>'."

[[ -d "$SDPInstallDir/Server/Unix/setup/hms" ]] || \
   bail "The HMS setup dir [$SDPInstallDir/Server/Unix/setup/hms] does not exist.  The version of the SDP package in $SDPInstallDir is not recent enough. Please acquire the latest version and redo changes to mkdirs.sh."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i OverallReturnStatus=0

if [[ "${P4U_LOG}" != off ]]; then
   touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   exec > >(tee ${P4U_LOG})
   exec 2>&1
   initlog
fi

#------------------------------------------------------------------------------
msg "Starting Preflight Checklist."
PreflightOK=1

SDPConfigScript="$SDPInstallDir/Server/Unix/setup/mkdirs.sh"
[[ -x "$SDPConfigScript" ]] ||\
   {
      PreflightOK=0
      errmsg "SDP config script not executable: [$SDPConfigScript]."
   }

vmsg "Verified: SDP config script is executable: [$SDPConfigScript]."

SDPHostsFile="/p4/common/config/sdp_hosts.cfg"
[[ -f "$SDPHostsFile" ]] ||\
   {
      PreflightOK=0
      errmsg "SDP hosts file missing: [$SDPHostsFile].\nCreate thaat file from this template: $SDPInstallDir/Server/Unix/p4/common/config/sdp_hosts.cfg\n"
   }

vmsg "Verified: SDP hosts file exists: [$SDPHostsFile]."

# Extact values already defined in mkdirs.sh.
DD=$(grep "^DD=" $SDPConfigScript|tail -1|cut -d '=' -f 2)
DB1=$(grep "^DB1=" $SDPConfigScript|tail -1|cut -d '=' -f 2)
DB2=$(grep "^DB2=" $SDPConfigScript|tail -1|cut -d '=' -f 2)

if [[ -z "$DB1" ]]; then
   # Backward compatibility for older form of mkdirs.sh, which
   # used MD= for /metadata rather than DB1 for /hxmetadata1.
   DB1=$(grep "^MD=" $SDPConfigScript|tail -1|cut -d '=' -f 2)
   DB2="$DB1"
fi

LG=$(grep "^LG=" $SDPConfigScript|tail -1|cut -d '=' -f 2)
OSUSER=$(grep "^OSUSER=" $SDPConfigScript|tail -1|cut -d '=' -f 2)
MAILFROM=$(grep "^MAILFROM=" $SDPConfigScript|tail -1|cut -d '=' -f 2)
ADMINPASS=$(grep "^ADMINPASS=" $SDPConfigScript|tail -1|cut -d '=' -f 2)

# Removed excess whitespace.
DB1=$(echo $DB1)
DD=$(echo $DD)
LG=$(echo $LG)
OSUSER=$(echo $OSUSER)

# Find all active SDP Instances.
ActiveInstances=""
for i in /p4/*; do
   [[ -L $i ]] || continue
   [[ -f $i/root/server.id ]] || continue
   ActiveInstances+=" ${i#/p4/}"
done

# Trim excess whitespace.
ActiveInstances=$(echo $ActiveInstances)

if [[ -n "$ActiveInstances" ]]; then
   msg "The following active SDP instances were detected: $ActiveInstances."
else
   PreflightOK=0
   errmsg "No Active SDP Instances Detected."
fi

DirList="/p4 /$DD/p4/common/bin /$LG/p4 /$DB1/p4"
[[ "$DB1" == "$DB2" ]] || DirList="$DirList /$DB2/p4"

for dir in $DirList; do
   if [[ -d "$dir" ]]; then
      vmsg "Verified: SDP directory exists: [$dir]."
   else
      PreflightOK=0
      errmsg "Missing SDP directory: [$dir]."
   fi
done

for i in $ActiveInstances; do
   DirList="/$DD/p4/$i/bin /$DD/p4/$i/checkpoints /$DD/p4/$i/depots /$LG/p4/$i/logs /$DB1/p4/$i/root /$DB2/p4/$i/offline_db"
   for dir in $DirList; do
      if [[ -d "$dir" ]]; then
         vmsg "Verified: SDP directory for instance $i exists: [$dir]."
      else
         PreflightOK=0
         errmsg "Missing SDP directory for instance $i: [$dir]."
      fi
   done
done

DirList="/$DD/p4/hms /$LG/p4/hms /$DB1/p4/hms"
[[ "$DB1" == "$DB2" ]] || DirList="$DirList /$DB2/p4/hms"
for dir in $DirList; do
   if [[ -d "$dir" ]]; then
      PreflightOK=0
      errmsg "HMS SDP directory exists but should not: [$dir]."
   else
      vmsg "Verified: HMS SDP directory doesn't (yet) exist: [$dir]."
   fi
done

if [[ $(id -u -n) != $OSUSER ]]; then
   PreflightOK=0
   errmsg "Must run as $OSUSER, not $(id -u -n)."
else
   vmsg "Verified: Running os OSUSER ($OSUSER)."
fi

# Test sudo access.  Ignore the NO_OP setting since we want
# to try this preflight check even in no-op mode
run "sudo /bin/ls /etc/sudoers.d > /dev/null" "Testing sudo access. If prompted for a password, do Ctrl-C. Then enable sudo access for the perforce user and try again." 0 1

if [[ $CMDEXITCODE -eq 0 ]]; then
   vmsg "Verified: OSUSER ($OSUSER) has sudo privs."
else
   PreflightOK=0
   errmsg "Could not verify sudo access for OSUSER $OSUSER.  Sudo access required."
fi

if [[ $PreflightOK -eq 1 ]]; then
   msg "Preflight Checklist Completed OK. Proceeding ..."
else
   bail "Aborting due to preflight checklist failures."
fi

if [[ $Interactive -ne 0 ]]; then
   input=""
   while [[ -z "$input" ]]; do
      echo -e "\nProceed [y/Y/n/N]?: "
      read -e input
      if [[ "${input}" == "Y" || $input == "y" ]]; then
         continue
      elif [[ "${input}" == "N" || $input == "n" ]]; then
         msg "Confirmation to proceed not received. Halting."
         exit 1
      fi
   done
fi

#------------------------------------------------------------------------------
msg "${H2}\nStep 1: Initialize HMS SDP instance as /p4/hms."

cd "$SDPInstallDir/Server/Unix/setup" ||\
   bail "Failed to cd to $SDPInstallDir/Server/Unix/setup."

vmsg "Operating in $PWD."

run "sudo $SDPConfigScript hms > mkdirs.hms.log 2>&1" \
   "Configuring new HMS instance of SDP in /p4/hms." 1 1 ||\
   bail "Failed to initialize HMS instance."

[[ -f $HMSVarsFile ]] ||\
   bail "After HMS initialization, expected file is missing: $HMSVarsFile."

run "chmod +w $HMSVarsFile" "Making HMS vars file writable."

if [[ $NO_OP -eq 0 ]]; then
   sed -e "s:export P4PORTNUM=.*:export P4PORTNUM=8378:g" \
      -e "s:export P4BROKERPORTNUM=.*:export P4BROKERPORTNUM=8373:g" \
      $HMSVarsFile > $TmpFile1 ||\
      bail "Failed to export."
   mv -f "$TmpFile1" "$HMSVarsFile" || bail "Failed to update $HMSVarsFile."
else
   msg "NO_OP: Would have updated $HMSVarsFile."
fi

#------------------------------------------------------------------------------
msg "${H2}\nStep 2: Populate HMS instance with static load data."

cd "$SDPInstallDir/Server/Unix/setup/hms" ||\
   bail "Failed to cd to $SDPInstallDir/Server/Unix/setup/hms."

run "/p4/hms/bin/p4d_hms_init start" "Starting new HMS instance of p4d." 1 1 ||\
   bail "Failed to start new HMS instance."

sleep 1

if [[ $NO_OP -eq 0 ]]; then
   vmsg "Loading HMS environment."
   source /p4/common/bin/p4_vars hms ||\
      bail "Failed to load HMS environment."
else
   msg "NO_OP: Woud load HMS environment."
fi

# Environment safety check:
if [[ "$P4ROOT" == "/p4/hms/root" ]]; then
   vmsg "Verified: HMS Environment looks OK."
else
   bail "HMS environment is not as expected, P4ROOT is $P4ROOT."
fi

sed -e "s:REPL_OSUSER:$OSUSER:g" \
   -e "s:REPL_MAILFROM:$MAILFROM:g" \
   superuser.user.p4t > superuser.user.p4s

run "p4 user -f -i < superuser.user.p4s" "Creating user $OSUSER." ||\
   bail "Failed to create user $OSUSER."

sed -e "s:REPL_MAILFROM:$MAILFROM:g" \
   swarm.user.p4t > swarm.user.p4s

run "p4 user -f -i < swarm.user.p4s" "Creating user swarm." ||\
   bail "Failed to create user swarm."

echo "$ADMINPASS" > $TmpFile1
echo "$ADMINPASS" > $TmpFile2
echo "$ADMINPASS" >> $TmpFile2

run "p4 passwd < $TmpFile2" "Setting password for $OSUSER." ||\
   bail "Failed to set password for $OSUSER."

run "p4 login -a < $TmpFile1" "Logging in super user." ||\
   bail "Failed to login."

run "p4 passwd swarm < $TmpFile2" "Setting password for swarm." ||\
   bail "Failed to set password for swarm"

for g in *.group.p4s; do
   run "p4 group -i < $g" "Loading group spec file $g." ||\
      bail "Failed to load group spec file $g with contents:\n$(cat $g)"
done

run "p4 triggers -i < triggers.p4s" "Loading triggers table." ||\
   bail "Failed to load triggers table with contents:\n$(cat triggers.p4s)"

run "p4 protect -i < protect.p4s" "Loading protections table." ||\
   bail "Failed to load protections table. with contents:\n$(cat protect.p4s)"

for d in *.depot.p4s; do
   run "p4 depot -i < $d" "Loading depot spec file $d." ||\
      bail "Failed to load depot spec file $d with contents:\n$(cat $d)"
done

# Stream specs need to be created in a certain order, respecting the
# parenting relationships; e.g. 'main' must be created before a child
# of 'main' can be created.  Rather than work out the parenting
# relationships to elegantly create streams in the correct order, here
# we take a simple brute-force approach of making several passes at
# creating the entire list of streams. Streams that can't be created on
# the first pass because their parent does't yet exist will succeed on a
# subsequent pass.  This simple approach results in harmless errors
# creating streams.  We arbitrarily select 3 passes, which will create up
# to 5 levels of streams.
for pass in {1..5}; do
   for s in *.stream.p4s; do
      run "p4 stream -i < $s" "Loading stream spec file $s." ||\
         msg "Info: Ignore this known-harmless error creating a stream spec."
   done
done

# Do one final pass at stream spec creation.  On this final passs, the stream
# specs should already exist, so problems loading stream specs at this point
# are treated as hard errors.
for s in *.stream.p4s; do
   run "p4 stream -i < $s" "Loading stream spec file $s." ||\
      bail "Failed to load stream spec file $s with contents:\n$(cat $s)"
done

msg "Create the deployment virtual stream for host $ThisHost."

echo -e "Stream: //sdp/deploy_${ThisHost}\n
Owner:	perforce\n
Name:	deploy_${ThisHost}\n
Parent:	//sdp/main\n
Type:	virtual\n
Description:
\tThis stream stream manages the deployment of SDP files
\ton $ThisHost.\n
Options:	allsubmit unlocked notoparent nofromparent mergedown\n
Paths:
\tshare Server/Unix/p4/common/..." > $TmpFile3

for i in hms $ActiveInstances; do
   echo -e "\tshare host/$ThisHost/p4/$i/bin/..." >> $TmpFile3
done

echo -e "Remapped:
\tServer/Unix/p4/common/... common/..." >> $TmpFile3

for i in hms $ActiveInstances; do
   echo -e "\thost/$ThisHost/p4/$i/bin/... $i/bin/..." >> $TmpFile3
done

run "p4 stream -i < $TmpFile3" "Loading generated deployment stream spec." ||\
   bail "Failed to load this generated stream spec:\n$(cat $TmpFile3)\n"

for r in *.remote.p4s; do
   run "p4 remote -i < $r" "Loading remote spec file $r." ||\
      bail "Failed to load remote spec file $r."
done

rm -f sdp.hostname.client.p4s
sed -e "s:REPL_SHORT_HOSTNAME:$ThisHost:g" \
   -e "s:REPL_HOSTNAME:$(hostname):g" sdp.hostname.client.p4t > sdp.hostname.client.p4s || warnmsg "Error generating client spec for deployment workspace."

for c in *.client.p4s; do
   run "p4 client -i < $c" "Loading client spec file $c." ||\
      bail "Failed to load client spec file $c with contents:\n$(cat $c)"
done

msg "Configuring server per SDP standard."
run "p4master_run hms $SDPInstallDir/Server/setup/configure_new_server.sh hms" ||\
   bail "Failed to configure the new server."

msg "Enabling DVCS features."
run "p4master_run hms p4 configure set server.allowpush=3" ||\
   bail "Failed to set server.allowpush configurable."
run "p4master_run hms p4 configure set server.allowfetch=3" ||\
   bail "Failed to set server.allowfetch configurable."


#------------------------------------------------------------------------------
msg "${H2}\nStep 3: Fetch SDP from The Workshop."

msg "Fetching the SDP."
run "p4 fetch -r workshop.sdp_main -m 1" ||\
   bail "Failed to fetch SDP from The Workshop."

#------------------------------------------------------------------------------
msg "${H2}\nStep 4: Populate local mainline."

msg "Branching files from Workshop SDP mainline //sdp/workshop_main to local mainline //sdp/main."
run "p4 populate -S //sdp/workshop_main" ||\
   bail "Failed to populate //sdp/main from //sdp/workshop_main."

msg "Generating P4CONFIG file /p4/.p4config.SDP."
run "p4master_run hms echo P4PORT=\$P4PORT > /p4/.p4config.SDP"
run "p4master_run hms echo P4USER=\$P4USER >> /p4/.p4config.SDP"
run "echo P4CLIENT=sdp.$ThisHost >> /p4/.p4config.SDP"
run "echo P4IGNORE=.p4ignore >> /p4/.p4config.SDP"

msg "Setting P4CONFIG=/p4/.p4config.SDP."
export P4CONFIG=/p4/.p4config.SDP

run "p4 set"
run "p4 flush" ||\
   bail "Failed to flush."

#------------------------------------------------------------------------------
msg "${H2}\nStep 5: Version key files in /p4/common and /p4/N/bin dirs."

DirList="/p4/common/bin /p4/common/etc"
for i in $ActiveInstances; do
   DirList+=" /p4/$i/bin"
done

msg "Reconciling to detect local differences."
for d in $DirList; do
   run "p4 -d $d rec" ||\
       bail "Failed to reconcile in $d."
done

#------------------------------------------------------------------------------
msg "${H2}\nStep 6: Do initial live checkpoint for new hms instance."

msg "Taking a live checkpoint."
run "p4master_run hms -c $P4CBIN/live_checkpoint.sh" ||\
   bail "Failed to initialize SDP offline checkpoint process with a live checkpoint."

#------------------------------------------------------------------------------
if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\nAll processing completed successfully.\n"
else
   msg "${H}\nProcessing completed, but with errors.  Scan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus