mkrep.sh #6

#!/usr/local/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export SDP_ENV=${SDP_ENV:-/p4/common/bin/p4_vars}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=/p4/hms/logs/mkrep.$(date +'%Y%m%d-%H%M').log
export VERBOSITY=${VERBOSITY:-3}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=${P4CONFIG:-.p4config}
export P4ENVIRO=/dev/null/.p4enviro

[[ -r "$P4U_ENV" ]] || {
   echo -e "\nError: Cannot load environment from: $P4U_ENV\n\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   source $bash_lib ||\
      { echo -e "\nFATAL: Failed to load bash lib [$bash_lib]. Aborting.\n"; exit 1; }
done

declare Version=1.0.11
declare -i SilentMode=0
export VERBOSITY=3

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit $OverallReturnStatus
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT -i <SDP_Instance_1>[,<SDP_Instance_2>[,...]] -t <Type> -s <Site_Tag> -r <Replica_Host> [-c <cfg>] [-L <log>] [-si] [-v<n>] [-n] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script creates makes a replica, and provides enough information to
	make it ready in all respects.

OPTIONS:
 -i <SDP_Instance_1>[,<SDP_Instance_2>[,...]]

 -t <Type>
	Specify the replica type tag.  The type corresponds to the 'Type:' and
	'Services:' field of the server spec, which describes the type of services
	offered by a given replica.

	Valid values are:
	* ro:   Read-Only standby replica.
	* rom:  Read-Only standby replica, Metadata only.
	* fr:   Forwarding Replica (Unfiltered).
	* fs:   Forwarding Standby (Unfiltered).
	* frm:  Forwarding Replica (Unfiltered,  Metdata only).
	* fsm:  Forwarding Standby (Unfiltered,  Metdata only).
	* ffr:  Filtered Forwarding Replica.  Not a valid failover target.
	* edge: Edge Server. Filtered by def'n.

	Replica with 'standby' are always unfiltered, and use the 'journalcopy'
	method of replication, which copies a byte-for-byte verbatim journal file
	rather than one that is merely logically equivalent.

	The tag has several purposes:
	1. Short Hand.  Each tag represents a combination of 'Type:' and fully
	qualified 'Services:' values used in server specs.

	2. Distillation. Only the most useful Type/Services combinations have a
	shorthand form.

	3. For forwarding replicas, the name includes the critical distinction of
	whether any replication filtering is used; as filtering of any kind disqualifies
	a replica from being a potential failover target.  (No such distinction is
	needed for edge servers, which are filtered by definition).

 -s <Site_Tag>
	Specify a geographic site tag indicating where the replica will physically be
	located.  Valid site tags are defined in the site tags file,
	$SiteTagsFile

	Current valid site tags are:
	$(cat $SiteTagsFile 2>&1)

 -r <Replica_Host>
	Specify the target replica host.

 -c <cfg>
	Specify the Helix Topology Config file.  The default is:
	$HelixTopologyCfg

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	$(dirname ${P4U_LOG}).

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

-si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -n	No-Op.  Prints commands instead of running them.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

DEPENDENCIES:
	This script depends on ssh keys being defined to allow the Perforce
	operating system user ($OSUER) to ssh to any necessary machines
	without a password.

	This script assumes the replica host already has the SDP fully
	configured.

FILES:
	This Site Tags file defines the list of valid geographic site tags:
	$SiteTagsFile

EXAMPLES:
	Prepare an edge server to run on host syc-helix-04:
	$THISSCRIPT -i acme -t edge -s syd -r syc-helix-04
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare ReplicaHost=Unset
declare ReplicaTypeTag=Unset
declare ReplicaType=
declare SiteTag=Unset
declare SiteTagsFile=${P4CCFG:-/p4/common/config}/SiteTags.cfg
declare SDPInstance=Unset
#declare -i Interactive=1
declare -i MetadataOnly=0
declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-r) ReplicaHost=$2; shiftArgs=1;;
      (-t) ReplicaTypeTag=$2; shiftArgs=1;;
      (-i) SDPInstance=$2; shiftArgs=1;;
      (-s) SiteTag=$2; shiftArgs=1;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

[[ $SDPInstance == Unset ]] && usageError "\nThe '-i <SDP_Instance>' parameter is required."
[[ $ReplicaHost == Unset ]] && usageError "\nThe '-r <Replica_Host>' parameter is required."
[[ $ReplicaTypeTag == Unset ]] && usageError "\nThe '-t <Type>' parameter is required."
[[ $SiteTag == Unset ]] && usageError "\nThe '-s <Site_Tag>' parameter is required."

case "$ReplicaTypeTag" in
   (ro) ReplicaType=standby;;                             # Read-Only Standby replica.
   (rom) ReplicaType=standby; MetadataOnly=1;;            # Read-Only Standby replica, Metadata only.
   (fr) ReplicaType=forwarding-replica;;                  # Forwarding Replica (Unfiltered).
   (fs) ReplicaType=forwarding-standby;;                  # Forwarding Standby (Unfiltered).
   (frm) ReplicaType=forwarding-replica; MetadataOnly=1;; # Forwarding Replica (Unfiltered), Metdata only.
   (fsm) ReplicaType=forwarding-standby; MetadataOnly=1;; # Forwarding Standby (Unfiltered).
   (ffr) ReplicaType=forwarding-replica;;                 # Filtered Forwarding Replica
   (edge) ReplicaType=edge-server;;                       # Edge Server. Filtered by def'n, cannot be Metdata only.
   (*) usageError "The specified replica type tag [$ReplicaTypeTag].";;
esac

declare -i tagFound=0
if [[ -r "$SiteTagsFile" ]]; then
   while read line; do
      [[ $line == "#*" ]] && continue
      [[ -z "$(echo $line)" ]] && continue
      [[ "$line" == *":"* ]] || continue
      tag=${line%%:*}

      if [[ $tag == $SiteTag ]]; then
         tagFound=1
         break
      fi
   done < $SiteTagsFile
else
   bail "Missing site tag configuration file [$SiteTagsFile]. Aborting."
fi

[[ $tagFound -eq 1 ]] ||\
   bail "Failed to find specified site tag [$SiteTag] inite tag configuration file [$SiteTagsFile]. Aborting."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i OverallReturnStatus=0
declare ServerSpec= ServerSpecFile=
declare ServiceUser= ServiceUserSpecFile
declare ServiceUsersGroup=ServiceUsers
declare TmpDir=$(mktemp -d)
declare ProtectsFile=$TmpDir/protect.p4s
declare GroupSpecFile=$TmpDir/group.$ServiceUsersGroup.p4s
declare TmpFile=$TmpDir/tmpFile.$THISSCRIPT

GARBAGE+=" $TmpDir"

if [[ "${P4U_LOG}" != off ]]; then
   touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee ${P4U_LOG})
      exec 2>&1
   else
      exec >${P4U_LOG}
      exec 2>&1
   fi

   initlog
fi

msg "Starting $THISSCRIPT v$Version at $(date)."

msg "${H}\nPart 0: Environment Setup and Preflight Checks."

msg "Loading SDP environment for instance $SDPInstance."
source "$SDP_ENV" "$SDPInstance"

msg "Overriding P4CONFIG, P4ENVIRO, P4ALIASES, etc."
export P4ENVIRO=/dev/null/.p4enviro
export P4ALIASES=/dev/null/.p4aliases
export P4CONFIG=$TmpDir/.p4config
export P4TICKETS=$TmpDir/.p4tickets
export P4TRUST=$TmpDir/.p4trust
export P4BIN=/p4/hms/bin/p4_hms
echo "P4PORT=$P4MASTERPORT" > $P4CONFIG

msg "Checking ssh access to master host $P4MASTER."
ssh -q "$P4MASTER" /bin/ls > /dev/null 2>&1 || bail "Failed to ssh to host $P4MASTER."

msg "Verified: ssh access to master host $P4MASTER is OK."

msg "Checking ssh access to replica host $ReplicaHost."
ssh -q "$ReplicaHost" /bin/ls > /dev/null 2>&1 || bail "Failed to ssh to host $ReplicaHost."

msg "Verified: ssh access to replica host $ReplicaHost is OK."

msg "${H}\nPart 1: Preparation: Login."

if [[ "$P4MASTERPORT" == "ssl:"* ]]; then
   runCmd "p4 trust -y"
fi

runCmd "$P4BIN -u $P4USER -p $P4MASTERPORT -s login -s" \
   "Checking whether user $P4USER is logged into port $P4MASTERPORT." 1 1 0

if [[ $? -ne 0 ]]; then
   msg "Trying login with:\n$P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $P4CCFG/.p4passwd.p4_hms.admin\n"
   if [[ $NO_OP -eq 0 ]]; then
      $P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $P4CCFG/.p4passwd.p4_hms.admin
   else
      msg "NO_OP: Would execute: $P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $P4CCFG/.p4passwd.p4_hms.admin"
   fi
fi

runCmd "$P4BIN -u $P4USER -p $P4MASTERPORT -s login -s" \
   "Verifying that user $P4USER can login to port $P4MASTERPORT." 1 1 0 ||\
   bail "Login verification failed."

msg "${H}\nPart 2: Define Server Spec."

ServerSpec=p4d_${ReplicaTypeTag}_${SiteTag}

ServerSpecFile=$TmpDir/$ServerSpec.server.p4s

echo -e "ServerID: $ServerSpec\n
Type: server\n
Name: $ServerSpec\n
Services: $ReplicaType\n
Description:" > $ServerSpecFile || bail "Failed to initialize server spec file [$ServerSpecFile]."

case "$ReplicaTypeTag" in
   (ro) Desc="Read-Only Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (rom) Desc="Read-Only Standby Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (fr) Desc="Forwarding Replica (Unfiltered) in ${SiteTag^^}.";;
   (fs) Desc="Forwarding Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (frm) Desc="Forwarding Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (fsm) Desc="Forwarding Standby Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (ffr) Desc="Filtered Forwarding Replica in ${SiteTag^^}.";;
   (edge) Desc="Edge server in ${SiteTag^^}.";;
   (*) bail "\nInternal Error: Unrecognized replica type tag [$ReplicaTypeTag].";;
esac

echo -e "\t$Desc\n" >> $ServerSpecFile || bail "Failed to complete server spec file [$ServerSpecFile]."

msg "Creating server spec $ServerSpec with these contents:"
msg ${H}
cat $ServerSpecFile
msg ${H}

if [[ $NO_OP -eq 0 ]]; then
   p4 server -i < $ServerSpecFile || "Failed to load server spec from file: $ServerSpecFile"
else
   msg "NO_OP: Would run: p4 server -i .LT. $ServerSpecFile"
fi

msg "${H}\nPart 3: Set configurables."

ServiceUser=svc_${ServerSpec}
ServiceUserSpecFile=$TmpDir/$ServiceUser.user.p4s
declare -i ConfigureOK=1

runCmd "p4 configure set $ServerSpec#P4TARGET=$P4MASTERPORT" || ConfigureOK=0
runCmd "p4 configure set $ServerSpec#db.replication=readonly" || ConfigureOK=0
runCmd "p4 configure set $ServerSpec#rpl.forward.all=1" || ConfigureOK=0
runCmd "p4 configure set $ServerSpec#rpl.compress=4" || ConfigureOK=0
runCmd "p4 configure set $ServerSpec#server=4" || ConfigureOK=0
runCmd "p4 configure set $ServerSpec#monitor=2" || ConfigureOK=0
runCmd "p4 configure set $ServerSpec#serviceUser=$ServiceUser" || ConfigureOK=0
runCmd "p4 configure set $ServerSpec#journalPrefix=$P4HOME/journals.rep/p4_${SDPInstance}" || ConfigureOK=0

if [[ $NO_OP -eq 0 ]]; then
   # use the '-L' flag to 'p4 pull' to enable 'journalcopy' if indicated by the replica
   # type, i.e. if it has Standby in the name.
   if [[ "$Desc" == *"Standby"* ]]; then
      vmsg "Executing: p4 configure set $ServerSpec#startup.1='pull -i 1 -L'"
      p4 configure set $ServerSpec#startup.1="pull -i 1 -L" || ConfigureOK=0
   else
      vmsg "Executing: p4 configure set $ServerSpec#startup.1='pull -i 1'"
      p4 configure set $ServerSpec#startup.1="pull -i 1" || ConfigureOK=0
   fi
else
   vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.1='pull -i 1'"
fi

if [[ $MetadataOnly -eq 0 ]]; then
   runCmd "p4 configure set $ServerSpec#lbr.replication=readonly" || ConfigureOK=0
   for i in $(seq 2 6); do
      if [[ $NO_OP -eq 0 ]]; then
         vmsg "Executing: p4 configure set $ServerSpec#startup.$i='pull -i 1 -u'"
         p4 configure set $ServerSpec#startup.$i="pull -i 1 -u" || ConfigureOK=0
      else
         vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.1='pull -i 1'"
      fi
   done
else
   runCmd "p4 configure set $ServerSpec#lbr.replication=ondemand" || ConfigureOK=0
fi

if [[ $ConfigureOK -eq 1 ]]; then
   msg "Verified:  All configurables were set OK."
   runCmd "p4 configure show allservers" "Showing all persistent configurables." 0 1 0
else
   bail "Errors encountered setting configurables.  See the output above. Aborting."
fi

msg "${H}\nPart 4: Create replica service user $ServiceUser."

echo -e "User: $ServiceUser\n
Email: ${MAILFROM#\#}\n
FullName: Replication Server User for $ServerSpec\n
Type: service\n
AuthMethod: perforce\n" > $ServiceUserSpecFile || bail "Failed to initialize user spec file [$ServiceUserSpecFile]."

vmsg "Contents of $ServiceUserSpecFile:"
vmsg "${H}"
[[ $VERBOSITY -gt 2 ]] && cat $ServiceUserSpecFile
vmsg "${H}"

if [[ $NO_OP -eq 0 ]]; then
   p4 user -f -i < $ServiceUserSpecFile || "Failed to load user spec from file: $ServiceUserSpecFile"
else
   msg "NO_OP: Would run: p4 user -f -i .LT. $ServiceUserSpecFile"
fi

PasswdFile=$TmpDir/.p4passwd
touch $PasswdFile || bail "Failed to initialize password file $PasswdFile."

chmod 600 $PasswdFile
cat $P4CCFG/.p4passwd.p4_hms.admin > $PasswdFile
cat $P4CCFG/.p4passwd.p4_hms.admin >> $PasswdFile

if [[ $NO_OP -eq 0 ]]; then
   msg "Setting password for service user $ServiceUser."
   p4 passwd $ServiceUser < $PasswdFile
else
   msg "NO_OP: Would run: p4 passwd $ServiceUser .LT. $PasswdFile"
fi

msg "${H}\nPart 5: Make replica service user a super user with unlimited timeout."

msg "Checking if Protections table references group $ServiceUsersGroup."

if [[ -n "$(p4 protects -g $ServiceUsersGroup)" ]]; then
   msg "Verified: Protections table references group $ServiceUsersGroup."
else
   msg "Adding protections table entry to reference group $ServiceUsersGroup."
   p4 protect -o | grep -v '^#' > $TmpFile || bail "Failed to update tmp file $TmpFile."
   echo -e "\tsuper group $ServiceUsersGroup * //..." >> $TmpFile || bail "Failed to update tmp file $TmpFile."
   grep -v '^\s*$' $TmpFile > $ProtectsFile || bail "Failed to update protects file $ProtectsFile."

   vmsg "Contents of $ProtectsFile:"
   vmsg "${H};"
   [[ $VERBOSITY -gt 3 ]] && cat $ProtectsFile
   vmsg "${H}"

   if [[ $NO_OP -eq 0 ]]; then
      p4 protect -i < $ProtectsFile
   else
      msg "NO_OP: Would run: p4 protect -i .LT. $ProtectsFile"
   fi
fi

msg "Checking if serivce user $ServiceUser is in service users group $ServiceUsersGroup."

if [[ -n "$(p4 groups $ServiceUser | grep ^$ServiceUsersGroup$)" ]]; then
   msg "Verified: Serivce user $ServiceUser is in service users group $ServiceUsersGroup."
else
   # This logic will create the group spec for service users if it does not already exist,
   # or add our new service user to the group if it already exists.  The 'p4 group -o'
   # command generate a valid group spec whether the spec actually exists on the server or
   # not.
   p4 group -o $ServiceUsersGroup | grep -v '^#' |\
      sed "s:43200:unlimited:g;\$ s/.*/\t$ServiceUser/" > $GroupSpecFile ||\
      bail "Failed to update group spec file $GroupSpecFile."

   vmsg "Contents of $GroupSpecFile:"
   vmsg "${H}"
   [[ $VERBOSITY -gt 3 ]] && cat $GroupSpecFile
   vmsg "${H}"

   if [[ $NO_OP -eq 0 ]]; then
      p4 group -i < $GroupSpecFile
   else
      msg "NO_OP: Would run: p4 group -i .LT. $GroupSpecFile"
   fi
fi

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\nAll processing completed successfully.\n"
   msg "\nNext steps:"
   msg "STEP 1. Login as ${OSUSER}@${P4MASTER}."
   msg "STEP 2. Execute this command:\n\tnohup dailY_checkpiont.sh $SDPInstance < /dev/null > /dev/null 2>&1 &\n"
   msg "STEP 3. Monitor the checkpoint.log file, and confirm that the journal rotation\nstep completes.  This generally occurs within the first few seconds or minutes\nof starting daily_checkpoint.sh, even though it may take hours to complete for large\ndata sets.  Optionally, wait for it to complete.\n"
   msg "STEP 4. Login as ${OSUSER}@${ReplicaHost}."
   msg "STEP 5. Set your environment with:\n\tcd /p4/common/bin\n\tsource p4_vars $SDPInstance\n"
   msg "STEP 6. Transfer the highest numbered completed checkpoint file from\n${P4MASTER}:${CHECKPOINTS}.  Completed checkpoint files have a corresponding\n*.md5 file.  If you waited for daily_checpoint.sh to complete in STEP 3, you\nneed only the latest checkpoint file.  Otherwise, you also need the journal\nfile that has the same number.  Copy those files to:\n$ReplicaHost:$CHECKPOINTS/.\n\nThe commands, as run from $ReplicaHost, will look like these sample commands:\n\tcd $CHECKPOINTS\n\tscp -p ${P4MASTER}:${CHECKPOINTS}/p4_${SDPInstance}.ckp.2550.gz .\n\tscp -p ${P4MASTER}:${CHECKPOINTS}/p4_${SDPInstance}.jnl.2550 .\n"
   msg "STEP 7. Create $P4ROOT/server.id file like so:\n\techo $ServerSpec > $P4ROOT/server.id\n"
   msg "STEP 8. Verify that you have enough disk space, e.g. with:\n\tdf -h $P4ROOT, at least 30x zipped checkpoint size is needed.\n"
   msg "STEP 9. Recover the checkpoint like so:\n\tcd $P4ROOT\n\t$P4DBIN -r $P4ROOT -z -jr ${CHECKPOINTS}/p4_${SDPInstance}.ckp.NNN.gz\n\nwhere NNN is the latest checkpoint from STEP 6.\n\nIf you did not wait for daily_checkpoint.sh to complete in STEP 3, then also replay the latest journal file copied in STEP 6, like so:\n\t$P4DBIN -r $P4ROOT -z -jr ${CHECKPOINTS}/${SDPInstance}.jnl.NNN"
   msg "STEP 10. Start the replica like so:\n\t${P4DBIN}_init start\n\nWait several seconds, then do:\n\t$P4CBIN/p4login\n\nReview the $LOGS/p4login.log, then check replication status with:\n\tp4 pull -lj\n"
   msg "STEP 11. Kick off a verify to pull over archive files:\n\tnohup p4verify.sh $SDPInstance < /dev/null > /dev/null 2>&1 &\n"
   msg "STEP 12. Wait about one minute, then check, $LOGS/p4verify.log to ensure it got\noff to a good start.  That will run for a while."

else
   msg "${H}\nProcessing completed, but with errors.  Scan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus