hms_ts_setup.sh #1

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the P4 Helm Management System (HMS), and also available online:
# https://workshop.perforce.com/view/hms/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

declare -i ErrorCount=0
declare -i WarningCount=0
declare -i Debug=0
declare -i NoOp=0
declare ThisScript=${0##*/}
declare ThisUser=
declare Version=3.2.0
declare CmdLine="$0 $*"
declare Log=
declare H1="=============================================================================="
declare H2="------------------------------------------------------------------------------"
declare RequiredUtils="date egrep grep id ls rsync tail wc"

declare SDPInstallDirDefault=
declare SDPInstallDir=Unset
declare SDPMgmtWs=
declare SDPMgmtP4Cfg=
declare SDPHostsFile=
declare SDPHostsSample=
declare HMSSetupDir="$PWD"
declare HMSInstallRoot="${HMSSetupDir%/p4/common/site/hms/setup}"
declare CustomSetupScript="${HMS_CUSTOM_SETUP_SCRIPT:-}"
declare -i Interactive=1
declare ActiveInstances=
declare ThisHost="${HOSTNAME%%.*}"
declare DirList=
declare TmpFile1=
declare TmpFile2=

#==============================================================================
# Local Functions

function msg () { echo -e "$*"; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function errmsg () { msg "\nError: ${1:-Unknown Error}\n"; ErrorCount+=1; }
function warnmsg () { msg "\nWarning: ${1:-Unknown Warning}\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "$ErrorCount"; }

#------------------------------------------------------------------------------
# Function: run
#
# Short: Run a command with optional description, honoring $Debug
# and $NoOp settings.
#
# Input:
# $1 - cmd.  The command to run.  The command is displayed first
#      if $Debug is not 0.
# $2 - desc.  Text description of command to run.
#      This parameter is optional.
# $3 - honorNoOpFlag.  Pass in 1 to mean "Yes, honor the $NoOp setting
#      and display (but don't run) commands if $NoOp is 1."  Otherwise
#      $NoOp is ignored, and the command is always run.
#      This parameter is optional; the default value is 1.
# $4 - alwaysShowOutputFlag.  If set to 1, show output regardless of $Debug
#      value.  Otherwise, only show output if $Debug is not 0.
#      This parameter is optional; the default value is 0.
# $5 - grepString.  If set, this changes the exit code behavior.
#      If the specified string exists in the output, a 0 is returned, else 1.
#      Strings suitable for use with 'egrep' are allowed.
#
# Description:
#    Display an optional description of a command, and then run the
# command.  This is affected by $NoOp and $Debug.  If $NoOp is 1,
# the command is shown, but not run, provided $honorNoOpFlag is 1.
# The description is not shown if $Debug is not set.
#
# Output is shown if either $alwaysShowOutputFlag is 1 or $Debug is not 0.
#------------------------------------------------------------------------------
function run () {
   dbg "CALL: run ($*)"
   local cmd=${1:-}
   local desc=${2:-}
   local -i honorNoOpFlag=${3:-1}
   local -i alwaysShowOutputFlag=${4:-0}
   local grepString=${5:-}
   local cmdScript=
   local cmdOut=
   local -i grepExit
   local -i cmdExitCode=0

   cmdScript=$(mktemp XXXXXXXX_cmd.sh)
   cmdOut="${cmdScript%.sh}.out"

   [[ -n "$desc" ]] && msg "$desc"

   if [[ "$honorNoOpFlag" -eq 1 && "$NoOp" -eq 1 ]]; then
      msg "NO_OP: Would run: \"$cmd\""
   else
      msg "Running: \"$cmd\"."

      echo -e "#!/bin/bash\n$cmd\n" > "$cmdScript"
      chmod +x "$cmdScript"
      [[ "$Debug" -ne 0 ]] && dbg "BEGIN Contents of cmdScript $cmdScript:\n$(cat "$cmdScript")\nEND Contents of cmdScript"
      $cmdScript > "$cmdOut" 2>&1
      cmdExitCode=$?

      if [[ -n "$grepString" ]]; then
         # shellcheck disable=SC2196
         egrep "$grepString" "$cmdOut" > /dev/null 2>&1
         grepExit=$?
         cmdExitCode="$grepExit"
      fi

      if [[ "$alwaysShowOutputFlag" -eq 1 || "$Debug" -ne 0 ]]; then
         cat "$cmdOut"
      fi

      # Be clean and tidy.
      rm -f "$cmdScript" "$cmdOut"

      # If a grep was requested, return the exit code from the egrep,
      # otherwise return the exit code of the command executed.  In
      # any case, $cmdExitCode contains the exit code of the command.
      if [[ -n "$grepString" ]]; then
         return "$grepExit"
      else
         return "$cmdExitCode"
      fi
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: terminate
# shellcheck disable=SC2317
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Stop logging.
   [[ "$Log" == off ]] || msg "Log is: $Log\n${H1}"

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ "$errorMessage" != Unset ]]; then
      echo -e "\n\nUsage Error:\n\n$errorMessage\n\n"
   fi

   echo "USAGE for $ThisScript v$Version:

$ThisScript -i <instance1>[,<instance2>...] [-y] [-L <log>] [-n] [-debug|-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ "$style" == -man ]]; then
      echo -e "
DESCRIPTION:
	The HMS \"Tight Ship\" setup script installs 
	HMS server, or adds an 'hms' SDP instance to an existing SDP
	installation.  into a tight-ship style installation on the master
	server machine.

	This is best done immediately after a running the SDP install_sdp.sh
       	script to initialize the hms instance on a dedicated p4hms machine
	with no data.

	A tight ship installation is one in which:
	* The key SDP folders, e.g. /p4/common and /p4/N/bin (for
	each instance) are versioned locally using SDP.<hostname>
	workspaces in a local HMS instance, /p4/hms, which this script
	creates and initializes.

	* The crontab for the Perforce user is versioned.

	* Any local configuration and/or customizations of the SDP
	are versioned.

	* The HMS instance is aware of The Workshop, a public
	P4 Server hosted by Perforce Software.  It fetches SDP
	updates from The Workshop into the local hms instance using
	P4 native DVCS features.

	This simplifies future updates of the SDP, allowing them
	to be done with a 'p4 fetch' followed by a 'p4 merge'
	on your local server (if the server can reach The
	Workshop directly; if not an alternative
	approach using an interim personal repo can be used to
	bridge 'air gap' networks).

PRE-REQUISITES:
	The following are pre-requisites to running $ThisScript:

	1. The SDP install_sdp.sh script must have been run to create
	the hms instance.

ADDITIONAL SETUP:
	The following setup is necessary before operating in a tight
	ship environment, but are not necessary before running this
	script.

	1. The SSH keys must be configured enabling the 'perforce' user
	to ssh without a password to all machines listed in sdp_hosts.cfg
	configured above, including the current host (i.e. it must be
	able to ssh to itself using the current hostname).

OPTIONS:
 -i <instance1>[,<instance2>...]
 	Specify a comma-delimited list of SDP instances to manage.

 -y	Continue past the preflight without an interactive prompt.

 -d <sdp_dir>
	Specify the root of the directory tree where the SDP package
	is available.  The default is: $SDPInstallDirDefault

	The version of the SDP installed must be recent enough to
	contain the 'hms' setup directory:
	$SDPInstallDirDefault/Server/Unix/setup/hms.

 -s	Specify '-s' if sudo access is available.  If sudo access
	is available, HMS can manage more things than it can without
	sudo access, e.g. Swarm.

	Adding '-s' adds a preflight check to verify sudo access.

 -d	Set debug verbosity.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	${HOME}/${ThisScript%.sh}.<timestamp>.log

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

 -n	No-Op.  Prints commands instead of running them.

 -D     Use bash 'set -x' extreme debugging verbosity. Implies -d.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

FILES:

EXAMPLES:
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-i) ActiveInstances="${2/,/ }"; shiftArgs=1;;
      (-d) SDPInstallDir="$2"; shiftArgs=1;;
      (-y) Interactive=0;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) msg "$ThisScript v$Version"; exit 2;;
      (-L) Log="$2"; shiftArgs=1;;
      (-n) NoOp=1;;
      (-debug) Debug=1;;
      (-D) Debug=1; set -x;; # Debug; use bash 'set -x' extreme debug mode.
      (*) usage -h "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ "$shiftArgs" -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

SDPInstallDirDefault="/p4"

[[ -n "$Log" ]] || \
   Log="${HOME}/${ThisScript/.sh/}.$(date +'%Y%m%d-%H%M%S').log"

[[ "$SDPInstallDir" == Unset ]] && SDPInstallDir="$SDPInstallDirDefault"

[[ -d "$SDPInstallDir" ]] || \
   bail "The SDP install dir [$SDPInstallDir] does not exist. Specify the correct value with '-d <sdp_dir>'."

[[ -d "$HMSSetupDir" ]] || \
   bail "The HMS setup dir [$HMSSetupDir] does not exist."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

for u in $RequiredUtils; do
   [[ -n "$(command -v "$u")" ]] || errmsg "Missing required utility: $u"
done

[[ "$ErrorCount" -eq 0 ]] || bail "Aborted early due to missing required utilitiies."

if [[ "$Log" != off ]]; then
   touch "$Log" || bail "Couldn't touch log file: $Log"

   # Redirect stdout and stderr to a log file.
   exec > >(tee "$Log")
   exec 2>&1

   msg "${H1}\nLog is: $Log"
fi

ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@${HOSTNAME%%.*} on $(date) as:\n\t$CmdLine"

TmpFile1=$(mktemp)

#------------------------------------------------------------------------------
msg "Starting Preflight Checklist."

SDPHostsFile="$SDPInstallDir/common/site/config/sdp_hosts.cfg"
SDPHostsSample="$HMSInstallRoot/p4/common/site/config/sdp_hosts.cfg.sample"

if [[ -f "$SDPHostsFile" ]]; then
   msg "Verified: SDP hosts file exists: [$SDPHostsFile]."
else
   if [[ -e "$SDPHostsSample" ]]; then
      msg "Generate SDP Hosts file [$SDPHostsFile] from sample [$SDPHostsSample]."
      TmpFile2=$(mktemp)
      sed -E \
         -e "s|ALL_SDP_HOSTS=.*|ALL_SDP_HOSTS=\"${HOSTNAME%%.*}\"|g" \
         -e "s|SDP_REPLICA_HOSTS=.*|SDP_REPLICA_HOSTS=|g" \
         -e "s|SDP_REMOTE_HOSTS=.*|SDP_REMOTE_HOSTS=|g" \
         -e "s|SDP_SYNC_HOSTS=.*|SDP_SYNC_HOSTS=\"\$ALL_SDP_HOSTS\"|g" \
         -e "s|SDP_STATUS_DIRS=.*|SDP_STATUS_DIRS=\"/p4/common/bin /p4/common/cloud /p4/common/config /p4/common/etc /p4/common/lib /p4/common/test\"|g" \
         "$SDPHostsSample" > "$TmpFile2" ||\
         errmsg "Could not generate SDP Hosts temp file [$TmpFile2] from sample [$SDPHostsSample]."

         dbg "Generated SDP hosts file:\n$(cat "$TmpFile2")"

         if [[ "$NoOp" -eq 0 ]]; then
            # The /p4/common/site should always exist, but /p4/common/site/config might not.
            # Create that directory if needed; use 'mkdir' but not 'mkdir -p' since we should
            # only need to create one directory level.
            [[ -d /p4/common/site/config ]] || mkdir /p4/common/site/config 
            mv -f "$TmpFile2" "$SDPHostsFile" ||\
               errmsg "Failed to move generated SDP hosts temp file [$TmpFile2] to [$SDPHostsFile]."
         else
            msg "NO_OP: Would update SDP hosts file [$SDPHostsFile] from temp file [$TmpFile2]."
         fi
   else
      errmsg "SDP hosts file missing: [$SDPHostsFile], and also the sample from which to generate one is missing: $SDPHostsSample\n"
   fi
fi

for dir in $DirList; do
   if [[ -d "$dir" ]]; then
      msg "Verified: SDP directory exists: [$dir]."
   else
      errmsg "Missing SDP directory: [$dir]."
   fi
done

if [[ $(id -u -n) != "$OSUSER" ]]; then
   errmsg "Must run as $OSUSER, not $(id -u -n)."
else
   msg "Verified: Running os OSUSER ($OSUSER)."
fi

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "Preflight Checklist Completed OK."
else
   bail "Aborting due to failed preflight checks."
fi

if [[ "$Interactive" -ne 0 ]]; then
   input=""
   while [[ -z "$input" ]]; do
      echo -e -n "\nEnter Y to proceed, N to stop [y/Y/n/N]:  "
      read -r -e input
      if [[ "${input}" == "Y" || $input == "y" ]]; then
         continue
      elif [[ "${input}" == "N" || $input == "n" ]]; then
         msg "Confirmation to proceed not received. Halting."
         exit 1
      else
         input=
      fi
   done
fi

#------------------------------------------------------------------------------
msg "${H2}\nStep 1: Populate HMS instance with static load data."

cd "$HMSSetupDir" ||\
   bail "Failed to cd to HMS setup dir: $HMSSetupDir"

msg "Operating in: $PWD"

if [[ "$NoOp" -eq 0 ]]; then
   msg "Loading HMS environment."
   # shellcheck disable=SC1091
   source /p4/common/bin/p4_vars hms ||\
      bail "Failed to load HMS environment."
else
   msg "NO_OP: Would load HMS environment."
fi

# Environment safety check:
if [[ "$P4ROOT" == "/p4/hms/root" ]]; then
   msg "Verified: HMS Environment looks OK."
else
   # In NO_OP mode, P4ROOT won't be set correctly, so just skip this test.
   if [[ "$NoOp" -eq 0 ]]; then
      bail "HMS environment is not as expected, P4ROOT is $P4ROOT."
   fi
fi

if [[ "$P4PORT" =~ ^ssl[46]*: && ! -f /p4/ssl/certificate.txt ]]; then
   run "/p4/hms/bin/p4d_hms -Gc" "Generating SSL Certificate" 1 1 ||\
      bail "Failed to generate SSL certificate."
fi

sed -e "s:REPL_MAILFROM:$MAILFROM:g" \
   swarm.user.p4t > swarm.user.p4s

if [[ "$P4PORT" =~ ^ssl[46]*: ]]; then
   run "p4 trust -f -y"
fi

run "p4 user -f -i < swarm.user.p4s" "Creating user swarm." ||\
   bail "Failed to create user swarm."

for g in *.group.p4t; do
   _gtmp=$(echo "$g" | cut -f1 -d.)
   rm -f "$_gtmp.group.p4s"
   sed -e "s:REPL_ADMINUSER:$P4USER:g" "$g" > "$_gtmp.group.p4s"
done

for g in *.group.p4s; do
   run "p4 group -i < $g" "Loading group spec file $g." ||\
      bail "Failed to load group spec file $g with contents:\n$(cat "$g")"
done

run "p4 triggers -i < triggers.p4s" "Loading triggers table." ||\
   bail "Failed to load triggers table with contents:\n$(cat triggers.p4s)"

for d in *.depot.p4t; do
   _dtmp=$(echo "$d" | cut -f1 -d.)
   rm -f "$_dtmp.depot.p4s"
   sed -e "s:REPL_ADMINUSER:$P4USER:g" "$d" > "$_dtmp.depot.p4s"
done

for d in *.depot.p4s; do
   run "p4 depot -i < $d" "Loading depot spec file $d." ||\
      bail "Failed to load depot spec file $d with contents:\n$(cat "$d")"
done

for s in *.stream.p4t; do
   _stmp=$(echo "$s" | cut -f1 -d.)
   rm -f "$_stmp.stream.p4s"
   sed -e "s:REPL_ADMINUSER:$P4USER:g" "$s" > "$_stmp.stream.p4s"
   # shellcheck disable=SC2072
   if [[ "$P4D_VERSION" > "2020.2" ]]; then
      echo -e "\nParentView: inherit\n" >> "$_stmp.stream.p4s"
   fi
done

# Stream specs need to be created in a certain order, respecting the
# parenting relationships; e.g. 'main' must be created before a child
# of 'main' can be created.  Rather than work out the parenting
# relationships to elegantly create streams in the correct order, here
# we take a simple brute-force approach of making several passes at
# creating the entire list of streams. Streams that can't be created on
# the first pass because their parent does't yet exist will succeed on a
# subsequent pass.  This simple approach results in harmless errors
# creating streams.  We arbitrarily select 5 passes, which will create up
# to 5 levels of streams.
# shellcheck disable=SC2034
for pass in {1..5}; do
   for s in *.stream.p4s; do
      run "p4 stream -i < $s" "Loading stream spec file $s." ||\
         msg "Info: Ignore this known-harmless error creating a stream spec."
   done
done

# Do one final pass at stream spec creation.  On this final passs, the stream
# specs should already exist, so problems loading stream specs at this point
# are treated as hard errors.
for s in *.stream.p4s; do
   run "p4 stream -i < $s" "Loading stream spec file $s." ||\
      bail "Failed to load stream spec file $s with contents:\n$(cat "$s")"
done

msg "Create the deployment virtual stream for host $ThisHost."

echo -e "Stream: //p4-sdp/deploy_${ThisHost}\n
Owner:	$P4USER\n
Name:	deploy_${ThisHost}\n
Parent:	//p4-sdp/main\n
Type:	virtual\n
Description:
\tThis stream manages the deployment of SDP files
\ton $ThisHost.\n
Options:	allsubmit unlocked notoparent nofromparent mergedown\n
Paths:
\tshare Server/Unix/.p4ignore
\tshare Server/Unix/p4/common/..." > "$TmpFile1"

for i in hms $ActiveInstances; do
   echo -e "\tshare host/$ThisHost/p4/$i/bin/..." >> "$TmpFile1"
done

echo -e "\nRemapped:
\tServer/Unix/.p4ignore .p4ignore
\tServer/Unix/p4/common/... common/..." >> "$TmpFile1"

for i in hms $ActiveInstances; do
   echo -e "\thost/$ThisHost/p4/$i/bin/... $i/bin/..." >> "$TmpFile1"
done

# shellcheck disable=SC2072
if [[ "$P4D_VERSION" > "2020.2" ]]; then
   echo -e "\nParentView: inherit" >> "$TmpFile1"
fi

run "p4 stream -i < $TmpFile1" "Loading generated deployment stream spec." ||\
   bail "Failed to load this generated stream spec:\n$(cat "$TmpFile1")\n"

rm -f "$TmpFile1"

for r in *.remote.p4s; do
   run "p4 remote -i < $r" "Loading remote spec file $r." ||\
      bail "Failed to load remote spec file $r."
done

for b in *.branch.p4t; do
   # Remove the generated Perforce spec file with a *.p4s extension, to be
   # generated from a template with a .p4t extension.
   rm -f "${b/p4t/p4s}"
   sed -e "s:REPL_ADMINUSER:$P4USER:g" \
      "$b" > "${b/p4t/p4s}"
   run "p4 branch -i < ${b/p4t/p4s}" "Loading branch spec file ${b/p4t/p4s}." ||\
      bail "Failed to load branch spec file ${b/p4t/p4s}."
done

rm -f sdp.hostname.client.p4s
sed -e "s:REPL_ADMINUSER:$P4USER:g" \
   -e "s:REPL_SHORT_HOSTNAME:$ThisHost:g" \
   -e "s:REPL_HOSTNAME:$(hostname):g" sdp.hostname.client.p4t > sdp.hostname.client.p4s ||\
   warnmsg "Error generating client spec for deployment workspace."

rm -f superuser.sdp_main.client.p4s
sed -e "s:REPL_P4USER:$P4USER:g" \
   -e "s:REPL_ADMINUSER:$P4USER:g" \
   -e "s:REPL_SHORT_HOSTNAME:$ThisHost:g" \
   -e "s:REPL_HOSTNAME:$(hostname):g" \
   superuser.sdp_main.client.p4t > superuser.sdp_main.client.p4s ||\
   warnmsg "Error generating client spec for //p4-sdp_main."

for c in *.client.p4s; do
   run "p4 client -i < $c" "Loading client spec file $c." ||\
      bail "Failed to load client spec file $c with contents:\n$(cat "$c")"
done

run "p4master_run hms p4 configure unset filesys.checklinks" ||\
   warnmsg "Failed to unset filesys.checklinks configurable."

msg "Enabling DVCS features."
run "p4master_run hms p4 configure set server.allowpush=3" ||\
   bail "Failed to set server.allowpush configurable."
run "p4master_run hms p4 configure set server.allowfetch=3" ||\
   bail "Failed to set server.allowfetch configurable."

#------------------------------------------------------------------------------
msg "${H2}\nStep 2: Fetch SDP from The Workshop."

msg "Fetching the SDP."
run "p4master_run hms p4 fetch" ||\
   bail "Failed to fetch SDP from The Workshop."

#------------------------------------------------------------------------------
msg "${H2}\nStep 3: Populate local mainline."

msg "Branching files from Workshop SDP mainline //p4-sdp/workshop_main to local mainline //p4-sdp/main."
run "p4 populate -S //p4-sdp/workshop_main" ||\
   bail "Failed to populate //p4-sdp/main from //p4-sdp/workshop_main."

msg "Generating P4CONFIG file /p4/.p4config.SDP."
run "p4master_run hms echo P4PORT=\$P4PORT > /p4/.p4config.SDP"
run "p4master_run hms echo P4USER=\$P4USER >> /p4/.p4config.SDP"
run "echo P4CLIENT=sdp.$ThisHost >> /p4/.p4config.SDP"
run "p4master_run hms echo P4TICKETS=\$P4TICKETS >> /p4/.p4config.SDP"
run "p4master_run hms echo P4TRUST=\$P4TRUST >> /p4/.p4config.SDP"
run "echo P4IGNORE=.p4ignore >> /p4/.p4config.SDP"
run "echo net.autotune=1 >> /p4/.p4config.SDP"

msg "Setting P4CONFIG=/p4/.p4config.SDP."
export P4CONFIG=/p4/.p4config.SDP

run "p4 set"
run "p4 flush" ||\
   bail "Failed to flush."

#------------------------------------------------------------------------------
msg "${H2}\nStep 4: Overlay HMS on the SDP."

run "rsync -av $HMSInstallRoot/p4/common/ /p4/common" \
   "Overlaying files in /p4/common" ||\
   bail "Failed to overlayfiles into: /p4/common"

#------------------------------------------------------------------------------
msg "${H2}\nStep 5: Version key files in /p4/common and /p4/N/bin dirs."

run "p4 -s sync -f /p4/.p4ignore" "Deploy /p4/.p4ignore." ||\
   bail "Failed to force-sync the /p4/.p4ignore file."

DirList="/p4/common/bin /p4/common/etc /p4/common/config /p4/common/site"
for i in hms $ActiveInstances; do
   DirList+=" /p4/$i/bin"
done

msg "Reconciling to detect local differences."
for d in $DirList; do
   run "p4 -d $d rec" ||\
       bail "Failed to reconcile in $d."
done

#------------------------------------------------------------------------------
msg "${H2}\nStep 6: Sync SDP management workspace."

SDPMgmtWs="$P4USER.${HOSTNAME%%.*}.sdp_main"
SDPMgmtP4Cfg=/p4/hms/tmp/sdp_main/.p4config

run "p4master_run hms p4 -c $SDPMgmtWs -s sync" ||\
   bail "Failed to sync SDP management workspace [$SDPMgmtWs]."

msg "Generating P4CONFIG file $SDPMgmtP4Cfg"
run "p4master_run hms echo P4PORT=\$P4PORT > $SDPMgmtP4Cfg"
run "p4master_run hms echo P4USER=\$P4USER >> $SDPMgmtP4Cfg"
run "echo P4CLIENT=$SDPMgmtWs >> $SDPMgmtP4Cfg"
run "p4master_run hms echo P4TICKETS=\$P4TICKETS >> $SDPMgmtP4Cfg"
run "p4master_run hms echo P4TRUST=\$P4TRUST >> $SDPMgmtP4Cfg"
run "echo P4IGNORE=.p4ignore >> $SDPMgmtP4Cfg"
run "echo net.autotune=1 >> $SDPMgmtP4Cfg"

if [[ "$NoOp" -eq 0 ]]; then
   [[ -r "$SDPMgmtP4Cfg" ]] || \
      bail "Failed to generate SDP management P4CONFIG file [$SDPMgmtP4Cfg]."
fi

#------------------------------------------------------------------------------
msg "${H2}\nStep 7: Do live checkpoint for hms instance."

msg "Taking a live checkpoint."
run "p4master_run hms -c $P4CBIN/live_checkpoint.sh" ||\
   bail "Failed to initialize SDP offline checkpoint process with a live checkpoint."

#------------------------------------------------------------------------------
msg "${H2}\nStep 8: Custom site-local setup."

if [[ -x "$CustomSetupScript" ]]; then
   run "$CustomSetupScript" \
      "Executing custom setup script specified with \$HMS_CUSTOM_SETUP_SCRIPT." ||\
      bail "Custom setup script returned non-zero exit code."
else
   msg "No custom setup script found."
fi

#------------------------------------------------------------------------------
if [[ "$ErrorCount" -eq 0 && "$WarningCount" -eq 0 ]]; then
   msg "${H2}\nAll processing completed successfully.\n"
elif [[ "$ErrorCount" -eq 0 ]]; then
   msg "${H2}\nProcessing completed with no errors but with $WarningCount warnings.  Scan above output carefully.\n" 
else
   msg "${H2}\nProcessing completed, but with $ErrorCount errors and $WarningCount warnings.  Scan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit "$ErrorCount"