mkrep.sh #15

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

if [[ ${BASH_VERSINFO[0]} -lt 4 ]]; then
   echo -e "\n\nERROR: $0 requires bash version 4.x+; current bash version is $BASH_VERSION."
   exit 1
fi


# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export SDP_ENV=${SDP_ENV:-/p4/common/bin/p4_vars}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=/p4/hms/logs/mkrep.$(date +'%Y%m%d-%H%M').log
export VERBOSITY=${VERBOSITY:-3}
declare SSH_Opts="-oBatchMode=yes"
declare _RND_PW=`date +%s | sha256sum | base64 | head -c 32`

mkrep_config="${0%/*}/mkrep.cfg"
source $mkrep_config || {
  echo  "Error: failed to source $mkrep_config"
  exit 1
}
# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=${P4CONFIG:-.p4config}
export P4ENVIRO=/dev/null/.p4enviro

[[ -r "$P4U_ENV" ]] || {
   echo -e "\nError: Cannot load environment from: $P4U_ENV\n\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   source $bash_lib ||\
      { echo -e "\nFATAL: Failed to load bash lib [$bash_lib]. Aborting.\n"; exit 1; }
done

declare Version=2.0.4
declare -i SilentMode=0
declare -i StartupCmdNumFirst StartupCmdNumLast
declare -i DoSSHChecks=1
export VERBOSITY=3

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # Don't litter.
   cleanTrash

   # With the trap removed, exit.
   exit $OverallReturnStatus
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT -i <SDP_Instance> -t <Type> -s <Site_Tag> -r <Replica_Host> [-ssh_opts=\"opts\"] [-skip_ssh] [-c <cfg>] [-L <log>] [-si] [-v<n>] [-n] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script creates makes a replica, and provides enough information to
	make it ready in all respects.

OPTIONS:
 -i <SDP_Instance>
	Specify the SDP Instance.

 -t <Type>
	Specify the replica type tag.  The type corresponds to the 'Type:' and
	'Services:' field of the server spec, which describes the type of services
	offered by a given replica.

	Valid values are:
	* ro:   Read-Only standby replica.
	* rom:  Read-Only standby replica, Metadata only.
	* fr:   Forwarding Replica (Unfiltered).
	* fs:   Forwarding Standby (Unfiltered).
	* frm:  Forwarding Replica (Unfiltered,  Metdata only).
	* fsm:  Forwarding Standby (Unfiltered,  Metdata only).
	* ffr:  Filtered Forwarding Replica.  Not a valid failover target.
	* edge: Edge Server. Filtered by def'n.

	Replica with 'standby' are always unfiltered, and use the 'journalcopy'
	method of replication, which copies a byte-for-byte verbatim journal file
	rather than one that is merely logically equivalent.

	The tag has several purposes:
	1. Short Hand.  Each tag represents a combination of 'Type:' and fully
	qualified 'Services:' values used in server specs.

	2. Distillation. Only the most useful Type/Services combinations have a
	shorthand form.

	3. For forwarding replicas, the name includes the critical distinction of
	whether any replication filtering is used; as filtering of any kind disqualifies
	a replica from being a potential failover target.  (No such distinction is
	needed for edge servers, which are filtered by definition).

 -s <Site_Tag>
	Specify a geographic site tag indicating where the replica will physically be
	located.  Valid site tags are defined in the site tags file,
	$SiteTagsFile

	Current valid site tags are:
	$(grep -v '^#' $SiteTagsFile 2>&1|grep -v '$^'|sed 's:^:\t:g')

 -r <Replica_Host>
	Specify the target replica host.

 -ssh_opts=\"opts\"
	Specify '-ssh_opts' to pass paraemters on to the ssh command.  For example, to
	specify ssh operation on non-standard port 2222, specify '-ssh_opts=\"-p 2222\"'.

 -skip_ssh
	Specify '-skip_ssh' to skip SSH checks duing the preflight check.

	This is useful if you only intend to do the metadata prepaation phase
	of startup prior to SSH being setup.

 -c <cfg>
	Specify the Helix Topology Config file.  The default is:
	$HelixTopologyCfg

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	$(dirname ${P4U_LOG}).

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

-si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -n	No-Op.  Prints commands instead of running them.

 -D     Set extreme debugging verbosity.

 -f	Full Mode Setup:  The completes an edge servers setup so no additional steps 
        are required.  This setup requires an ssh connection from the master to the 
        edge to be in place first.  It also requires the depot log journal and /p4
        mounts to be in place and setup as expected.  This setup assumes a standard
        SDP and HMS setup.   HMS is required to sync the configuration data over.  

-se     Stream Exists:  If you are setting up a second instance on an edge you will
        already have a hms deployment stream.   In this scenario one will manually have
        to add the new SDP Instance to the stream before running full mode setup.
        This flag notes that you have done this.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

DEPENDENCIES:
	This script depends on ssh keys being defined to allow the Perforce
	operating system user ($OSUER) to ssh to any necessary machines
	without a password.

	This script assumes the replica host already has the SDP fully
	configured.

FILES:
	This Site Tags file defines the list of valid geographic site tags:
	$SiteTagsFile

EXAMPLES:
	Prepare an edge server to run on host syc-helix-04:
	$THISSCRIPT -i acme -t edge -s syd -r syc-helix-04
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare ReplicaHost=Unset
declare ReplicaTypeTag=Unset
declare ReplicaType=
declare SiteTag=Unset
declare SiteTagsFile=${P4CCFG:-/p4/common/config}/SiteTags.cfg
declare SDPInstance=Unset
#declare -i Interactive=1
declare -i MetadataOnly=0
declare -i shiftArgs=0
declare -i FullSetup=0
declare -i UseExistingStream=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-r) ReplicaHost=$2; shiftArgs=1;;
      (-t) ReplicaTypeTag=$2; shiftArgs=1;;
      (-i) SDPInstance=$2; shiftArgs=1;;
      (-s) SiteTag=$2; shiftArgs=1;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-ssh_opts) export $SSH_Opts=$2; shiftArgs=1;;
      (-skip_ssh) DoSSHChecks=0;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-se) UseExistingStream=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-f) FullSetup=1;;
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

[[ $SDPInstance == Unset ]] && usageError "\nThe '-i <SDP_Instance>' parameter is required."
[[ $ReplicaHost == Unset ]] && usageError "\nThe '-r <Replica_Host>' parameter is required."
[[ $ReplicaTypeTag == Unset ]] && usageError "\nThe '-t <Type>' parameter is required."
[[ $SiteTag == Unset ]] && usageError "\nThe '-s <Site_Tag>' parameter is required."
[[ $DoSSHChecks -eq 0 && $FullSetup -eq 1 ]] && \
   usageError "The '-skip_ssh' and '-f' (full setup) options are mutually exclusive."

# Set log file to point at SDP_INSTANCE.
export P4U_LOG=/p4/${SDPInstance}/logs/mkrep.$(date +'%Y%m%d-%H%M').log

case "$ReplicaTypeTag" in
   (ro) ReplicaType=standby;;                             # Read-Only Standby replica.
   (rom) ReplicaType=standby; MetadataOnly=1;;            # Read-Only Standby replica, Metadata only.
   (fr) ReplicaType=forwarding-replica;;                  # Forwarding Replica (Unfiltered).
   (fs) ReplicaType=forwarding-standby;;                  # Forwarding Standby (Unfiltered).
   (frm) ReplicaType=forwarding-replica; MetadataOnly=1;; # Forwarding Replica (Unfiltered), Metdata only.
   (fsm) ReplicaType=forwarding-standby; MetadataOnly=1;; # Forwarding Standby (Unfiltered).
   (ffr) ReplicaType=forwarding-replica;;                 # Filtered Forwarding Replica
   (edge) ReplicaType=edge-server;;                       # Edge Server. Filtered by def'n, cannot be Metdata only.
   (*) usageError "The specified replica type tag [$ReplicaTypeTag].";;
esac

declare -i tagFound=0
if [[ -r "$SiteTagsFile" ]]; then
   while read line; do
      [[ $line == "#*" ]] && continue
      [[ -z "$(echo $line)" ]] && continue
      [[ "$line" == *":"* ]] || continue
      tag=${line%%:*}

      if [[ $tag == $SiteTag ]]; then
         tagFound=1
         break
      fi
   done < $SiteTagsFile
else
   bail "Missing site tag configuration file [$SiteTagsFile]. Aborting."
fi

[[ $tagFound -eq 1 ]] ||\
   bail "Failed to find specified site tag [$SiteTag] inite tag configuration file [$SiteTagsFile]. Aborting."

if [[ "$ReplicaTypeTag" != "edge" && $FullSetup -eq 1 ]] ; then
   bail "FullSetup is only supported for edge configurations currently."
fi

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM


declare -i OverallReturnStatus=0
declare ServerSpec= ServerSpecFile= ShortServerSpec=
declare ServiceUser= ServiceUserSpecFile
declare ServiceUsersGroup=ServiceUsers
declare TmpDir=$(mktemp -d)
declare ProtectsFile=$TmpDir/protect.p4s
declare GroupSpecFile=$TmpDir/group.$ServiceUsersGroup.p4s
declare TmpFile=$TmpDir/tmpFile.$THISSCRIPT

GARBAGE+=" $TmpDir"

touch ${TmpFile} || bail "Couldn't touch tmp file [$TmpFile]."

if [[ "${P4U_LOG}" != off ]]; then
   touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee ${P4U_LOG})
      exec 2>&1
   else
      exec >${P4U_LOG}
      exec 2>&1
   fi

   initlog
fi

msg "Starting $THISSCRIPT v$Version at $(date)."

msg "${H}\nPart 0: Environment Setup and Preflight Checks."

msg "Loading SDP environment for instance $SDPInstance."
source "$SDP_ENV" "$SDPInstance"
source /p4/common/bin/backup_functions.sh
check_vars
set_vars

msg "Overriding P4CONFIG, P4ENVIRO, P4ALIASES, etc."
export P4ENVIRO=/dev/null/.p4enviro
export P4ALIASES=/dev/null/.p4aliases
export P4CONFIG=$TmpDir/.p4config
export P4TICKETS=$TmpDir/.p4tickets
export P4TRUST=$TmpDir/.p4trust
export P4BIN=/p4/${SDPInstance}/bin/$P4SERVER
echo "P4PORT=$P4MASTERPORT" > $P4CONFIG

# Should be full and HMS.
if [[ $FullSetup -eq 1 ]] ; then
  if [[ ! -f $P4USER_PW_FILE ]] ; then
    bail "P4USER_PW_FILE not set correctly."
  fi
  $_ADMIN_PW=`cat $P4USER_PW_FILE`
  source "$SDP_ENV" "hms"
  if [[ ! -f /p4/hms/bin/p4_hms ]] ; then
    bail "HMS is not setup.  Could not find /p4/hms/bin/p4_hms"
  fi
  if [[ ! -f /p4/.p4config.SDP ]] ; then
    bail "HMS not setup.  Could not find .p4config.SDP"
  fi

run "$P4BIN -u $P4USER -p $P4MASTERPORT -s login -s" \
   "Checking whether user $P4USER is logged into port $P4MASTERPORT." 1 1 0

if [[ $? -ne 0 ]]; then
   msg "Trying login with:\n$P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $P4CCFG/.p4passwd.$P4SERVER.admin\n"
   if [[ $NO_OP -eq 0 ]]; then
      $P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $P4CCFG/.p4passwd.$P4SERVER.admin
   else
      msg "NO_OP: Would execute: $P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $P4CCFG/.p4passwd.$P4SERVER.admin"
   fi
fi

run "$P4BIN -u $P4USER -p $P4MASTERPORT -s login -s" \
   "Verifying that user $P4USER can login to port $P4MASTERPORT." 1 1 0 ||\
   bail "Login verification failed."

  msg "Verified HMS setup."

  HMS_StreamSpec="deploy_$ReplicaHost"
  HMS_StreamSpecFile="$TmpDir/${HMS_StreamSpec}.stream.p4s"

  if [[ `$P4BIN streams | grep $HMS_StreamSpec ` && $UseExistingStream -eq 0 ]]; then
	bail "Stream spec $HMS_StreamSpec already exists.\n  Please manually add the $SDPInstance and use -se option."
  fi
if [[ $UseExistingStream -eq 0 ]] ; then
echo -e "\n
Stream: //sdp/$HMS_StreamSpec\n
Owner: $P4USER\n
Name: $HMS_StreamSpec\n
Parent:  //sdp/main\n
Type: virtual\n
Description:\n
  This stream manages the deployment of SDP files on $ReplicaHost\n
Options: allsubmit unlocked notoparent nofromparent mergedown\n
Paths:\n
  share Server/Unix/.p4ignore\n
  share Server/Unix/p4/common/...\n
  share host/$ReplicaHost/p4/${SDPInstance}/bin/...\n
Remapped:\n
  Server/Unix/.p4ignore .p4ignore\n
  Server/Unix/p4/common/... common/...\n
  host/$ReplicaHost/p4/${SDPInstance}/bin/... ${SDPInstance}/bin/...\n
" > $HMS_StreamSpecFile

msg "Creating Stream Spec $HMS_StreamSpec with these contents:"
msg ${H}
cat $HMS_StreamSpecFile
msg ${H}

if [[ $NO_OP -eq 0 ]]; then
   p4 stream -i < $HMS_StreamSpecFile || bail "Failed to lead stream spec from file $HMS_StreamSpecFile"
else
   msg "NO_OP: Would run: p4 stream -i .LT. $HMS_StreamSpecFile"
fi
else
  msg "Using existing stream $HMS_StreamSpec"
fi
  HMS_ClientSpec="sdp.$ReplicaHost"
  HMS_ClientSpecFile="$TmpDir/${HMS_ClientSpec}.client.p4s"
echo -e "\n
Client: $HMS_ClientSpec\n
Owner: $P4USER\n
Description: $ReplicaHost HMS deployment client\n
Root: /p4\n
Options: noallwrite noclobber nocompress unlocked modtime rmdir\n
SubmitOptions: leaveunchanged\n
LineEnd: local\n
Stream: //sdp/$HMS_StreamSpec\n
" > $HMS_ClientSpecFile

msg "Creating Client Spec $HMS_ClientSpec with these contents:"
msg ${H}
cat $HMS_ClientSpecFile
msg ${H}

if [[ $NO_OP -eq 0 ]]; then
   p4 client -i < $HMS_ClientSpecFile || bail "Failed to lead client spec from file $HMS_ClientSpecFile"
else
   msg "NO_OP: Would run: p4 client -i .LT. $HMS_ClientSpecFile"
fi
    
   _p4config_SDP="P4PORT=$P4MASTERPORT\nP4USER=p4admin\nP4CLIENT=$HMS_ClientSpec\nP4TICKETS=/p4/${SDPInstance}/.p4tickets\nP4TRUST=/p4/${SDPInstance}/.p4trust\nP4IGNORE=.p4ignore"

  #Saving for login on host
  HMSP4MASTERPORT=$P4MASTERPORT
  HMSP4USER=$P4USER

  #Reset to $SDPInstance
  source "$SDP_ENV" "$SDPInstance"
fi

if [[ $DoSSHChecks -eq 1 ]]; then
   msg "Checking ssh access to master host $P4MASTER."
   ssh $SSH_Opts -q "$P4MASTER" /bin/ls > /dev/null 2>&1 || bail "Failed to ssh to host $P4MASTER."

   msg "${H}\nCreating SSH Command."
   export SSH_CMD_R="ssh $SSH_Opts -q $ReplicaHost"
   export SSH_CMD_M="ssh $SSH_Opts -q $P4MASTER"
   vmsg "SSH_CMD_R=$SSH_CMD_R"
   vmsg "SSH_CMD_M=$SSH_CMD_M"

   msg "Checking ssh access to master host $P4MASTER."
   $SSH_CMD_M /bin/ls > /dev/null 2>&1 || bail "Failed to ssh to host $P4MASTER."
   msg "Verified: ssh access to master host $P4MASTER is OK."

   msg "Checking ssh access to replica host $ReplicaHost."
   $SSH_CMD_R /bin/ls > /dev/null 2>&1 || bail "Failed to ssh to host $ReplicaHost."
   msg "Verified: ssh access to replica host $ReplicaHost is OK."

   if [[ $FullSetup -eq 1 ]]; then
      $SSH_CMD_R touch /p4 > /dev/null 2>&1 || bail "ssh does not have write access to /p4."
      msg "Verified: ssh has write access to /p4."
      $SSH_CMD_R touch $DB1 > /dev/null 2>&1 || bail "ssh does not have write access to $DB1."
      msg "Verified: ssh has write access to $DB1."
      $SSH_CMD_R touch $DB2 > /dev/null 2>&1 || bail "ssh does not have write access to $DB2."
      msg "Verified: ssh has write access to $DB2."
      $SSH_CMD_R touch $DD > /dev/null 2>&1 || bail "ssh does not have write access to $DD."
      msg "Verified: ssh has write access to $DD."
      $SSH_CMD_R touch $LG > /dev/null 2>&1 || bail "ssh does not have write access to $LG."
      msg "Verified: ssh has write access to $LG."
      $SSH_CMD_R touch $CN > /dev/null 2>&1 || bail "ssh does not have write access to $CN."
      msg "Verified: ssh has write access to $CN."
      $SSH_CMD_R "[[ ! -f /tmp/p4 ]]" || bail "/tmp/p4 already exists"
      msg "Verified: tmp p4 command does not exist."

      $SSH_CMD_R "[[ ! \${BASH_VERSINFO[0]} -lt 4 ]]" || bail "requires bash version 4.x+ on $ReplicaHost."
      msg "Verified: Bash 4.x on $ReplicaHost"
      $SSH_CMD_R "[[ ! -d /p4/$SDPInstance/root ]]" || bail "/p4/$SDPInstance/root exists on $ReplicaHost!  Helix already setup?"
      msg "Verified: /p4/$SDPInstance/root DNE on $ReplicaHost"
   fi
else
   msg "Skipping SSH preflight checks due to -skip_ssh."
fi

msg "${H}\nPart 1: Preparation: Login."

if [[ "$P4MASTERPORT" == "ssl:"* ]]; then
   run "p4 trust -y"
fi

run "$P4BIN -u $P4USER -p $P4MASTERPORT -s login -s" \
   "Checking whether user $P4USER is logged into port $P4MASTERPORT." 1 1 0

if [[ $? -ne 0 ]]; then
   msg "Trying login with:\n$P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $P4CCFG/.p4passwd.$P4SERVER.admin\n"
   if [[ $NO_OP -eq 0 ]]; then
      $P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $P4CCFG/.p4passwd.$P4SERVER.admin
   else
      msg "NO_OP: Would execute: $P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $P4CCFG/.p4passwd.$P4SERVER.admin"
   fi
fi

run "$P4BIN -u $P4USER -p $P4MASTERPORT -s login -s" \
   "Verifying that user $P4USER can login to port $P4MASTERPORT." 1 1 0 ||\
   bail "Login verification failed."

msg "${H}\nPart 2: Define Server Spec."
#TODO Need to make ArchiveDataFilter appear here through some option
ShortServerSpec="${ReplicaTypeTag}_${SiteTag}"
ServerSpec="p4d_${ShortServerSpec}"

ServerSpecFile="$TmpDir/$ServerSpec.server.p4s"

echo -e "ServerID: $ServerSpec\n
Type: server\n
Name: $ServerSpec\n
Services: $ReplicaType\n
ExternalAddress: $ReplicaHost:$P4MASTERPORT\n
Description:" > $ServerSpecFile || bail "Failed to initialize server spec file [$ServerSpecFile]."

case "$ReplicaTypeTag" in
   (ro) Desc="Read-Only Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (rom) Desc="Read-Only Standby Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (fr) Desc="Forwarding Replica (Unfiltered) in ${SiteTag^^}.";;
   (fs) Desc="Forwarding Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (frm) Desc="Forwarding Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (fsm) Desc="Forwarding Standby Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (ffr) Desc="Filtered Forwarding Replica in ${SiteTag^^}.";;
   (edge) Desc="Edge server in ${SiteTag^^}.";;
   (*) bail "\nInternal Error: Unrecognized replica type tag [$ReplicaTypeTag].";;
esac

echo -e "\t$Desc\n" >> $ServerSpecFile || bail "Failed to complete server spec file [$ServerSpecFile]."

msg "Creating server spec $ServerSpec with these contents:"
msg ${H}
cat $ServerSpecFile
msg ${H}

if [[ $NO_OP -eq 0 ]]; then
   p4 server -i < $ServerSpecFile || "Failed to load server spec from file: $ServerSpecFile"
else
   msg "NO_OP: Would run: p4 server -i .LT. $ServerSpecFile"
fi

msg "${H}\nPart 3: Set configurables."

ServiceUser=svc_${ServerSpec}
ServiceUserSpecFile=$TmpDir/$ServiceUser.user.p4s
declare -i ConfigureOK=1

run "p4 configure set $ServerSpec#P4TARGET=$P4MASTERPORT" || ConfigureOK=0
run "p4 configure set $ServerSpec#db.replication=readonly" || ConfigureOK=0
run "p4 configure set $ServerSpec#rpl.forward.all=1" || ConfigureOK=0
run "p4 configure set $ServerSpec#rpl.compress=4" || ConfigureOK=0
run "p4 configure set $ServerSpec#server=4" || ConfigureOK=0
run "p4 configure set $ServerSpec#monitor=2" || ConfigureOK=0
run "p4 configure set $ServerSpec#serviceUser=$ServiceUser" || ConfigureOK=0

# For master/commit serveres, the journalPrefix is always:
# /p4/N/checkpoints/p4_N
#
# Regardless of what host that folder appears on, checkpoints
# from there will have originated on the master server.
#
# For 'standby' replicas that use journalcopy, journalPrefix is:
# /p4/N/journals.rep/p4_N

# The journals.rep folder is on the /hxlogs volume, suitable for fast
# writes, and on a volume that is not typically backed up -- which
# is OK for active/live journals.  Journalcopy 'standby' replicas use
# journalPrefix to determine where to write active journals there,
# a unique feature of 'standby' replicas.
#
# Warning: This means don't take a checkpoint on a 'standby'
# replica, as it will land on a volume that may not be large enough
# to store it, and which is not backed up. This restriction is
# acceptable with the SDP, since we don't need to take checkpoints
# on standby replicas.

# For edge servers and forwarding replicas, set journalPrefix to something
# like:
# /p4/N/checkpoints.bos_edge/p4_N.bos_edge
#
# The checkpoints.* folders are on the /hxdepots volume, which is
# always backed up.  Unlike 'standby' replicas, all other replicas
# use journalPrefix only on journal rotation and checkpoint creation.
# We want journalPrefix pointing to a directoy on th /hxdepots
# volume for anything other than a journalcopy replica.
#
# Incorporating the SeverID (server spec name) allows multiple
# workspace servers (edg servers used for horiozontal scaling)
# to share storage on a SAN, while providing each edge a separate
# location for checkpoints.
#
# In the context of journalPrefix value and resulting
# checkpoint/journal file names, the leading 'p4d_' in the server
# spec name is redundant, and so is trimmed.

case $ReplicaType in
   (*standby)
      run "p4 configure set $ServerSpec#journalPrefix=$P4HOME/journals.rep" || ConfigureOK=0
   ;;
   (*)
      run "p4 configure set $ServerSpec#journalPrefix=$P4HOME/checkpoints.${ShortServerSpec}/p4_${SDPInstance}.${ShortServerSpec}" || ConfigureOK=0
   ;;
esac

if [[ $NO_OP -eq 0 ]]; then
   # use the '-L' flag to 'p4 pull' to enable 'journalcopy' if indicated by the replica
   # type, i.e. if it has Standby in the name.
   if [[ "$Desc" == *"Standby"* ]]; then
      vmsg "Executing: p4 configure set $ServerSpec#startup.1='pull -i 1 -L'"
      p4 configure set $ServerSpec#startup.1="pull -i 1 -L" || ConfigureOK=0
      p4 configure set $ServerSpec#startup.2="journalcopy -i 1" || ConfigureOK=0
      StartupCmdNumFirst=3
      StartupCmdNumLast=7
   else
      vmsg "Executing: p4 configure set $ServerSpec#startup.1='pull -i 1'"
      p4 configure set $ServerSpec#startup.1="pull -i 1" || ConfigureOK=0
      StartupCmdNumFirst=2
      StartupCmdNumLast=12
   fi
else
   if [[ "$Desc" == *"Standby"* ]]; then
      vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.1=\"pull -i 1 -L\""
      vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.2=\"journalcopy -i 1\""
      StartupCmdNumFirst=3
      StartupCmdNumLast=7
   else
      vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.1=\"pull -i 1\""
      StartupCmdNumFirst=2
      StartupCmdNumLast=6
   fi
fi

if [[ $MetadataOnly -eq 0 ]]; then
   run "p4 configure set $ServerSpec#lbr.replication=readonly" || ConfigureOK=0
   for i in $(seq $StartupCmdNumFirst $StartupCmdNumLast); do
      if [[ $NO_OP -eq 0 ]]; then
         vmsg "Executing: p4 configure set $ServerSpec#startup.$i='pull -i 1 -u'"
         p4 configure set $ServerSpec#startup.$i="pull -i 1 -u" || ConfigureOK=0
      else
         vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.1='pull -i 1'"
      fi
   done
else
   run "p4 configure set $ServerSpec#lbr.replication=ondemand" || ConfigureOK=0
fi

if [[ $ConfigureOK -eq 1 ]]; then
   msg "Verified:  All configurables were set OK."
   run "p4 configure show allservers" "Showing all persistent configurables." 0 1 0
else
   bail "Errors encountered setting configurables.  See the output above. Aborting."
fi

msg "${H}\nPart 4: Create replica service user $ServiceUser."

echo -e "User: $ServiceUser\n
Email: ${MAILFROM#\#}\n
FullName: Replication Server User for $ServerSpec\n
Type: service\n
AuthMethod: perforce\n" > $ServiceUserSpecFile || bail "Failed to initialize user spec file [$ServiceUserSpecFile]."

vmsg "Contents of $ServiceUserSpecFile:"
vmsg "${H}"
[[ $VERBOSITY -gt 2 ]] && cat $ServiceUserSpecFile
vmsg "${H}"

if [[ $NO_OP -eq 0 ]]; then
   p4 user -f -i < $ServiceUserSpecFile || "Failed to load user spec from file: $ServiceUserSpecFile"
else
   msg "NO_OP: Would run: p4 user -f -i .LT. $ServiceUserSpecFile"
fi

PasswdFile=$P4CCFG/.p4passwd.$P4SERVER.$ServiceUser
touch $PasswdFile || bail "Failed to initialize password file $PasswdFile."

chmod 600 $PasswdFile
echo $_RND_PW > $PasswdFile
echo $_RND_PW >> $PasswdFile

if [[ $NO_OP -eq 0 ]]; then
   msg "Setting password for service user $ServiceUser."
   p4 passwd $ServiceUser < $PasswdFile
else
   msg "NO_OP: Would run: p4 passwd $ServiceUser .LT. $PasswdFile"
fi

msg "${H}\nPart 5: Make replica service user a super user with unlimited timeout."

msg "Checking if Protections table references group $ServiceUsersGroup."

if [[ -n "$(p4 protects -g $ServiceUsersGroup)" ]]; then
   msg "Verified: Protections table references group $ServiceUsersGroup."
else
   msg "Adding protections table entry to reference group $ServiceUsersGroup."
   #p4 protect -o | grep -v '^#' > $TmpFile || bail "Failed to update tmp file $TmpFile."
   #echo -e "\tsuper group $ServiceUsersGroup * //..." >> $TmpFile || bail "Failed to update tmp file $TmpFile."
   #grep -v '^\s*$' $TmpFile > $ProtectsFile || bail "Failed to update protects file $ProtectsFile."

   vmsg "Contents of $ProtectsFile:"
   vmsg "${H};"
   [[ $VERBOSITY -gt 3 ]] && cat $ProtectsFile
   vmsg "${H}"

   if [[ $NO_OP -eq 0 ]]; then
      p4 protect -i < $ProtectsFile
   else
      msg "NO_OP: Would run: p4 protect -i .LT. $ProtectsFile"
   fi
fi

msg "Checking if serivce user $ServiceUser is in service users group $ServiceUsersGroup."

if [[ -n "$(p4 groups $ServiceUser | grep ^$ServiceUsersGroup$)" ]]; then
   msg "Verified: Serivce user $ServiceUser is in service users group $ServiceUsersGroup."
else
   # This logic will create the group spec for service users if it does not already exist,
   # or add our new service user to the group if it already exists.  The 'p4 group -o'
   # command generate a valid group spec whether the spec actually exists on the server or
   # not.
   p4 group -o $ServiceUsersGroup | grep -v '^#' |\
      sed "s:43200:unlimited:g;\$ s/.*/\t$ServiceUser/" > $GroupSpecFile ||\
      bail "Failed to update group spec file $GroupSpecFile."

   vmsg "Contents of $GroupSpecFile:"
   vmsg "${H}"
   [[ $VERBOSITY -gt 3 ]] && cat $GroupSpecFile
   vmsg "${H}"

   if [[ $NO_OP -eq 0 ]]; then
      p4 group -i < $GroupSpecFile
   else
      msg "NO_OP: Would run: p4 group -i .LT. $GroupSpecFile"
   fi
fi

if [[ $FullSetup -eq 1 ]] ; then
   msg "${H}\nPart 6: Running Full Setup :)\n"
   msg "Part 6.1 Setup pathing:\n"
   _dirs=(/$DD/p4/$SDPInstance/seed /$DD/p4/$SDPInstance/bin /$DD/p4/$SDPInstance/cache /$DD/p4/$SDPInstance/checkpoints /$DD/p4/$SDPInstance/depots /$DD/p4/$SDPInstance/checkpoints.${ShortServerSpec} /$DD/p4/$SDPInstance/seed /$LG/p4/$SDPInstance/journals.rep /$LG/p4/$SDPInstance/tmp /$LG/p4/$SDPInstance/logs /$DB1/p4/$SDPInstance/db1 /$DB2/p4/$SDPInstance/db2 /$CN/p4/common /$CN/p4/common/config) 
   for i in "${_dirs[@]}" ; do
     if [[ $NO_OP -eq 0 ]]; then
	$SSH_CMD_R "if [[ ! -d $i ]] ; then mkdir -p $i ; fi" || bail "Couldn't create $i"
	msg "Created $i on $ReplicaHost"
     else
        msg "NO_OP: Would run mkdir -p $i"
     fi
   done
   _ln=("/$DD/p4/$SDPInstance /p4/$SDPInstance" "/$CN/p4/common /p4/common" "/$LG/p4/$SDPInstance/journals.rep /p4/$SDPInstance/journals.rep" "/$LG/p4/$SDPInstance/tmp /p4/$SDPInstance/tmp" "/$LG/p4/$SDPInstance/logs /p4/$SDPInstance/logs" "/$DB1/p4/$SDPInstance/db1 /p4/$SDPInstance/root" "/$DB2/p4/$SDPInstance/db2 /p4/$SDPInstance/offline_db")
   for i in "${_ln[@]}" ; do
      if [[ $NO_OP -eq 0 ]]; then
	$SSH_CMD_R "if [[ ! -L `echo $i | cut -d' ' -f2` ]] ; then ln -s $i ; fi" || bail "Couldn't create symlink $i"
	msg "Created symlink $i on $ReplicaHost"
      else
        msg "NO_OP: Would run ln -s $i"
      fi
   done
   _chmod=(/$DD/p4/$SDPInstance /$LG/p4/$SDPInstance /$DB1/p4/$SDPInstance /$DB2/p4/$SDPInstance /$CN/p4/common)
   for i in "${_chmod[@]}" ; do
      if [[ $NO_OP -eq 0 ]]; then
	$SSH_CMD_R "chmod 700 -R $i" || bail "Couldn't chmod 700 $i"
	msg "Chmod 700 $i on $ReplicaHost"
      else
        msg "NO_OP: Would run chmod 700 -R $i"
      fi
   done

   msg "Part 6.1.1: Syncing the SDP using HMS"
   _hms_password=`cat /p4/common/config/.p4passwd.p4_hms.admin`
   if [[ $NO_OP -eq 0 ]] ; then
      $SSH_CMD_R "echo -e '$_p4config_SDP' > /p4/.p4config.SDP"
      msg "Wrote /p4/.p4config.SDP with:"
      msg "$_p4config_SDP"
      $SSH_CMD_R "echo $_hms_password > /p4/common/config/.p4passwd.p4_hms.admin" || bail "Couldn't write p4_hms password"
      msg "Wrote /p4/common/config/.p4passwd.p4_hms.admin"
      scp $P4BIN ${ReplicaHost}:/tmp/p4 || bail "Failed to copy /tmp/p4 to $ReplicaHost" || bail "Could not scp p4"
      $SSH_CMD_R "chmod +w /tmp/p4"
      msg "Copyied p4 over to allow initial sync"
      $SSH_CMD_R "export P4CONFIG=/p4/.p4config.SDP && /tmp/p4 -u $HMSP4USER -p $HMSP4MASTERPORT -s login < /p4/common/config/.p4passwd.p4_hms.admin" || bail "Could not login hms"
      $SSH_CMD_R "cd /p4 && export P4CONFIG=/p4/.p4config.SDP && /tmp/p4 sync -f" || bail "Could not sync hms"
      msg "Synced HMS!"
   else
      msg "Would write .p4config.SDP with:"
      msg "$_p4config_SDP"
      msg "Would write /p4/common/config/.p4passwd.p4_hms.admin"
      msg "NO_OP: $SSH_CMD_R echo $_hms_password > /p4/common/config/.p4passwd.p4_hms.admin"
      msg "NO_OP: scp $P4BIN ${ReplicaHost}:/tmp/p4 "
      msg "Copyied p4 over to allow initial sync"
      msg "NO_OP: $SSH_CMD_R /tmp/p4 -u $HMSP4USER -p $HMSP4MASTERPORT -s login < /p4/common/config/.p4passwd.p4_hms.admin"
      msg "NO_OP: $SSH_CMD_R cd /p4 && /tmp/p4 sync -f" 
      msg "Synced HMS! :-)"
   fi

   msg "Part 6.1.2: Linking binaries in SDP"
   _ln=("/p4/common/bin/p4_bin /p4/$SDPInstance/bin/p4_$SDPInstance" "/p4/common/bin/p4broker_${SDPInstance}_bin /p4/$SDPInstance/bin/p4broker_$SDPInstance" "/p4/common/bin/p4d_${SDPInstance}_bin /p4/$SDPInstance/bin/p4d_$SDPInstance")
   for i in "${_ln[@]}" ; do
      if [[ $NO_OP -eq 0 ]]; then
	$SSH_CMD_R "if [[ ! -L `echo $i | cut -d' ' -f2` ]] ; then ln -s $i ; fi" || bail "Couldn't create symlink $i"
	msg "Created symlink $i on $ReplicaHost"
      else
        msg "NO_OP: Would run ln -s $i"
      fi
   done
   if [[ $NO_OP -eq 0 ]] ; then
     $SSH_CMD_R "sed s/REPL_SDP_INSTANCE/$SDPInstance/g /p4/common/etc/init.d/p4broker_instance_init.template > /p4/$SDPInstance/bin/p4broker_${SDPInstance}_init"
     $SSH_CMD_R "sed s/REPL_SDP_INSTANCE/$SDPInstance/g /p4/common/etc/init.d/p4d_instance_init.template > /p4/$SDPInstance/bin/p4d_${SDPInstance}_init"
     $SSH_CMD_R "chmod +x /p4/$SDPInstance/bin/*_init"
   else 
     msg "NO_OP: sed s/REPL_SDP_INSTANCE/$SDPInstance/g /p4/common/etc/init.d/p4broker_instance_init.template > /p4/$SDPInstance/bin/p4broker_${SDPInstance}_init"
     msg "NO_OP: sed s/REPL_SDP_INSTANCE/$SDPInstance/g /p4/common/etc/init.d/p4d_instance_init.template > /p4/$SDPInstance/bin/p4d_${SDPInstance}_init"
   fi

   if [[ $NO_OP -eq 0 ]] ; then
     $SSH_CMD_R "export P4CONFIG=/p4/.p4config.SDP && cd /p4/$SDPInstance/bin && /tmp/p4 reconcile ... && /tmp/p4 add ... && /tmp/p4 submit -d 'added bins'"
   else
     msg "NO_OP: export P4CONFIG=/p4/.p4config.SDP && cd /p4/$SDPInstance/bin && /tmp/p4 reconcile ... && /tmp/p4 add ... && /tmp/p4 submit -d 'added bins'"
  fi

   
   msg "\nPart 6.2 Rotating Journal\n"
   if [[ $NO_OP -eq 0 ]]; then
      /p4/common/bin/p4login
      LOGFILE=$LOGS/checkpoint.log
      rotate_last_run_logs
      check_uid
      check_dirs
      get_journalnum
      #get_offline_journal_num
      truncate_journal
      #replay_journals_to_offline_db
   else
      msg "NO_OP: Would Rotate Journal"
   fi
   get_journalnum
   #We want current not next, and we didn't do a full checkpoint
   JOURNALNUM=$(($JOURNALNUM-1))
   msg "Using Journal Number $JOURNALNUM"
   #Get Latest Checkpoint
   LastCheckpoint=$(ls -t ${CHECKPOINTS}/p4_${SDPInstance}.ckp.*.gz | head -1)
   LastCheckpointFile=`basename $LastCheckpoint`
   msg "Using Checkpoint: $LastCheckpoint"
   CHECKPOINTNUM=`echo $LastCheckpointFile | sed s/.*ckp.// | sed s/.gz//`
   msg "Will copy Journals:"
   for (( j=$CHECKPOINTNUM ; $j <= $JOURNALNUM ; j++ ));do
     msg "\tp4_${SDPInstance}.jnl.${j}"
   done

   msg "\nPart 6.3 Copying Checkpoint and Journal to the Replica\n"
   for (( j=$CHECKPOINTNUM ; $j <= $JOURNALNUM ; j++ ));do
     _scp_cmd="scp ${CHECKPOINTS}/p4_${SDPInstance}.jnl.${j} ${ReplicaHost}:${DD}/p4/${SDPInstance}/seed/"
     if [[ $NO_OP -eq 0 ]] ; then
	$_scp_cmd || bail "Couldn't copy jnl to $ReplicaHost"
        msg "Copied journal over"
        vmsg "$_scp_cmd"
     else
        msg "NO_OP: $_scp_cmd" 
     fi
   done
   _scp_cmd="scp $LastCheckpoint ${ReplicaHost}:${DD}/p4/${SDPInstance}/seed/"
   if [[ $NO_OP -eq 0 ]] ; then
	$_scp_cmd || bail "Couldn't copy ckp to $ReplicaHost"
        msg "Copied journal over"
        vmsg "$_scp_cmd"
   else
     msg "NO_OP: $_scp_cmd"
   fi

   msg "Part 6.4 Set Server ID"
   if [[ $NO_OP -eq 0 ]] ; then
      $SSH_CMD_R "echo $ServerSpec > $P4ROOT/server.id"
      $SSH_CMD_R "echo $ServerSpec > $OFFLINE_DB/server.id"
      msg "Set server.id to $ServerSpec in $P4ROOT and $OFFLINE_DB"
   else
      msg "NO_OP: Would set Server ID to $ServerSpec on $ReplicaHost in $P4ROOT and $OFFLINE_DB"
   fi

   msg "Part 6.5 Recover the checkpoint"
   if [[ $NO_OP -eq 0 ]] ; then
       msg "$P4DBIN -r $P4ROOT -z -jr /${DD}/p4/${SDPInstance}/seed/$LastCheckpointFile"
       $SSH_CMD_R "$P4DBIN -r $P4ROOT -z -jr /${DD}/p4/${SDPInstance}/seed/$LastCheckpointFile" || bail "Couldn't load checkpoint"
       msg "$P4DBIN -r $OFFLINE_DB -z -jr /${DD}/p4/${SDPInstance}/seed/$LastCheckpointFile"
       $SSH_CMD_R "$P4DBIN -r $OFFLINE_DB -z -jr /${DD}/p4/${SDPInstance}/seed/$LastCheckpointFile" || bail "Couldn't load checkpoint"
       for (( j=$CHECKPOINTNUM ; $j <= $JOURNALNUM ; j++ ));do
         msg "$P4DBIN -r $OFFLINE_DB -jr /${DD}/p4/${SDPInstance}/seed/p4_${SDPInstance}.jnl.${j}"
         $SSH_CMD_R "$P4DBIN -r $OFFLINE_DB -jr /${DD}/p4/${SDPInstance}/seed/p4_${SDPInstance}.jnl.${j}" || bail "Couldn't replay journal $j"
         msg "$P4DBIN -r $P4ROOT -jr /${DD}/p4/${SDPInstance}/seed/p4_${SDPInstance}.jnl.${j}"
         $SSH_CMD_R "$P4DBIN -r $P4ROOT -jr /${DD}/p4/${SDPInstance}/seed/p4_${SDPInstance}.jnl.${j}" || bail "Couldn't replay journal $j"
       done
       msg "echo Usable > $OFFLINE_DB/offline_db_usable.txt"
       $SSH_CMD_R "echo Usable > $OFFLINE_DB/offline_db_usable.txt"
   else
       msg "NO_OP: $P4DBIN -r $P4ROOT -z -jr /${DD}/p4/${SDPInstance}/seed/$LastCheckpointFile"
       msg "NO_OP: $P4DBIN -r $OFFLINE_DB -z -jr /${DD}/p4/${SDPInstance}/seed/$LastCheckpointFile"
       for (( j=$CHECKPOINTNUM ; $j <= $JOURNALNUM ; j++ ));do
         msg "NO_OP: $P4DBIN -r $OFFLINE_DB -jr /${DD}/p4/${SDPInstance}/seed/p4_${SDPInstance}.jnl.${j}"
         msg "NO_OP: $P4DBIN -r $P4ROOT -jr /${DD}/p4/${SDPInstance}/seed/p4_${SDPInstance}.jnl.${j}"
       done
       msg "NO_OP: echo Usable > $OFFLINE_DB/offline_db_usable.txt"
   fi

   msg "Part 6.6 Create the service and admin  users password file"
   if [[ $NO_OP -eq 0 ]] ; then
      msg "echo $_RND_PW > $P4CCFG/.p4passwd.${P4SERVER}.service"
      $SSH_CMD_R "echo $_RND_PW > $P4CCFG/.p4passwd.${P4SERVER}.service"
      $SSH_CMD_R "echo $_ADMIN_PW > $P4CCFG/.p4passwd.${P4SERVER}.admin"
   else
      msg "NO_OP: echo $_RND_PW > $P4CCFG/.p4passwd.${P4SERVER}.service"
      msg "NO_OP: echo $_ADMIN_PW > $P4CCFG/.p4passwd.${P4SERVER}.admin"
   fi

   msg "Part 6.7 Log in the service user"
   if [[ $NO_OP -eq 0 ]] ; then
      $SSH_CMD_R "source /p4/common/bin/p4_vars $SDPInstance && /tmp/p4 -u $ServiceUser -p $P4MASTERPORT login < $P4CCFG/.p4passwd.${P4SERVER}.service"
      vmsg "source /p4/common/bin/p4_vars $SDPInstance && /tmp/p4 -u $ServiceUser -p $P4MASTERPORT login < $P4CCFG/.p4passwd.${P4SERVER}.service"
   else
      msg "NO_OP: source /p4/common/bin/p4_vars $SDPInstance && /tmp/p4 -u $ServiceUser -p $P4MASTERPORT login < $P4CCFG/.p4passwd.${P4SERVER}.service"
   fi

   msg "${H}\nAll processing completed!\n"
   msg "Now log into your new replica and start it with ${P4BIN}_init start and then verify things look good with pull -lj"
   msg "After things look good you can start a p4verify.sh to start seeding the files"

fi 

if [[ $OverallReturnStatus -eq 0 && $FullSetup -eq 0 ]]; then
   msg "${H}\nAll processing completed successfully.\n"
   msg "\nNext steps:"
   msg "STEP 1. Login as ${OSUSER}@${P4MASTER}."
   msg "STEP 1.1 Set your environment with:\n\tcd /p4/common/bin\n\tsource p4_vars $SDPInstance\n"
   msg "STEP 1.2 If an Archive Filter is needed add it now into the server spec.  This is usually only needed for Export Controlled locations.\n\tp4 server $ServerSpec\n"
   msg "STEP 2. Execute this command:\n\tnohup daily_checkpoint.sh $SDPInstance < /dev/null > /dev/null 2>&1 &\n"
   msg "STEP 3. Monitor the checkpoint.log file, and confirm that the journal rotation\nstep completes.  This generally occurs within the first few seconds or minutes\nof starting daily_checkpoint.sh, even though it may take hours to complete for large\ndata sets.  Optionally, wait for it to complete.\n"
   msg "STEP 4. Login as ${OSUSER}@${ReplicaHost}."
   msg "STEP 5. Set your environment with:\n\tcd /p4/common/bin\n\tsource p4_vars $SDPInstance\n"
   msg "STEP 5.1 Create local service user password file\n\t echo $_RND_PW > $P4CCFG/.p4passwd.${P4SERVER}.service"
   msg "STEP 6. Transfer the highest numbered completed checkpoint file from\n${P4MASTER}:${CHECKPOINTS}.  Completed checkpoint files have a corresponding\n*.md5 file.  If you waited for daily_checkpoint.sh to complete in STEP 3, you\nneed only the latest checkpoint file.  Otherwise, you also need the journal\nfile that has the same number.  Copy those files to:\n$ReplicaHost:$CHECKPOINTS/.\n\nThe commands, as run from $ReplicaHost, will look like these sample commands:\n\tcd $CHECKPOINTS\n\tscp -p ${P4MASTER}:${CHECKPOINTS}/p4_${SDPInstance}.ckp.2550.gz .\n\tscp -p ${P4MASTER}:${CHECKPOINTS}/p4_${SDPInstance}.jnl.2550 .\n\nIf you have enabled ssh access from the commit server the commands will instead be from the commit:\n\tscp ${CHECKPOINTS}/p4_${SDPInstance}.ckp.2550.gz ${ReplicaHost}:${CHECKPOINTS}/\n\tscp ${CHECKPOINTS}/p4_${SDPInstance}.jnl.2550 ${ReplicaHost}:${CHECKPOINTS}/\n"
   msg "STEP 7. Create $P4ROOT/server.id file like so:\n\techo $ServerSpec > $P4ROOT/server.id\n"
   msg "STEP 7.1 Create $OFFLINE_DB/server.id file like so:\n\techo $ServerSpec > $OFFLINE_DB/server.id\n"
   msg "STEP 8. Verify that you have enough disk space, e.g. with:\n\tdf -h $P4ROOT, at least 30x zipped checkpoint size is needed.\n"
   msg "STEP 9. Recover the checkpoint like so:\n\tcd $P4ROOT\n\t$P4DBIN -r $P4ROOT -z -jr ${CHECKPOINTS}/p4_${SDPInstance}.ckp.NNN.gz\n\nwhere NNN is the latest checkpoint from STEP 6.\n\nIf you did not wait for daily_checkpoint.sh to complete in STEP 3, then also replay the latest journal file copied in STEP 6, like so:\n\t$P4DBIN -r $P4ROOT -z -jr ${CHECKPOINTS}/${SDPInstance}.jnl.NNN"
   msg "STEP 9.1 Recover the checkpoint to the offline_db:\n\tcd $OFFLINE_DB\n\t$P4DBIN -r $OFFLINE_DB -z -jr ${CHECKPOINTS}/p4_${SDPInstance}.ckp.NNN.gz\n\nwhere NNN is the latest checkpoint from STEP 6."
   msg "STEP 9.2 Mark the Offline_db as usable:\n\techo Usable > offline_db_usable.txt"
PasswdFile=$P4CCFG/.p4passwd.$SDPInstance.$ServiceUser
   msg "STEP 10.-1 Loggin the service user to the master:\n\t$P4DBIN -u $ServiceUser -p $P4MASTERPORT login < $P4CCFG/.p4passwd.$SDPInstance.service " 
   msg "STEP 10. Start the replica like so:\n\t${P4DBIN}_init start\n\nWait several seconds, then do:\n\t$P4CBIN/p4login -v -service\n\nReview the $LOGS/p4login.log, then check replication status with:\n\tp4 pull -lj\n"
   msg "STEP 11. Kick off a verify to pull over archive files:\n\tnohup p4verify.sh $SDPInstance < /dev/null > /dev/null 2>&1 &\n"
   msg "STEP 12. Wait about one minute, then check, $LOGS/p4verify.log to ensure it got\noff to a good start.  That will run for a while."

else
   msg "${H}\nProcessing completed, but with errors.  Scan above output carefully.\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus