mkrep.sh #37

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

if [[ ${BASH_VERSINFO[0]} -lt 4 ]]; then
   echo -e "\\n\\nERROR: $0 requires bash version 4.x+; current bash version is $BASH_VERSION."
   exit 1
fi

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export SDP_ENV=${SDP_ENV:-/p4/common/bin/p4_vars}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=Unset
export VERBOSITY=${VERBOSITY:-3}
declare SSH_Opts="-oBatchMode=yes"
declare RandomPassword=

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=${P4CONFIG:-.p4config}
export P4ENVIRO=/dev/null/.p4enviro

[[ -r "$P4U_ENV" ]] || {
   echo -e "\\nError: Cannot load environment from: $P4U_ENV\\n\\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   # shellcheck disable=SC1090
   source "$bash_lib" ||\
      { echo -e "\\nFATAL: Failed to load bash lib [$bash_lib]. Aborting.\\n"; exit 1; }
done

declare Version=2.4.8
declare -i SilentMode=0
declare -i StartupCmdNumFirst StartupCmdNumLast
declare -i DoSSHChecks=1
declare -i PreflightOK=1
RandomPassword=$(date +%s | sha256sum | base64 | head -c 32)
declare JournalPrefix=
export VERBOSITY=3

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # Don't litter.
   cleanTrash

   # With the trap removed, exit.
   exit "$OverallReturnStatus"
}

function bail {
   declare msg="${1:-Unknown Error}"
   declare -i rc

   rc="${2:-1}"
   echo -e "\\n$THISSCRIPT (line: ${BASH_LINENO[0]}): FATAL: $msg\\n\\n" >&2

   exit "$rc"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   # tag::includeManual[]
   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT -i <SDP_Instance> -t <Type> -s <Site_Tag> -r <Replica_Host> [-f <From_ServerID>] [-p] [-ssh_opts=\"opts\"] [-skip_ssh] [-L <log>] [-si] [-v<n>] [-n] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script creates makes a replica, and provides enough information to
	make it ready in all respects.

OPTIONS:
 -i <SDP_Instance>
	Specify the SDP Instance.

 -t <Type>
	Specify the replica type tag. The type corresponds to the 'Type:' and
	'Services:' field of the server spec, which describes the type of services
	offered by a given replica.

	Valid values are:
	* ha:   High Availability mandatory standby replica, for 'p4 failover' (P4D 2018.2+)
	* ham:  High Availability metadata-only mandatory standby replica, for 'p4 failover' (P4D 2018.2+)
	* ro:   Read-Only standby replica.
	* rom:  Read-Only standby replica, Metadata only.
	* fr:   Forwarding Replica (Unfiltered).
	* fs:   Forwarding Standby (Unfiltered).
	* frm:  Forwarding Replica (Unfiltered, Metadata only).
	* fsm:  Forwarding Standby (Unfiltered, Metadata only).
	* ffr:  Filtered Forwarding Replica.  Not a valid failover target.
	* edge: Edge Server. Filtered by definition.

	Replicas with 'standby' are always unfiltered, and use the 'journalcopy'
	method of replication, which copies a byte-for-byte verbatim journal file
	rather than one that is merely logically equivalent.

	The tag has several purposes:
	1. Short Hand. Each tag represents a combination of 'Type:' and fully
	qualified 'Services:' values used in server specs.

	2. Distillation. Only the most useful Type/Services combinations have a
	shorthand form.

	3. For forwarding replicas, the name includes the critical distinction of
	whether any replication filtering is used; as filtering of any kind disqualifies
	a replica from being a potential failover target. (No such distinction is
	needed for edge servers, which are filtered by definition).

 -s <Site_Tag>
	Specify a geographic site tag indicating the location and/or data center where
	the replica will physically be located. Valid site tags are defined in the site
	tags file:

	$SiteTagsFile

	Current valid site tags defined in this file are:
	$(grep -v '^#' "$SiteTagsFile" 2>&1|grep -v '$^'|sed 's:^:\t:g')

 -r <Replica_Host>
	Specify the target replica host.

 -f <From_ServerID>
	Specify ServerID of the P4TARGET server from which we are replicating.
	This is used to populate the 'ReplicatingFrom' field of the server
	spec. The value must be a valid ServerID.

	By default, this is determined dynamically checking the ServerID of the
	master server. This option should be used if the target is something
	other than the master. For example, to create an HA replica of an edge
	server, you might specify something like '-f p4d_edge_syd'.

 -p	This script performs a check to ensure that the Protections table grants
	super access to the group $ServiceUsersGroup.

	By default, an error is displayed if the check fails, i.e. if super user
	access for the group $ServiceUsersGroup cannot be verified. This is
	because, by default, we want to avoid making changes to the Protections
	table. Some sites have local policies or custom automation that requires
	site-specific procedures to update the Protections table.

	If '-p' is specified, an attempt is made to append the Protections table
	an entry like:

	super group $ServiceUsersGroup * //...

 -ssh_opts=\"opts\"
	Specify '-ssh_opts' to pass paraemters on to the ssh command. For
	example, to specify ssh operation on non-standard port 2222, specify
	'-ssh_opts=\"-p 2222\"'.

 -skip_ssh
	Specify '-skip_ssh' to skip the SSH access preflight check.

	This is useful if you only intend to do the metadata preparation phase
	of creating a new replica, prior to SSH being setup or perhaps even
	prior to the hardware being available.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging. By default, all output (stdout and stderr) goes in the logs
	directory referenced by \$LOGS.

	NOTE: This script is self-logging. That is, output displayed on the screen
	is simultaneously captured in the log file. Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

-si	Operate silently. All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal. This cannot be used with
	'-L off'.

 -n	No-Op. Prints commands instead of running them.

 -D	Set extreme debugging verbosity.

 -f	Full Mode Setup: The completes an edge servers setup so no additional steps 
	are required. This setup requires an ssh connection from the master to the 
	edge to be in place first. It also requires the depot log journal and /p4
	mounts to be in place and setup as expected. This setup assumes a standard
	SDP setup.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

DEPENDENCIES:
	This script depends on ssh keys being defined to allow the Perforce
	operating system user ($OSUSER) to ssh to any necessary machines
	without a password.

	This script assumes the replica host already has the SDP fully
	configured.

FILES:
	This Site Tags file defines the list of valid geographic site tags:
	$SiteTagsFile

EXAMPLES:
	Prepare an edge server to run on host syc-helix-04:
	$THISSCRIPT -i acme -t edge -s syd -r syc-helix-04
"
   # end::includeManual[]
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare ReplicaHost=Unset
declare ReplicaTypeTag=Unset
declare ReplicaType=
declare FromServerID=Unset
declare -i CreateFromServerID=0
declare -i CreateMasterServiceUser=0
declare SiteTag=Unset
declare SiteTagsFile="${P4CCFG:-/p4/common/config}/SiteTags.cfg"
declare SDPInstance=Unset
#declare -i Interactive=1
declare -i MetadataOnly=0
declare -i shiftArgs=0
declare -i UpdateProtections=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-r) ReplicaHost="$2"; shiftArgs=1;;
      (-t) ReplicaTypeTag="$2"; shiftArgs=1;;
      (-i) SDPInstance="$2"; shiftArgs=1;;
      (-s) SiteTag="$2"; shiftArgs=1;;
      (-f) FromServerID="$2"; shiftArgs=1;;
      (-p) UpdateProtections=1;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-ssh_opts) SSH_Opts="$2"; shiftArgs=1;;
      (-skip_ssh) DoSSHChecks=0;;
      (-L) export P4U_LOG="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && "$P4U_LOG" == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

[[ "$P4U_LOG" == Unset ]] && \
   P4U_LOG=${LOGS:-/tmp}/mkrep.$(date +'%Y%m%d-%H%M').log

[[ $SDPInstance == Unset ]] && usageError "\\nThe '-i <SDP_Instance>' parameter is required."
[[ $ReplicaHost == Unset ]] && usageError "\\nThe '-r <Replica_Host>' parameter is required."
[[ $ReplicaTypeTag == Unset ]] && usageError "\\nThe '-t <Type>' parameter is required."
[[ $SiteTag == Unset ]] && usageError "\\nThe '-s <Site_Tag>' parameter is required."

# Set log file to point at SDP_INSTANCE.
P4U_LOG="/p4/${SDPInstance}/logs/mkrep.$(date +'%Y%m%d-%H%M%S').log"

case "$ReplicaTypeTag" in
   (ha) ReplicaType=standby;;                             # HA Mandatory Standby replica.
   (ham) ReplicaType=standby; MetadataOnly=1;;            # HA Mandatory Standby replica.
   (ro) ReplicaType=standby;;                             # Read-Only Standby replica.
   (rom) ReplicaType=standby; MetadataOnly=1;;            # Read-Only Standby replica, Metadata only.
   (fr) ReplicaType=forwarding-replica;;                  # Forwarding Replica (Unfiltered).
   (fs) ReplicaType=forwarding-standby;;                  # Forwarding Standby (Unfiltered).
   (frm) ReplicaType=forwarding-replica; MetadataOnly=1;; # Forwarding Replica (Unfiltered), Metdata only.
   (fsm) ReplicaType=forwarding-standby; MetadataOnly=1;; # Forwarding Standby (Unfiltered).
   (ffr) ReplicaType=forwarding-replica;;                 # Filtered Forwarding Replica
   (edge) ReplicaType=edge-server;;                       # Edge Server. Filtered by def'n, cannot be Metdata only.
   (*) usageError "The specified replica type tag [$ReplicaTypeTag] is invalid.";;
esac

msg "DEBUG: F=$FromServerID\\nRTT=$ReplicaTypeTag\\n"
if [[ "$FromServerID" != "Unset" && "$ReplicaTypeTag" == "ha" ]]; then
   if [[ "$FromServerID" != "p4d_edge"* ]]; then
      usageError "An HA replica can only be created from the master or edge server. So if '-f <FromServerID>' is specified with '-t ha', the '-f' value must refer to an edge server, with a value prefixed with 'p4d_edge_'."
   fi
fi

declare -i tagFound=0
if [[ -r "$SiteTagsFile" ]]; then
   while read -r line; do
      [[ $line == "#*" ]] && continue
      # shellcheck disable=SC2086 disable=SC2116
      [[ -z "$(echo $line)" ]] && continue
      [[ "$line" == *":"* ]] || continue
      tag=${line%%:*}

      if [[ "$tag" == "$SiteTag" ]]; then
         tagFound=1
         break
      fi
   done < "$SiteTagsFile"
else
   bail "Missing site tag configuration file [$SiteTagsFile]. Aborting."
fi

[[ $tagFound -eq 1 ]] ||\
   bail "Failed to find specified site tag [$SiteTag] inite tag configuration file [$SiteTagsFile]. Aborting."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i OverallReturnStatus=0
declare ServerSpec=
declare ServerSpecFile=
declare ShortServerSpec=
declare ServiceUser=
declare ServiceUserSpecFile=
declare ServiceUsersGroup=ServiceUsers
declare TmpDir=
declare TmpFile=
declare ProtectsFile=
declare GroupSpecFile=
declare AdminPasswdFile=
declare ServicePasswdFile=

TmpDir=$(mktemp -d)
TmpFile="$TmpDir/tmpFile.$THISSCRIPT"
ProtectsFile="$TmpDir/protect.p4s"
GroupSpecFile="$TmpDir/group.$ServiceUsersGroup.p4s"

GARBAGE+=" $TmpDir"

touch "${TmpFile}" || bail "Couldn't touch tmp file [$TmpFile]."

if [[ "${P4U_LOG}" != off ]]; then
   touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "${P4U_LOG}")
      exec 2>&1
   else
      exec >"${P4U_LOG}"
      exec 2>&1
   fi

   initlog
fi

msg "Starting $THISSCRIPT v$Version at $(date)."

msg "${H}\\nPart 0: Environment Setup and Preflight Checks."

msg "Loading SDP environment for instance $SDPInstance."
# shellcheck disable=SC1090
source "$SDP_ENV" "$SDPInstance"
# shellcheck disable=SC1091
source /p4/common/bin/backup_functions.sh
check_vars
set_vars

msg "Using isolated shell environment, overriding P4CONFIG/P4ENVIRO/P4ALIASES, etc."
export P4ENVIRO=/dev/null/.p4enviro
export P4ALIASES=/dev/null/.p4aliases
export P4CONFIG="$TmpDir/.p4config"
export P4TICKETS="$TmpDir/.p4tickets"
export P4TRUST="$TmpDir/.p4trust"
export P4BIN="/p4/${SDPInstance}/bin/$P4SERVER"

echo "P4PORT=$P4PORT" > "$P4CONFIG"

if [[ "$P4PORT" == "ssl:"* ]]; then
   run "$P4BIN trust -y" "Trusting port $P4PORT" ||\
      bail "Failed to trust P4PORT $P4PORT."
fi

run "$P4BIN login -a < $SDP_ADMIN_PASSWORD_FILE" \
   "Logging $P4USER into port $P4PORT" ||\
   bail "Failed to login."

# If the '-f <from_serverid>' was specified, use it.  If not, get the ServerID of the
# master/commit server, one with a Services value of 'commit-server' or simply 'standard'.
# As a last resort, e.g. if the master server does not have a server spec, use 'p4 info'
# on the current server.
# From the list of server specs, find the master/commit server spec.
msg "Preflight check for master ServerID."
if [[ "$FromServerID" == Unset ]]; then
   FromServerID=$("$P4BIN" -ztag -F "%ServerID% %Services%" servers |\
      grep -E ' (standard|commit-server)')

   if [[ -n "$FromServerID" ]]; then
      FromServerID=${FromServerID%% *}
      msg "Verified: Master ServerID ($FromServerID) exists."
      if [[ -n "$("$P4BIN" -ztag -F %Update% user -o "svc_${FromServerID}")" ]]; then
         msg "Verified: Serivce user for ServerID $FromServerID exists: svc_${FromServerID}"
      else
         msg "Serivce user for ServerID $FromServerID will be created: svc_${FromServerID}"
         CreateMasterServiceUser=1
      fi
   else
      if [[ -n "${P4MASTER_ID:-}" ]]; then
         FromServerID="$P4MASTER_ID"
         CreateFromServerID=1
         CreateMasterServiceUser=1
         msg "Server spec $FromServerID and service user will be created: svc_${FromServerID}"
      else
         errmsg "No server spec with Services value of 'commit-server' or 'standard' found., and P4MASTER_ID is not set.\\n"
         PreflightOK=0
      fi
   fi
fi

msg "Checking if Protections table references group $ServiceUsersGroup."
if [[ "$("$P4BIN" protects -g $ServiceUsersGroup -m)" == "super" ]]; then
   msg "Verified: Protections table grants super access to group $ServiceUsersGroup."
else
   if [[ "$UpdateProtections" -eq 1 ]]; then
      msg "Protections does not grant access to group $ServiceUsersGroup as required, but '-p' was specified to mitigate this."
   else
      errmsg "Protections does not grant access to group $ServiceUsersGroup as required,\\nand '-p' not specified. Specify '-p' or adust the Protections table to add\\nthis line near the bottom:\\n\\n\\tsuper group $ServiceUsersGroup * //...\\n"
      PreflightOK=0
   fi
fi

# Version check: Require P4D 2018.1+ for using 'standby' replica.
# shellcheck disable=SC2072
if [[ "$ReplicaType" == *"standby" && "$P4D_VERSION" > "2018.1" ]]; then
   msg "P4D is 2018.1+, as recommended for standby replicas."
elif [[ "$P4D_VERSION" > "2016.2" ]]; then
   msg "P4D is 2016.2+, as supported for $ReplicaType replicas."
else
   bail "P4D must be 2018.1+ if using journalcopy replicas, P4D_VERSION is $P4D_VERSION.."
   PreflightOK=0
fi

# Version check: Require P4D 2018.2 for using 'ha' replica, i.e. a 2018.2-style 'mandatory'
# standby replica with ReplicatingFrom field set.
if [[ "$ReplicaTypeTag" == "ha"* ]]; then
   # shellcheck disable=SC2072
   if [[ "$P4D_VERSION" > "2018.2" ]]; then
      msg "P4D is 2018.2+, as required for 'ha' type replicas that use 'p4 failover'."
   else
      errmsg "P4D must be 2018.2+ if using HA replicas, P4D_VERSION is $P4D_VERSION. Aborting."
      PreflightOK=0
   fi
fi

if [[ $DoSSHChecks -eq 1 ]]; then
   msg "${H}\\nDoing SSH Preflight Checks."

   msg "Checking ssh access to master host $P4MASTER."
   # shellcheck disable=SC2086
   if ssh $SSH_Opts -q "$P4MASTER" /bin/ls > /dev/null 2>&1; then
      msg "Verified: SSH access to master host $P4MASTER is OK."
   else
      errmsg "Failed to ssh to master host $P4MASTER."
      PreflightOK=0
   fi

   msg "Checking ssh access to replica host $ReplicaHost."
   # shellcheck disable=SC2086
   if ssh $SSH_Opts -q "$ReplicaHost" /bin/ls > /dev/null 2>&1; then
      msg "Verified: SSH access to replica host $ReplicaHost is OK."
   else
      errmsg "Failed to ssh to host $ReplicaHost."
      PreflightOK=0
   fi
else
   msg "Skipping SSH preflight checks due to -skip_ssh."
fi

if  [[ "$PreflightOK" -eq 1 ]]; then
   msg "\\nPreflight checks passed. Continuing."
else
   bail "\\nPreflight checks did not pass. Aborting."
fi

#--------------------------------------------------------------
msg "${H}\\nPart 1: Define Server Spec."

if [[ "$CreateFromServerID" -eq 1 ]]; then
   msg "Creating required master server spec [$FromServerID]."

   ServerSpecFile="${TmpDir}/${FromServerID}.server.p4s"

   echo -e "ServerID: $FromServerID\\n
Type: server\\n
Name: $FromServerID\\n
Services: commit-server\\n
Description:\\n\\tMaster server." > "$ServerSpecFile" || bail "Failed to initialize server spec file [$ServerSpecFile]."

   if [[ $NO_OP -eq 0 ]]; then
      "$P4BIN" -s server -i < "$ServerSpecFile" ||\
         bail "Failed to load server spec from file: $ServerSpecFile\\n$(cat "$ServerSpecFile")\\n"
   else
      msg "NO_OP: Would run: $P4BIN -s server -i .LT. $ServerSpecFile"
   fi
fi

ShortServerSpec="${ReplicaTypeTag}_${SiteTag}"
ServerSpec="p4d_${ShortServerSpec}"

ServerSpecFile="$TmpDir/$ServerSpec.server.p4s"

if [[ "$P4PORT" == "ssl:" ]]; then
   SSLPrefix=ssl:
else
   SSLPrefix=
fi

# Strip off ssl: and host: element from front of value to just leave numeric port.
ReplicaPortNum=${P4MASTERPORT##*:}

if [[ "$ReplicaTypeTag" == "ha"* ]]; then
   echo -e "ServerID: $ServerSpec\\n
Type: server\\n
Name: $ServerSpec\\n
Options: nomandatory\\n
ReplicatingFrom: $FromServerID\\n
Services: $ReplicaType\\n
ExternalAddress: ${SSLPrefix}${ReplicaHost}:${ReplicaPortNum}\\n
Description:" > "$ServerSpecFile" || bail "Failed to initialize server spec file [$ServerSpecFile]."

warnmsg "Server $ServerSpec is intended to have the Options: field value set to 'mandatory'.  However, it is initially configured as 'nomandatory' to prevent undue stalling of a global topology while a fresh new standby replica gets caught up.  As of 2019.1, new standby replicas cannot be made 'mandatory' until they are online.

After this replica is brought online and is seen to be replicating properly (and up-to-date), consider making it a 'mandatory' replica.

A 'mandatory' replica is one that you can trust is at least as current as all other replicas, and this is the optimal candidate for a smooth failover from the master server.  However, if the 'mandatory' replica fails, it cannot be easily ignored -- it stalls, the global topology does.

Should that ever occur, you can modify the server spec manually on the master server, changing the 'mandatory' value to 'nomandatory'. That will enable global replication to move on.  It should then be a high priority to figure out what went wrong with the replica so that it can be brought back online.\\n"

elif [[ "$ReplicaType" == *"standby" ]]; then
   echo -e "ServerID: $ServerSpec\\n
Type: server\\n
Name: $ServerSpec\\n
Options: nomandatory\\n
ReplicatingFrom: $FromServerID\\n
Services: $ReplicaType\\n
ExternalAddress: ${SSLPrefix}${ReplicaHost}:${ReplicaPortNum}\\n
Description:" > "$ServerSpecFile" || bail "Failed to initialize server spec file [$ServerSpecFile]."
else
   echo -e "ServerID: $ServerSpec\\n
Type: server\\n
Name: $ServerSpec\\n
Services: $ReplicaType\\n
ExternalAddress: ${SSLPrefix}${ReplicaHost}:${ReplicaPortNum}\\n
Description:" > "$ServerSpecFile" || bail "Failed to initialize server spec file [$ServerSpecFile]."
fi

case "$ReplicaTypeTag" in
   (ha) Desc="High Availability Mandatory Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (ham) Desc="High Availability Metadata-only Mandatory Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (ro) Desc="Read-Only Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (rom) Desc="Read-Only Standby Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (fr) Desc="Forwarding Replica (Unfiltered) in ${SiteTag^^}.";;
   (fs) Desc="Forwarding Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (frm) Desc="Forwarding Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (fsm) Desc="Forwarding Standby Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (ffr) Desc="Filtered Forwarding Replica in ${SiteTag^^}.";;
   (edge) Desc="Edge server in ${SiteTag^^}.";;
   (*) bail "\\nInternal Error: Unrecognized replica type tag [$ReplicaTypeTag].";;
esac

echo -e "\\t$Desc\\n" >> "$ServerSpecFile" || bail "Failed to complete server spec file [$ServerSpecFile]."

msg "Creating server spec $ServerSpec with these contents:"
msg "${H}"
cat "$ServerSpecFile"
msg "${H}"

if [[ $NO_OP -eq 0 ]]; then
   "$P4BIN" -s server -i < "$ServerSpecFile" ||\
      bail "Failed to load server spec from file: $ServerSpecFile\\n$(cat "$ServerSpecFile")\\n"
else
   msg "NO_OP: Would run: $P4BIN -s server -i .LT. $ServerSpecFile"
fi

#--------------------------------------------------------------
msg "${H}\\nPart 2: Set configurables."

ServiceUser="svc_${ServerSpec}"
ServiceUserSpecFile="${TmpDir}/${ServiceUser}.user.p4s"
declare -i ConfigureOK=1

run "$P4BIN configure set $ServerSpec#P4TARGET=$P4MASTERPORT" || ConfigureOK=0
run "$P4BIN configure set $ServerSpec#db.replication=readonly" || ConfigureOK=0
run "$P4BIN configure set $ServerSpec#rpl.forward.all=1" || ConfigureOK=0
run "$P4BIN configure set $ServerSpec#rpl.compress=4" || ConfigureOK=0
run "$P4BIN configure set $ServerSpec#server=4" || ConfigureOK=0
run "$P4BIN configure set $ServerSpec#monitor=2" || ConfigureOK=0
run "$P4BIN configure set $ServerSpec#serviceUser=$ServiceUser" || ConfigureOK=0

if [[ "$ReplicaType" == *"standby" ]] ; then
   run "$P4BIN configure set $ServerSpec#rpl.journalcopy.location=1" || ConfigureOK=0
fi

# For master/commit servers, the journalPrefix is always:
# /p4/N/checkpoints/p4_N
#
# Regardless of what host that folder appears on, checkpoints
# from there will have originated on the master server.
#
# For replicas of all types, including edge servers, set journalPrefix
# to something like:
# /p4/N/checkpoints.bos_edge/p4_N.bos_edge
#
# where 'bos_edge' is a sample ServerID, less the 'p4d_' prefix.
# In the context of journalPrefix value and resulting checkpoint/journal
# file names, the leading 'p4d_' in the ServerID is redundant, and thus
# trimmed.
#
# The checkpoints and checkpoints.* folders are on the /hxdepots volume,
# which is always backed up.
#
# Incorporating the SeverID (server spec name) allows multiple
# "workspace servers" (edge servers used for horizontal scaling)
# to share storage on a SAN, while providing each edge a separate
# location for checkpoints.
#

# shellcheck disable=SC2153
JournalPrefix="$P4HOME/checkpoints.${ShortServerSpec}/p4_${SDPInstance}.${ShortServerSpec}"
run "$P4BIN configure set $ServerSpec#journalPrefix=$JournalPrefix" || ConfigureOK=0

if [[ $NO_OP -eq 0 ]]; then
   # For 'journalcopy' replicas, i.e. those with 'Standby' in the name,
   # startup.1 is the 'journalcopy' command to pull the raw P4JOURNAL file from
   # the P4TARGET server, and startup.2 is a 'pull' command with the -L' flag
   # to replay P4JOURNAL records into the db.

   # With the SDP, the pulled journal appears as a file $LOGS/journal.N, where N
   # is the journal counter. The rpl.journalcopy.location=1 setting enables this
   # desired behavior.

   # For non-journalcopy replicas (including any filtered replicas, including
   # edge servers that are filtered by nature), startup.1 is a pull
   # command that both pulls journal chunks and replays them into the database.
   if [[ "$Desc" == *"Standby"* ]]; then
      vmsg "Executing: $P4BIN configure set $ServerSpec#startup.1='journalcopy -i 0'"
      # shellcheck disable=SC2140
      "$P4BIN" configure set "$ServerSpec#startup.1"="journalcopy -i 0" || ConfigureOK=0
      vmsg "Executing: $P4BIN configure set $ServerSpec#startup.2='pull -i 1 -L'"
      # shellcheck disable=SC2140
      "$P4BIN" configure set "$ServerSpec#startup.2"="pull -i 1 -L" || ConfigureOK=0
      StartupCmdNumFirst=3
      StartupCmdNumLast=7
   else
      vmsg "Executing: $P4BIN configure set $ServerSpec#startup.1='pull -i 1'"
      # shellcheck disable=SC2140
      "$P4BIN" configure set "$ServerSpec#startup.1"="pull -i 1" || ConfigureOK=0
      StartupCmdNumFirst=2
      StartupCmdNumLast=6
   fi
else
   if [[ "$Desc" == *"Standby"* ]]; then
      vmsg "NO_OP: Would execute: $P4BIN configure set $ServerSpec#startup.1=\"journalcopy -i 0\""
      vmsg "NO_OP: Would execute: $P4BIN configure set $ServerSpec#startup.2=\"pull -i 1 -L\""
      StartupCmdNumFirst=3
      StartupCmdNumLast=7
   else
      vmsg "NO_OP: Would execute: $P4BIN configure set $ServerSpec#startup.1=\"pull -i 1\""
      StartupCmdNumFirst=2
      StartupCmdNumLast=6
   fi
fi

if [[ $MetadataOnly -eq 0 ]]; then
   run "$P4BIN configure set $ServerSpec#lbr.replication=readonly" || ConfigureOK=0
   for i in $(seq $StartupCmdNumFirst $StartupCmdNumLast); do
      if [[ $NO_OP -eq 0 ]]; then
         vmsg "Executing: $P4BIN configure set $ServerSpec#startup.$i='pull -i 1 -u'"
         # shellcheck disable=SC2140
         "$P4BIN" configure set "$ServerSpec#startup.$i"="pull -i 1 -u" || ConfigureOK=0
      else
         vmsg "NO_OP: Would execute: $P4BIN configure set $ServerSpec#startup.1='pull -i 1'"
      fi
   done
else
   run "$P4BIN configure set $ServerSpec#lbr.replication=shared" || ConfigureOK=0
fi

if [[ $ConfigureOK -eq 1 ]]; then
   msg "Verified:  All configurables were set OK."
   run "$P4BIN configure show allservers" "Showing all persistent configurables." 0 1 0
else
   bail "Errors encountered setting configurables.  See the output above. Aborting."
fi

#--------------------------------------------------------------
msg "${H}\\nPart 3: Create replica service user $ServiceUser."

echo -e "User: $ServiceUser\\n
Email: ${MAILFROM#\#}\\n
FullName: Replication Server User for $ServerSpec\\n
Type: service\\n
AuthMethod: perforce\\n" > "$ServiceUserSpecFile" || bail "Failed to initialize user spec file [$ServiceUserSpecFile]."

vmsg "Contents of $ServiceUserSpecFile:"
vmsg "${H}"
[[ $VERBOSITY -gt 2 ]] && cat "$ServiceUserSpecFile"
vmsg "${H}"

if [[ $NO_OP -eq 0 ]]; then
   "$P4BIN" user -f -i < "$ServiceUserSpecFile" || \
      bail "Failed to load user spec from file: $ServiceUserSpecFile\\n$(cat "$ServiceUserSpecFile")\\n"
else
   msg "NO_OP: Would run: $P4BIN user -f -i .LT. $ServiceUserSpecFile"
fi

AdminPasswdFile="$P4CCFG/.p4passwd.$P4SERVER.admin"
ServicePasswdFile="$P4CCFG/.p4passwd.$P4SERVER.$ServiceUser"
touch "$ServicePasswdFile" || bail "Failed to initialize password file $ServicePasswdFile."

chmod 600 "$ServicePasswdFile"
echo "$RandomPassword" > "$ServicePasswdFile"
echo "$RandomPassword" >> "$ServicePasswdFile"

if [[ $NO_OP -eq 0 ]]; then
   msg "Setting password for service user $ServiceUser."
   "$P4BIN" passwd "$ServiceUser" < "$ServicePasswdFile"
else
   msg "NO_OP: Would run: $P4BIN passwd $ServiceUser .LT. $ServicePasswdFile"
fi

if [[ "$CreateMasterServiceUser" -eq 1 ]]; then
   MasterServiceUserSpecFile="${TmpDir}/${FromServerID}.user.p4s"
   MasterServiceUser=svc_${FromServerID}
   msg "Creating service $MasterServiceUser for master server spec."

   echo -e "User: $MasterServiceUser\\n
Email: ${MAILFROM#\#}\\n
FullName: Server User for $FromServerID\\n
Type: service\\n
AuthMethod: perforce\\n" > "$MasterServiceUserSpecFile" ||\
      bail "Failed to initialize user spec file [$MasterServiceUserSpecFile]."

   vmsg "Contents of $MasterServiceUserSpecFile:"
   vmsg "${H}"
   [[ $VERBOSITY -gt 2 ]] && cat "$MasterServiceUserSpecFile"
   vmsg "${H}"

   if [[ $NO_OP -eq 0 ]]; then
      p4 user -f -i < "$MasterServiceUserSpecFile" || \
         bail "Failed to load user spec from file: $MasterServiceUserSpecFile\\n$(cat "$MasterServiceUserSpecFile")\\n"
   else
      msg "NO_OP: Would run: p4 user -f -i .LT. $MasterServiceUserSpecFile"
   fi

   MasterServicePasswdFile="$P4CCFG/.p4passwd.$P4SERVER.$MasterServiceUser"
   touch "$MasterServicePasswdFile" ||\
      bail "Failed to initialize password file $MasterServicePasswdFile."

   chmod 600 "$MasterServicePasswdFile"
   echo "$RandomPassword" > "$MasterServicePasswdFile"
   echo "$RandomPassword" >> "$MasterServicePasswdFile"

   if [[ $NO_OP -eq 0 ]]; then
      msg "Setting password for master service user $MasterServiceUser."
      p4 passwd "$MasterServiceUser" < "$MasterServicePasswdFile"
   else
      msg "NO_OP: Would run: p4 passwd $MasterServiceUser .LT. $MasterServicePasswdFile"
   fi
fi

#--------------------------------------------------------------
msg "${H}\\nPart 4: Make replica service user a super user with unlimited timeout."

msg "Checking if Protections table references group $ServiceUsersGroup."

if [[ "$UpdateProtections" -eq 1 ]]; then
   msg "Adding protections table entry to reference group $ServiceUsersGroup."
   p4 protect -o | grep -v '^#' > "$ProtectsFile" ||\
      bail "Failed to dump protections to tmp file: $ProtectsFile"

   echo -e "\\tsuper group $ServiceUsersGroup * //..." >> "$ProtectsFile" ||\
      bail "Failed to update tmp file $TmpFile."
      p4 protect -i < "$ProtectsFile" ||\
         bail "Failed to update Protecttions table from file $TmpFile."

   vmsg "Contents of $ProtectsFile:"
   vmsg "${H};"
   [[ $VERBOSITY -gt 3 ]] && cat "$ProtectsFile"
   vmsg "${H}"

   if [[ $NO_OP -eq 0 ]]; then
      p4 protect -i < "$ProtectsFile"
   else
      msg "NO_OP: Would run: p4 protect -i .LT. $ProtectsFile"
   fi
fi

msg "Checking if serivce user $ServiceUser is in service users group $ServiceUsersGroup."

# shellcheck disable=SC2143
if [[ -n $(p4 groups "$ServiceUser" | grep "^$ServiceUsersGroup$") ]]; then
   msg "Verified: Serivce user $ServiceUser is in service users group $ServiceUsersGroup."
else
   # This logic will create the group spec for service users if it does not already exist,
   # or add our new service user to the group if it already exists.  The 'p4 group -o'
   # command generates a valid group spec whether the spec actually exists on the server or
   # not.
   if [[ "$CreateMasterServiceUser" -eq 1 ]]; then
      msg "Adding service users $ServiceUser and $MasterServiceUser to group $ServiceUsersGroup."
      p4 group -o "$ServiceUsersGroup" | grep -v '^#' |\
         sed "s:43200:unlimited:g;\$ s/.*/\t$ServiceUser\n\t$MasterServiceUser/" > "$GroupSpecFile" ||\
         bail "Failed to update group spec file $GroupSpecFile."
   else
      msg "Adding service user $ServiceUser to group $ServiceUsersGroup."
      p4 group -o "$ServiceUsersGroup" | grep -v '^#' |\
         sed "s:43200:unlimited:g;\$ s/.*/\t$ServiceUser/" > "$GroupSpecFile" ||\
         bail "Failed to update group spec file $GroupSpecFile."
   fi

   vmsg "Contents of $GroupSpecFile:"
   vmsg "${H}"
   [[ $VERBOSITY -gt 3 ]] && cat "$GroupSpecFile"
   vmsg "${H}"

   if [[ $NO_OP -eq 0 ]]; then
      p4 -s group -i < "$GroupSpecFile" ||\
         bail "Failed to load group spec from file: $GroupSpecFile\\n$(cat "$GroupSpecFile")\\n"
   else
      msg "NO_OP: Would run: p4 -s group -i .LT. $GroupSpecFile"
   fi
fi

#--------------------------------------------------------------
if [[ $OverallReturnStatus -eq 0 ]]; then
   declare -i N=1
   declare SampleCheckpoint=
   msg "${H}\\nAll processing completed successfully\\n\\nNext steps:"
   msg "\\n=== PART 1 - Create Seed Checkpoint ===\\n"
   msg "STEP $N. Login as ${OSUSER}@${P4MASTER}."
   N+=1

   msg "STEP $N. Set your shell environment with:\\n\\tcd /p4/common/bin\\n\\tsource p4_vars $SDPInstance\\n"
   N+=1

   if [[ "$ReplicaTypeTag" == "ffr" ]]; then
      msg "STEP $N. Define replication filtering.  If you choose to filter by using ArchiveDataFilter and/or ClientDataFilter fields of the server spec, make those changes:\\n\\tp4 server $ServerSpec\\n\\nAlternately, if you choose to filter by database table, use 'p4 configure' commands to modify the $ServerSpec#startup.<n> settings related to the ServerID, adding the '-T' flag to the single 'pull' startup command that pulls metadata.\\n"
      N+=1
   fi

   if [[ "$ReplicaTypeTag" == "edge" || "$ReplicaTypeTag" == "ffr" ]]; then
      msg "STEP $N. Do a journal rotation to update offline_db:\\n\\trotate_journal.sh ${SDPInstance}\\n\\nThis should take only a few minutes even for large data sets."
      N+=1
   fi

   if [[ "$ReplicaTypeTag" == "edge" ]]; then
      SampleCheckpoint="/p4/${SDPInstance}/checkpoints/p4_${SDPInstance}.${ServerSpec#p4d_}.seed.NNNN.gz"
      msg "STEP $N. As this is an edge server, create the special edge seed checkpoint:\\n\\tnohup edge_dump.sh ${SDPInstance} ${ServerSpec} < /dev/null > /dev/null 2>&1 &\\n"
      N+=1
      msg "STEP $N. Monitor the log until successful completion:\\n\\ttail -f \$(ls -t \$LOGS/edge_dump.*.log|head -1)\\n"
      N+=1
   elif [[ "$ReplicaTypeTag" == "ffr" ]]; then
      SampleCheckpoint="/p4/${SDPInstance}/checkpoints/p4_${SDPInstance}.${ServerSpec#p4d_}.seed.ckp.gz"
      msg "STEP $N. As this is a filtered replica, create the special filtered seed checkpoint:\\n\\tnohup p4d_${SDPInstance} -r /p4/${SDPInstance}/offline_db -P $ServerSpec -J off -Z -jd $SampleCheckpoint < /dev/null > \$LOGS/seed.$ServerSpec.log 2>&1 &\\n"
      N+=1
      msg "STEP $N. Monitor the log until successful completion:\\n\\ttail -f \$LOGS/seed.${ServerSpec}.log\\n"
      N+=1
   else
      SampleCheckpoint="/p4/${SDPInstance}/checkpoints/p4_${SDPInstance}.ckp.NNNN.gz"
      msg "STEP $N. Create a new regular checkpoint to seed the replica.  Execute this command:\\n\\tnohup daily_checkpoint.sh $SDPInstance < /dev/null > /dev/null 2>&1 &\\n\\nNote: This step can be skipped if you choose to wait until the next regular daily checkpoint is created before proceeding on to PART 2."
      N+=1
   msg "STEP $N. Monitor the checkpoint.log file until successful completion:\\n\\ttail -f \$LOGS/checkpoint.log\\n"
      N+=1
   fi

   msg "\\n=== PART 2 - Load Checkpoint on Replica ===\\n"
   msg "STEP $N. Login as ${OSUSER}@${ReplicaHost}."
   N+=1

   msg "STEP $N. Set your environment with:\\n\\tcd /p4/common/bin\\n\\tsource p4_vars $SDPInstance\\n"
   N+=1

   if [[ "$ReplicaTypeTag" == "edge" ]]; then
      msg "STEP $N. Copy the edge seed checkpoint file created in the steps above from\\n${P4MASTER}:${CHECKPOINTS}.  Successfully completed checkpoint files have a\\ncorresponding *.md5 file which must also be copied. That might look like:\\n\\tcd $CHECKPOINTS\\n\\tscp -p $P4MASTER:${SampleCheckpoint/gz/md5}\\n\\tscp -p ${P4MASTER}:${SampleCheckpoint}\\n\\nReplace NNNN with the appropriate journal counter number.\\n"
      N+=1
   elif [[ "$ReplicaTypeTag" == "ffr" ]]; then
      msg "STEP $N. Copy the filtered replica seed checkpoint file and created in the steps above from\\n${P4MASTER}:${CHECKPOINTS}.  Successfully completed checkpoint files have a corresponding\\n*.md5 file which must also be copied. That might look like:\\n\\tcd $CHECKPOINTS\\n\\tscp -p ${P4MASTER}:${SampleCheckpoint}.md5\\n\\tscp -p ${P4MASTER}:${SampleCheckpoint}\\n"
      N+=1
   else
      msg "STEP $N. Copy the latest regular checkpoint file created in the steps above from\\n${P4MASTER}:${CHECKPOINTS}.  Successfully completed checkpoint files have a corresponding\\n*.md5 file which must also be copied. That might look like this:\\n\\tcd \$CHECKPOINTS\\n\\tscp -p ${P4MASTER}:${SampleCheckpoint/gz/md5}\\n\\tscp -p ${P4MASTER}:${SampleCheckpoint}\\n\\nReplace NNNN with the appropriate journal counter number.\\n"
      N+=1
   fi

   msg "STEP $N. Create $P4ROOT/server.id file like so:\\n\\techo $ServerSpec > $P4ROOT/server.id\\n"
   N+=1

   if [[ "$ReplicaTypeTag" == "ha"* ]]; then
      msg "STEP $N. If this machine is to be a potential target for a 'p4 failover', it will need a license\\nfile. This excludes filtered replicas and edge servers. The IP address in the\\nlicense file should match that returned by running the command 'hostname -I' on the\\nreplica server machine. The license file should be copied to this file on that\\nmachine: $P4ROOT/license\\n"
      N+=1
   fi

   msg "STEP $N. Verify that you have enough disk space, e.g. with:\\n\\tdf -h $P4ROOT, at least 30x zipped checkpoint size is needed.\\n"
   N+=1

   msg "STEP $N: Login super super user and replication service user to P4TARGET server, like so:\\n\\tp4 -p $P4MASTERPORT login -a < $AdminPasswdFile\\n\\tp4 -p $P4MASTERPORT login -a $ServiceUser\\n"
   N+=1

   # shellcheck disable=SC2153
   msg "STEP $N. Load the checkpoint like so:\\n\\tnohup load_checkpoint.sh $SampleCheckpoint -i ${SDPInstance} -y < /dev/null > /dev/null 2>&1 &\\n"
   N+=1

   msg "STEP $N. Monitor the log until successful completion:\\n\\ttail -f \$(ls -t \$LOGS/load_checkpoint.*.log|head -1)\\n"
   N+=1

   msg "STEP $N. OPTIONAL: Kick off a verify to pull over archive files:\\n\\tnohup p4verify.sh $SDPInstance < /dev/null > /dev/null 2>&1 &\\n\\nWait about one minute, then check that it is off to a good start:\\n\\ttail \$LOGS/p4verify.log. That may run for a long while depending on the scale of the versioned file tree."
   N+=1

else
   msg "${H}\\nProcessing completed, but with errors.  Scan above output carefully.\\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

# See the terminate() function, which is really where this script exits.
exit "$OverallReturnStatus"