mkrep.sh #31

  • //
  • guest/
  • perforce_software/
  • sdp/
  • dev/
  • Server/
  • Unix/
  • p4/
  • common/
  • bin/
  • mkrep.sh
  • View
  • Commits
  • Open Download .zip Download (33 KB)
#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

if [[ ${BASH_VERSINFO[0]} -lt 4 ]]; then
   echo -e "\\n\\nERROR: $0 requires bash version 4.x+; current bash version is $BASH_VERSION."
   exit 1
fi

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export SDP_ENV=${SDP_ENV:-/p4/common/bin/p4_vars}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=Unset
export VERBOSITY=${VERBOSITY:-3}
declare SSH_Opts="-oBatchMode=yes"
declare RandomPassword=

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.
export P4CONFIG=${P4CONFIG:-.p4config}
export P4ENVIRO=/dev/null/.p4enviro

[[ -r "$P4U_ENV" ]] || {
   echo -e "\\nError: Cannot load environment from: $P4U_ENV\\n\\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   # shellcheck disable=SC1090
   source "$bash_lib" ||\
      { echo -e "\\nFATAL: Failed to load bash lib [$bash_lib]. Aborting.\\n"; exit 1; }
done

declare Version=2.2.0
declare -i SilentMode=0
declare -i StartupCmdNumFirst StartupCmdNumLast
declare -i DoSSHChecks=1
RandomPassword=$(date +%s | sha256sum | base64 | head -c 32)
declare JournalPrefix=
export VERBOSITY=3

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # Don't litter.
   cleanTrash

   # With the trap removed, exit.
   exit "$OverallReturnStatus"
}

function bail {
   declare msg="${1:-Unknown Error}"
   declare -i rc

   rc="${2:-1}"
   echo -e "\\n$THISSCRIPT (line: ${BASH_LINENO[0]}): FATAL: $msg\\n\\n" >&2

   exit "$rc"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT -i <SDP_Instance> -t <Type> -s <Site_Tag> -r <Replica_Host> [-p] [-ssh_opts=\"opts\"] [-skip_ssh] [-L <log>] [-si] [-v<n>] [-n] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
        This script creates makes a replica, and provides enough information to
        make it ready in all respects.

OPTIONS:
 -i <SDP_Instance>
        Specify the SDP Instance.

 -t <Type>
        Specify the replica type tag.  The type corresponds to the 'Type:' and
        'Services:' field of the server spec, which describes the type of services
        offered by a given replica.

        Valid values are:
        * ha:   High Availability mandatory standby replica, for 'p4 failover' (P4D 2018.2+)
        * ham:  High Availability metadata-only mandatory standby replica, for 'p4 failover' (P4D 2018.2+)
        * ro:   Read-Only standby replica.
        * rom:  Read-Only standby replica, Metadata only.
        * fr:   Forwarding Replica (Unfiltered).
        * fs:   Forwarding Standby (Unfiltered).
        * frm:  Forwarding Replica (Unfiltered,  Metadata only).
        * fsm:  Forwarding Standby (Unfiltered,  Metadata only).
        * ffr:  Filtered Forwarding Replica.  Not a valid failover target.
        * edge: Edge Server. Filtered by definition.

        Replicas with 'standby' are always unfiltered, and use the 'journalcopy'
        method of replication, which copies a byte-for-byte verbatim journal file
        rather than one that is merely logically equivalent.

        The tag has several purposes:
        1. Short Hand.  Each tag represents a combination of 'Type:' and fully
        qualified 'Services:' values used in server specs.

        2. Distillation. Only the most useful Type/Services combinations have a
        shorthand form.

        3. For forwarding replicas, the name includes the critical distinction of
        whether any replication filtering is used; as filtering of any kind disqualifies
        a replica from being a potential failover target.  (No such distinction is
        needed for edge servers, which are filtered by definition).

 -s <Site_Tag>
        Specify a geographic site tag indicating where the replica will physically be
        located.  Valid site tags are defined in the site tags file,
        $SiteTagsFile

        Current valid site tags are:
        $(grep -v '^#' "$SiteTagsFile" 2>&1|grep -v '$^'|sed 's:^:\t:g')

 -r <Replica_Host>
        Specify the target replica host.

 -p        This script performs a check to ensure that the Protections table grants
        super access to the group $ServiceUsersGroup.

        By default, an error is displayed if the check fails, i.e. if super user
        access for the group $ServiceUsersGroup cannot be verified.  This is
        becuase, by default, we want to avoid making changes to the Protections
        table in case the customer has local policies or custom automation that
        requires site-specific procedures to update the Protections table.

        If '-p' is specified, an attempt is made to append the Protections table
        an entry like:

        super group $ServiceUsersGroup * //...

 -ssh_opts=\"opts\"
        Specify '-ssh_opts' to pass paraemters on to the ssh command.  For example, to
        specify ssh operation on non-standard port 2222, specify '-ssh_opts=\"-p 2222\"'.

 -skip_ssh
        Specify '-skip_ssh' to skip SSH checks duing the preflight check.

        This is useful if you only intend to do the metadata prepaation phase
        of startup prior to SSH being setup.

 -v<n>        Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
        Specify the path to a log file, or the special value 'off' to disable
        logging.  By default, all output (stdout and stderr) goes in the logs
	directory referenced by \$LOGS.

        NOTE: This script is self-logging.  That is, output displayed on the screen
        is simultaneously captured in the log file.  Do not run this script with
        redirection operators like '> log' or '2>&1', and do not use 'tee.'

-si        Operate silently.  All output (stdout and stderr) is redirected to the log
        only; no output appears on the terminal.  This cannot be used with
        '-L off'.
      
 -n        No-Op.  Prints commands instead of running them.

 -D     Set extreme debugging verbosity.

 -f        Full Mode Setup:  The completes an edge servers setup so no additional steps 
        are required.  This setup requires an ssh connection from the master to the 
        edge to be in place first.  It also requires the depot log journal and /p4
        mounts to be in place and setup as expected.  This setup assumes a standard
        SDP setup.

HELP OPTIONS:
 -h        Display short help message
 -man        Display man-style help message
 -V        Dispay version info for this script and its libraries.

DEPENDENCIES:
        This script depends on ssh keys being defined to allow the Perforce
        operating system user ($OSUSER) to ssh to any necessary machines
        without a password.

        This script assumes the replica host already has the SDP fully
        configured.

FILES:
        This Site Tags file defines the list of valid geographic site tags:
        $SiteTagsFile

EXAMPLES:
        Prepare an edge server to run on host syc-helix-04:
        $THISSCRIPT -i acme -t edge -s syd -r syc-helix-04
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare ReplicaHost=Unset
declare ReplicaTypeTag=Unset
declare ReplicaType=
declare SiteTag=Unset
declare SiteTagsFile=${P4CCFG:-/p4/common/config}/SiteTags.cfg
declare SDPInstance=Unset
#declare -i Interactive=1
declare -i MetadataOnly=0
declare -i shiftArgs=0
declare -i UpdateProtections=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-r) ReplicaHost=$2; shiftArgs=1;;
      (-t) ReplicaTypeTag=$2; shiftArgs=1;;
      (-i) SDPInstance=$2; shiftArgs=1;;
      (-s) SiteTag=$2; shiftArgs=1;;
      (-p) UpdateProtections=1;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-ssh_opts) SSH_Opts="$2"; shiftArgs=1;;
      (-skip_ssh) DoSSHChecks=0;;
      (-L) export P4U_LOG="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && "$P4U_LOG" == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

[[ "$P4U_LOG" == Unset ]] && \
   P4U_LOG=${LOGS:-/tmp}/mkrep.$(date +'%Y%m%d-%H%M').log

[[ $SDPInstance == Unset ]] && usageError "\\nThe '-i <SDP_Instance>' parameter is required."
[[ $ReplicaHost == Unset ]] && usageError "\\nThe '-r <Replica_Host>' parameter is required."
[[ $ReplicaTypeTag == Unset ]] && usageError "\\nThe '-t <Type>' parameter is required."
[[ $SiteTag == Unset ]] && usageError "\\nThe '-s <Site_Tag>' parameter is required."

# Set log file to point at SDP_INSTANCE.
P4U_LOG="/p4/${SDPInstance}/logs/mkrep.$(date +'%Y%m%d-%H%M%S').log"

case "$ReplicaTypeTag" in
   (ha) ReplicaType=standby;;                             # HA Mandatory Standby replica.
   (ham) ReplicaType=standby; MetadataOnly=1;;            # HA Mandatory Standby replica.
   (ro) ReplicaType=standby;;                             # Read-Only Standby replica.
   (rom) ReplicaType=standby; MetadataOnly=1;;            # Read-Only Standby replica, Metadata only.
   (fr) ReplicaType=forwarding-replica;;                  # Forwarding Replica (Unfiltered).
   (fs) ReplicaType=forwarding-standby;;                  # Forwarding Standby (Unfiltered).
   (frm) ReplicaType=forwarding-replica; MetadataOnly=1;; # Forwarding Replica (Unfiltered), Metdata only.
   (fsm) ReplicaType=forwarding-standby; MetadataOnly=1;; # Forwarding Standby (Unfiltered).
   (ffr) ReplicaType=forwarding-replica;;                 # Filtered Forwarding Replica
   (edge) ReplicaType=edge-server;;                       # Edge Server. Filtered by def'n, cannot be Metdata only.
   (*) usageError "The specified replica type tag [$ReplicaTypeTag] is invalid.";;
esac

declare -i tagFound=0
if [[ -r "$SiteTagsFile" ]]; then
   while read -r line; do
      [[ $line == "#*" ]] && continue
      # shellcheck disable=SC2086 disable=SC2116
      [[ -z "$(echo $line)" ]] && continue
      [[ "$line" == *":"* ]] || continue
      tag=${line%%:*}

      if [[ "$tag" == "$SiteTag" ]]; then
         tagFound=1
         break
      fi
   done < "$SiteTagsFile"
else
   bail "Missing site tag configuration file [$SiteTagsFile]. Aborting."
fi

[[ $tagFound -eq 1 ]] ||\
   bail "Failed to find specified site tag [$SiteTag] inite tag configuration file [$SiteTagsFile]. Aborting."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i OverallReturnStatus=0
declare ServerSpec=
declare ServerSpecFile=
declare ShortServerSpec=
declare ServiceUser=
declare ServiceUserSpecFile=
declare ServiceUsersGroup=ServiceUsers
declare TmpDir=$(mktemp -d)
declare ProtectsFile="$TmpDir/protect.p4s"
declare GroupSpecFile="$TmpDir/group.$ServiceUsersGroup.p4s"
declare TmpFile="$TmpDir/tmpFile.$THISSCRIPT"

GARBAGE+=" $TmpDir"

touch "${TmpFile}" || bail "Couldn't touch tmp file [$TmpFile]."

if [[ "${P4U_LOG}" != off ]]; then
   touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee "${P4U_LOG}")
      exec 2>&1
   else
      exec >"${P4U_LOG}"
      exec 2>&1
   fi

   initlog
fi

msg "Starting $THISSCRIPT v$Version at $(date)."

msg "${H}\\nPart 0: Environment Setup and Preflight Checks."

msg "Loading SDP environment for instance $SDPInstance."
# shellcheck disable=SC1090
source "$SDP_ENV" "$SDPInstance"
# shellcheck disable=SC1091
source /p4/common/bin/backup_functions.sh
check_vars
set_vars

# Version check: Require P4D 2018.1 for using 'standby' replica.
# shellcheck disable=SC2072
if [[ "$ReplicaType" == *"standby" && "$P4D_VERSION" > "2018.1" ]]; then
   msg "P4D is 2018.1+, as recommended for standby replicas."
elif [[ "$P4D_VERSION" > "2016.2" ]]; then
   msg "P4D is 2016.2+, as supported for $ReplicaType replicas."
else
   bail "P4D must be 2018.1+ if using journalcopy replicas, P4D_VERSION is $P4D_VERSION. Aborting."
fi

# Version check: Require P4D 2018.2 for using 'ha' replica, i.e. a 2018.2-style 'mandatory'
# standby replica with ReplicatingFrom field set.
if [[ "$ReplicaTypeTag" == "ha"* ]]; then
   # shellcheck disable=SC2072
   if [[ "$P4D_VERSION" > "2018.2" ]]; then
      msg "P4D is 2018.2+, as required for 'ha' type replicas that use 'p4 failover'."
   else
      bail "P4D must be 2018.2+ if using HA replicas, P4D_VERSION is $P4D_VERSION. Aborting."
   fi
fi

msg "Overriding P4CONFIG, P4ENVIRO, P4ALIASES, etc."
export P4ENVIRO=/dev/null/.p4enviro
export P4ALIASES=/dev/null/.p4aliases
export P4CONFIG="$TmpDir/.p4config"
export P4TICKETS="$TmpDir/.p4tickets"
export P4TRUST="$TmpDir/.p4trust"
export P4BIN="/p4/${SDPInstance}/bin/$P4SERVER"
echo "P4PORT=$P4MASTERPORT" > "$P4CONFIG"

run "$P4BIN -u $P4USER -p $P4MASTERPORT -s login -s" \
   "Verifying that user $P4USER can login to port $P4MASTERPORT." 1 1 0
if [[ $? -ne 0 ]]; then
   msg "Trying login with:\n$P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $SDP_ADMIN_PASSWORD_FILE\n"
   if [[ $NO_OP -eq 0 ]]; then
      $P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $SDP_ADMIN_PASSWORD_FILE
   else
      msg "NO_OP: Would execute: $P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $SDP_ADMIN_PASSWORD_FILE"
   fi
fi

if [[ $DoSSHChecks -eq 1 ]]; then
   msg "Checking ssh access to master host $P4MASTER."
   # shellcheck disable=SC2086
   ssh $SSH_Opts -q "$P4MASTER" /bin/ls > /dev/null 2>&1 || bail "Failed to ssh to host $P4MASTER."

   msg "${H}\\nCreating SSH Command."
   export SSH_CMD_R="ssh $SSH_Opts -q $ReplicaHost"
   export SSH_CMD_M="ssh $SSH_Opts -q $P4MASTER"
   vmsg "SSH_CMD_R=$SSH_CMD_R"
   vmsg "SSH_CMD_M=$SSH_CMD_M"

   msg "Checking ssh access to master host $P4MASTER."
   $SSH_CMD_M /bin/ls > /dev/null 2>&1 || bail "Failed to ssh to host $P4MASTER."
   msg "Verified: ssh access to master host $P4MASTER is OK."

   msg "Checking ssh access to replica host $ReplicaHost."
   $SSH_CMD_R /bin/ls > /dev/null 2>&1 || bail "Failed to ssh to host $ReplicaHost."
   msg "Verified: ssh access to replica host $ReplicaHost is OK."
else
   msg "Skipping SSH preflight checks due to -skip_ssh."
fi

#--------------------------------------------------------------
msg "${H}\\nPart 1: Preparation: Login."

if [[ "$P4MASTERPORT" == "ssl:"* ]]; then
   run "p4 trust -y"
fi

run "$P4BIN -u $P4USER -p $P4MASTERPORT -s login -s" \
   "Checking whether user $P4USER is logged into port $P4MASTERPORT." 1 1 0

if [[ "$CMDEXITCODE" -ne 0 ]]; then
   msg "Trying login with:\\n$P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $SDP_ADMIN_PASSWORD_FILE\\n"
   if [[ $NO_OP -eq 0 ]]; then
      "$P4BIN" -u "$P4USER" -p "$P4MASTERPORT" -s login -a < "$SDP_ADMIN_PASSWORD_FILE"
   else
      msg "NO_OP: Would execute: $P4BIN -u $P4USER -p $P4MASTERPORT -s login -a < $SDP_ADMIN_PASSWORD_FILE"
   fi
fi

run "$P4BIN -u $P4USER -p $P4MASTERPORT -s login -s" \
   "Verifying that user $P4USER can login to port $P4MASTERPORT." 1 1 0 ||\
   bail "Login verification failed."

#--------------------------------------------------------------
msg "${H}\\nPart 2: Define Server Spec."
#TODO Need to make ArchiveDataFilter appear here through some option
ShortServerSpec="${ReplicaTypeTag}_${SiteTag}"
ServerSpec="p4d_${ShortServerSpec}"

ServerSpecFile="$TmpDir/$ServerSpec.server.p4s"

# strip of ssl: or an host: element from front of value to just leave port - so strip twice
ReplicaPort=${P4MASTERPORT#*:}
ReplicaPort=${ReplicaPort#*:}

if [[ "$ReplicaTypeTag" == "ha"* ]]; then
   echo -e "ServerID: $ServerSpec\\n
Type: server\\n
Name: $ServerSpec\\n
Options: nomandatory\\n
ReplicatingFrom: master.${SDPInstance}\\n
Services: $ReplicaType\\n
ExternalAddress: $ReplicaHost:$ReplicaPort\\n
Description:" > "$ServerSpecFile" || bail "Failed to initialize server spec file [$ServerSpecFile]."

warnmsg "Server $ServerSpec is intended to have the Options: field value set to 'mandatory'.  However, it is initially configured as 'nomandatory' to prevent undue stalling of a global topology while a fresh new standby replica gets caught up.  As of 2019.1, new standby replicas cannot be made 'mandatory' until they are online.

After this replica is brought online, consider making it a 'mandatory' replica.

A 'mandatory' replica is one that you can trust is at least as current as all other replicas, and this is the optimal candidate for a smooth failover from the master server.  However, if the 'mandatory' replica fails, it cannot be easily ignored -- it stalls, the global topology does.

Should that ever occur, you can modify the server spec manually on the master serer, changing the 'mandatory' value to 'nomandatory'. That will enable global replication to move on.  It should then be a high priority to figure out what went wrong with the replica so that it can be brought back online.\\n"

elif [[ "$ReplicaType" == *"standby" ]]; then
   echo -e "ServerID: $ServerSpec\\n
Type: server\\n
Name: $ServerSpec\\n
Options: nomandatory\\n
ReplicatingFrom: master.${SDPInstance}\\n
Services: $ReplicaType\\n
ExternalAddress: $ReplicaHost:$ReplicaPort\\n
Description:" > "$ServerSpecFile" || bail "Failed to initialize server spec file [$ServerSpecFile]."
else
   echo -e "ServerID: $ServerSpec\\n
Type: server\\n
Name: $ServerSpec\\n
Services: $ReplicaType\\n
ExternalAddress: $ReplicaHost:$ReplicaPort\\n
Description:" > "$ServerSpecFile" || bail "Failed to initialize server spec file [$ServerSpecFile]."
fi

case "$ReplicaTypeTag" in
   (ha) Desc="High Availability Mandatory Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (ham) Desc="High Availability Metadata-only Mandatory Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (ro) Desc="Read-Only Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (rom) Desc="Read-Only Standby Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (fr) Desc="Forwarding Replica (Unfiltered) in ${SiteTag^^}.";;
   (fs) Desc="Forwarding Standby Replica (Unfiltered) in ${SiteTag^^}.";;
   (frm) Desc="Forwarding Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (fsm) Desc="Forwarding Standby Replica (Unfiltered, Metadata Only) in ${SiteTag^^}.";;
   (ffr) Desc="Filtered Forwarding Replica in ${SiteTag^^}.";;
   (edge) Desc="Edge server in ${SiteTag^^}.";;
   (*) bail "\\nInternal Error: Unrecognized replica type tag [$ReplicaTypeTag].";;
esac

echo -e "\\t$Desc\\n" >> "$ServerSpecFile" || bail "Failed to complete server spec file [$ServerSpecFile]."

msg "Creating server spec $ServerSpec with these contents:"
msg "${H}"
cat "$ServerSpecFile"
msg "${H}"

if [[ $NO_OP -eq 0 ]]; then
   p4 server -i < "$ServerSpecFile" || bail "Failed to load server spec from file: $ServerSpecFile"
else
   msg "NO_OP: Would run: p4 server -i .LT. $ServerSpecFile"
fi

#--------------------------------------------------------------
msg "${H}\\nPart 3: Set configurables."

ServiceUser=svc_${ServerSpec}
ServiceUserSpecFile=$TmpDir/$ServiceUser.user.p4s
declare -i ConfigureOK=1

run "p4 configure set $ServerSpec#P4TARGET=$P4MASTERPORT" || ConfigureOK=0
run "p4 configure set $ServerSpec#db.replication=readonly" || ConfigureOK=0
run "p4 configure set $ServerSpec#rpl.forward.all=1" || ConfigureOK=0
run "p4 configure set $ServerSpec#rpl.compress=4" || ConfigureOK=0
run "p4 configure set $ServerSpec#server=4" || ConfigureOK=0
run "p4 configure set $ServerSpec#monitor=2" || ConfigureOK=0
run "p4 configure set $ServerSpec#serviceUser=$ServiceUser" || ConfigureOK=0

if [[ "$ReplicaType" == *"standby" ]] ; then
   run "p4 configure set $ServerSpec#rpl.journalcopy.location=1" || ConfigureOK=0
fi

# For master/commit servers, the journalPrefix is always:
# /p4/N/checkpoints/p4_N
#
# Regardless of what host that folder appears on, checkpoints
# from there will have originated on the master server.
#
# For replicas of all types, including edge servers, set journalPrefix
# to something like:
# /p4/N/checkpoints.bos_edge/p4_N.bos_edge
#
# where 'bos_edge' is a sample ServerID, less the 'p4d_' prefix.
# In the context of journalPrefix value and resulting checkpoint/journal
# file names, the leading 'p4d_' in the ServerID is redundant, and thus
# trimmed.
#
# The checkpoints and checkpoints.* folders are on the /hxdepots volume,
# which is always backed up.
#
# Incorporating the SeverID (server spec name) allows multiple
# "workspace servers" (edge servers used for horizontal scaling)
# to share storage on a SAN, while providing each edge a separate
# location for checkpoints.
#

# shellcheck disable=SC2153
JournalPrefix="$P4HOME/checkpoints.${ShortServerSpec}/p4_${SDPInstance}.${ShortServerSpec}"
run "p4 configure set $ServerSpec#journalPrefix=$JournalPrefix" || ConfigureOK=0

if [[ $NO_OP -eq 0 ]]; then
   # For 'journalcopy' replicas, i.e. those with 'Standby' in the name,
   # startup.1 is the 'journalcopy' command to pull the raw P4JOURNAL file from
   # the P4TARGET server, and startup.2 is a 'pull' command with the -L' flag
   # to replay P4JOURNAL records into the db.

   # With the SDP, the pulled journal appears as a file $LOGS/journal.N, where N
   # is the journal counter. The rpl.journalcopy.location=1 setting enables this
   # desired behavior.

   # For non-journalcopy replicas (including any filtered replicas, including
   # edge servers that are filtered by nature), startup.1 is a pull
   # command that both pulls journal chunks and replays them into the database.
   if [[ "$Desc" == *"Standby"* ]]; then
      vmsg "Executing: p4 configure set $ServerSpec#startup.1='journalcopy -i 0'"
      # shellcheck disable=SC2140
      p4 configure set "$ServerSpec#startup.1"="journalcopy -i 0" || ConfigureOK=0
      vmsg "Executing: p4 configure set $ServerSpec#startup.2='pull -i 1 -L'"
      # shellcheck disable=SC2140
      p4 configure set "$ServerSpec#startup.2"="pull -i 1 -L" || ConfigureOK=0
      StartupCmdNumFirst=3
      StartupCmdNumLast=7
   else
      vmsg "Executing: p4 configure set $ServerSpec#startup.1='pull -i 1'"
      # shellcheck disable=SC2140
      p4 configure set "$ServerSpec#startup.1"="pull -i 1" || ConfigureOK=0
      StartupCmdNumFirst=2
      StartupCmdNumLast=6
   fi
else
   if [[ "$Desc" == *"Standby"* ]]; then
      vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.1=\"journalcopy -i 0\""
      vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.2=\"pull -i 1 -L\""
      StartupCmdNumFirst=3
      StartupCmdNumLast=7
   else
      vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.1=\"pull -i 1\""
      StartupCmdNumFirst=2
      StartupCmdNumLast=6
   fi
fi

if [[ $MetadataOnly -eq 0 ]]; then
   run "p4 configure set $ServerSpec#lbr.replication=readonly" || ConfigureOK=0
   for i in $(seq $StartupCmdNumFirst $StartupCmdNumLast); do
      if [[ $NO_OP -eq 0 ]]; then
         vmsg "Executing: p4 configure set $ServerSpec#startup.$i='pull -i 1 -u'"
         # shellcheck disable=SC2140
         p4 configure set "$ServerSpec#startup.$i"="pull -i 1 -u" || ConfigureOK=0
      else
         vmsg "NO_OP: Would execute: p4 configure set $ServerSpec#startup.1='pull -i 1'"
      fi
   done
else
   run "p4 configure set $ServerSpec#lbr.replication=shared" || ConfigureOK=0
fi

if [[ $ConfigureOK -eq 1 ]]; then
   msg "Verified:  All configurables were set OK."
   run "p4 configure show allservers" "Showing all persistent configurables." 0 1 0
else
   bail "Errors encountered setting configurables.  See the output above. Aborting."
fi

#--------------------------------------------------------------
msg "${H}\\nPart 4: Create replica service user $ServiceUser."

echo -e "User: $ServiceUser\\n
Email: ${MAILFROM#\#}\\n
FullName: Replication Server User for $ServerSpec\\n
Type: service\\n
AuthMethod: perforce\\n" > "$ServiceUserSpecFile" || bail "Failed to initialize user spec file [$ServiceUserSpecFile]."

vmsg "Contents of $ServiceUserSpecFile:"
vmsg "${H}"
[[ $VERBOSITY -gt 2 ]] && cat "$ServiceUserSpecFile"
vmsg "${H}"

if [[ $NO_OP -eq 0 ]]; then
   p4 user -f -i < "$ServiceUserSpecFile" || \
      bail "Failed to load user spec from file: $ServiceUserSpecFile"
else
   msg "NO_OP: Would run: p4 user -f -i .LT. $ServiceUserSpecFile"
fi

PasswdFile="$P4CCFG/.p4passwd.$P4SERVER.$ServiceUser"
touch "$PasswdFile" || bail "Failed to initialize password file $PasswdFile."

chmod 600 "$PasswdFile"
echo "$RandomPassword" > "$PasswdFile"
echo "$RandomPassword" >> "$PasswdFile"

if [[ $NO_OP -eq 0 ]]; then
   msg "Setting password for service user $ServiceUser."
   p4 passwd "$ServiceUser" < "$PasswdFile"
else
   msg "NO_OP: Would run: p4 passwd $ServiceUser .LT. $PasswdFile"
fi

#--------------------------------------------------------------
msg "${H}\\nPart 5: Make replica service user a super user with unlimited timeout."

msg "Checking if Protections table references group $ServiceUsersGroup."

if [[ "$(p4 protects -g $ServiceUsersGroup -m)" == "super" ]]; then
   msg "Verified: Protections table grants super access to group $ServiceUsersGroup."
else
   if [[ "$UpdateProtections" -eq 1 ]]; then
      msg "Adding protections table entry to reference group $ServiceUsersGroup."
      p4 protect -o | grep -v '^#' > "$ProtectsFile" ||\
         bail "Failed to dump protections to tmp file: $ProtectsFile"
      echo -e "\\tsuper group $ServiceUsersGroup * //..." >> "$ProtectsFile" ||\
         bail "Failed to update tmp file $TmpFile."
         p4 protect -i < "$ProtectsFile" || bail "Failed to update Protecttions table from file $TmpFile."

      vmsg "Contents of $ProtectsFile:"
      vmsg "${H};"
      [[ $VERBOSITY -gt 3 ]] && cat "$ProtectsFile"
      vmsg "${H}"

      if [[ $NO_OP -eq 0 ]]; then
         p4 protect -i < "$ProtectsFile"
      else
         msg "NO_OP: Would run: p4 protect -i .LT. $ProtectsFile"
      fi
   else
      bail "The Protections table must grant super access to the group: $ServiceUsersGroup\\nConsider using the '-p' flag to update the table automatically, or else follow your site-specifc procedures for updating the Protections table to grant super access to this group."
   fi
fi

msg "Checking if serivce user $ServiceUser is in service users group $ServiceUsersGroup."

# shellcheck disable=SC2143
if [[ -n $(p4 groups "$ServiceUser" | grep "^$ServiceUsersGroup$") ]]; then
   msg "Verified: Serivce user $ServiceUser is in service users group $ServiceUsersGroup."
else
   # This logic will create the group spec for service users if it does not already exist,
   # or add our new service user to the group if it already exists.  The 'p4 group -o'
   # command generate a valid group spec whether the spec actually exists on the server or
   # not.
   p4 group -o $ServiceUsersGroup | grep -v '^#' |\
      sed "s:43200:unlimited:g;\$ s/.*/\t$ServiceUser/" > "$GroupSpecFile" ||\
      bail "Failed to update group spec file $GroupSpecFile."

   vmsg "Contents of $GroupSpecFile:"
   vmsg "${H}"
   [[ $VERBOSITY -gt 3 ]] && cat "$GroupSpecFile"
   vmsg "${H}"

   if [[ $NO_OP -eq 0 ]]; then
      p4 group -i < "$GroupSpecFile"
   else
      msg "NO_OP: Would run: p4 group -i .LT. $GroupSpecFile"
   fi
fi

#--------------------------------------------------------------
if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\\nAll processing completed successfully.\\n"
   msg "\\nNext steps:"
   msg "STEP 1. Login as ${OSUSER}@${P4MASTER}."
   msg "STEP 1.1 Set your environment with:\\n\\tcd /p4/common/bin\\n\\tsource p4_vars $SDPInstance\\n"
   msg "STEP 1.2 If an Archive Filter is needed add it now into the server spec.  This is usually only needed for Export Controlled locations.\\n\\tp4 server $ServerSpec\\n"
   msg "STEP 2. Execute this command:\\n\\tnohup daily_checkpoint.sh $SDPInstance < /dev/null > /dev/null 2>&1 &\\n"
   msg "STEP 3. Monitor the checkpoint.log file, and confirm that the journal rotation\\nstep completes.  This generally occurs within the first few seconds or minutes\\nof starting daily_checkpoint.sh, even though it may take hours to complete for large\\ndata sets.  Optionally, wait for it to complete.\\n"
   msg "STEP 4. Login as ${OSUSER}@${ReplicaHost}."
   msg "STEP 5. Set your environment with:\\n\\tcd /p4/common/bin\\n\\tsource p4_vars $SDPInstance\\n"
   msg "STEP 5.1 Create local service user password file\\n\\t echo '$RandomPassword' > $P4CCFG/.p4passwd.${P4SERVER}.service"
   msg "STEP 6. Transfer the highest numbered completed checkpoint file from\\n${P4MASTER}:${CHECKPOINTS}.  Completed checkpoint files have a corresponding\\n*.md5 file.  If you waited for daily_checkpoint.sh to complete in STEP 3, you\\nneed only the latest checkpoint file.  Otherwise, you also need the journal\\nfile that has the same number.  Copy those files to:\\n$ReplicaHost:$CHECKPOINTS/.\\n\\nThe commands, as run from $ReplicaHost, will look like these sample commands:\\n\\tcd $CHECKPOINTS\\n\\tscp -p ${P4MASTER}:${CHECKPOINTS}/p4_${SDPInstance}.ckp.2550.gz .\\n\\tscp -p ${P4MASTER}:${CHECKPOINTS}/p4_${SDPInstance}.jnl.2550 .\\n\\nIf you have enabled ssh access from the commit server the commands will instead be from the commit:\\n\\tscp ${CHECKPOINTS}/p4_${SDPInstance}.ckp.2550.gz ${ReplicaHost}:${CHECKPOINTS}/\\n\\tscp ${CHECKPOINTS}/p4_${SDPInstance}.jnl.2550 ${ReplicaHost}:${CHECKPOINTS}/\\n"
   msg "STEP 7. Create $P4ROOT/server.id file like so:\\n\\techo $ServerSpec > $P4ROOT/server.id\\n"
   msg "STEP 7.1 Create $OFFLINE_DB/server.id file like so:\\n\\techo $ServerSpec > $OFFLINE_DB/server.id\\n"
   msg "STEP 8. Verify that you have enough disk space, e.g. with:\\n\\tdf -h $P4ROOT, at least 30x zipped checkpoint size is needed.\\n"
   # shellcheck disable=SC2153
   msg "STEP 9. Recover the checkpoint like so:\\n\\tcd $P4ROOT\\n\\t$P4DBIN -r $P4ROOT -z -jr ${CHECKPOINTS}/p4_${SDPInstance}.ckp.NNN.gz\\n\\nwhere NNN is the latest checkpoint from STEP 6.\\n\\nIf you did not wait for daily_checkpoint.sh to complete in STEP 3, then also replay the latest journal file copied in STEP 6, like so:\\n\\t$P4DBIN -r $P4ROOT -z -jr ${CHECKPOINTS}/${SDPInstance}.jnl.NNN"
   msg "STEP 9.1 Recover the checkpoint to the offline_db:\\n\\tcd $OFFLINE_DB\\n\\t$P4DBIN -r $OFFLINE_DB -z -jr ${CHECKPOINTS}/p4_${SDPInstance}.ckp.NNN.gz\\n\\nwhere NNN is the latest checkpoint from STEP 6."
   msg "STEP 9.2 Mark the Offline_db as usable:\\n\\techo Usable > offline_db_usable.txt"
   PasswdFile=$P4CCFG/.p4passwd.$SDPInstance.$ServiceUser
   msg "STEP 10.1 Login the service user to the master:\\n\\t$P4DBIN -u $ServiceUser -p $P4MASTERPORT login < $P4CCFG/.p4passwd.p4_$SDPInstance.service " 
   # shellcheck disable=SC2153
   msg "STEP 10.2 Start the replica like so:\\n\\t${P4DBIN}_init start\\n\\nWait several seconds, then do:\\n\\t$P4CBIN/p4login -v -service\\n\\nReview the $LOGS/p4login.log, then check replication status with:\\n\\tp4 pull -lj\\n"
   msg "STEP 11. Kick off a verify to pull over archive files:\\n\\tnohup p4verify.sh $SDPInstance < /dev/null > /dev/null 2>&1 &\\n"
   msg "STEP 12. Wait about one minute, then check, $LOGS/p4verify.log to ensure it got\\noff to a good start.  That will run for a while."

else
   msg "${H}\\nProcessing completed, but with errors.  Scan above output carefully.\\n" 
fi

# Illustrate using $SECONDS to display runtime of a script.
msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

# See the terminate() function, which is really where this script exits.
exit "$OverallReturnStatus"
# Change User Description Committed
#65 29533 C. Thomas Tyler Corrected journalPrefix for a standard 'ha' replica in new logic that
better handles 'ham' replicas.

Fix to unreleased dev-branch version.
#64 29470 C. Thomas Tyler Fixed bad ServerID if '-f <FromServerID>' was specified for master ServerID.

Added preflight check for running on commit server (P4MASTER_ID == SERVERID).

Fixed bug with infer_type_tag() needed if '-f <FromServerID>' is not edge.

Simplified docs for '-f' option.

#review-29471 @lee_marzke
#63 29336 C. Thomas Tyler Tweaks to output (error from copy/paste).
This is a harmless non-functional change.
#62 29303 C. Thomas Tyler mkrep.sh v3.1.0:
* Added infer_type_tag() function with logic to determine if P4TARGET is
filtered or not.
* Discouraged use of 'ro' and 'rom' types (as they don't support 'p4 failover').
* Clarified usage to indicate daisy chaining is acceptable from
forwarding replicas.
* Added preflight check to avoid accidental overwrite of existing server spec.
* Added '-os' option to overwrite existing server spec intentionally.
* Re-added test for mkrep.sh to default test suite after fixing the test.
* Added '-N' option to fully support server spec naming standard that allows
for things like 'p4d_edge2_syd' and 'p4d_edge3_syd'. Added docs for same.

#review-29304
#61 28619 C. Thomas Tyler Fixed bug in mkrep.sh SSL detection logic.

#review-28620
#60 28595 C. Thomas Tyler For mkrep.sh: Added hooks to support custom pre- and post-processing.
#59 28535 C. Thomas Tyler Fixed typo ('super super') and added some missing dots in the
instructions for calling scp.
#58 28207 C. Thomas Tyler mkrep.sh no longer requires the '-i <SDP_Instance>' parameter if the
SDP_INSTANCE variable is defined.

#review-28205
#57 28195 C. Thomas Tyler Refined location of SiteTags.cfg.sample file.
#56 28193 C. Thomas Tyler Renamed sample files (e.g.
SiteTags.cfg) in SDP tarball tree, appending
a .sample tag, to make rsync overlay of /p4/common/config safe.

Updated related docs in mkrep.sh referring to the sample file, to improve
an error message to guide the user to find the sample file.

#review-28194
#55 27987 C. Thomas Tyler Doc correction.
Non-functional change.
#54 27897 C. Thomas Tyler Fixed typo in last change.

#review-27898 @d_benedict
#53 27895 C. Thomas Tyler Clarified intent of '-r' flag value in docs.
This is a doc-only change.

#review-27896 @d_benedict
#52 27894 C. Thomas Tyler Non-functional internal comment correction in mkrep.sh.
Comments
implied that HA replicas must be configured as 'Mandatory' (i.e.
using the 'mandatory' value in the 'Options:' field of the server
spec. The use of 'mandatory' is a choice with trade-offs, not an
absolute.
#51 27722 C. Thomas Tyler Refinements to @27712:
* Resolved one out-of-date file (verify_sdp.sh).
* Added missing adoc file for which HTML file had a change (WorkflowEnforcementTriggers.adoc).
* Updated revdate/revnumber in *.adoc files.
* Additional content updates in Server/Unix/p4/common/etc/cron.d/ReadMe.md.
* Bumped version numbers on scripts with Version= def'n.
* Generated HTML, PDF, and doc/gen files:
  - Most HTML and all PDF are generated using Makefiles that call an AsciiDoc utility.
  - HTML for Perl scripts is generated with pod2html.
  - doc/gen/*.man.txt files are generated with .../tools/gen_script_man_pages.sh.

#review-27712
#50 27640 C. Thomas Tyler Clarify -f flag is only required for edge servers as upstream.

#review-27612
#49 27634 C. Thomas Tyler Fixed typo in doc; non-functional change.
#48 27633 C. Thomas Tyler Tweak in error message; changed from:
   So if '-f <FromServerID>' is specified with '-t ha'
to:
   So if '-f <FromServerID>' is specified with '-t ha[m]'

The error message references the '-t ha', but could also be
'-t ham' (for a metadata-only HA replica).
#47 27266 C. Thomas Tyler mkrep.sh v2.7.2:
* Added smarts to drop info about pulling archvie files if replica
  is setup as a metadata-only replica.
* Fixed grammar error in instructions.
#46 27265 C. Thomas Tyler mkrep.sh v2.7.1:
* Fixed issue with setting P4TARGET.
* Fixed doc typos.
* Corrected bad use of 'mandatory'.
#45 27250 C. Thomas Tyler Adjusted JournalPrefix standard to account for shared /hxdepots.

The JournalPrefix standard now allows for unfiltered replicas
(such as HA/DR replicas) to use same journalPrefix value as filtered
replicas and edge servers, using per-ServerID checkpoints folder,
if they share the same /hxdepots (e.g. NFS-mounted) with the master
(e.g. when lbr.replication=shared).

Related code change made to mkdirs.sh and mkrep.sh to support the
tweaks to the standard.

#review-27251
#44 27103 C. Thomas Tyler Removed 'Mandatory' from Description in ServerID in mkrep.sh.

The word 'Mandatory' was removed from the generated Description field
of the server spec for the HA server.  The use of 'Mandatory' is a
valid choice, but since the Description field isn't updated
dynamically and 'Mandatory' isn't a required part of the nature of
an HA replica, it was removed from the Description.
#43 27071 C. Thomas Tyler Added Example 3, creating a replica of an edge server.

Now fully supports workflow for creating a replica of an edge server.

Fixed issue where P4TARGET was set incorrectly when '-f <FromServerID>' is specified.

Updated output to more clearly delinate when automated processing completes and
when humans pick up the baton.

Updated generated guidance to include steps needed when creating a replica
of an edge server.

General doc enhancements, including adding REPLICA SERVER MACHINE SETUP section.

#review-27072
#42 27066 C. Thomas Tyler mkrep.sh v2.5.2:
* Corrected guidance suggesting logging a service user with 'p4 login -a';
the '-a' flag doesn't apply to service users. Also added a note on why the
SDP 'p4login' can't be used in that step.
#41 27053 C. Thomas Tyler Corrected Verion id issue.
#40 27050 C. Thomas Tyler mkrep.sh v2.5.0:
* Added doc references to SDP Server Spec and journalPrefix standards.
* Updated journalPrefix value definition to follow the standard.
* Removed '-si' "Silent Mode" option, a carry over from the template, as
there is no need for it in this script.
* Removed display of current SiteTags from '-man' output as it won't generate relibly for AsciiDoc.
* Removed '-ssh_opts' and '-skip_ssh' flags from doc.
* Removed 'mandatory' from definition of HA replica. This is now deemed a valid conifuration, but not part of the definition of HA.
* Removed extraneous text related to 'full setup mode' (from a custom
installed version). [A future full setup may be added relying on the Helix Installer.]
* Removed including of backup_functions.sh, and dropped logic handled better by the 'p4login' script.
* Generated passwords for service users are now discareded.
* Fixed various bugs in error messages.

To Do in a follow-on changelist:
* Update test_MultiSDP.py (which tets mkrep.sh)

#review-27051
#39 27010 C. Thomas Tyler Fixed issue generating man page on Mac where 'sha256sum' is not
as readily available.  Since this is just used to generate a
random password, fallback logic is now used: 'sha256sum' is used
if available, else 'md5sum' is used if it is afailable, else
'sum' is used as a last resort.
#38 26718 Robert Cowham Rename P4MASTER to P4MASTERHOST for clarity with comments in:
- mkdirs.cfg/mkdirs.sh
- p4_<instance>.vars
- other files which reference
Remove unnecessary sed for p4p.template
#37 26637 Robert Cowham Include script help within doc
Requires a couple of tags in the scripts themselves.
#36 26445 C. Thomas Tyler mkrep.sh v2.4.x:
    * ServerID for master server spec now created if needed.
    * Service user for master ServerID created and password set if needed.
    * Enhanced preflight checks to do all checks before bailing.
    * Moved check for ServiceUsers group in Protections into preflight.
#35 26381 C. Thomas Tyler mkrep.sh v2.3.2:
* Added '-f <From_ServerID>' flag to specify the ServerID from which
to replicate, a value used for the ReplicatingFrom: field.
* Added safety check to check that if '-t ha' is used with '-f',
the value for '-f' refers to an edge server.
* Added logic to replace the default value for ReplicatingFrom,
replacing naming-convention based logic with logic to dynamically
determine the value. This is done by finding the server spec with
Services of 'commit-server' or or 'standard', and using it. Or, if
that bares no fruit, doing 'p4 info' on the current server to extract
the ServerID.
* Enhancements to guidance info for creating edge servers and
filtered forwarding replicas, with details on using edge_dump.sh
or generating filtered replica checkpoints.
* Doc content and formatting improvements.
* Maintained shellcheck compliance.
#34 26139 Robert Cowham Allow mkrep.sh to create a replica from a replica.
#33 26089 Robert Cowham Fix message which refers to p4d instead of p4 for login.
Means it can't be copy/pasted currently.
#32 25800 C. Thomas Tyler mkrep.sh will no longer allow a standby replica to initially be
'mandtory' on startup, as required by p4d 2019.1.

If it was intended to be mandatory (i.e. has an 'ha' in the
ServerID), a new warning message id displayed describing why
that converting it to be mandatory should be considered after
it is caught up, and why.

#review @michael_shields
#31 25796 C. Thomas Tyler mkrep.sh will no longer allow a standby replica to initially be
'mandtory' on startup, as required by p4d 2019.1.

If it was intended to be mandatory (i.e. has an 'ha' in the
ServerID), a new warning message id displayed describing why
that converting it to be mandatory should be considered after
it is caught up, and why.

#review @michael_shields
#30 25567 Robert Cowham Remove -b 5 on journalcopy threads - not required
#29 25561 Robert Cowham Fix minor typos
#28 25557 Robert Cowham Fix failing test
#27 25550 C. Thomas Tyler Split HMS from SDP, such that HMS will live on as a separate product,
layered on the SDP.

See: https://swarm.workshop.perforce.com/jobs/SDP-356

Many of the files deleted here have already been copied to HMS.  See
these HMS changes:

https://swarm.workshop.perforce.com/changes/25531

https://swarm.workshop.perforce.com/changes/25533

And this branch spec: SDP_Split_2019.2_HMS

In mkrep.sh, HMS-awareness is removed from mkrep.sh, a core SDP script, and
the '-f' (Full Setup) flag has been temporarily removed.  This useful
functionality will be restored in an HMS script.  This is a big change,
so shellcheck v0.6.0 was implemented to support it.
#26 25258 Robert Cowham Rework the containers in preparation for multi container testing
mkrep changes:
Remove -c cfg option which was unused anyway
Converted tabs to spaces
Fixed logic error causing forwarding replicas to bail due to unsupported p4d version
Standby servers require ReplicatingFrom field
Add seconds to log file name (useful for testing to avoid overwriting files)
#25 25210 Robert Cowham Bail on server error
#review @tom_tyler
#24 25204 Robert Cowham Fix problem with creating server spec

#review @tom_tyler
#23 25161 C. Thomas Tyler Tweaked replication per input from Perforce Engineering: The 'startup.2'
command for a standby replica that does a 'p4 pull -i 1 -L' is a local
operation, so the '-b 5' network retry flags were nonsensical and have
been removed.

#review-25162 @michael_shields
#22 25156 C. Thomas Tyler Fixed logic issue handling replicas, so P4D2018.2+ is required only for 'ha'
type replicas.
Also tweaked to ensure ReplicationFrom: and Options: fields are not applied
to server specs for possibly older P4D versions that don't support them.

#review-25157 @robert_cowham
#21 25152 Robert Cowham If bailing then include line no of error from calling script
#20 24869 C. Thomas Tyler Various enhancements to mkrep.sh, including adding support
for 2018.2 'ha' failover replicas.

Moved default location for mkrep.cfg to /p4/common/config.

Now references global SDP variable $SDP_ADMIN_PASSWORD_FILE to
define location of the admin password file.
#19 24248 C. Thomas Tyler Removed all references to journals.rep value for
journalPrefix for journalcopy replicas.

P4D 2018.1+ is now required for creating standby
replicas.

Note that the SDP validator (new in a coming SDP release)
will help spot journals.rep usage that should be
converted when upgrading to 2018.1 fo P4D/SDP.

To Do: Rewite the 'seed' stuff to use mkdirs.sh instead.
#18 24243 C. Thomas Tyler Enhanced mkrep.sh handling for journalcopy/standby replicas.

Changed lbr.replication value for metadata-only replicas from
'ondemand' to 'shared'. There is no behavior change as the those
are synonyms, but 'shared' more clearly conveys the intent.

Adjusted number of pull threads to be 5 for all full replicas
(i.e. not metadata-only replicas).
#17 23885 C. Thomas Tyler Fixed typo detected in code review after submit.
#16 23881 C. Thomas Tyler Set rpl.journalcopy.location=1 for 2018.1+ standby replicas.

#review @michael_shields
#15 23522 C. Thomas Tyler mkrep.sh tweaks:
* Documented existing '-ssh_opts' option.
* Implemented and documented new '-skip_ssh' option.
* Enhanced cosmetics of showing current geographic site tags.
* Upated Version to 2.0.2 reflect big changes with Chris Geen's
  last uptate, plus this small update.
* Added command line parsing logic to make new '-skip_ssh'
  logic mutually exclusive with '-f' (full setup) flag.

Bypassing pre-commit review; there are still more changes to make:
* Make it so it benefits from 'Tight Ship' style of management
if used (as it does not), but doesn't not introduce a dependency
on it.  The last change caused a dependency on Tight Ship
managment; we want to remove that dependency, but keep the benefit
of Tight Ship, if used.  Tight Ship means using a '/p4/hms' SDP
instance to version changes to things like SDP config files in
/p4/common/config.

#review-23523 @cgeen
#14 23453 cgeen Automation of the edge server creation with mkrep script.
 Two new variables into the script -f and -se.  Details in the man.
 A valid HMS instance is needed to use the full automation, this is vallidated first.
 This works by doing a journal rotate and then syncing the last checkpoint over and the
 subsequent journals.   Currently it only deals with new edge servers and not replicas or edge replicas.

Reviewer Comment: I love what this is doing, I'm approving so I can play with it. Separately we're working on net suite enhancements, and that will need to be updated to handle this.
#13 23266 C. Thomas Tyler Fixes and Enhancements:
* Enabled daily_checkpoint.sh operate on edge servers, to
keep /p4/N/offline_db current on those hosts for site-local
recovery w/o requiring a site-local replica (though having
a site-local replica can still be useful).
* Disabled live_checkpoint.sh for edge servers.
* More fully support topologies using edge severs, in both
geographically distributed and horizaontal scaling "wokspace
server" solutions.
* Fix broken EDGESERVER value definition.
* Modified name of SDP counter that gets set when a checkpoint is taken
to incorporate ServerID, so now the counter name will look like
lastSDPCheckpoint.master.1, or lastSDPCheckpoint.p4d_edge_sfo, rather
than just lastSDPCheckpoint.

There will be multiple such counters in a topology that uses edge
servers, and/or which takes checkpoints on replicas.

* Added comments for all functions.

For the master server, journalPrefix remains:
/p4/N/checkpoints/p4_N

The /p4/N/checkpoints is reserved for writing by the
master/commit server only.

For non-standby (possibly filtered) replicas and edge serves,
journalPrefix is:
/p4/N/checkpoints.<ShortServerID>/p4_N.<ShortServerID>

Here, ShortServerID is just the ServerID with the 'p4d_' prefix
trimmed, since it is redundant in this context.  See mkrep.sh,
which enshines a ServerID (server spec) naming standard, with
values like 'p4d_fr_bos' (forwarding replica in Boston) and
p4d_edge_blr (Edge server in Bangalore).  So the journalPrefix
for the p4d_edge_bos replica would be:
/p4/N/checkpoints.edge_bos/p4_N.edge_bos

For "standby" (aka journalcopy) replicas, journalPrefix is set
to /p4/N/journals.rep. which is written to the $LOGS volume, due
to the nature of standby replicas using journalPrefix to write
active server logs to pre-rotated journals.

Some take-away to be updated in docs:
* The /p4/N/checkpoints folder must be reserved for checkpoints that
originate on the master. It should be safe to rsync this folder
(with --delete if desired) to any replica or edge server.  This is
consistent with the current SDP.
* I want to change 'journals.rep' to 'checkpoints.<ShortServerID>'
for non-standby replicas, to ensure that checkpoints and journals
taken on those hosts are written to a volume where they are backed
up.
* In sites with multiple edge serves, some sharing achive files
('workspace servers'), multiple edge servers will share the same
SAN. So we one checkpoints dir per ServerID, and we want that
dir to be on the /hxdepots volume.

Note that the journalPrefix for replicas was a fixed /p4/N/journals.rep.
This was on the /hxlogs volume - a presumably fast-for-writes volume,
but typically NOT backed up and not very large. This change puts it
under /p4/N/checkpoints.* for edge servers and non-standby replicas,
but ensures other replica types and edge servers can generate
checkpoints to a location that is backed up and has plenty of storage
capacity.  For standby replicas only (which cannot be filtered),
the journalPrefix remains /p4/N/journals.rep on the /hxlogs volume.
#12 23230 cgeen Updates to the mkrep.sh script.
 Adds ssh_opts for automation, by default in batch.  Updates logging to point at $SDPInstance as hms might not be installed.  Users the instance pw file as it might differ from hms.
#11 22628 C. Thomas Tyler Fixed minor order-of-processing bug resulting in a harmless error
appearing at the end of script processing as cleanTrash() was
called to clean garbage files.  The run() function was called to
clean garbage files/dirs just as a directory that function depended
on got cleaned up.

The fix was applied to scripts that used libcore.sh, including the
template.sh template script.

Also corrected comments in p4u_env.sh.

Bypassing pre-commit review as this has been well tested.

#review-22629
#10 21962 C. Thomas Tyler Updated various scripts to use run() and rrun() functions in favor
of predecessor runCmd() and runRemoteCmd().  The older functions won't
be removed to avoid breaking scripts that rely on their behavior and
have no issues with them.  The newer fuctions are more scalable and
avoid erroneous "Argument list too long" from bash due to buffer
overruns when used with commands with large amounts of output.

Enhanced runRemoteCmd() to clean up after itself, as it generated files
in /tmp that didn't get automatically cleaned up.  If used in scripts
called very often (e.g. every 5 minutes in a crontab), this leads to
significant issues with /tmp filling up with garbage files over a
period of several weeks.

Enhanced test_utils.sh to test new run() and rrun() calls.
#9 21921 C. Thomas Tyler HMS change, to simplify setup on compliant platforms (i.e.
those using bash 4.x).
The shebang line for all bash scripts has been standardized to /bin/bash, and
bash checks for $BASH_VERSION added where needed.  References to having a
custom-built /usr/local/bin/bash have been removed.

As a trade-off, non-compliant platforms (way-old Linux and Mac using bash 3.x) will
have a bit more work to do to work with shebang lines.
#8 21030 C. Thomas Tyler mkrep.sh: Added missing 'journalcopy' startup command for
standby replicas.

Corrected typo in script name.
#7 20797 C. Thomas Tyler Added more replica types, forwarding-standby and the metadata-only
variant.
Added '-L' to 'p4 pull' in startup.1 command for standby replicas.
#6 20777 C. Thomas Tyler Tweaked HMS mkrep.sh script to avoid using p4login, as p4login assumes
it is being run on a server where there the given SDP instance is configured.
Normally the way p4login is used, that's a safe assumption.  But in this case
mkrep.sh is intended to be called on the HMS server, which may not actually
host the SDP instance for which a replica is being made. They will have their
p4_N.vars files in /p4/common/config, but may not have a /p4/N/root or even
/p4/N directory on the HMS server itself.

This introduces a dependency, deemed acceptable for HMS purposes, that all
instances that use this mkrep.sh script have the same password for the
'perforce' super user as the hms instance, as it uses only the
/p4/common/config/.p4passwd.p4_hms.admin password file (for the /p4/hms
instance).
#5 20708 C. Thomas Tyler Per discussion: s/checkpoints.rep/journals.rep/g

This directory name changed, used in the journalPrefix configurable, is
intended to clarify that the should be targeted to for a FAST volume
for use with journalcopy, rather than the LARGE volume as would be
implied when using a directory with "checkpoints" in the name.
#4 20430 C. Thomas Tyler Tweak to mkrep.sh:
Tweaked to use p4login script.
#3 20363 C. Thomas Tyler Removed references to legacy names for checkpoint scripts.
No functional changes.  Bypassing pre-commit code review.

#review-20364
#2 20170 Russell C. Jackson (Rusty) Moved password and users into the config directory to allow for instance specific
users and passwords. Ran into a case where two different teams were sharing the same
server hardware and needed this type of differentiation. Surprised that we haven't hit
this sooner.

Also defaulted mkdirs to use the numeric ports since this is the most common
installation.
#1 20154 C. Thomas Tyler Added mkrep.sh script to create a replica.

This current version does most of the steps of creating a replica in
an SDP environment, benefitting from server spec naming standards and
conventions.  It then provides guidance on the remaining steps
(e.g. transferring the checkpoint to the replica host, etc.)

It greatly reduces the complexity of creating a replica by
distilling the many potential replica types to just those few commonly
used, 'fr' (forwarding replica), 'ro' (read-only replica), and 'edge'.

The script is self-documenting with a '-h' (short usage) and '-man'
(detailed man page) options.

This first version is functional but is definitely a work in progress.

It will evolve as part of the HMS project, but this part of HMS lives
in the stock SDP.