backup_functions.sh #70

  • //
  • guest/
  • perforce_software/
  • sdp/
  • main/
  • Server/
  • Unix/
  • p4/
  • common/
  • bin/
  • backup_functions.sh
  • View
  • Commits
  • Open Download .zip Download (65 KB)
#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
set -u

#==============================================================================
# Global Variables.
export P4DInitScript=
export P4DRef=
export P4DSystemdServiceFile=
export P4BrokerInitScript=
export P4BrokerRef=
export P4BrokerSystemdServiceFile=
export P4ProxyInitScript=
export P4ProxyRef=
export P4ProxySystemdServiceFile=
export OFFLINE_DB=
export EDGESERVER=
export STANDBYSERVER=
declare -i DoParallelCheckpoints=

#==============================================================================
# Common functions used in various SDP scripts.

#------------------------------------------------------------------------------
# Verify key variables in the shell environment exist, or else abort.
#
# If checks in this function fail, this function an 'echo' and 'exit 1'
# rather than calling 'log' or 'die', as this function is generally called
# early in processing, before the log is initialized.
#------------------------------------------------------------------------------
function check_vars () {

   local CheckVarsPreflightOK=1
   CommonVars="SDP_INSTANCE P4HOME P4PORT P4ROOT P4JOURNAL P4BIN P4DBIN P4TICKETS P4TRUST KEEPCKPS KEEPJNLS KEEPLOGS CHECKPOINTS LOGS OSUSER"
   InstanceVars="P4MASTER_ID P4MASTERPORT"

   # First, check vars that should be set in /p4/common/bin/p4_vars.
   for var in $CommonVars; do
      # Detect unset variables, using ':-' to avoid 'unbound variable' errors.
      # shellcheck disable=SC1083
      if [[ -z "$(eval echo \${"$var":-})" ]]; then
         echo "Error: Required variable \$$var is NOT set. It should be set in /p4/common/bin/p4_vars."
         CheckVarsPreflightOK=0
      fi
   done

   # Next, check vars that should be set in /p4/common/config/p4_N.instance.
   # For some variables, provide additional details that help help users correct
   # the problem.
   for var in $InstanceVars; do
      # shellcheck disable=SC1083
      if [[ -z "$(eval echo \${"$var":-})" ]]; then
         echo "Error: Required variable \$$var is NOT set. It should be set in /p4/common/config/p4_N.vars, where N is the SDP instance name."

         if [[ "$var" == "P4MASTER_ID" ]]; then
            echo "The value for P4MASTER_ID should be the name of the ServerID of the master server."
         fi

         CheckVarsPreflightOK=0
      fi
   done

   if [[ "$CheckVarsPreflightOK" -eq 0 ]]; then
      echo "Use p4master_run or source p4_vars before calling this script."
      echo "Aborting to to errors in shell environment preflight checks."
      exit 1
   fi
}

#------------------------------------------------------------------------------
#  is_edge ($ServerID, $RootDir)
#
# Determine if a given ServerID is an edge server or not, checking a given
# database root dir (e.g. $P4ROOT or $OFFLINE_DB).
#
# Input:
# $1 - ServerID (required)
# $2 - RootDir (optional, defaults to $P4ROOT)
#
# Output YES if an edge server, NO otherwise.
#
#------------------------------------------------------------------------------
function is_edge () {
   local ServerID=${1:-Unset}
   local RootDir=${2:-$P4ROOT}
   local ServicesData=
   local EdgeCheck=

   # Extract a slice of db.server referencing the given ServerID,
   # and then grab the field containing Services data.
   ServicesData=$("$P4DBIN" -r "$RootDir" -J off -L /dev/null -k db.server -jd - 2>&1 |\
      $GREP "@db.server@ @${ServerID}@" | "$CUT" -d '@' -f 13)

   # Do a bitwise operation to determine if the ServicesData value indicates
   # this is an edge server.
   if [[ -n "$ServicesData" ]]; then
      EdgeCheck=$((ServicesData & 4096))

      if [[ "$EdgeCheck" -gt 0 ]]; then
         echo YES
      else
         echo NO
      fi
   else
      echo NO
   fi
}

#------------------------------------------------------------------------------
# is_replica ($ServerID, $RootDir)
#
# Determine if a given ServerID is a replica server or not, checking a given
# database root dir (e.g. $P4ROOT or $OFFLINE_DB).
#
# Input:
# $1 - ServerID (required)
# $2 - RootDir (optional, defaults to $P4ROOT)
#
# Output YES if an standby server, NO otherwise.
#
#------------------------------------------------------------------------------
is_replica () {
   local ServerID="${1:-Unset}"
   local RootDir="${2:-$P4ROOT}"
   local ServicesData=

   # Extract a slice of db.server referencing the given ServerID,
   # and then grab the field containing Services data.
   ServicesData=$("$P4DBIN" -r "$RootDir" -J off -L /dev/null -k db.server -jd - 2>&1 |\
      "$GREP" "@db.server@ @${ServerID}@" | "$CUT" -d '@' -f 13 | tr -d ' ')

   # Do a check to see if the ServicesData value indicates
   # this is an standby server.
   if [[ -n "$ServicesData" ]]; then
      if [[ "$ServicesData" -eq '2533' ]]; then
         echo YES
      else
         echo NO
      fi
   else
      echo NO
   fi
}

#------------------------------------------------------------------------------
# is_standby ($ServerID, $RootDir)
#
# Determine if a given ServerID is an standby server or not, checking a given
# database root dir (e.g. $P4ROOT or $OFFLINE_DB).
#
# Input:
# $1 - ServerID (required)
# $2 - RootDir (optional, defaults to $P4ROOT)
#
# Output YES if an standby server, NO otherwise.
#
#------------------------------------------------------------------------------
function is_standby () {
   local ServerID="${1:-Unset}"
   local RootDir="${2:-$P4ROOT}"
   local ServicesData=

   # Extract a slice of db.server referencing the given ServerID,
   # and then grab the field containing Services data.
   ServicesData=$("$P4DBIN" -r "$RootDir" -J off -L /dev/null -k db.server -jd - 2>&1 |\
      "$GREP" "@db.server@ @${ServerID}@" | "$CUT" -d '@' -f 13 | tr -d ' ')

   # Do a check to see if the ServicesData value indicates
   # this is an standby server.
   if [[ -n "$ServicesData" ]]; then
      if [[ "$ServicesData" -eq '35141' || "$ServicesData" -eq '35301' ]]; then
         echo YES
      else
         echo NO
      fi
   else
      echo NO
   fi
}

#------------------------------------------------------------------------------
# Set variables for use in various scripts:
# OFFLINE_DB=path to offline db directory
# EDGESERVER=1 if this is an edge server, 0 otherwise.
# STANDBYSERVER=1 if this is a standby server, 0 otherwise.
#
# This must be called after loading the standard shell environment by
# doing:
# source /p4/common/bin/p4_vars N
#
# This sets P4HOME, SERVERID, etc. needed by this function.
#------------------------------------------------------------------------------
function set_vars () {

   P4DInitScript="$P4HOME/bin/p4d_${SDP_INSTANCE}_init"
   P4DRef="${P4DInitScript%_init}"
   P4DSystemdServiceFile="/etc/systemd/system/p4d_${SDP_INSTANCE}.service"
   P4BrokerInitScript="$P4HOME/bin/p4broker_${SDP_INSTANCE}_init"
   P4BrokerRef="${P4BrokerInitScript%_init}"
   P4BrokerSystemdServiceFile="/etc/systemd/system/p4broker_${SDP_INSTANCE}.service"
   P4ProxyInitScript="$P4HOME/bin/p4p_${SDP_INSTANCE}_init"
   P4ProxyRef="${P4ProxyInitScript%_init}"
   P4ProxySystemdServiceFile="/etc/systemd/system/p4p_${SDP_INSTANCE}.service"

   # shellcheck disable=SC2072
   if [[ -n "${DO_PARALLEL_CHECKPOINTS:-}" && "$DO_PARALLEL_CHECKPOINTS" != "0" && "$P4D_VERSION" > "2022.2" ]]; then
      DoParallelCheckpoints=1
      if [[ "$DO_PARALLEL_CHECKPOINTS" =~ ^[0-9]+$ ]]; then
         if [[ "$DO_PARALLEL_CHECKPOINTS" == 1 ]]; then
            Threads=4
         else
            # Strip any leading zeros from the DO_PARALLEL_CHECKPOINTS value.
            Threads=$(echo $DO_PARALLEL_CHECKPOINTS|bc)
         fi
      else
         Threads=4
      fi
   else
      DoParallelCheckpoints=0
   fi
   OFFLINE_DB="${P4HOME}/offline_db"

   # shellcheck disable=SC2153
   if [[ -n "$SERVERID" ]]; then
      if [[ "$(is_edge "$SERVERID")" == YES ]]; then
         export EDGESERVER=1
      else
         export EDGESERVER=0
      fi
   else
      export EDGESERVER=0
   fi

   if [[ -n "$SERVERID" ]]; then
      if [[ "$(is_replica "$SERVERID")" == YES ]]; then
         export REPLICASERVER=1
         # Get commit server from P4TARGET setting in database
      else
         export REPLICASERVER=0
      fi
   else
      export REPLICASERVER=0
   fi

   if [[ -n "$SERVERID" ]]; then
      if [[ "$(is_standby "$SERVERID")" == YES ]]; then
         export STANDBYSERVER=1
         # Get commit server from P4TARGET setting in database
      else
         export STANDBYSERVER=0
      fi
   else
      export STANDBYSERVER=0
   fi

   # Ensure that SDP_ADMIN_PASSWORD_FILE is set, using existing value if set (e.g.
   # in p4_vars), otherwise set it to the SDP standard value.
   export SDP_ADMIN_PASSWORD_FILE="${SDP_ADMIN_PASSWORD_FILE:-Unset}"
   if [[ "$SDP_ADMIN_PASSWORD_FILE" == Unset ]]; then
      export SDP_ADMIN_PASSWORD_FILE="$P4CCFG/.p4passwd.${P4SERVER}.admin"
   fi
}

#------------------------------------------------------------------------------
# Check if user is running as required OS user.
#------------------------------------------------------------------------------
function check_uid () {
   user=$(id -un)
   if [[ "${user}" != "${OSUSER}" ]]; then
      die "Must be run by user: ${OSUSER}. Abort!"
   fi
}

#------------------------------------------------------------------------------
# Function log() - echo message to logfile or stdout.
#
# If $LOGFILE is defined, write message to the log file only; nothing goes to
# stdout.  Prepend a datestamp.
# If $LOGFILE isn't defined, just echo to stdout, w/o timestamp or.
# In all cases, support '-e' formatting.
# Input:
# $1 - message to log (must be quoted).
#------------------------------------------------------------------------------
function log () {
   if [[ "${LOGFILE:-Unset}" != Unset ]]; then
      echo -n "$(date)" >> "$LOGFILE" 2>&1
      echo -e " $0: $*" >> "$LOGFILE" 2>&1
   else
      echo -e "$@"
   fi
}

#------------------------------------------------------------------------------
# Get old logfile timestamp.
function get_old_log_timestamp () {
   local log=${1:-}
   local oldLogTimestamp=
   [[ -n "$log" ]] || return

   if [[ "$(uname -s)" == "Darwin" ]]; then
      # shellcheck disable=SC2012
      oldLogTimestamp="$(stat -l -t '%Y%m%d-%H%M%S' "$log" 2>&1 | awk '{print $6}')"
   else
      # shellcheck disable=SC2012
      oldLogTimestamp="$(ls -l --time-style +'%Y%m%d-%H%M%S' "$log" 2>&1 | awk '{print $6}')"
   fi

   [[ -n "$oldLogTimestamp" ]] || oldLogTimestamp=$(date +'%Y%m%d-%H%M%S')

   echo "$oldLogTimestamp"
}

#------------------------------------------------------------------------------
# Decide depending on our mail utility, how to specify sender (if we need to).
# Mail on some platforms sets sender by default.
# If the mail utility returns what looks like a version identifier
# when given the '-V' flag, use a '-S' flag.  If it does not return a
# version identifier, don't set a mail sender option.
# Allow GNU Mailutils alternative flag instead.
#------------------------------------------------------------------------------
function get_mail_sender_opt () {
   local mail_sender_opt=
   local mail_ver=
   if [[ -n "$MAILFROM" ]]; then
      mail_ver=$($SDPMAIL -V 2>&1)
      # shellcheck disable=SC2076
      if [[ "$mail_ver" =~ "GNU Mailutils" ]]; then
         mail_sender_opt="-aFrom:$MAILFROM"
      elif  [[ "$mail_ver" =~ ^[0-9]+\.[0-9] ]]; then
         mail_sender_opt="-S from=$MAILFROM"
      fi
   fi
   echo "$mail_sender_opt"
}

#------------------------------------------------------------------------------
# Email the log file by $LOGFILE.
#------------------------------------------------------------------------------
function mail_log_file () {
   local subject=$1
   local mail_sender_opt
   mail_sender_opt=$(get_mail_sender_opt)
   $SDPMAIL -s "$subject" "$mail_sender_opt" "$MAILTO" < "$LOGFILE"
}

#------------------------------------------------------------------------------
# Deliver the $LOGFILE via AWS SNS.
#------------------------------------------------------------------------------
function sns_log_file () {
   # AWS SNS has a 100 character limit for subject field
   local subject="$1"
   local short_subject=
   # shellcheck disable=SC2116
   short_subject="$(echo "${subject:0:100}")"
   echo -e "Sending alert and log file contents to administrator via AWS SNS." >&2
   aws --region "$AWS_DEFAULT_REGION" sns publish --topic-arn "$SNS_ALERT_TOPIC_ARN" --subject "$short_subject" --message "$(cat "$LOGFILE")"
}

#------------------------------------------------------------------------------
# Function die() - log message, send email/SNS, and exit.
# If $LOGFILE is defined, write message to the log file, email/SNS log,
# and exit.
# If $LOGFILE is not defined, write message to the stdout, and skip
# email/SNS.
# If in terminal session, display message to stderr as well.
#------------------------------------------------------------------------------
function die () {
   # mail the error (with more helpful subject line than cron)
   log "ERROR!!! - $HOSTNAME $P4SERVER $0: $*"

   if [[ "${LOGFILE:-Unset}" != Unset ]]; then
       if [[ "${SNS_ALERT_TOPIC_ARN:-Unset}" != Unset ]]; then
           log "Using SNS for log file delivery..."
           sns_log_file "ERROR!!! - $HOSTNAME $P4SERVER $0: $*"
       else
           log "Using email for log file delivery..."
           mail_log_file "ERROR!!! - $HOSTNAME $P4SERVER $0: $*"
       fi
   fi
   
   # if running from terminal, also send to stderr
   if tty >/dev/null; then
      echo -e "$@" >&2
   fi
   
   rm -f "${LOGS}/ckp_running.txt"
   
   exit 1
}

#------------------------------------------------------------------------------
# Convert various byte values (K,M,G,%) to bytes
# Pass in values such as 1024K, 512M, 1G or 10%
#------------------------------------------------------------------------------
function convert_to_bytes () {
   local value=$1
   local totalsize=${2:-Undefined}
   local size=
   local unit=

   # Break up value into size (numeric) and unit (K,M,G)
   size=$("$GREP" -Eo '[[:alpha:]%]+|[0-9]+' <<< "$value" | head -1)
   unit=$("$GREP" -Eo '[[:alpha:]%]+|[0-9]+' <<< "$value" | tail -1)

   # Based on unit, convert to bytes
   case "$unit" in
      K)
         echo $((size * 1024))
         ;;
      M)
         echo $((size * 1024**2))
         ;;
      G)
         echo $((size * 1024**3))
         ;;
      %)
         echo $((totalsize * size / 100))
         ;;
   esac
}

#------------------------------------------------------------------------------
# Write a semaphore file, $LOGS/ckp_running.txt.  This file is written at
# the start of processing, and removed upon successful completion.  It
# prevents multiple concurrent operations from being launched accidentally
# e.g. by multiple human admins, or a human inadvertently competing with a
# cron job.
#
# It is also intended to get human admins to determine the root cause of
# checkpoint failures.
#------------------------------------------------------------------------------
function ckp_running() {
   if [[ -f "${LOGS}/ckp_running.txt" ]]; then
      die "Last checkpoint not complete. Check the backup process or contact support."
   fi
   echo "Checkpoint running." > "${LOGS}/ckp_running.txt"
}

#------------------------------------------------------------------------------
# Remove the ckp_running.txt semaphore file when checkpoint processing is
# complete.
#------------------------------------------------------------------------------
function ckp_complete() {
   rm -f "${LOGS}/ckp_running.txt"
}

#------------------------------------------------------------------------------
# Ensure key directories are writable. Abort if they are not.
#------------------------------------------------------------------------------
function check_dirs () {
   # Check that key dirs are writable
   local -i dirsOK=1
   dirList="$OFFLINE_DB $CHECKPOINTS $LOGS"
   [[ "$EDGESERVER" -eq 1 ]] && dirList+=" ${CHECKPOINTS}.${SERVERID#p4d_}"
   for dir in $dirList; do
      if [[ ! -d "$dir" || ! -w "$dir" ]]; then
         log "Error: Dir $dir does not exist or is not writable."
         dirsOK=0
      fi
   done
   [[ "$dirsOK" -eq 1 ]] || die "Some expected dirs are missing or not writable. Aborting."
}

#------------------------------------------------------------------------------
# Add the results of df -h or df -m to the log file.
#------------------------------------------------------------------------------
function check_disk_space () {
   log "Checking disk space..."
   $P4BIN diskspace >> "$LOGFILE" 2>&1
}

#------------------------------------------------------------------------------
# Check value of journal; ensure it is an integer.
#------------------------------------------------------------------------------
function check_journalnum () {
   local JNLNUM=${1:-Unset}
   re='^[0-9]+$'
   if ! [[ $JNLNUM =~ $re ]] ; then
      die "The journal counter value [$JNLNUM] is invalid. It must be numeric."
   fi
}

#------------------------------------------------------------------------------
# Check the checkpoints directory for the oldest checkpoint
#------------------------------------------------------------------------------
function get_ckpnum () {
   if [[ "$EDGESERVER" -eq 0 ]]; then
      # shellcheck disable=SC2034 disable=SC2012 disable=SC2016
      OLDESTCHECKPOINT=$(ls -1tr "${CHECKPOINTS}/" | "$GREP" ckp | "$GREP" -v md5 | head -n 1 | "$AWK" -F '.ckp.' '{ print $(2) }' | tr -d '.gz')
   else
      # shellcheck disable=SC2034 disable=SC2012 disable=SC2016
      OLDESTCHECKPOINT=$(ls -1tr "${CHECKPOINTS}.${SERVERID#p4d_}/" | "$GREP" ckp | "$GREP" -v md5 | head -n 1 | "$AWK" -F '.ckp.' '{ print $(2) }' | tr -d '.gz')
   fi
}

#------------------------------------------------------------------------------
# Determine journal counter by checking counter in db.counters.
#------------------------------------------------------------------------------
get_journalnum () {
   # get the current journal and checkpoint serial numbers.
   local nextCheckpointNum
   if [[ -r "$P4ROOT/db.counters" ]]; then
      nextCheckpointNum=$("$P4DBIN" -r "$P4ROOT" -k db.counters -jd - 2>&1 | grep @journal@ | cut -d '@' -f 8)

      if [[ -n "$nextCheckpointNum" ]]; then
         check_journalnum "$nextCheckpointNum"
         JOURNALNUM="$nextCheckpointNum"
      else
         # Special case: If db.counters is empty, then we have a new/empty data
         # set, so just set the value to 0.
         JOURNALNUM=0
      fi
   else
      # Special case: If db.counters doesn't exist, then we have a new/empty
      # data set, so just set the value to 0.
      JOURNALNUM=0
   fi

   # If we are on an edge server, the journal has already rotated, so we have to decrement the value
   # so that we replay the correct journal file and create the correct checkpoint number on the
   # edge server.
   #
   # In the case of a standby server, the journal rotation occurs on the master server,
   # so we don't need to increment the journal number again, so we decrement by 1.
   # Also, when replaying the journals to the offline db, we don't want to play to the live journal
   # because it is still being replicated.
   if [[ "$EDGESERVER" -eq 1 || "$REPLICASERVER" -eq 1 || "$STANDBYSERVER" -eq 1 ]]; then
      JOURNALNUM=$((JOURNALNUM - 1))
   fi
   CHECKPOINTNUM=$((JOURNALNUM + 1))
}

#------------------------------------------------------------------------------
# Determine journal space usage and minimum disk space requirement
#------------------------------------------------------------------------------
get_journal_stats () {
   # Get minimum disk space required on server journal filesystem before server rejects commands
   # This will return the configured and default value, but grab the configured value which shows first
   # If a configured value is not present, it will use the default value
   # shellcheck disable=SC2034 disable=SC2016
   P4JOURNALMIN=$("$P4BIN" configure show filesys.P4JOURNAL.min | "$AWK" '{ print $1 }' | $CUT -d'=' -f2 | head -1)
   # Get current journal free disk space
   # shellcheck disable=SC2034
   P4JOURNALFREE=$("$P4BIN" -ztag -F "%freeBytes%" diskspace P4JOURNAL)
   # Get total available disk space for journal
   # shellcheck disable=SC2034
   P4JOURNALTOTAL=$("$P4BIN" -ztag -F "%totalBytes%" diskspace P4JOURNAL)
}

#------------------------------------------------------------------------------
# Verify that the offline databases are usable by checking the existence
# of a 'offline_db_usable.txt' file that is written only when databases
# are in a known-good state, following successful recovery from a checkpoint.
#------------------------------------------------------------------------------
check_offline_db_usable () {
   # Check it is OK
   if [[ ! -f "$OFFLINE_DB/offline_db_usable.txt" ]]; then
      die "Offline database not in a usable state. Check the backup process."
   fi

   if [[ ! -f "$OFFLINE_DB/db.counters" ]]; then
      die "Offline database not found. Consider creating it with live_checkpoint.sh. Be aware that live_checkpoint.sh locks the live system and may take a long time. Aborting."
   fi
}

#------------------------------------------------------------------------------
# Determine journal counter in offline databases.
#------------------------------------------------------------------------------
get_offline_journal_num () {
   # Get the journal number of the offline database
   check_offline_db_usable
   OFFLINEJNLNUM=$("$P4DBIN" -r "$OFFLINE_DB" -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") || die "Cannot get the offline journal number. Abort!"
   check_journalnum "$OFFLINEJNLNUM"
   log "Offline journal number is: $OFFLINEJNLNUM"
}

#------------------------------------------------------------------------------
# Cleanup old checkpoint and numbered journal files.
#------------------------------------------------------------------------------
remove_old_checkpoints_and_journals () {
   local CheckpointsDir=
   local StandbyReplicaJournalsDir=
   local FilePrefix=
   local JournalPrefix=
   local File=

   if [[ "$KEEPCKPS" -eq 0 ]]; then
      log "Skipping cleanup of old checkpoints because KEEPCKPS is set to 0."
   else
      log "Deleting obsolete checkpoints and journals. Keeping latest $KEEPCKPS per KEEPCKPS setting in p4_vars."
      # For the master server, we can safely rely on the SDP standard that the journalPrefix
      # is of the form '/p4/N/checkpoints/p4_N'. For replicas and edge servers, determine
      # the values dynamically based on the current journalPrefix value for the given ServerID.
      if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then
         # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N
         CheckpointsDir="${CHECKPOINTS}"
         FilePrefix="${P4SERVER}"
      else
         JournalPrefix="$("$P4DBIN" -r "$P4ROOT" -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)"
         if [[ -n "$JournalPrefix" ]]; then
            CheckpointsDir="${JournalPrefix%/*}"
            FilePrefix="${JournalPrefix##*/}"
         else
            log "Warning: Could not determine journalPrefix for ServerID $SERVERID."
            CheckpointsDir="${CHECKPOINTS}"
            FilePrefix="${P4SERVER}"
         fi
      fi

      if [[ -d "$CheckpointsDir" ]]; then
         if [[ "$DoParallelCheckpoints" -eq 1 ]]; then
            # Remove parallel checkpoint dirs.
            for Dir in $(find "${CheckpointsDir}/" -type d -name "${FilePrefix}.ckp.*" | sort -r | "$AWK" "NR > $KEEPCKPS"); do
               log "rm -rf $Dir"
               rm -rf "$Dir"
            done
         else
            # Remove selected checkpoint files based on the KEEPCKPS
            # setting regardless of whether compressed or not.
            # We multiply KEEPCKP by 2 for the ckp files because of the md5 files.
            # shellcheck disable=SC2012
            for File in $(ls -t "${CheckpointsDir}/${FilePrefix}".ckp.* 2>/dev/null | "$AWK" "NR > ($KEEPCKPS * 2)"); do
               log "rm -f $File"
               rm -f "$File"
            done
         fi

         # Use KEEPJNLS to allow for separate journal rotation at a higher
         # frequency.
         # shellcheck disable=SC2012
         for File in $(ls -t "${CheckpointsDir}/${FilePrefix}".jnl.* 2>/dev/null | "$AWK" "NR > $KEEPJNLS"); do
            log "rm -f $File"
            rm -f "$File"
         done
      fi

      StandbyReplicaJournalsDir="${P4HOME}/journals.rep"
      if [[ -d "$StandbyReplicaJournalsDir" ]]; then
         # shellcheck disable=SC2012
         for File in $(ls -t "$StandbyReplicaJournalsDir/${FilePrefix}".ckp.* 2>/dev/null | "$AWK" "NR > ($KEEPCKPS * 2)"); do
            log "rm -f $File"
            rm -f "$File"
         done

         # shellcheck disable=SC2012
         for File in $(ls -t "${StandbyReplicaJournalsDir}/${FilePrefix}".jnl.* 2>/dev/null | "$AWK" "NR > $KEEPJNLS"); do
            log "rm -f $File"
            rm -f "$File"
         done
      fi

      # This is a workaround to cleanup $LOGS/journal.NNN files on standby replicas.
      # These files are normally removed by p4d during journal rotation on the standby
      # replica. Use only if standby journals are not removed due to a standby replica
      # sharing /hxdepots with its P4TARGET server. To use this workround, add this
      # line to the end of the /p4/common/config/p4_N.vars file:
      #
      # export SDP_REMOVE_STANDBY_JOURNALS=1
      #
      if [[ "${SDP_REMOVE_STANDBY_JOURNALS:-0}" == 1 && "$(is_standby "$SERVERID")" == YES ]]; then
         log "Removing excess journal.NNN files due to SDP_REMOVE_STANDBY_JOURNALS=1."
         # shellcheck disable=SC2012
         for File in $(ls -t "${LOGS}/journal."* 2>/dev/null | "$AWK" "NR > $KEEPJNLS"); do
            # Process only files named 'journal.NNN' in $LOGS.
            [[ "$File" =~ /journal.[0-9]+$ ]] || continue
            log "rm -f $File"
            rm -f "$File"
         done
      fi
   fi
}

#------------------------------------------------------------------------------
# Function: is_server_up ($server)
#
# Input:
# $1 - server, one of 'p4d', 'p4p', or 'p4broker'
#
# Output: None
#
# Return Codes:
# 0: Server is up.
# 1: Server is down.
# 2: Bad usage.
#
# Server up/down status is checked using the appropriate init script.
#------------------------------------------------------------------------------
function is_server_up () {
   local server="${1:-Unset}"

   case "$server" in
      (p4d)
         "$P4DInitScript" status > /dev/null 2>&1
         return $?
      ;;
      (p4broker)
         "$P4BrokerInitScript" status > /dev/null 2>&1
         return $?
      ;;
      (p4p)
        "$P4ProxyInitScript" status > /dev/null 2>&1
         return $?
      ;;
      (Unset)
         log "Internal Error: is_server_up(): No server type specified."
         return  2
      ;;
      (*)
         log "Internal Error: is_server_up(): Unknown server specified: $server"
         return 2
      ;;
   esac
}

#------------------------------------------------------------------------------
# Shutdown p4d using systemd if configured for systemd. Otherwise call the
# underlying init script directly.
#
# Log the shutdown activity.
#
#------------------------------------------------------------------------------
stop_p4d () {
   log "Shutting down the ${P4DBIN##*/} server."
   local -i maxStopDelay=${SDP_MAX_STOP_DELAY_P4D:-43200}
   local -i stopVerified=0
   local -i i=0
   local -i useSystemd=0
   local serviceName=

   if [[ -n "$(command -v systemctl)" ]]; then
      serviceName="${P4DBIN##*/}"
      if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then
         useSystemd=1
      fi
   fi

   if [[ "$useSystemd" -eq 1 ]]; then
      { sudo systemctl stop "${P4DBIN##*/}"; } >> "$LOGFILE" 2>&1 ||\
         die "Failed to execute: sudo systemctl stop ${P4DBIN##*/}"

      # With systemd, we must independently confirm service stop,
      # waiting if needed.
      stopVerified=0
      i=0; while [[ "$i" -lt "$maxStopDelay" ]]; do
         if is_server_up p4d; then
            sleep 1
         else
            stopVerified=1
            break
         fi
         i+=1
      done
   else
      "$P4DInitScript" stop >> "$LOGFILE" 2>&1
      stopVerified=1
   fi

   if [[ "$stopVerified" -eq 1 ]]; then
      log "Stopped ${P4DBIN##*/} server."
      return 0
   else
      log "Error: Server ${P4DBIN##*/} did not stop after $maxStopDelay seconds. Tailing $P4LOG:"
      tail "$P4LOG" >> "$LOGFILE" 2>&1
      die "Aborting due to failed p4d stop."
   fi
}

#------------------------------------------------------------------------------
# Shutdown p4broker using systemd if configured for systemd. Otherwise call the
# underlying init script directly.
#
# Log the shutdown activity.
#
#------------------------------------------------------------------------------
stop_p4broker () {
   log "Shutting down the ${P4BROKERBIN##*/} server."
   local -i maxStopDelay=${SDP_MAX_STOP_DELAY_P4BROKER:-600}
   local -i stopVerified=0
   local -i i=0
   local -i useSystemd=0
   local serviceName=

   if [[ -n "$(command -v systemctl)" ]]; then
      serviceName="${P4BROKERBIN##*/}"
      if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then
         useSystemd=1
      fi
   fi

   if [[ "$useSystemd" -eq 1 ]]; then
      { sudo systemctl stop "${P4BROKERBIN##*/}"; } >> "$LOGFILE" 2>&1 ||\
         die "Failed to execute: sudo systemctl stop ${P4BROKERBIN##*/}"

      # With systemd, we must independently confirm service stop,
      # waiting if needed.
      stopVerified=0
      i=0; while [[ "$i" -lt "$maxStopDelay" ]]; do
         if is_server_up p4broker; then
            sleep 1
         else
            stopVerified=1
            break
         fi
         i+=1
      done
   else
      "$P4BrokerInitScript" stop >> "$LOGFILE" 2>&1
      stopVerified=1
   fi

   if [[ "$stopVerified" -eq 1 ]]; then
      log "Stopped ${P4BROKERBIN##*/} server."
      return 0
   else
      die "Server ${P4BROKERBIN##*/} did not stop after $maxStartDelay seconds."
   fi
}

#------------------------------------------------------------------------------
# Shutdown p4p using systemd if configured for systemd. Otherwise call the
# underlying init script directly.
#
# Log the shutdown activity.
#
#------------------------------------------------------------------------------
stop_p4p () {
   log "Shutting down the ${P4PBIN##*/} server."
   local -i maxStopDelay=${SDP_MAX_STOP_DELAY_P4P:-600}
   local -i stopVerified=0
   local -i i=0
   local -i useSystemd=0
   local serviceName=

   if [[ -n "$(command -v systemctl)" ]]; then
      serviceName="${P4PBIN##*/}"
      if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then
         useSystemd=1
      fi
   fi

   if [[ "$useSystemd" -eq 1 ]]; then
      { sudo systemctl stop "${P4PBIN##*/}"; } >> "$LOGFILE" 2>&1 ||\
         die "Failed to execute: sudo systemctl stop ${P4PBIN##*/}"

      # With systemd, we must independently confirm service stop,
      # waiting if needed.
      stopVerified=0
      i=0; while [[ "$i" -lt "$maxStopDelay" ]]; do
         if is_server_up p4p; then
            sleep 1
         else
            stopVerified=1
            break
         fi
         i+=1
      done
   else
      "$P4ProxyInitScript" stop >> "$LOGFILE" 2>&1
      stopVerified=1
   fi

   if [[ "$stopVerified" -eq 1 ]]; then
      log "Stopped ${P4PBIN##*/} server."
      return 0
   else
      die "Server ${P4PBIN##*/} did not stop after $maxStopDelay seconds."
   fi
}

#------------------------------------------------------------------------------
# Start p4d using systemd if configured for systemd. Otherwise call the
# underlying init script directly.
#
# Log the startup activity.
#
# This is a do-or-die function. It returns success upon successful server
# startup, or else dies.
#------------------------------------------------------------------------------
function start_p4d () {
   log "Starting the ${P4DBIN##*/} server."
   local -i maxStartDelay=${SDP_MAX_START_DELAY_P4D:-120}
   local -i startVerified=0
   local -i i=0
   local -i useSystemd=0
   local serviceName=

   if [[ -n "$(command -v systemctl)" ]]; then
      serviceName="${P4DBIN##*/}"
      if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then
         useSystemd=1
      fi
   fi

   if [[ "$useSystemd" -eq 1 ]]; then
      { sudo systemctl start "${P4DBIN##*/}"; } ||\
         die "Failed to execute: sudo systemctl start ${P4DBIN##*/}"
   else
      "$P4DInitScript" start >> "$LOGFILE" 2>&1
   fi

   # Confirm that p4d started, waiting if needed.
   startVerified=0
   i=0; while [[ "$i" -lt "$maxStartDelay" ]]; do
      if is_server_up p4d; then
         startVerified=1
         break
      else
         sleep 1
      fi
      i+=1
   done

   if [[ "$startVerified" -eq 1 ]]; then
      log "Server ${P4DBIN##*/} started successfully."
      return 0
   else
      log "Error: Server ${P4DBIN##*/} did not start after $maxStartDelay seconds. Tailing $P4LOG:"
      tail "$P4LOG" >> "$LOGFILE" 2>&1
      die "Aborting due to failed p4d start."
   fi
}

#------------------------------------------------------------------------------
# Start p4broker using systemd if configured for systemd. Otherwise call the
# underlying init script directly.
#
# Log the startup activity.
#
# This is a do-or-die function. It returns success upon successful server
# startup, or else dies.
#------------------------------------------------------------------------------
function start_p4broker () {
   log "Starting the ${P4BROKERBIN##*/} server."
   local -i maxStartDelay=${SDP_MAX_START_DELAY_P4BROKER:-60}
   local -i startVerified=0
   local -i i=0
   local -i useSystemd=0
   local serviceName=

   if [[ -n "$(command -v systemctl)" ]]; then
      serviceName="${P4BROKERBIN##*/}"
      if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then
         useSystemd=1
      fi
   fi

   if [[ "$useSystemd" -eq 1 ]]; then
      { sudo systemctl start "${P4BROKERBIN##*/}"; } ||\
         die "Failed to execute: sudo systemctl start ${P4BROKERBIN##*/}"
   else
      "$P4BrokerInitScript" start >> "$LOGFILE" 2>&1
   fi

   # Confirm that p4broker started, waiting if needed.
   startVerified=0
   i=0; while [[ "$i" -lt "$maxStartDelay" ]]; do
      if is_server_up p4broker; then
         startVerified=1
         break
      else
         sleep 1
      fi
      i+=1
   done

   if [[ "$startVerified" -eq 1 ]]; then
      log "Server ${P4BROKERBIN##*/} started successfully."
      return 0
   else
      die "Server ${P4BROKERBIN##*/} did not start after $maxStartDelay seconds."
   fi
}

#------------------------------------------------------------------------------
# Start p4p using systemd if configured for systemd. Otherwise call the
# underlying init script directly.
#
# Log the startup activity.
#
# This is a do-or-die function. It returns success upon successful server
# startup, or else dies.
#------------------------------------------------------------------------------
function start_p4p () {
   log "Starting the ${P4PBIN##*/} server."
   local -i maxStartDelay=${SDP_MAX_START_DELAY_P4P:-60}
   local -i startVerified=0
   local -i i=0
   local -i useSystemd=0
   local serviceName=

   if [[ -n "$(command -v systemctl)" ]]; then
      serviceName="${P4PBIN##*/}"
      if [[ -n "$(systemctl is-enabled "$serviceName" 2>/dev/null)" ]]; then
         useSystemd=1
      fi
   fi

   if [[ "$useSystemd" -eq 1 ]]; then
      { sudo systemctl start "${P4PBIN##*/}"; } ||\
         die "Failed to execute: sudo systemctl start ${P4PBIN##*/}"
   else
      "$P4ProxyInitScript" start >> "$LOGFILE" 2>&1
   fi

   # Confirm that p4p started, waiting if needed.
   startVerified=0
   i=0; while [[ "$i" -lt "$maxStartDelay" ]]; do
      if is_server_up p4p; then
         startVerified=1
         break
      else
         sleep 1
      fi
      i+=1
   done

   if [[ "$startVerified" -eq 1 ]]; then
      log "Server ${P4PBIN##*/} started successfully."
      return 0
   else
      die "Server ${P4PBIN##*/} did not start after $maxStartDelay seconds."
   fi
}

#------------------------------------------------------------------------------
# Do a front-door 'p4d admin journal' command to rotate the current/active
# journal file on the master server, starting a fresh new P4JOURNAL file.
#
# In a distributed topology with replicas/edge servers, this function must
# be called only on the master/commit server.
#------------------------------------------------------------------------------
function truncate_journal () {
   local CheckpointFile="${CHECKPOINTS}/${P4SERVER}.ckp.${CHECKPOINTNUM}.gz"
   local JournalFile="${CHECKPOINTS}/${P4SERVER}.jnl.${JOURNALNUM}"

   if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then
      [[ -f "$CheckpointFile" ]] && \
         die "Checkpoint $CheckpointFile already exists, check the backup process."
      [[ -f "$JournalFile" ]] && \
         die "Journal $JournalFile already exists, check the backup process."

      log "Truncating journal..."
      # During journal rotation, either by a front-door 'p4 admin journal' or a
      # back-door 'p4d -jj', p4d does a copy-then-delete rather than an mv at
      # the OS level.  During rotation, the perforce server will pause
      # responses to clients (just as with a checkpoint), but this should be
      # for a short period of time even for large data sets, as the journal
      # typically represents a single day of metadata.
      # Curly braces capture output of 'time'.
      "$P4CBIN"/p4login -p "$P4MASTERPORT"
      { time "$P4BIN" -p "$P4MASTERPORT" admin journal; } >> "$LOGFILE" 2>&1 || { die "Journal rotation failed. Abort!"; }
      # The test below waits until the journal file exists in the checkpoints directory before proceeding.
      test=1
      while [[ $test != 0 ]]; do
         sleep 5
         if [[ -f "$JournalFile" ]]; then
            test=0
         fi
      done
      "$P4CBIN"/p4login
   else
      log "Warning: The truncate_journal () function should only be called on the master server. It is ignored on edge and replica replica servers."
   fi
}

#------------------------------------------------------------------------------
# Call 'p4d -jj' to rotate the current/active journal file on the master
# server from an edge server, starting a fresh new P4JOURNAL file.
#
# In a distributed topology with edge and standby servers, this function can be
# used to trigger a journal rotation on master/commit server. It's not meant to
# be used from the master server itself.
#------------------------------------------------------------------------------
function truncate_journal_on_master () {
   # Increment Edge journal number since the journal will increment on the master after calling journal rotation
   local EdgeJournalNum=$((JOURNALNUM + 1))
   local StandbyJournalNum=$((JOURNALNUM + 2)) # If using journalcopy, have to add 2 since live journal is in checkpoints folder
   local JournalFile=

   if [[ "$EDGESERVER" -eq 1 ]]; then
      # Refer to ckp/jnl files starting like (example ServerID=p4d_edge_nyc):
      # /p4/N/checkpoints.edge_nyc/p4_N.edge_nyc
      JournalFile="${CHECKPOINTS}.${SERVERID#p4d_}/${P4SERVER}.${SERVERID#p4d_}.jnl.${EdgeJournalNum}"
   elif [[ "$STANDBYSERVER" -eq 1 ]]; then
      JournalFile="${CHECKPOINTS}/${P4SERVER}.jnl.${StandbyJournalNum}"
   fi

   if [[ "$SERVERID" != "$P4MASTER_ID" ]]; then
      [[ -f "$JournalFile" ]] && \
         die "Journal $JournalFile already exists, check the backup process."

      log "Truncating journal on ${P4MASTERPORT}."
      # 'p4d -jj' does a copy-then-delete, instead of a simple mv.
      # During 'p4d -jj' the perforce server will hang the responses to clients,
      # this should be for a very short period of time even for large data
      # sets, as the journal represents a single day of metadata.
      # Curly braces capture output of 'time'.
      "$P4CBIN"/p4login -p "$P4MASTERPORT"
      { time "$P4BIN" -p "$P4MASTERPORT" admin journal; } >> "$LOGFILE" 2>&1 || { die "Journal rotation failed. Abort!"; }
      # The test below waits until the journal file exists in the checkpoints directory before proceeding.
      test=1
      while [[ $test != 0 ]]; do
         sleep 5
         if [[ -f "$JournalFile" ]]; then
            test=0
         fi
      done
      "$P4CBIN"/p4login -service
   else
      log "Warning: truncate_journal_on_master () function should not be called on a master server. Ignoring."
   fi
}

#------------------------------------------------------------------------------
# Similar to truncate_journal() above, p4d_truncate_journal() is intended to be
# usable form the p4d_base init script, to allow journal rotation on p4d
# start.  As it may be called from the init script, it may be called on the
# master, a replica, or the edge. However, it should will only do the journal
# rotation if called on the master.
#------------------------------------------------------------------------------
function p4d_truncate_journal () {
   local JournalFile="${CHECKPOINTS}/${P4SERVER}.jnl.${JOURNALNUM}"

   if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then
      [[ -f "$JournalFile" ]] && \
         die "Journal $JournalFile already exists, check the backup process."
      log "Rotating journal prior to starting p4d."
      "$P4DBIN" -r "$P4ROOT" -J "$P4JOURNAL" -jj >> "$LOGFILE" 2>&1 ||\
         die "Failed to rotate journal. Aborting p4d server start."
   else
      log "Warning: The p4d_truncate_journal() function has no effect if called on a server other than the master. Ignoring."
   fi
}

#------------------------------------------------------------------------------
# Replay any and all numbered journal files into the offline databases.
#------------------------------------------------------------------------------
function replay_journals_to_offline_db () {
   local CheckpointsDir=
   local FilePrefix=
   local NumberedJournal=
   local JournalPrefix=

   log "Replay any unreplayed journals to the offline database."
   check_offline_db_usable

   # For the master server, we can safely rely on the SDP standard that the
   # journalPrefix is of the form '/p4/N/checkpoints/p4_N'. For replicas and
   # edge servers, determine the values dynamically based on the current journal
   # Prefix value for the given ServerID.
   if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then
      # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N
      CheckpointsDir="${CHECKPOINTS}"
      FilePrefix="${P4SERVER}"
   else
      JournalPrefix="$("$P4DBIN" -r "$P4ROOT" -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)"
      if [[ -n "$JournalPrefix" ]]; then
         CheckpointsDir="${JournalPrefix%/*}"
         FilePrefix="${JournalPrefix##*/}"
      else
         log "Warning: Could not determine journalPrefix for ServerID $SERVERID."
         CheckpointsDir="${CHECKPOINTS}"
         FilePrefix="${P4SERVER}"
      fi
   fi

   for (( j=OFFLINEJNLNUM; j <= JOURNALNUM; j++ )); do
      NumberedJournal="${CheckpointsDir}/${FilePrefix}.jnl.${j}"
      log "Replay journal $NumberedJournal to offline db."
      rm -f "${OFFLINE_DB}/offline_db_usable.txt" >> "$LOGFILE" 2>&1
      # Curly braces capture output of 'time'.
      { time "$P4DBIN" -r "$OFFLINE_DB" -jr -f "$NumberedJournal"; } >> "$LOGFILE" 2>&1 || { die "Offline journal replay failed. Abort!"; }
      echo "Offline db file restored successfully." > "${OFFLINE_DB}/offline_db_usable.txt"
   done
}

#------------------------------------------------------------------------------
# Replay the live, active P4JOURNAL file into the offline database.
#------------------------------------------------------------------------------
function replay_active_journal_to_offline_db () {
   log "Replay active journal to offline db."

   local ActiveJournal=

   # On a standby server, the current/active journal is named /p4/N/logs/journal.<jnlNum>.
   # On the master and other server types, the active journal is $P4JOURNAL.
   if [[ "$STANDBYSERVER" -eq 1 ]]; then
      local _JNLNUM
      _JNLNUM=$("$P4DBIN" -r "$P4ROOT" -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") || die "Cannot get $P4ROOT journal number. Abort!"
      ActiveJournal="$LOGS/journal.$_JNLNUM"
   else
      ActiveJournal="$P4JOURNAL"
   fi

   # Curly braces capture output of 'time'.
   { time "$P4DBIN" -r "$OFFLINE_DB" -jr -f "${ActiveJournal}"; } >> "$LOGFILE" 2>&1 || { die "Active Journal replay failed. Abort!"; }

}

#------------------------------------------------------------------------------
# Recreate offline databases from the latest checkpoint.
#------------------------------------------------------------------------------
function recreate_offline_db_files () {
   local CheckpointCmd=
   local CheckpointsDir=
   local ParallelCheckpointDir=
   local ParallelCheckpointOKFile=
   local FilePrefix=
   local LastCheckpointMD5=
   local LastCheckpoint=

   # For the master server, we can safely rely on the SDP standard that the journalPrefix
   # is of the form '/p4/N/checkpoints/p4_N'. For replicas and edge servers, determine
   # the values dynamically based on the current journalPrefix value for the given ServerID.
   if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then
      # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N
      CheckpointsDir="${CHECKPOINTS}"
      FilePrefix="${P4SERVER}"
   else
      JournalPrefix="$("$P4DBIN" -r "$P4ROOT" -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)"
      if [[ -n "$JournalPrefix" ]]; then
         CheckpointsDir="${JournalPrefix%/*}"
         FilePrefix="${JournalPrefix##*/}"
      else
         log "Warning: Could not determine journalPrefix for ServerID $SERVERID."
         CheckpointsDir="${CHECKPOINTS}"
         FilePrefix="${P4SERVER}"
      fi
   fi

   # Make sure we have at least one checkpoint available.
   if [[ "$DoParallelCheckpoints" -eq 1 ]]; then
      if [[ -z "$(ls "${CheckpointsDir}"/*.OK 2>/dev/null)" ]]; then
         ckp_complete

         if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then
            die "No parallel checkpoint dirs found in $CheckpointsDir with prefix ${FilePrefix}.  Consider running 'live_checkpoint.sh $SDP_INSTANCE'."
         else
            die "No parallel checkpoint dirs found in $CheckpointsDir with prefix ${FilePrefix}."
         fi
      fi
   else
      if [[ -z "$(ls "${CheckpointsDir}/${FilePrefix}".ckp.*.md5 2>/dev/null)" ]]; then
         ckp_complete

         if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then
            die "No checkpoints found in $CheckpointsDir with prefix $FilePrefix.  Consider running 'live_checkpoint.sh $SDP_INSTANCE'."
         else
            die "No checkpoints found in $CheckpointsDir with prefix $FilePrefix."
         fi
      fi
   fi
   
   # shellcheck disable=SC2012
   if [[ "$DoParallelCheckpoints" -eq 1 ]]; then
      ParallelCheckpointOKFile="$(ls -t "${CheckpointsDir}"/*.OK 2>/dev/null|head -1)"
      [[ -n "$ParallelCheckpointOKFile" ]] || \
         die "Could not find valid parallel checkpoint dir for latest checkpoint. Abort!"
   else
      LastCheckpointMD5=$(ls -t "${CheckpointsDir}/${FilePrefix}".ckp.*.md5 | head -1)
      [[ -n "$LastCheckpointMD5" ]] || \
         die "Could not find *.md5 file for latest checkpoint. Abort!"
   fi

   # shellcheck disable=SC2129
   rm -f "${OFFLINE_DB}"/offline_db_usable.txt >> "$LOGFILE" 2>&1
   rm -f "${OFFLINE_DB}"/db.* >> "$LOGFILE" 2>&1
   rm -f "${OFFLINE_DB}"/save/db.* >> "$LOGFILE" 2>&1

   if [[ "$DoParallelCheckpoints" -eq 1 ]]; then
      ParallelCheckpointDir="${ParallelCheckpointOKFile%.OK}"
      log "Recovering from last parallel checkpoint dir, $ParallelCheckpointDir."
      CheckpointCmd="$P4DBIN -r $OFFLINE_DB -z -N $Threads -jrp ${ParallelCheckpointDir}"
   else
      # Account for the idiosyncrasy that MD5 files for checkpoints may look
      # like p4_N.ckp.gz.md5 or p4_N.ckp.md5.
      if [[ "$LastCheckpointMD5" == *".gz.md5" ]]; then
         LastCheckpoint="${LastCheckpointMD5%.md5}"
      else
         LastCheckpoint="${LastCheckpointMD5%.md5}.gz"
      fi

      [[ -r "$LastCheckpoint" ]] || \
         die "Missing last checkpoint file: $LastCheckpoint. Abort!"

      log "Recovering from last full checkpoint, $LastCheckpoint."
      CheckpointCmd="$P4DBIN -r $OFFLINE_DB -jr -z ${LastCheckpoint}"
   fi

   log "Running: $CheckpointCmd"
   # Curly braces capture output of 'time'.
   { time $CheckpointCmd; } >> "$LOGFILE" 2>&1 || { die "Restore of checkpoint to $OFFLINE_DB failed!"; }

   echo "Offline db file restored successfully." > "${OFFLINE_DB}/offline_db_usable.txt"
}

#------------------------------------------------------------------------------
# Take a live checkpoint from db.* files in P4ROOT.
#------------------------------------------------------------------------------
function checkpoint () {
   local CheckpointCmd=
   local CheckpointsDir=
   local FilePrefix=
   local ParallelCheckpointDir=
   local ParallelCheckpointOKFile=

   log "Create a new checkpoint from live db files in $P4ROOT."

   if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then
      # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N
      CheckpointsDir="${CHECKPOINTS}"
      FilePrefix="${P4SERVER}"
   else
      die "Live checkpoints can only be run on the master server."
   fi

   if [[ "$DoParallelCheckpoints" -eq 1 ]]; then
      ParallelCheckpointDir="${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}"
      ParallelCheckpointOKFile="${ParallelCheckpointDir}.OK"
      CheckpointCmd="$P4DBIN -r $P4ROOT -Z -N $Threads -jcp ${CheckpointsDir}/${FilePrefix}"
   else
      CheckpointCmd="$P4DBIN -r $P4ROOT -jc -Z ${CheckpointsDir}/${FilePrefix}"
   fi

   log "Running: $CheckpointCmd"

   # Curly braces capture output of 'time'.
   { time $CheckpointCmd; } >> "$LOGFILE" 2>&1 || { die "ERROR - New live checkpoint failed!"; }

   if [[ "$DoParallelCheckpoints" -eq 1 ]]; then
      touch "$ParallelCheckpointOKFile"
      log "Live Parallel checkpoint completed OK. Writing: $ParallelCheckpointOKFile"
   else
      log "Live checkpoint completed OK."
   fi
}

#------------------------------------------------------------------------------A
# Take a checkpoint from the ROOTDIR, typically either /p4/N/root or
# /p4/N/offline_db.
#------------------------------------------------------------------------------
function dump_checkpoint () {
   local CheckpointCmd=
   local CheckpointsDir=
   local NewCheckpoint=
   local NewCheckpointMD5=
   local FilePrefix=
   local JournalPrefix=
   local -i DoSnapshot=0
   local -i SnapshotOK=1
   local -i CheckpointOK=1
   local ParallelCheckpointDir=
   local ParallelCheckpointOKFile=

   # shellcheck disable=SC2153
   log "Dump out new checkpoint from db files in $ROOTDIR."

   # For the master server, we can safely rely on the SDP standard that the journalPrefix
   # is of the form '/p4/N/checkpoints/p4_N'. For replicas and edge servers, determine
   # the values dynamically based on the current journalPrefix value for the given ServerID.
   if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then
      # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N
      CheckpointsDir="${CHECKPOINTS}"
      FilePrefix="${P4SERVER}"
   else
      JournalPrefix="$("$P4DBIN" -r "$P4ROOT" -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)"
      if [[ -n "$JournalPrefix" ]]; then
         CheckpointsDir="${JournalPrefix%/*}"
         FilePrefix="${JournalPrefix##*/}"
      else
         log "Warning: Could not determine journalPrefix for ServerID $SERVERID."
         CheckpointsDir="${CHECKPOINTS}"
         FilePrefix="${P4SERVER}"
      fi
   fi

   NewCheckpoint="${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}.gz"
   NewCheckpointMD5="${NewCheckpoint}.md5"

   if [[ -r "$NewCheckpoint" && -r "$NewCheckpointMD5" ]]; then
      log "Warning: Skipping generation of existing checkpoint $NewCheckpoint.\\nVerified MD5 file exists: $NewCheckpointMD5."
      return
   fi

   if [[ "$DoParallelCheckpoints" -eq 1 ]]; then
      ParallelCheckpointDir="${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}"
      ParallelCheckpointOKFile="${ParallelCheckpointDir}.OK"
      CheckpointCmd="$P4DBIN -r $ROOTDIR -z -N $Threads -jdp $ParallelCheckpointDir"
   else
      CheckpointCmd="$P4DBIN -r $ROOTDIR -jd -z ${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}.gz"
   fi

   log "Running: $CheckpointCmd"

   # Curly braces capture output of 'time'.
   if { time $CheckpointCmd; } >> "$LOGFILE" 2>&1; then
      CheckpointOK=1
      if [[ "$DoParallelCheckpoints" -eq 1 ]]; then
         log "Parallel checkpoint dump completed OK. Writing: $ParallelCheckpointOKFile"
         touch "$ParallelCheckpointOKFile"
      else
         log "Checkpoint dump completed OK."
      fi
   else
      CheckpointOK=0
   fi

   if [[ -n "${SNAPSHOT_SCRIPT:-}" ]]; then
      DoSnapshot=1
      log "Calling site-specific snapshot script: $SNAPSHOT_SCRIPT"
      if "$SNAPSHOT_SCRIPT" >> "$LOGFILE" 2>&1; then
         SnapshotOK=1
      else
         SnapshotOK=0
      fi
   fi

   if [[ "$DoSnapshot" -eq 0 ]]; then
      if [[ "$CheckpointOK" -eq 1 ]]; then
         log "New checkpoint dump succeeded."
      else
         die "New checkpoint dump FAILED."
      fi
   else
      if [[ "$CheckpointOK" -eq 0 && "$SnapshotOK" -eq 0 ]]; then
         die "Both checkpoint dump and snapshot FAILED."
      elif [[ "$CheckpointOK" -eq 1 && "$SnapshotOK" -eq 0 ]]; then
         die "New checkpoint dump succeeded, but snapshot FAILED."
      elif [[ "$CheckpointOK" -eq 0 && "$SnapshotOK" -eq 1 ]]; then
         die "New checkpoint dump FAILED, but snapshot succeeded."
      else
         log "New checkpoint dump and snapshot succeeded."
      fi
   fi
}

#------------------------------------------------------------------------------
# Compare journal numbers between live and offline databases, to ensure
# they can be safely swapped out.
#------------------------------------------------------------------------------
function compare_journal_numbers () {

   local _OFFLINEJNLNUM
   _OFFLINEJNLNUM=$("$P4DBIN" -r "$OFFLINE_DB" -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") || die "Cannot get $OFFLINE_DB journal number. Abort!"
   check_journalnum "$_OFFLINEJNLNUM"

   # Get the journal number of the root database
   if [[ ! -f "$P4ROOT/db.counters" ]]; then
      die "$P4ROOT database not found. Something is seriously wrong since the server was just running a minute ago! Contact support-helix-core@perforce.com"
   fi
   local _JNLNUM
   _JNLNUM=$("$P4DBIN" -r "$P4ROOT" -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") || die "Cannot get $P4ROOT journal number. Abort!"
   check_journalnum "$_JNLNUM"

   if [[ "$_JNLNUM" -gt "$_OFFLINEJNLNUM" ]]; then
      log "$P4ROOT journal number is: $_JNLNUM"
      log "$OFFLINE_DB journal number is: $_OFFLINEJNLNUM"
      die "$OFFLINE_DB journal number is less than $P4ROOT, cannot switch."
   fi
}

#------------------------------------------------------------------------------
# Swap out live db.* database files in P4ROOT with those in offline_db.
#------------------------------------------------------------------------------
function switch_db_files () {
   local verifyCmd="$P4CBIN/verify_sdp.sh -skip crontab,excess,init,license,masterid,version -L off"

   log "Calling 'verify_sdp.sh' before swapping db.* files:\\n$verifyCmd"
   $verifyCmd >> "$LOGFILE" 2>&1 ||\
      die "Error: Cannot confirm all is well with $P4CBIN/verify_sdp.sh. Aborting"

   # Compare the Offline and Master journal numbers before switching to make
   # sure they match.
   compare_journal_numbers

   log "Switching root and offline_db links."
   [[ -d "${P4ROOT}"/save ]] || mkdir -p "${P4ROOT}"/save >> "$LOGFILE" 2>&1

   # shellcheck disable=SC2129
   echo "P4ROOT is not available during switch_db_files() processing." > "$P4ROOT/P4ROOT_not_usable.txt" 2>> "$LOGFILE"
   echo "P4ROOT is not available during switch_db_files() processing." > "$OFFLINE_DB/P4ROOT_not_usable.txt" 2>> "$LOGFILE"

   # shellcheck disable=SC2129
   rm -f "${P4ROOT}"/save/db.* >> "$LOGFILE" 2>&1
   rm -rf "${P4ROOT}"/server.locks >> "$LOGFILE" 2>&1
   mv "${P4ROOT}"/db.* "${P4ROOT}"/save/. >> "$LOGFILE" 2>&1

   if [[ -r "$P4ROOT"/license ]]; then
      mv "${P4ROOT}"/license "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1
   fi

   if [[ -n "$(ls "$P4ROOT"/license* 2>/dev/null)" ]]; then
      mv "${P4ROOT}"/license* "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1
   fi

   if [[ -r "${P4ROOT}"/rdb.lbr ]]; then
      mv "${P4ROOT}"/rdb.lbr "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1
   fi

   if [[ -n "$(ls "$P4ROOT"/state* 2>/dev/null)" ]]; then
      mv "${P4ROOT}"/state* "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1
   fi

   if [[ -r "${P4ROOT}"/server.id ]]; then
      mv "${P4ROOT}"/server.id "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1
   fi

   if [[ -n "$(ls "$P4ROOT"/server.id* 2>/dev/null)" ]]; then
      mv "${P4ROOT}"/server.id* "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1
   fi

   rm -f "${OFFLINE_DB}/offline_db_usable.txt" >> "$LOGFILE" 2>&1
   LinkOfflineDB="$(readlink "$OFFLINE_DB")"
   LinkP4ROOT="$(readlink "$P4ROOT")"
   unlink "$OFFLINE_DB"
   unlink "$P4ROOT"

   ln -s "$LinkOfflineDB" "$P4ROOT" >> "$LOGFILE" 2>&1 ||\
      die "Link of $LinkOfflineDB to $P4ROOT failed."

   ln -s "$LinkP4ROOT" "$OFFLINE_DB" >> "$LOGFILE" 2>&1 ||\
      die "Link of $LinkP4ROOT to $OFFLINE_DB failed."

   rm -f "$P4ROOT/P4ROOT_not_usable.txt" >> "$LOGFILE" 2>&1
   rm -f "$OFFLINE_DB/P4ROOT_not_usable.txt" >> "$LOGFILE" 2>&1
}

#------------------------------------------------------------------------------
# Function: trim_log_file ($LogToTrim, $MaxLines)
#
# For log files expected to be short, keep them at a max size.
#
# When they get too big, trim them from the top first, so the most recent
# output is retained.
#
# Thus function treats its processing as non-essential; most errors are
# silently ignored and output discarded. Only error output related to replacing
# the original log is retained in LOGFILE or displayed.
#------------------------------------------------------------------------------
function trim_log_file () {
   local LogToTrim="${1:-}"
   local MaxLines="${2:-5000}"
   local TmpFile=
   local Lines=

   [[ -w "$LogToTrim" ]] || return

   # Abort if MaxLines isn't numeric.
   [[ "$MaxLines" =~ ^[0-9]+$ ]] || return

   TmpFile="${LogToTrim}.trimming.$$.$RANDOM"
   Lines=$(wc -l "$LogToTrim")
   Lines=${Lines%% *}

   # Confirm Lines is a number, else just abort.
   [[ "$Lines" =~ ^[0-9]+$ ]] || return

   # If the file isn't big enough to need trimming, abort.
   [[ "$Lines" -gt "$MaxLines" ]] || return

   log "Trimming log $LogToTrim from $Lines to $MaxLines lines."

   # If the trimming fails, discard output and just return.
   if tail -"$MaxLines" "$LogToTrim" > "$TmpFile" 2>/dev/null; then
      if [[ -n "${LOGFILE:-}" ]]; then
         mv -f "$TmpFile" "$LogToTrim" >> "$LOGFILE" 2>&1
      else
         mv -f "$TmpFile" "$LogToTrim"
      fi
   else
      return
   fi
}

#------------------------------------------------------------------------------
# Rotate specified log files, and compress with gzip.
#------------------------------------------------------------------------------
function rotate_log_file () {
   local LogToRotate="${1:-}"
   local GzExt="${2:-}"
   local -i i=1
   local Datestamp=
   local RotatedLog=
   local RotatedZippedLog=

   [[ -n "$LogToRotate" ]] || return

   if [[ -n "${LOGFILE:-}" ]]; then
      pushd "$LOGS" > /dev/null 2>> "$LOGFILE" || die "Could not cd to: $LOGS"
   else
      pushd "$LOGS" > /dev/null || die "Could not cd to: $LOGS"
   fi

   Datestamp=$(date +'%Y-%m-%d_%H-%M-%S')
   RotatedLog="${LogToRotate}.${Datestamp}"

   if [[ -f "${LogToRotate}" ]]; then
      if [[ -n "${LOGFILE:-}" ]]; then
         mv -f "${LogToRotate}" "${RotatedLog}" >> "$LOGFILE" 2>&1

         if [[ -n "$GzExt" ]]; then
            RotatedZippedLog="${RotatedLog}${GzExt}"

            # If needed, move existing zipped log aside.
            if [[ -e "$RotatedZippedLog" ]]; then
               while [[ -e "${LogToRotate}.${Datestamp}.${i}${GzExt}" ]]; do
                  i+=1
               done
               log "Moving pre-existing $RotatedZippedLog aside to ${LogToRotate}.${Datestamp}.${i}${GzExt}" >> "$LOGFILE" 2>&1
               mv -f "$RotatedZippedLog" "${LogToRotate}.${Datestamp}.${i}${GzExt}" >> "$LOGFILE" 2>&1
            fi

            gzip "$RotatedLog" >> "$LOGFILE" 2>&1
         fi
      else
         mv -f "${LogToRotate}" "${RotatedLog}"

         if [[ -n "$GzExt" ]]; then
            RotatedZippedLog="${RotatedLog}${GzExt}"

            # If needed, move existing zipped log aside.
            if [[ -e "$RotatedZippedLog" ]]; then
               while [[ -e "${LogToRotate}.${Datestamp}.${i}${GzExt}" ]]; do
                  i+=1
               done
               log "Moving pre-existing $RotatedZippedLog aside to ${LogToRotate}.${Datestamp}.${i}${GzExt}"
               mv -f "$RotatedZippedLog" "${LogToRotate}.${Datestamp}.${i}${GzExt}"
            fi

            gzip "$RotatedLog"
         fi
      fi
   fi

   if [[ -n "${LOGFILE:-}" ]]; then
      popd > /dev/null 2>> "$LOGFILE" || die "Could not cd to: $OLDPWD"
   else
      popd > /dev/null || die "Could not cd to: $OLDPWD"
   fi
}

#------------------------------------------------------------------------------
# At the start of each run for live_checkpoint.sh, daily_checkpoint.sh, and
# recreate_db_checkpoint.sh, before *any* logging activity occurs, rotate the
# logs from the most recent prior run, always named "checkpoint.log" or "log".
#------------------------------------------------------------------------------
function rotate_last_run_logs () {
   # Rotate prior log file for the current script.
   rotate_log_file "$LOGFILE"

   # Rotate prior server log.
   rotate_log_file "log" ".gz"

   # Rotate prior broker log.
   rotate_log_file "p4broker.log" ".gz"

   # Rotate prior audit log.
   rotate_log_file "audit.log" ".gz"

   # Rotate prior monitor metrics log.
   rotate_log_file "monitor_metrics.log" ".gz"
   
   # Rotate prior p4triggers log.
   rotate_log_file "p4triggers.log" ".gz"

}

#------------------------------------------------------------------------------
# Remove log files matching a specified name prefix, preserving a specified
# number of the recent logs.
#------------------------------------------------------------------------------
function remove_log_files () {
   REMOVE_LOGNAME=$1
   KEEPNUM=$2

   # shellcheck disable=SC2012
   for I_LOGFILE in $(ls -t "${REMOVE_LOGNAME:?}"* 2>/dev/null | $AWK "NR > $KEEPNUM"); do
      log "rm -f $I_LOGFILE"
      rm -f "$I_LOGFILE"
   done
}

#------------------------------------------------------------------------------
# Remove old logs.
#------------------------------------------------------------------------------
function remove_old_logs () {
   # Remove old Checkpoint Logs
   # Use KEEPJNLS rather than KEEPLOGS, so we keep the same number
   # of checkpoint logs as we keep checkpoints.
   pushd "$LOGS" > /dev/null 2>> "$LOGFILE" || die "Could not cd to: $LOGS"

   if [[ "$KEEPJNLS" -eq 0 ]]; then
      log "Skipping cleanup of old checkpoint logs because KEEPJNLS is set to 0."
   else
      log "Deleting old checkpoint logs.  Keeping latest $KEEPJNLS, per KEEPJNLS setting in p4_vars."
      remove_log_files "checkpoint.log" "$KEEPJNLS"
   fi

   if [[ "$KEEPLOGS" -eq 0 ]]; then
      log "Skipping cleanup of old server logs because KEEPLOGS is set to 0."
   else
      log "Deleting old server logs.  Keeping latest $KEEPLOGS, per KEEPLOGS setting in p4_vars."
      remove_log_files "log" "$KEEPLOGS"
      remove_log_files "p4broker.log" "$KEEPLOGS"
      remove_log_files "broker_rotate.log" "$KEEPLOGS"
      remove_log_files "audit.log" "$KEEPLOGS"
      remove_log_files "sync_replica.log" "$KEEPLOGS"
      remove_log_files "replica_status.log" "$KEEPLOGS"
      remove_log_files "replica_cleanup.log" "$KEEPLOGS"
      remove_log_files "request_checkpoint.log" "$KEEPLOGS"
      remove_log_files "recreate_offline_db.log" "$KEEPLOGS"
      remove_log_files "edge_shelf_replicate.log" "$KEEPLOGS"
      remove_log_files "upgrade.log" "$KEEPLOGS"
      remove_log_files "p4login" "$KEEPLOGS"
      remove_log_files "p4verify.log" "$KEEPLOGS"
      remove_log_files "journal_watch.log" "$KEEPLOGS"
      remove_log_files "refresh_P4ROOT_from_offline_db.log" "$KEEPLOGS"
      remove_log_files "purge_revisions.log" "$KEEPLOGS"
      remove_log_files "monitor_metrics.log" "$KEEPLOGS"
   fi
   popd > /dev/null 2>>"$LOGFILE" || die "Could not cd to: $OLDPWD"
}

#------------------------------------------------------------------------------
# Set the SDP Checkpoint counter to indicate last successful SDP checkpoint
# operation. For standby servers, set the SDP Checkpoint counter on the master.
#------------------------------------------------------------------------------
function set_counter() {
   "$P4CBIN/p4login"

   if [[ "$EDGESERVER" -eq 1 || "$STANDBYSERVER" -eq 1 ]]; then
      "$P4BIN" -u "$P4USER" -p "$P4MASTERPORT" counter "LastSDPCheckpoint.$SERVERID" "$(date +'%s (%Y/%m/%d %H:%M:%S %z %Z)')" > /dev/null
   else
      "$P4BIN" -u "$P4USER" -p "$P4PORT" counter "LastSDPCheckpoint.$SERVERID" "$(date +'%s (%Y/%m/%d %H:%M:%S %z %Z)')" > /dev/null
   fi
}
# Change User Description Committed
#80 31566 C. Thomas Tyler Released SDP 2024.2.31564 (2025/05/14).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#79 31204 Will Kreitzmann Released SDP 2024.2.31193 (2025/01/17).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#78 31077 C. Thomas Tyler Released SDP 2024.2.31075 (2024/12/20).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#77 30915 C. Thomas Tyler Released SDP 2024.1.30913 (2024/11/20).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#76 30388 C. Thomas Tyler Released SDP 2024.1.30385 (2024/06/11).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#75 30297 C. Thomas Tyler Released SDP 2023.2.30295 (2024/05/08).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#74 30043 C. Thomas Tyler Released SDP 2023.2.30041 (2023/12/22).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#73 29954 C. Thomas Tyler Released SDP 2023.1.29949 (2023/12/01).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#72 29891 C. Thomas Tyler Released SDP 2023.1.29699 (2023/07/11).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#71 29612 C. Thomas Tyler Released SDP 2023.1.29610 (2023/05/25).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#70 29443 C. Thomas Tyler Released SDP 2022.2.29441 (2023/02/27).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#69 29401 C. Thomas Tyler Released SDP 2022.2.29399 (2023/02/06).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#68 29252 C. Thomas Tyler Released SDP 2022.2.29250 (2022/12/08).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#67 29143 C. Thomas Tyler Released SDP 2022.1.29141 (2022/10/29).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#66 28858 C. Thomas Tyler Released SDP 2022.1.28855 (2022/05/27).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#65 28651 C. Thomas Tyler Released SDP 2021.2.28649 (2022/03/03).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#64 28412 C. Thomas Tyler Released SDP 2021.2.28410 (2021/11/24).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#63 28240 C. Thomas Tyler Released SDP 2021.1.28238 (2021/11/12).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#62 27761 C. Thomas Tyler Released SDP 2020.1.27759 (2021/05/07).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#61 27463 C. Thomas Tyler Released SDP 2020.1.27457 (2021/02/17).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#60 27331 C. Thomas Tyler Released SDP 2020.1.27325 (2021/01/29).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#59 26496 C. Thomas Tyler Released SDP 2019.3.26494 (2020/04/23).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#58 26480 C. Thomas Tyler Released SDP 2019.3.26478 (2020/04/12).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#57 26475 C. Thomas Tyler Released SDP 2019.3.26472 (2020/04/10).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#56 26470 C. Thomas Tyler Released SDP 2019.3.26468 (2020/04/10).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#55 26411 C. Thomas Tyler Released SDP 2019.3.26407 (2020/03/28).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#54 26403 C. Thomas Tyler Released SDP 2019.3.26400 (2020/03/28).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#53 26246 C. Thomas Tyler Released SDP 2019.3.26239 (2020/01/08).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#52 26161 C. Thomas Tyler Released SDP 2019.3.26159 (2019/11/06).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#51 25596 C. Thomas Tyler Released SDP 2019.2.25594 (2019/05/02).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#50 25380 C. Thomas Tyler Released SDP 2019.1.25374 (2019/03/21).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#49 25279 C. Thomas Tyler Released SDP 2019.1.25276 (2019/03/06).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#48 25245 C. Thomas Tyler Released SDP 2019.1.25238 (2019/03/02).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#47 23331 C. Thomas Tyler Released SDP 2017.4.23329 (2017/12/05).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#46 23044 C. Thomas Tyler Released SDP 2017.3.23041 (2017/10/24).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#45 23006 C. Thomas Tyler Released SDP 2017.3.23003 (2017/10/19).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#44 22685 Russell C. Jackson (Rusty) Update main with current changes from dev.
#43 22185 C. Thomas Tyler Released SDP 2017.2.22177 (2017/05/17).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#42 21723 C. Thomas Tyler Released SDP 2017.1.21720 (2017/02/17).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#41 21338 C. Thomas Tyler Released SDP 2016.2.21328 (2016/12/16).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#40 21193 Russell C. Jackson (Rusty) Update main from dev.
#39 20974 C. Thomas Tyler Released SDP 2016.2.20972 (2016/11/01).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#38 20858 C. Thomas Tyler Released SDP 2016.2.20856 (2016/10/04).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#37 20767 C. Thomas Tyler Released SDP 2016.2.20755 (2016/09/29).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#36 20353 C. Thomas Tyler Released SDP 2016.1.20348.
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev',
with selective removal of changes related to work-in-progress
changes.
#35 19898 C. Thomas Tyler Released SDP/MultiArch/2016.1/19888 (2016/07/07).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#34 19835 C. Thomas Tyler Released Rev.
SDP/MultiArch/2016.1/19768 (2016/06/24).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#33 19694 C. Thomas Tyler Released SDP/MultiArch/2016.1/19661 (2016/06/08).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#32 19414 C. Thomas Tyler Released SDP/MultiArch/2016.1/19410 (2016/05/17).
#31 18961 C. Thomas Tyler Released: SDP/MultiArch/2016.1/18958 (2016/04/08).
#30 18619 Russell C. Jackson (Rusty) Updating main with current changes.
#29 18530 Russell C. Jackson (Rusty) Update main from dev.
#28 16155 Russell C. Jackson (Rusty) Removed check code that probably hasn't ever worked.
 Deleted mirror_ldap* since that functionality is built into the server now.
#27 15856 C. Thomas Tyler Replaced the big license comment block with a shortened
form referencing the LICENSE file included with the SDP
package, and also by the URL for the license file in
The Workshop.
#26 15784 Russell C. Jackson (Rusty) Added missing bracket.
#25 15780 C. Thomas Tyler Per Robert:  Style police causing problems :)
Fixed!
#24 15777 C. Thomas Tyler No functional changes.

Style Policing only on bash scripts only.

Normalized indentation and line breaks, removed offending tabs, and general whitespace
usage.
#23 15609 C. Thomas Tyler Pushing SDP 2015.1.15607 (2015/09/02).
#22 15197 Russell C. Jackson (Rusty) Corrected versions from testing.
#21 15193 Russell C. Jackson (Rusty) Added semaphore file to indicate state of the offline database and added check into
 the backup process to fail if the state of the offline db is not good.
#20 15190 Russell C. Jackson (Rusty) Added a semaphore file to prevent the checkpoint process from running if another one
 hasn't finished.

 Added a check to make sure the journal number is numeric.
#19 13928 dsp Set lastSDPCounter after a successfull SDP checkpoint

       p4 admin checkpoint sets lastCheckpointAction, which is useful for monitoring, in particular
       when checkpoint age should be observed from the outside through p4. However the SDP is using
       p4d directly to create checkpoints and will not set checkpoints. In order to distinguish human
       actions from the SDP cronjobs, set a new counter lastSDPCounter in a similar format.
#18 13908 C. Thomas Tyler Pushing SDP 2015.1.13906.
#17 12171 Russell C. Jackson (Rusty) Merge in changes to remove the need for p4master_run.
#16 11950 Russell C. Jackson (Rusty) Made die function record ERROR!!!
$HOSTNAME and $P4SERVER in subject.

 Cleaned up message passed to die command and corrected a typo.
#15 11929 Russell C. Jackson (Rusty) Updated die function to just pass parameter to mail_log_file instead of echo.
#14 11919 Russell C. Jackson (Rusty) Added a SERVERID variable to p4_vars and updated backup_functions to use it.

 Changed the location and the names of the config files so that they could
 live in /p4/common/config (You're welcome Tom). The files names are:

 p4_$INSTANCE.vars
 p4_$INSTANCE.p4review.cfg

 p4_$INSTANCE.vars will now set P4REPLICA to FALSE if SERVERID matches
 MASTERNAME, otherwise it is TRUE.

 This change means that a user must change server.id now in order to change
 the role of the server rather than changing the instance vars file. This
 makes more sense to a user that is reading the admin guide about server.id
 rather than overwriting the file based on a setting that isn't in the admin guide.

 Change mkdirs to reflect all of the above changes.
#13 11908 adrian_waters Use set -u to trap unbounded variables
#12 11886 Russell C. Jackson (Rusty) Changed $prog to $0 so that we don't have to set prog in the calling functions.
#11 11766 Robert Cowham Missed a reference to $MAIL in @11758
Tweaked run_tests.sh to output more error messages on failure. Though
this still doesn't show output of individual failed commands.
#10 11758 Russell C. Jackson (Rusty) Change MAIL variable to SDPMAIL to avoid conflicts with customer variables.

 Changed sdp_sync.sh to use get_mail_opts from backup_functions to avoid
 duplicate functions.
#9 11730 Russell C. Jackson (Rusty) Moved P4SERVER variable to p4_vars so that all scripts can use it properly.
 replica_status.sh referenced it, but it wasn't working since it was only in
 backup_functions.sh
#8 11710 Russell C. Jackson (Rusty) Changed die function to call new email function.

Added su to OSUSER functionality to p4master_run to avoid problems with
 people running scripts manually as root by mistake.
#7 11707 Robert Cowham Refactored sending of mail to a common function.
Make the setting of "MAILFROM" work for Ubuntu (GNU Mailutils) as well as CentOS
#6 11570 Russell C. Jackson (Rusty) Brought in changes from Mark Foundry to add -S $MAILFROM to mail commands.

 Changed sync_replica.sh and weekly_sync_replica.sh to use $LOGFILE for consistency.

 Added mail command to both files as well.
#5 11540 Russell C. Jackson (Rusty) Converted to unix format.
#4 11534 Russell C. Jackson (Rusty) Added -f to -jr to cover offline obliterates where the entries are already removed from the offline database.
#3 11524 Russell C. Jackson (Rusty) Released updated version of the SDP from Dev.
#2 11130 Robert Cowham Check for the existence of offline database and log error message if not found.
#1 10148 C. Thomas Tyler Promoted the Perforce Server Deployment Package to The Workshop.