load_checkpoint.sh #9

#!/bin/bash
#------------------------------------------------------------------------------
### WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
### THIS SCRIPT SHOULD BE USED WITH GREAT CAUTION.
#------------------------------------------------------------------------------
set -u

# This script loads a specified checkpoint simultaneously into /p4/N/root
# and /p4/N/offline_db.

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME="${P4U_HOME:-/p4/common/bin}"
export P4U_LIB="${P4U_LIB:-/p4/common/lib}"
export P4U_ENV="$P4U_LIB/p4u_env.sh"
export P4U_LOG=Unset
export VERBOSITY=${VERBOSITY:-5}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH="$P4U_HOME:$PATH:~/bin:."
export P4CONFIG="${P4CONFIG:-.p4config}"

[[ -r "$P4U_ENV" ]] || {
   echo -e "\\nError: Cannot load environment from: $P4U_ENV\\n\\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   # shellcheck disable=SC1090
   source "$bash_lib" ||\
      { echo -e "\\nFATAL: Failed to load bash lib [$bash_lib]. Aborting.\\n"; exit 1; }
done

declare ThisScript=${0##*/}
declare CmdLine="$0 $*"
declare ThisUser=
declare Version=2.4.3
declare -i SilentMode=0
declare -i GenSSLCertsIfNeeded=1
declare -i StartWithoutLicenseFile=0
declare -i i=0
declare Checkpoint=Unset
declare SDPInstance=
declare SDPInstanceVarsFile=
declare OfflineDB=
declare OfflineDBUsableFile=
declare CaseMode=
declare CaseFlag=
declare CompressFlag=
declare P4BrokerCfg=
declare P4DPids=
declare SetServerID=
declare ThisHost=${HOSTNAME%%.*}
declare P4DInitScript=
declare P4DSystemdServiceFile=
declare P4BrokerInitScript=
declare P4BrokerSystemdServiceFile=
declare Datestamp=
declare -i JournalCount=0
declare -a Journals

#==============================================================================
# Local Functions

function user_confirmation_and_warning () {
   local interactive=${1:-1}
   local input=""
   msg "\\n
==============================================================================
WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING\\n

This script will remove files in P4ROOT and the offline_db folder,
including db.* files, state* files, and others.

It is STRONGLY recommended that these files be preserved first unless
it is absolutely certain they will never potentially be useful in any
recovery scenario.

This is being run on host $HOSTNAME at $(date).

WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
==============================================================================
\\n\\n"

   if [[ $interactive -eq 0 ]]; then
      msg "Bypassing interactive confirmation due to -y on command line."
      return 0
   fi

   while [[ -z "$input" ]]; do
      echo -e -n "\\nEnter Y to proceed, N to stop [y/Y/n/N]:  "
      read -r -e input
      if [[ "${input^^}" == "Y" ]]; then
         continue
      elif [[ "${input^^}" == "N" ]]; then
         msg "Confirmation to proceed not received."
         return 1
      else
         input=
      fi
   done

   return 0
}


#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   vvmsg "$ThisScript: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   stoplog

   # Don't litter.
   cleanTrash

   # With the trap removed, exit.
   exit "$OverallReturnStatus"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ "$errorMessage" != Unset ]]; then
      echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n"
   fi

   echo "USAGE for $ThisScript v$Version:

$ThisScript <checkpoint> [<jnl.1> [<jnl.2> ...]] [-i <instance>] [-s <ServerID>] [-c] [-l] [-r] [-b] [-y] [-L <log>] [-si] [-v<n>] [-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	This script loads a specified checkpoint into /p4/N/root and /p4/N/offline_db,
	where 'N' is the SDP instance name.

	At the start of processing, preflight checks are done. Preflight checks
	include:
	* The specified checkpoint and corresponding *.md5 file must exist.
	* The \$P4ROOT/server.id file must exist, unless '-s' is specified.
	* The \$P4ROOT/license file must exist, unless '-l' is specified.
	* Basic SDP structure and key files must exist.

  	If the preflight passes, the p4d_N service is shutdown, and also the
	p4broker_N service is shutdown if configured.

	Next, the specified checkpoint is loaded. Upon completion, the Helix Core
	server process, p4d_N, is started.

	If the server to be started is a replica, the serviceUser configured for
	the replica is logged into the P4TARGET server.  Any needed 'p4 trust' and
	'p4 login' commands are done to enable replication.

	Note that this part of the processing will fail if the correct super user
	password is not stored in the standard SDP password file,

	/p4/common/config/.p4passwd.p4_N.admin

	After starting the server, a local 'p4 trust' is done if needed, and then
	a 'p4login -service -v' and 'p4login -v'.

	By default, the p4d_N service is started, but the p4broker_N service is not.
	Specify '-b' to restart both services.

ARGUMENTS AND OPTIONS:
 <checkpoint>
	Specify the path to the checkpoint file to load. Exactly one checkpoint
	must be specified.

	The file may be a compressed or uncompressed checkpoint, and it may be a case
	sensitive or case-insensitive checkpoint.  The checkpoint file must have a
	corresponding *.md5 checksum file in the same directory, with one of two name
	variations: If the checkpoint file is /somewhere/foo.gz, the checksum file may
	be named /somewhere/foo.gz.md5 or /somewhere/foo.md5.

<jnl.1> [<jnl.2> ...]
	Specify the path to the one or more journal files to replay after the
	checkpoint, in the correct sequence order.
	
 -i <instance>
	Specify the SDP instance. This can be omitted if SDP_INSTANCE is already
	defined.

 -s <ServerID>
	Specify the ServerID.  This value is written into \$P4ROOT/server.id file.

	If no \$P4ROOT/server.id file exists, this flag is required.

	If the \$P4ROOT/server.id file exists, this argument is not needed. If this
	'-s <ServerID>' is given and a \$P4ROOT/server.id file exists, the value in
	the file must match the value specified with this argument.

 -c	Specify that SSL certificates are required, and not to be generated with
	'p4d_N -Gc'.

	By default, if '-c' is not supplied and SSL certs are not available, certs
	are generated automatically with 'p4d_N -Gc'.

 -l	Specify that the server is to start without a license file. By default, if
	there is no \$P4ROOT/license file, this script will abort.  Note that if '-l'
	is specified and a license file is actually needed, the attempt this script makes
	to start the server after loading the checkpoint will fail.

 -r	Specify '-r' to replay only to P4ROOT.  By default, this script replays both
	to P4ROOT and the offline_db.

 -b	Specify '-b' to start the a p4broker process (if configured).  By default
	the p4d process is started after loading the checkpoint, but the p4broker
	process is not. This can be useful to ensure the human administrator has
	an opportunity to do sanity checks before enabling the broker to allow
	access by end users (if the broker is deployed for this usage).

 -y	Use the '-y' flag to bypass an interactive warning and confirmation
	prompt.

 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).  The default is 5.

 -L <log>
	Specify the path to a log file.  By default, all output (stdout and stderr)
	goes to:
	/p4/<instance>/logs/${ThisScript%.sh}.<timestamp>.log

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

 -si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.
      
 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Display version info for this script and its libraries.

EXAMPLES:
	EXAMPLE 1: Non-interactive usage (bash syntax) to load a checkpoint:

	nohup $P4CBIN/load_checkpoint.sh /p4/1/checkpoints/p4_1.ckp.4025.gz -i 1 -y < /dev/null > /dev/null 2>&1 &

	Then, monitor with:
	tail -f \$(ls -t \$LOGS/load_checkpoint.*.log|head -1)\\n

	EXAMPLE 2: Non-interactive usage (bash syntax) to loading a checkpoint and subsequent journals:

	nohup $P4CBIN/load_checkpoint.sh /p4/1/checkpoints/p4_1.ckp.4025.gz /p4/1/checkpoints/p4_1.jnl.4025 /p4/1/checkpoints/p4_1.jnl.4026 -i 1 -y < /dev/null > /dev/null 2>&1 &

	Then, monitor with:
	tail -f \$(ls -t \$LOGS/load_checkpoint.*.log|head -1)\\n

	EXAMPLE 3: Interactive usage to load a checkpoint with no license file.

	$P4CBIN/load_checkpoint.sh /p4/1/checkpoints/p4_1.ckp.4025.gz -i 1 -l

	With interactive usage, logging still occurs; all output to the screen is
	captured.

	Note that non-interactive usage with nohup is recommended for checkpoints
	with a long replay duration, to make operation more reliable in event of a
	terminal disconnect.  Alternately, running interactively in a 'screen'
	session (if 'screen' is available) provides similar protection against
	terminal disconnects.
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i LoadOfflineDB=1
declare -i StartBrokerWhenDone=0
declare -i Interactive=1
declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-i) SDPInstance="$2"; shiftArgs=1;;
      (-s) SetServerID="$2"; shiftArgs=1;;
      (-c) GenSSLCertsIfNeeded=0;;
      (-l) StartWithoutLicenseFile=1;;
      (-r) LoadOfflineDB=0;;
      (-b) StartBrokerWhenDone=1;;
      (-y) Interactive=0;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-L) export P4U_LOG="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown flag ($1).";;
      (*)
         if [[ "$Checkpoint" == Unset ]]; then
            Checkpoint="$1"
         else
            Journals[$JournalCount]="$1"
            JournalCount+=1
         fi
      ;;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "${P4U_LOG:-Unset}" == Unset ]] && \
   P4U_LOG="${LOGS:-/tmp}/${ThisScript%.sh}.$(date +'%Y%m%d-%H%M%S').log"

if [[ -z "$SDPInstance" ]]; then
   if [[ "${SDP_INSTANCE:-Unset}" != Unset ]]; then
      SDPInstance="$SDP_INSTANCE"
   else
      usage -h "SDP Instance parameter is missing and SDP_INSTANCE is undefined." 
   fi
fi

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i OverallReturnStatus=0

# Initialize logging.
touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

# Redirect stdout and stderr to a log file.
if [[ "$SilentMode" -eq 0 ]]; then
   exec > >(tee "${P4U_LOG}")
   exec 2>&1
else
   exec >"${P4U_LOG}"
   exec 2>&1
fi

initlog

ThisUser=$(id -n -u)
msg "Started $ThisScript v$Version as $ThisUser@$ThisHost on $(date) as:\\n$CmdLine\\n"

# shellcheck disable=SC2153
SDPInstanceVarsFile="/p4/common/config/p4_${SDPInstance}.vars"

[[ "$Checkpoint" == Unset ]] && usageError "Missing checkpoint parameter."
[[ -r "$SDPInstanceVarsFile" ]] ||\
   bail "Missing SDP instance vars file. Typo in instance name? Expected file: $SDPInstanceVarsFile"
[[ -r "$Checkpoint" ]] ||\
   bail "Specified checkpoint file does not exist: $Checkpoint"

# shellcheck disable=SC1091
source /p4/common/bin/p4_vars "$SDPInstance"
# shellcheck disable=SC1091
source /p4/common/bin/ps_functions.sh ||\
   bail "Failed to load ps_functions.sh."
#shellcheck disable=SC1091
source /p4/common/bin/edge_vars ||\
   bail "Failed to load edge_vars."
declare CheckpointMD5=

# Set or check ServerID value.
if [[ -n "$SetServerID" ]]; then
   if [[ -f "$P4ROOT/server.id" ]]; then
      if grep -q "$SetServerID" "$P4ROOT/server.id"; then
         msg "Verified: $P4ROOT/server.id value matches set value: $SetServerID"
      else
         bail "Existing $P4ROOT/server.id file is not set to $SetServerID, but to $(cat "$P4ROOT"/server.id). Aborting."
      fi
   else
      msg "Setting ServerID $SetServerID in $P4ROOT/server.id"
      echo "$SetServerID" > "$P4ROOT/server.id" ||\
         bail "Failed to write $P4ROOT/server.id. Aborting"
   fi
fi

# shellcheck disable=SC2153
P4DInitScript="$P4HOME/bin/p4d_${SDP_INSTANCE}_init"
P4DSystemdServiceFile="/etc/systemd/system/p4d_${SDP_INSTANCE}.service"
P4BrokerInitScript="$P4HOME/bin/p4broker_${SDP_INSTANCE}_init"
P4BrokerSystemdServiceFile="/etc/systemd/system/p4broker_${SDP_INSTANCE}.service"

user_confirmation_and_warning "$Interactive" ||\
   bail "User confirmation not verified. Aborting."

cd "$P4ROOT" || bail "Could not cd to P4ROOT [$P4ROOT]."

[[ "$PWD" == "/p4/${SDPInstance}/root" ]] || bail "Unexpected P4ROOT value of: $P4ROOT"

CheckpointMD5="${Checkpoint}.md5"
if [[ -r "$CheckpointMD5" ]]; then
   msg "Verified: MD5 file for checkpoint exists: $CheckpointMD5"
else
   CheckpointMD5="${Checkpoint/.gz/.md5}"
   if [[ -r "$CheckpointMD5" ]]; then
      msg "Verified: MD5 file for checkpoint exists: $CheckpointMD5"
   else
      bail "Could not find MD5 file for checkpoint. Tried ${Checkpoint}.md5 and ${Checkpoint/.gz/.md5}"
   fi
fi

msg "Checking for license file."
if [[ ! -r "$P4ROOT/license" ]]; then
   if [[ "$StartWithoutLicenseFile" -eq 1 ]]; then
      warnmsg "No $P4ROOT/license file found. Continuing due to '-l'."
   else
      bail "No $P4ROOT/license file found. If none needed, specify '-l' and try again."
   fi
fi

if [[ "$P4PORT" == "ssl:"* && ! -r "$P4SSLDIR/certificate.txt" ]]; then
   if [[ "$GenSSLCertsIfNeeded" -eq 1 ]]; then
      if [[ ! -d "$P4SSLDIR" ]]; then
         mkdir -p "$P4SSLDIR" ||\
            bail "Failed to create P4SSLDIR [$P4SSLDIR]."
      fi

      chmod 700 "$P4SSLDIR" ||\
         bail "Failed to do 'chmod 700 $P4SSLDIR'."

      "$P4DBIN" -Gc ||\
         bail "There was a problem generating SSL certificates with '$P4DBIN -Gc'."
   else
      bail "The P4PORT has SSL is enabled, but there is no certificate file\\n[$P4SSLDIR/certificate.txt], and the '-c' flag as specified requiring SSL certificates to be in place.\\n"
   fi
fi

[[ -x "$P4DInitScript" ]] || bail "Aborting: P4D Init script is missing or not executable: $P4DInitScript"

msg "Checking to see if broker is configured."
if [[ -r "/p4/common/config/p4_${SDPInstance}.broker.${ThisHost}.cfg" ]]; then
   P4BrokerCfg="/p4/common/config/p4_${SDPInstance}.broker.${ThisHost}.cfg"
   msg "Broker configured with: $P4BrokerCfg"
elif [[ -r "/p4/common/config/p4_${SDPInstance}.broker.cfg" ]]; then
   P4BrokerCfg="/p4/common/config/p4_${SDPInstance}.broker.cfg"
   msg "Broker configured with: $P4BrokerCfg"
else
   msg "No broker config detected."
fi

if [[ -x "$P4BrokerInitScript" && -r "$P4BrokerCfg" ]]; then
   msg "Checking p4broker status."
   if "$P4BrokerInitScript" status; then
      msg "Shutting down p4broker."
      if [[ -r "$P4BrokerSystemdServiceFile" ]]; then
         sudo systemctl stop "${P4BROKERBIN##*/}" ||\
            bail "Failed to execute: sudo systemctl stop ${P4BROKERBIN##*/}"
      else
         "$P4BrokerInitScript" stop
      fi
   else
      msg "Verified: p4broker is down."
   fi
fi

msg "Checking p4d status."
P4DPids=$(get_pids "$P4DBIN")

if [[ -n "$P4DPids" ]]; then
   msg "Shutting down p4d."
   if [[ -r "$P4DSystemdServiceFile" ]]; then
      sudo systemctl stop "${P4DBIN##*/}" ||\
         bail "Failed to execute: sudo systemctl stop ${P4DBIN##*/}"
   else
      "$P4DInitScript" stop
   fi
else
   msg "Verified: No processes are running for: $P4DBIN"
fi

echo "P4ROOT is not available during load_checkpoint.sh processing." > "$P4ROOT/P4ROOT_not_usable.txt"

# Remove db.* and state* files, but not server.id or license files.
Cmd="/bin/rm -f db.* state*"
msg "Cleaning up databases with this command in $PWD:\\n$Cmd"
$Cmd || bail "Failed to clean up old databases and state files in $P4ROOT"

# Move P4LOG aside if it exists.
if [[ -r "$P4LOG" ]]; then
   Cmd="mv $P4LOG $P4LOG.moved.$(date +'%Y%m%d-%H%M')"
   msg "Moving P4LOG aside with this command:\\n$Cmd"
   $Cmd || bail "Failed to move P4LOG aside."
fi

# Move P4JOURNAL aside if it exists.
if [[ -r "$P4JOURNAL" ]]; then
   Cmd="mv $P4JOURNAL $P4JOURNAL.moved.$(date +'%Y%m%d-%H%M')"
   msg "Moving P4JOURNAL aside with this command:\\n$Cmd"
   $Cmd || bail "Failed to move P4JOURNAL aside."
fi

# Move journal.NNN files (for standby replicas) aside if they exist.
cd "$LOGS" || bail "Could not cd to: $LOGS"

Datestamp=$(date +'%Y%m%d-%H%M')
# shellcheck disable=SC2045
for f in $(ls journal.* 2>/dev/null); do
   [[ "$f" =~ ^journal.[0-9]+$ ]] || continue
   Cmd="mv $f $f.moved.$Datestamp"
   msg "Moving file $f aside with this command:\\n$Cmd"
   $Cmd || bail "Failed to move file $f aside."
done
cd - > /dev/null || bail "Could not cd to: $OLDPWD"

# For checkpoint replay operations, determine whether case sensitivity
# flag is needed, and whether '-z' is needed.
if [[ "$Checkpoint" == *".gz" ]]; then
   if [[ "$(uname -s)" == "Darwin" ]]; then
      CaseMode=$(zcat < "$Checkpoint" | head -1 | grep -E '^@nx@ (0|2) ' | cut -d ' ' -f5)
   else
      CaseMode=$(zcat "$Checkpoint" | head -1 | grep -E '^@nx@ (0|2) ' | cut -d ' ' -f5)
   fi

   if [[ "$CaseMode" == "2" ]]; then
      CaseFlag='-C1'
      msg "Case-insensitive checkpoint detected."
   else
      msg "Case-sensitive checkpoint detected."
   fi

   # In P4D 2018.1, the '-z' flag should be dropped, as the need for
   # compression/decompression is determined by P4D when replying,
   # and thus '-z' should not be used (though it is supported by P4D
   # for backward compatibility, and we use it here if using an older
   # server.
   # shellcheck disable=SC2072
   [[ "$P4D_VERSION" < "2018.1" ]] && CompressFlag='-z'
else
   CaseMode=$(head -1 "$Checkpoint"| grep -E '^@nx@ (0|2) ' | cut -d ' ' -f5)
   if [[ "$CaseMode" == "2" ]]; then
      CaseFlag='-C1'
      msg "Case-insensitive checkpoint detected."
   else
      msg "Case-sensitive checkpoint detected."
   fi
fi

Cmd="$P4DBIN -r $P4ROOT $CompressFlag $CaseFlag -jr $Checkpoint"

msg "Replaying checkpoint to P4ROOT with this command:\\n$Cmd"
$Cmd || bail "Checkpoint replay to P4ROOT failed."

if [[ "$JournalCount" -gt 0 ]]; then
   for jnl in "${Journals[@]}"; do
      Cmd="$P4DBIN -r $P4ROOT -f -jr $jnl"
      msg "Replaying journal to P4ROOT with this command:\\n$Cmd"
      $Cmd || bail "Replay of journal to P4ROOT failed for journal: $jnl"
   done
fi

rm -f "$P4ROOT/P4ROOT_not_usable.txt"

Cmd="$P4DBIN -r $P4ROOT -t localhost:0.0.0.0 -xu"
msg "Ensuring databases are upgraded with this command:\\n$Cmd"
$Cmd || bail "Database upgrade in P4ROOT failed."

if [[ "$LoadOfflineDB" -eq 1 ]]; then
   OfflineDB=${P4ROOT/root/offline_db}
   OfflineDBUsableFile="$OfflineDB/offline_db_usable.txt"

   cd "$OfflineDB" || bail "Could not cd to offline_db dir [$OfflineDB]."

   [[ "$PWD" == "/p4/${SDPInstance}/offline_db" ]] || bail "Unexpected OfflineDB value of: $OfflineDB"

   # Remove db.* and state* files, but not server.id or license files.
   Cmd="/bin/rm -f db.* state* $OfflineDBUsableFile"
   msg "Cleaning up databases and offline_db_usable.txt file with this command in $PWD:\\n$Cmd"
   $Cmd || bail "Failed to clean up old databases and state files in $OfflineDB"

   Cmd="$P4DBIN -r $OfflineDB $CompressFlag $CaseFlag -jr $Checkpoint"

   msg "Replaying checkpoint to offline_db with this command:\\n$Cmd"
   $Cmd || bail "Checkpoint replay to offline_db failed."

   Cmd="$P4DBIN -r $OfflineDB -t localhost:0.0.0.0 -xu"
   msg "Ensuring databases in offline_db are upgraded with this command:\\n$Cmd"
   $Cmd || bail "Database upgrade in offline_db failed."

   echo "Database restored successfully." > "$OfflineDBUsableFile" ||\
      bail "Failed to write this file: $OfflineDBUsableFile"
fi

msg "Starting p4d."
if [[ -r "$P4DSystemdServiceFile" ]]; then
   sudo systemctl start "${P4DBIN##*/}" ||\
      bail "Failed to execute: sudo systemctl start ${P4DBIN##*/}"
else
   "$P4DInitScript" start
fi

# Delay a bit to give p4d time to start.
i=0; while [[ "$i" -lt 10 ]]; do
   sleep 2
   if p4 info -s > /dev/null 2>&1; then
      break
   fi
   i+=1
done

if p4 info -s > /dev/null 2>&1; then
   msg "The P4D Server started OK."
else
   msg "Warning: The P4D Server did not start."
   OverallReturnStatus=1
fi

msg "Logging in."
msg "Running: $P4CBIN/p4login -v"
"$P4CBIN/p4login" -v
msg "Running: $P4CBIN/p4login -p $P4MASTERPORT -v"
"$P4CBIN/p4login" -p "$P4MASTERPORT" -v

# Now that we have loaded a checkpoint, re-load the shell environment to
# ensure that P4REPLICA is set correctly, as some db.config settings are
# determined dynamically.

# shellcheck disable=SC1091
source /p4/common/bin/p4_vars "$SDPInstance"
if [[ "$P4REPLICA" == TRUE ]]; then
   msg "Logging in service user."
   msg "Running: $P4CBIN/p4login -v -service"
   "$P4CBIN/p4login" -v -service

   sleep 3
   Cmd="$P4BIN -s pull -lj"
   msg "Checking replication status with this command:\\n$Cmd"
   $Cmd || bail "Replica is not replicating properly."
fi

msg "\\nAll processing completed successfully.\\n"

if [[ -x "$P4BrokerInitScript" && -r "$P4BrokerCfg" ]]; then
   if [[ "$StartBrokerWhenDone" -eq 1 ]]; then
      if [[ -r "$P4BrokerSystemdServiceFile" ]]; then
         sudo systemctl start "${P4BROKERBIN##*/}" ||\
            bail "Failed to execute: sudo systemctl start ${P4BROKERBIN##*/}"
      else
         "$P4BrokerInitScript" start
      fi

      sleep 1
      msg "Running: $P4CBIN/p4login -p $P4BROKERPORT -v"
      "$P4CBIN/p4login" -p "$P4BROKERPORT" -v
   else
      if [[ -r "$P4BrokerSystemdServiceFile" ]]; then
         msg "NOT starting broker. Start it manually when ready with:\\n\\tsudo systemctl start ${P4BROKERBIN##*/}\\n"
      else
         msg "NOT starting broker. Start it manually when ready with:\\n\\t$P4BrokerInitScript start\\n"
      fi
   fi
fi

msg "Checkpoint load processing took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

exit "$OverallReturnStatus"