load_checkpoint.sh #1

#!/bin/bash
#------------------------------------------------------------------------------
### WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
### THIS SCRIPT SHOULD BE USED WITH GREAT CAUTION.
#------------------------------------------------------------------------------
set -u

# This script loads a specified checkpoint simultaneously into  /p4/N/root
# and /p4/N/offline_db.

function msg () { echo -e "$*"; }
function bail () { msg "${1:-Unknown Error}\n"; exit ${2:-1}; }
function usage () {
   msg "Sample Usage (bash syntax):
\tcd /p4/common/bin
\tsource ./p4_vars 1
\tnohup ./load_checkpoint.sh 1 /p4/1/checkpoints/p4_1.ckp.4025.gz -y < /dev/null > \$LOGS/load_checkpoint.\$(date +'%Y%m%d-%H%M').log 2>&1 &\n
Then, monitor with:
\ttailf \$(ls -t \$LOGS/load_checkpoint.*.log|head -1)\n"
   exit 1
}

function usageError () { msg "\nUsage Error: ${1:-Bad Usage}\n"; usage; }

export SDP_INSTANCE=${1:-Unset} 
[[ "$SDP_INSTANCE" == Unset ]] && \
   usageError "Missing required SDP Instance parameter." 

function user_confirmation_and_warning () {
   local interactive=${1:-1}
   local input=""
   msg "\n
==============================================================================
WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING\n

This script will remove files in P4ROOT and the offline_db folder,
including db.* files, state* files, and others.

It is STRONGLY recommended that these files be preseved first unless
it is absolutely certain they will never potentially be useful in any
recovery scenario.

This is being run on host $HOSTNAME at $(date).

WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
==============================================================================
\n\n"

   if [[ $interactive -eq 0 ]]; then
      msg "Bypassing interactive confirmation due to -y on command line."
      return 0
   fi

   while [[ -z "$input" ]]; do
      echo -e -n "\nEnter Y to proceed, N to stop [y/Y/n/N]:  "
      read -e input
      if [[ "${input^^}" == "Y" ]]; then
         continue
      elif [[ "${input^^}" == "N" ]]; then
         msg "Confirmation to proceed not received."
         return 1
      else
         input=
      fi
   done

   return 0
}


declare Version=1.0.9
declare Checkpoint=${2:-Unset}
declare UserAutoConfirm=${3:-Unset}
declare SDPInstanceVarsFile="/p4/common/config/p4_${SDP_INSTANCE}.vars"
declare OfflineDB=
declare OfflineDBUsableFile=
declare ExcludedTables=db.have,db.working,db.resolve,db.locks,db.revsh,db.workingx,db.resolvex
declare P4DInitScript="/p4/${SDP_INSTANCE}/bin/p4d_${SDP_INSTANCE}_init"
declare P4DSystemdServiceFile="/etc/systemd/system/p4d_${SDP_INSTANCE}.service"
declare P4BrokerInitScript="/p4/${SDP_INSTANCE}/bin/p4broker_${SDP_INSTANCE}_init"
declare P4BrokerSystemdServiceFile="/etc/systemd/system/p4broker_${SDP_INSTANCE}.service"
declare P4BrokerCfg=
declare P4DPids=
declare ThisHost=$(hostname -s)
declare -i Interactive=1

[[ "$Checkpoint" == Unset ]] && usageError "Missing checkpoint parameter."
[[ -r "$SDPInstanceVarsFile" ]] ||\
   bail "Missing SDP instance vars file. Typo in instance name? Expected: $SDPInstanceVarsFile"
[[ -r "$Checkpoint" ]] ||\
   bail "Specified checkpoint file does not exist: $Checkpoint"

[[ "$UserAutoConfirm" == "-y" ]] && Interactive=0

source /p4/common/bin/p4_vars $SDP_INSTANCE
source /p4/common/bin/ps_functions.sh ||\
   bail "Failed to load ps_functions.sh."
declare CheckpointMD5=

msg "Started ${0##*/} v$Version at $(date)"

user_confirmation_and_warning "$Interactive" ||\
   bail "User confirmation not verified. Aborting."

cd "$P4ROOT"

[[ "$PWD" == "/p4/${SDP_INSTANCE}/root" ]] || bail "Unexpected P4ROOT value of: $P4ROOT"

CheckpointMD5="${Checkpoint}.md5"
if [[ -r "$CheckpointMD5" ]]; then
   msg "Verified: MD5 file for checkpoint exists: $CheckpointMD5"
else
   CheckpointMD5="${Checkpoint/.gz/.md5}"
   if [[ -r "$CheckpointMD5" ]]; then
      msg "Verified: MD5 file for checkpoint exists: $CheckpointMD5"
   else
      bail "Could not find MD5 file for checkpoint. Tried ${Checkpoint}.md5 and ${Checkpoint/.gz/.md5}"
   fi
fi

[[ -x "$P4DInitScript" ]] || bail "Aborting: P4D Init script is missing or not executable: $P4DInitScript"

if [[ -r "/p4/common/config/p4_${SDP_INSTANCE}.broker.${ThisHost}.cfg" ]]; then
   P4BrokerCfg="/p4/common/config/p4_${SDP_INSTANCE}.broker.${ThisHost}.cfg"
else
   P4BrokerCfg="/p4/common/config/p4_${SDP_INSTANCE}.broker.cfg"
fi

if [[ -x "$P4BrokerInitScript" && -r "$P4BrokerCfg" ]]; then
   msg "Checking p4broker status."
   $P4BrokerInitScript status
   if [[ $? -eq 0 ]]; then
      msg "Shutting down p4broker."
      if [[ -r "$P4BrokerSystemdServiceFile" ]]; then
         sudo systemctl stop ${P4BROKERBIN##*/} ||\
            bail "Failed to execute: sudo systemctl stop ${P4BROKERBIN##*/}"
      else
         $P4BrokerInitScript stop
      fi
   else
      msg "Verified: p4broker is down."
   fi
fi

msg "Checking p4d status."
P4DPids=$(get_pids "$P4DBIN")

if [[ -n "$P4DPids" ]]; then
   msg "Shutting down p4d."
   if [[ -r "$P4DSystemdServiceFile" ]]; then
      sudo systemctl stop ${P4DBIN##*/} ||\
         bail "Failed to execute: sudo systemctl stop ${P4DBIN##*/}"
   else
      $P4DInitScript stop
   fi
else
   msg "Verified: No processes are running for: $P4DBIN"
fi

# Remove db.* and state* files, but not server.id or license files.
Cmd="/bin/rm -f db.* state*"
msg "Cleaning up databases with this command in $PWD:\n$Cmd"
$Cmd || bail "Failed to clean up old databases and state files in $P4ROOT"

# Move P4LOG and P4JOURAL aside if they exist.
if [[ -r "$P4LOG" ]]; then
   Cmd="mv $P4LOG $P4LOG.moved.$(date +'%Y%m%d-%H%M')"
   msg "Moving P4LOG aside with this command:\n$Cmd"
   $Cmd || bail "Failed to move P4LOG aside."
fi

if [[ -r "$P4JOURNAL" ]]; then
   Cmd="mv $P4JOURNAL $P4JOURNAL.moved.$(date +'%Y%m%d-%H%M')"
   msg "Moving P4JOURNAL aside with this command:\n$Cmd"
   $Cmd || bail "Failed to move P4JOURNAL aside."
fi

if [[ "${SERVERID^^}" == "P4D_EDGE_"* ]]; then
   Cmd="$P4DBIN -r $P4ROOT -K $ExcludedTables -z -jr $Checkpoint"
else
   Cmd="$P4DBIN -r $P4ROOT -z -jr $Checkpoint"
fi

msg "Replaying checkpoint to P4ROOT with this command:\n$Cmd"
$Cmd || bail "Checkpoint replay to P4ROOT failed."

Cmd="$P4DBIN -r $P4ROOT -xu"
msg "Ensuring databases are upgraded with this command:\n$Cmd"
$Cmd || bail "Database upgrade in P4ROOT failed."

OfflineDB=${P4ROOT/root/offline_db}
OfflineDBUsableFile="$OfflineDB/offline_db_usable.txt"

cd "$OfflineDB"

[[ "$PWD" == "/p4/${SDP_INSTANCE}/offline_db" ]] || bail "Unexpected OfflineDB value of: $OfflineDB"

# Remove db.* and state* files, but not server.id or license files.
Cmd="/bin/rm -f db.* state* $OfflineDBUsableFile"
msg "Cleaning up databases and offline_db_usable.txt file with this command in $PWD:\n$Cmd"
$Cmd || bail "Failed to clean up old databases and state files in $OfflineDB"

if [[ "${SERVERID^^}" == "P4D_EDGE_"* ]]; then
   Cmd="$P4DBIN -r $OfflineDB -K $ExcludedTables -z -jr $Checkpoint"
else
   Cmd="$P4DBIN -r $OfflineDB -z -jr $Checkpoint"
fi

msg "Replaying checkpoint to offline_db with this command:\n$Cmd"
$Cmd || bail "Checkpoint replay to offline_db failed."

Cmd="$P4DBIN -r $OfflineDB -xu"
msg "Ensuring databases in offline_db are upgraded with this command:\n$Cmd"
$Cmd || bail "Database upgrade in offline_db failed."

echo "Database restored successfully." > $OfflineDBUsableFile ||\
   bail "Failed to write this filfe: $OfflineDBUsableFile"

msg "Starting p4d."
if [[ -r "$P4DSystemdServiceFile" ]]; then
   sudo systemctl start ${P4DBIN##*/} ||\
      bail "Failed to execute: sudo systemctl start ${P4DBIN##*/}"
else
   $P4DInitScript start
fi
sleep 1

msg "Logging in."
$P4CBIN/p4login -v

# Now that we have loaded a checkpoint, re-load the shell environment to
# ensure that P4REPLICA is set correctly, as some db.config settings are
# determined dynamically.
source /p4/common/bin/p4_vars $SDP_INSTANCE
if [[ "$P4REPLICA" == TRUE ]]; then
   msg "Logging in service user."
   p4login -v -service

   sleep 3
   Cmd="$P4BIN -s pull -lj"
   msg "Checking replication status with this command:\n$Cmd"
   $Cmd || bail "Replica is not replicating properly."
fi

msg "\nAll processing completed successfully.\n"

if [[ -x "$P4BrokerInitScript" && -r "$P4BrokerCfg" ]]; then
   if [[ -r "$P4BrokerSystemdServiceFile" ]]; then
      msg "NOT stating broker. Start it manually when ready with:\n\tsudo systemctl start ${P4BROKERBIN##*/}\n"
   else
      msg "NOT stating broker. Start it manually when ready with:\n\t$P4BrokerInitScript start\n"
   fi
fi

msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"