recover_edge.sh #20

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

# This script is designed to rebuild an Edge server from a seed checkpoint from
# the commit server WHILE KEEPING THE EXISTING EDGE SPECIFIC DATA.
#
# You have to first copy the seed checkpoint from the commit server, created with
# edge_dump.sh, to the edge server before running this script.  (Alternately,
# a full checkpoint from the commit server can be used so long as the edge server
# spec does not specify any filtering, e.g. does not make use of the
# ArchiveDataFilter or RevisionDataFilter fields of the server spec.)

# Then run this script on the edge server with the SDP instance name and full
# path of the commit seed checkpoint as parameters. The checkpoint can be
# a file or directory (for parallel checkpoints).
#
# Example 1: Recover for SDP instance 1, with a checkpoint file copied
# from the commit server in the usual place:
#  ./recover_edge.sh 1 /p4/1/checkpoints/p4_1.edge_syd.seed.ckp.9188.gz
#
#
# Example 2: Recover for SDP instance abc, with a checkpoint directory
# (from a parallel checkpoint) stored in a non-SDP location:
#  ./recover_edge.sh abc /home/perforce/xfer_ckp/p4_abc.edge_syd.seed.ckp.9188

set -u

declare ExcludedTables=
declare CheckpointTables=
declare Cmd=
declare EdgeCheckpointsDir=
declare EdgeDumpPrefix=
declare NewEdgeDumpPrefix=
declare -i DoEdgeSeedReplayParallel=0

# These two Create*Checkpoint values are set in the set_vars() function in
# backup_functions.sh:
# declare -i CreateParallelCheckpoint=0
# declare -i CreateMultifileParallelCheckpoint=0
declare -i Threads=
declare Timestamp=
declare MovedFile=
declare LogLink=

function usage () {
   echo -e "Usage:\\n\\t${0##*/} <SDP_Instance> <EdgeSeedCheckpoint>\\n"
   exit 1
}

[[ $# -ne 2 || ${1:-Unset} == -h ]] && usage

export SDP_INSTANCE=${SDP_INSTANCE:-Undefined}
export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ $SDP_INSTANCE == Undefined ]]; then
   echo -e "Usage Error: Instance parameter not supplied."
   usage
fi

declare EdgeSeedCheckpoint=${2:-Unset}
if [[ "$EdgeSeedCheckpoint" == Unset ]]; then
   echo -e "Usage Error: EdgeSeedCheckpoint parameter not supplied.  Usage:\n\t${0##*/} <SDP_Instance> <EdgeSeedCheckpoint>\n"
   echo "You must supply the Perforce instance as the second parameter to this script."
   exit 1
fi

# shellcheck disable=SC1091
source /p4/common/bin/p4_vars "$SDP_INSTANCE"
# shellcheck disable=SC1091
source /p4/common/bin/backup_functions.sh
# shellcheck disable=SC1091
source /p4/common/bin/edge_vars
# shellcheck disable=SC1091
source /p4/common/bin/log_functions.sh

export LOGFILE=
LOGFILE="$LOGS/recover_edge.$(date +'%Y%m%d-%H%M%S').log"

# The LogLink symlink has no timestamp. It points to the most recent log file.
LogLink="$LOGS/recover_edge.log"


[[ -n "$ExcludedTables" && -n "$CheckpointTables" ]] ||\
   die "Values for \$ExcludedTables and/or \$CheckpointTables not defined in $P4CBIN/edge_vars."

######### Start of Script ##########

if [[ -e "$LogLink" ]]; then
   if [[ -L "$LogLink" ]]; then
      rm -f "$LogLink"
   else
      # If the name that should be a symlink is not a symlink, move it aside before
      # creating the symlink.
      OldLogTimestamp=$(get_old_log_timestamp "$LogLink")
      mv -f "$LogLink" "${LogLink%.log}.${OldLogTimestamp}.log"
   fi
fi

# Point LogLink symlink to current log. Use a subshell so the 'cd' doesn't persist.
( cd "$LOGS" && ln -s "${LOGFILE##*/}" "${LogLink##*/}"; )


EdgeCheckpointsDir="${CHECKPOINTS}.${SERVERID#p4d_}"
EdgeDumpPrefix="$EdgeCheckpointsDir/${P4SERVER}.$(date +'%Y%m%d-%H%M%S').edge_dump"
if [[ -r "$EdgeSeedCheckpoint" ]]; then
   # If the specified edge checkpoint is a directory, use paralell replay options.
   if [[ -d "$EdgeSeedCheckpoint" ]]; then
      DoEdgeSeedReplayParallel=1

      # shellcheck disable=SC2072
      [[ "$P4D_VERSION" > "2022.2" ]] ||\
         die "The specifed edge seed checkpoint [$EdgeSeedCheckpoint] is a directory, but P4D version [$P4D_VERSION] is not new enough to handle parallel checkpoint directories. It must be 2023.1+. Aborting."

      # shellcheck disable=SC2072
      # If we're doing parallel checkpoints (because we detected that the specified edge checkpoint is a directory),
      # determine the preferred number of threads from SDP configuration. Default to 4 otherwise.
      if [[ -n "${DO_PARALLEL_CHECKPOINTS:-}" && "$DO_PARALLEL_CHECKPOINTS" != "0" && "$DO_PARALLEL_CHECKPOINTS" =~ ^[1-9]{1}[0-9]*$ ]]; then
         Threads="$DO_PARALLEL_CHECKPOINTS"
      else
         Threads=4
      fi
   else
      DoEdgeSeedReplayParallel=0
   fi
else
   die "The specifed edge seed checkpoint [$EdgeSeedCheckpoint] does not exist. Aborting."
fi

echo "Processing. This may take a while depending on checkpoint duration."
echo "Log file is: $LOGFILE"

check_vars
set_vars
ckp_running

log "Remove offline db"
rm -f "$OFFLINE_DB"/db.* > "$LOGFILE" 2>&1

# With -K filter out the various Edge-specific tables to be replaced with 
# current live versions.

log "Phase 1: Recover edge seed from commit server into offline_db."
if [[ "$DoEdgeSeedReplayParallel" -eq 1 ]]; then
   log "Recover checkpoint directory from commit server into offline_db, skipping tables not used on the edge."
   Cmd="$P4DBIN -r $OFFLINE_DB -K $ExcludedTables -N ${Threads:-4} -z -jrp $EdgeSeedCheckpoint"
else
   log "Recover checkpoint file from commit server into offline_db, skipping tables not used on the edge."
   Cmd="$P4DBIN -r $OFFLINE_DB -K $ExcludedTables -z -jr $EdgeSeedCheckpoint"
fi

log "Running: $Cmd"
$Cmd >> "$LOGFILE" 2>&1 || die "Failed to recover from $EdgeSeedCheckpoint."

log "Phase 2: Shutdown the edge server."
stop_p4d

log "Phase 3: Create dump of local edge tables."
# With -k (lowecase) we filter to include only edge-specific tables from the edge's live P4ROOT.
Cmd="$P4DBIN -r $P4ROOT -k $CheckpointTables -z -jd $EdgeDumpPrefix"
log "Creating a dump file of the edge specific data from P4ROOT."

log "Running: $Cmd"
$Cmd >> "$LOGFILE" 2>&1 ||\
   die "Failed to dump with this command: $Cmd"

log "Phase 4: Blend edge dump into offline_db (where seed was replayed earlier)."
log "Phase 4: Recover the edge dump into offline_db."
Cmd="$P4DBIN -r $OFFLINE_DB -jr ${EdgeDumpPrefix}.gz"
log "Running: $Cmd"

$Cmd >> "$LOGFILE" 2>&1 ||\
   die "Failed to recover from edge dump with this command: $Cmd"

log "Phase 5: Swap Tables and Restart Edge Replication"

if [[ -r "$P4LOG" ]]; then
   Timestamp="$(date +'%Y-%m-%d-%H%M%S')"
   MovedFile="${P4LOG}.moved.${Timestamp}"
   log "Moving P4LOG [$P4LOG] aside to [$MovedFile]."
   mv -f "$P4LOG" "$MovedFile"
else
   log "No P4LOG [$P4LOG] found. Skipping move of P4LOG."
fi

if [[ -r "$P4JOURNAL" ]]; then
   Timestamp="$(date +'%Y-%m-%d-%H%M%S')"
   MovedFile="${P4JOURNAL}.moved.${Timestamp}"
   log "Moving P4JOURNAL [$P4JOURNAL] aside to [$MovedFile]."
   mv -f "$P4JOURNAL" "$MovedFile"
else
   log "No P4JOURNAL [$P4JOURNAL] found. Skipping move of P4JOURNAL."
fi

log "Reset replication state and clear the P4ROOT folder db files."
# shellcheck disable=SC2129
rm -f "$P4ROOT"/db.* >> "$LOGFILE" 2>&1
rm -f "$P4ROOT"/state >> "$LOGFILE" 2>&1
rm -f "$P4ROOT"/rdb.lbr >> "$LOGFILE" 2>&1
rm -f "$P4JOURNAL" >> "$LOGFILE" 2>&1

log "Move the rebuilt database to P4ROOT"
mv "$OFFLINE_DB"/db.* "$P4ROOT"/. >> "$LOGFILE" 2>&1

log "Start the edge server back up."
start_p4d

log "Phase 6: Recreate the offline_db."
# With -K (uppercase), we filter to exclude edge-specific data from the commit.
log "Phase 6A: Load seed from commit server into offline_db."
if [[ "$DoEdgeSeedReplayParallel" -eq 1 ]]; then
   log "Recover checkpoint directory from commit server into offline_db, skipping tables not used on the edge."
   Cmd="$P4DBIN -r $OFFLINE_DB -K $ExcludedTables -N ${Threads:-4} -z -jrp $EdgeSeedCheckpoint"
else
   log "Recover checkpoint file from commit server into offline_db, skipping tables not used on the edge."
   Cmd="$P4DBIN -r $OFFLINE_DB -K $ExcludedTables -jr $EdgeSeedCheckpoint"
fi
log "Running: $Cmd"
$Cmd >> "$LOGFILE" 2>&1 ||\
   die "Edge recovered OK, but could not replay edge seed into offline_db."

log "Phase 6B: Load local edge dump into offline_db."
Cmd="$P4DBIN -r $OFFLINE_DB -jr ${EdgeDumpPrefix}.gz"

log "Running: $Cmd"
$Cmd >> "$LOGFILE" 2>&1 ||\
   die "Edge recovered OK, but could not replay edge tables into offline_db."

echo "Offline db file restored successfully." > "${OFFLINE_DB}/offline_db_usable.txt"

log "Phase 7: Create a new edge checkpoint from offline_db."
get_offline_journal_num

NewEdgeDumpPrefix="$EdgeCheckpointsDir/${P4SERVER}.${SERVERID#p4d_}.ckp.$((OFFLINEJNLNUM+1))"

# CreateParallelCheckpoint is defined in set_vars in backup_functions.sh.
# shellcheck disable=SC2154
if [[ "$CreateParallelCheckpoint" -eq 1 ]]; then
   # CreateMultifileParallelCheckpoint is defined in set_vars in backup_functions.sh.
   # shellcheck disable=SC2154
   if [[ "$CreateMultifileParallelCheckpoint" -eq 1 ]]; then
      Cmd="$P4DBIN -r $OFFLINE_DB -z -N ${Threads:-4} -jdpm $NewEdgeDumpPrefix"
      log "Creating a dump directory of the edge specific data from offline_db."
   else
      Cmd="$P4DBIN -r $OFFLINE_DB -z -N ${Threads:-4} -jdp $NewEdgeDumpPrefix"
      log "Creating a dump directory of the edge specific data from offline_db."
   fi
else
   Cmd="$P4DBIN -r $OFFLINE_DB -z -jd $NewEdgeDumpPrefix"
   log "Creating a dump file of the edge specific data from offline_db."
fi

log "Running: $Cmd"
$Cmd >> "$LOGFILE" 2>&1 ||\
   die "Edge recovered OK, but could not create new edge seed checkpoint from offline_db."

ckp_complete
log "End $P4SERVER Recover Edge"
mail_log_file "$HOSTNAME $P4SERVER Recover Edge log."