load_checkpoint.sh #5

  • //
  • guest/
  • perforce_software/
  • sdp/
  • dev/
  • Server/
  • Unix/
  • p4/
  • common/
  • bin/
  • load_checkpoint.sh
  • View
  • Commits
  • Open Download .zip Download (9 KB)
#!/bin/bash
#------------------------------------------------------------------------------
### WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
### THIS SCRIPT SHOULD BE USED WITH GREAT CAUTION.
#------------------------------------------------------------------------------
set -u

# This script loads a specified checkpoint simultaneously into  /p4/N/root
# and /p4/N/offline_db.

function msg () { echo -e "$*"; }
function bail () { msg "${1:-Unknown Error}\n"; exit ${2:-1}; }
function usage () {
   msg "Sample Usage (bash syntax):
\tcd /p4/common/bin
\tsource ./p4_vars 1
\tnohup ./load_checkpoint.sh 1 /p4/1/checkpoints/p4_1.ckp.4025.gz -y < /dev/null > \$LOGS/load_checkpoint.\$(date +'%Y%m%d-%H%M').log 2>&1 &\n
Then, monitor with:
\ttailf \$(ls -t \$LOGS/load_checkpoint.*.log|head -1)\n"
   exit 1
}

function usageError () { msg "\nUsage Error: ${1:-Bad Usage}\n"; usage; }

export SDP_INSTANCE=${1:-Unset} 
[[ "$SDP_INSTANCE" == Unset ]] && \
   usageError "Missing required SDP Instance parameter." 

function user_confirmation_and_warning () {
   local interactive=${1:-1}
   local input=""
   msg "\n
==============================================================================
WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING\n

This script will remove files in P4ROOT and the offline_db folder,
including db.* files, state* files, and others.

It is STRONGLY recommended that these files be preseved first unless
it is absolutely certain they will never potentially be useful in any
recovery scenario.

This is being run on host $HOSTNAME at $(date).

WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
==============================================================================
\n\n"

   if [[ $interactive -eq 0 ]]; then
      msg "Bypassing interactive confirmation due to -y on command line."
      return 0
   fi

   while [[ -z "$input" ]]; do
      echo -e -n "\nEnter Y to proceed, N to stop [y/Y/n/N]:  "
      read -e input
      if [[ "${input^^}" == "Y" ]]; then
         continue
      elif [[ "${input^^}" == "N" ]]; then
         msg "Confirmation to proceed not received."
         return 1
      else
         input=
      fi
   done

   return 0
}


declare Version=1.0.8
declare Checkpoint=${2:-Unset}
declare UserAutoConfirm=${3:-Unset}
declare SDPInstanceVarsFile="/p4/common/config/p4_${SDP_INSTANCE}.vars"
declare OfflineDB=
declare OfflineDBUsableFile=
declare ExcludedTables=db.have,db.working,db.resolve,db.locks,db.revsh,db.workingx,db.resolvex
declare P4DInitScript="/p4/${SDP_INSTANCE}/bin/p4d_${SDP_INSTANCE}_init"
declare P4DSystemdServiceFile="/etc/systemd/system/p4d_fuel.service"
declare P4BrokerInitScript="/p4/${SDP_INSTANCE}/bin/p4broker_${SDP_INSTANCE}_init"
declare P4BrokerSystemdServiceFile="/etc/systemd/system/p4broker_fuel.service"
declare P4BrokerCfg=
declare P4DPids=
declare ThisHost=$(hostname -s)
declare -i Interactive=1

[[ "$Checkpoint" == Unset ]] && usageError "Missing checkpoint parameter."
[[ -r "$SDPInstanceVarsFile" ]] ||\
   bail "Missing SDP instance vars file. Typo in instance name? Expected: $SDPInstanceVarsFile"
[[ -r "$Checkpoint" ]] ||\
   bail "Specified checkpoint file does not exist: $Checkpoint"

[[ "$UserAutoConfirm" == "-y" ]] && Interactive=0

source /p4/common/bin/p4_vars $SDP_INSTANCE
source /p4/common/bin/ps_functions.sh ||\
   bail "Failed to load ps_functions.sh."
declare CheckpointMD5=

msg "Started ${0##*/} v$Version at $(date)"

user_confirmation_and_warning "$Interactive" ||\
   bail "User confirmation not verified. Aborting."

cd "$P4ROOT"

[[ "$PWD" == "/p4/${SDP_INSTANCE}/root" ]] || bail "Unexpected P4ROOT value of: $P4ROOT"

CheckpointMD5="${Checkpoint}.md5"
if [[ -r "$CheckpointMD5" ]]; then
   msg "Verified: MD5 file for checkpoint exists: $CheckpointMD5"
else
   CheckpointMD5="${Checkpoint/.gz/.md5}"
   if [[ -r "$CheckpointMD5" ]]; then
      msg "Verified: MD5 file for checkpoint exists: $CheckpointMD5"
   else
      bail "Could not find MD5 file for checkpoint. Tried ${Checkpoint}.md5 and ${Checkpoint/.gz/.md5}"
   fi
fi

[[ -x "$P4DInitScript" ]] || bail "Aborting: P4D Init script is missing or not executable: $P4DInitScript"

if [[ -r "/p4/common/config/p4_${SDP_INSTANCE}.broker.${ThisHost}.cfg" ]]; then
   P4BrokerCfg="/p4/common/config/p4_${SDP_INSTANCE}.broker.${ThisHost}.cfg"
else
   P4BrokerCfg="/p4/common/config/p4_${SDP_INSTANCE}.broker.cfg"
fi

if [[ -x "$P4BrokerInitScript" && -r "$P4BrokerCfg" ]]; then
   msg "Checking p4broker status."
   $P4BrokerInitScript status
   if [[ $? -eq 0 ]]; then
      msg "Shutting down p4broker."
      if [[ -r "$P4BrokerSystemdServiceFile" ]]; then
         sudo systemctl stop ${P4BROKERBIN##*/} ||\
            bail "Failed to execute: sudo systemctl stop ${P4BROKERBIN##*/}"
      else
         $P4BrokerInitScript stop
      fi
   else
      msg "Verified: p4broker is down."
   fi
fi

msg "Checking p4d status."
P4DPids=$(get_pids "$P4DBIN")

if [[ -n "$P4DPids" ]]; then
   msg "Shutting down p4d."
   if [[ -r "$P4DSystemdServiceFile" ]]; then
      sudo systemctl stop ${P4DBIN##*/} ||\
         bail "Failed to execute: sudo systemctl stop ${P4DBIN##*/}"
   else
      $P4DInitScript stop
   fi
else
   msg "Verified: No processes are running for: $P4DBIN"
fi

# Remove db.* and state* files, but not server.id or license files.
Cmd="/bin/rm -f db.* state*"
msg "Cleaning up databases with this command in $PWD:\n$Cmd"
$Cmd || bail "Failed to clean up old databases and state files in $P4ROOT"

# Move P4LOG and P4JOURAL aside if they exist.
if [[ -r "$P4LOG" ]]; then
   Cmd="mv $P4LOG $P4LOG.moved.$(date +'%Y%m%d-%H%M')"
   msg "Moving P4LOG aside with this command:\n$Cmd"
   $Cmd || bail "Failed to move P4LOG aside."
fi

if [[ -r "$P4JOURNAL" ]]; then
   Cmd="mv $P4JOURNAL $P4JOURNAL.moved.$(date +'%Y%m%d-%H%M')"
   msg "Moving P4JOURNAL aside with this command:\n$Cmd"
   $Cmd || bail "Failed to move P4JOURNAL aside."
fi

if [[ "${SERVERID^^}" == "P4D_EDGE_"* ]]; then
   Cmd="$P4DBIN -r $P4ROOT -K $ExcludedTables -z -jr $Checkpoint"
else
   Cmd="$P4DBIN -r $P4ROOT -z -jr $Checkpoint"
fi

msg "Replaying checkpoint to P4ROOT with this command:\n$Cmd"
$Cmd || bail "Checkpoint replay to P4ROOT failed."

Cmd="$P4DBIN -r $P4ROOT -xu"
msg "Ensuring databases are upgraded with this command:\n$Cmd"
$Cmd || bail "Database upgrade in P4ROOT failed."

OfflineDB=${P4ROOT/root/offline_db}
OfflineDBUsableFile="$OfflineDB/offline_db_usable.txt"

cd "$OfflineDB"

[[ "$PWD" == "/p4/${SDP_INSTANCE}/offline_db" ]] || bail "Unexpected OfflineDB value of: $OfflineDB"

# Remove db.* and state* files, but not server.id or license files.
Cmd="/bin/rm -f db.* state* $OfflineDBUsableFile"
msg "Cleaning up databases and offline_db_usable.txt file with this command in $PWD:\n$Cmd"
$Cmd || bail "Failed to clean up old databases and state files in $OfflineDB"

if [[ "${SERVERID^^}" == "P4D_EDGE_"* ]]; then
   Cmd="$P4DBIN -r $OfflineDB -K $ExcludedTables -z -jr $Checkpoint"
else
   Cmd="$P4DBIN -r $OfflineDB -z -jr $Checkpoint"
fi

msg "Replaying checkpoint to offline_db with this command:\n$Cmd"
$Cmd || bail "Checkpoint replay to offline_db failed."

Cmd="$P4DBIN -r $OfflineDB -xu"
msg "Ensuring databases in offline_db are upgraded with this command:\n$Cmd"
$Cmd || bail "Database upgrade in offline_db failed."

echo "Database restored successfully." > $OfflineDBUsableFile ||\
   bail "Failed to write this filfe: $OfflineDBUsableFile"

msg "Starting p4d."
if [[ -r "$P4DSystemdServiceFile" ]]; then
   sudo systemctl start ${P4DBIN##*/} ||\
      bail "Failed to execute: sudo systemctl start ${P4DBIN##*/}"
else
   $P4DInitScript start
fi
sleep 1

msg "Logging in."
$P4CBIN/p4login -v

# Now that we have loaded a checkpoint, re-load the shell environment to
# ensure that P4REPLICA is set correctly, as some db.config settings are
# determined dynamically.
source /p4/common/bin/p4_vars $SDP_INSTANCE
if [[ "$P4REPLICA" == TRUE ]]; then
   msg "Logging in service user."
   p4login -v -service

   sleep 3
   Cmd="$P4BIN -s pull -lj"
   msg "Checking replication status with this command:\n$Cmd"
   $Cmd || bail "Replica is not replicating properly."
fi

msg "\nAll processing completed successfully.\n"

if [[ -x "$P4BrokerInitScript" && -r "$P4BrokerCfg" ]]; then
   if [[ -r "$P4BrokerSystemdServiceFile" ]]; then
      msg "NOT stating broker. Start it manually when ready with:\n\tsudo systemctl start ${P4BROKERBIN##*/}\n"
   else
      msg "NOT stating broker. Start it manually when ready with:\n\t$P4BrokerInitScript start\n"
   fi
fi

msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"
# Change User Description Committed
#31 29316 C. Thomas Tyler Enhanced robustness of start_p4d() and stop_p4d() interaction with systemd,
and also in similar logic in load_checkpoint.sh.

#review-29317
#30 29307 C. Thomas Tyler load_checkpoint.sh: Added '-delay' option as a workaround for server job
job079842.

#review-29308
#29 29195 C. Thomas Tyler Fixed order-of-operations issue seen without standard shell environment.
#28 29193 C. Thomas Tyler load_checkpoint.sh v2.7.0:

* If live P4JOURNAL is specified to replay, don't move it.
* Added '-k' option to keep rather than delete db.* files.

Updated test case to exercise new new options.

With these changes, this script, originally focused on
loading checkpoints on replicas/edge servers, is now
also suited to support recovery.

See also: SDP-582, for a separate script focused
on recovery scenarios, which can call this script as needed.

#review-29164 @robert_cowham
#27 28836 C. Thomas Tyler load_checkpoint.sh v2.6.2:
* Added logic to avoid errors if broker startes slowly.
* Format clarification to usage message for '-verify' option.
* Fixed typos in doc and code comments.
#26 28641 C. Thomas Tyler Tweaked scripts to support IPv6 SSL prefixes.

Added test script to test bash code snippets. First test is the
snippet to check if SSL is enabled, and if so get the SSL prefix.
#25 28587 C. Thomas Tyler load_checkpoint.sh v2.6.0:
* Added '-verify' option to call p4verify.sh from load_checkpoint.sh.

Default is to do a 'fast verify' if possible (if p4d is 2021.1+).  Other
options to p4verify.sh can be passed.

#review-28588
#24 28585 C. Thomas Tyler load_checkpoint.sh v2.5.2:
* Added logic to load edge servers.
* Added '-t <Type>' flag to specifiy replica types, with logic to
infer the value from '-s <ServerID>' if the SDP Server Spec Naming
Standard is followed.
* Fixed bug with failing to detect start of server with an untrusted
SSL connection.
* Refined logic for handling license file checks, eliminating needless
warnings and errors.
* Refined docs, with extra info for handling edge servers and
filtered forwarding replicas.

#review-28586
#23 28584 C. Thomas Tyler load_checkpoint.sh v2.4.4:
* Enhanced error message for case where '-s <ServerID>' disagrees with server.id file.
#22 28219 C. Thomas Tyler Fixed issue with slow p4d start causing 'p4login' failures.

#review-28220
#21 28203 C. Thomas Tyler Optimization to avoid referencing Journals array if empty.
#20 28168 C. Thomas Tyler Enhanced load_checkpoint.sh to accept a list of journal files to replay
following the checkpoint.

Fixed issue with 'zcat' on Mac.
#19 27722 C. Thomas Tyler Refinements to @27712:
* Resolved one out-of-date file (verify_sdp.sh).
* Added missing adoc file for which HTML file had a change (WorkflowEnforcementTriggers.adoc).
* Updated revdate/revnumber in *.adoc files.
* Additional content updates in Server/Unix/p4/common/etc/cron.d/ReadMe.md.
* Bumped version numbers on scripts with Version= def'n.
* Generated HTML, PDF, and doc/gen files:
  - Most HTML and all PDF are generated using Makefiles that call an AsciiDoc utility.
  - HTML for Perl scripts is generated with pod2html.
  - doc/gen/*.man.txt files are generated with .../tools/gen_script_man_pages.sh.

#review-27712
#18 27064 C. Thomas Tyler Fixed issue where 'source p4_vars' hangs if load_checkpoint.sh is running.

Added new semaphore file, $P4ROOT/P4ROOT_not_usable.txt.  This is used in
a way similar to 'offline_db_usable.txt' in the offline_db, except that this
file only exists when the databases in P4ROOT are not usable. This is the
opposite of how offline_db_usable.txt works, because P4ROOT is expected to
be usable 99.9% fo the time.  p4d_base will refuse to start p4d if this file
exists, protecting against possible operator errors (like trying to start
p4d when a checkpoint is still loading).

Added check_file_dne() function to verify_sdp.sh to confirm a named file does not exist.
Added checks in verify_sdp.sh that P4ROOT_not_usable.txt does not exist in P4ROOT
or offline_db.

Modified switch_db_files() (called by refresh_P4ROOT_from_offline_db.sh) to properly
use the new P4ROOT_not_usable.txt safety file.

Fixed bugs in p4d_base that could cause p4d_init.log to be overwritten if error output
was generated.

Removed call to 'backup_functions.sh' in p4d_base, as on balance it added more complexity
than needed.

#review-27065
#17 26940 C. Thomas Tyler Fixed issue with load_checkpoint.sh not clearing journal.NNN file when working on a standby (journalcopy) replica, which prevented replication from automatically starting.
 These files are now moved aside.

Removed undocumented edge processing logic determined to be unnecessary.

Minor doc improvements.
#16 26886 C. Thomas Tyler Fixed issue in load_checkpoint.sh where the critical 'p4d -xu' step
can fail due to lack of a license file with enough users for the
data set.  With this fix (borrowed from upgrade.sh), the critical
upgrade step succeeds, and the user is not left with a failure in
the middle of processing that would require manual cleanup.

The p4d server will still refuse to start without a proper license
later in the process, but at that we're at a cleaner break point in
processing, and the failure mode is clearer to admins.
#15 26825 C. Thomas Tyler Fixed order of operations issue when '-s <ServerID>' was specified.
#14 26649 Robert Cowham More SDP Doc tidy up.
Removed some command summary files.
#13 26457 C. Thomas Tyler Fixed typo for '-L' flag.
#12 26444 C. Thomas Tyler load_checkpoint.sh 2.3.x:
* Patch for bug with missing variable initializations.
* Does trusts and logins for services users as needed after loading checkpoint.
* Enhanced preflight check. ServerID file now required (can be specified)
* Now generates SSL certs if needed.
* Enhanced docs.
#11 26400 C. Thomas Tyler Added refresh_P4ROOT_from_offline_db.sh.

Updated backup_functions.sh to support functionality for db refresh.

Upgrade start_p4d() and stop_p4d() to use systemd if available, else
use the underlying SysV init scripts.

Updated verify_sdp.sh to be called from other scripts (sans
its own logging).  Added many checks to verify_sdp.sh to
support P4ROOT/offline_db swap.

Logic in P4ROOT/offline_db swap is more careful about what gets
swapped.

Added start_p4broker() and stop_p4broker() that behave similarly.

More shellcheck compliance.

#review-26401
#10 25949 C. Thomas Tyler Fixed typo in variable name.
#9 25938 C. Thomas Tyler Fixed bug in safety check.
#8 25920 C. Thomas Tyler Added new 'edge_vars' file to dynamically set list of edge-specific
db tables based on current P4D version.

Updated edge_dump.sh, recover_edge.sh, and load_checkpoint.sh
to use the new edge_vars file.

Made edge_dump.sh and recover_edge.sh shellcheck v0.6.0 compliant,
along with load_checkpoint.sh.
#7 25790 C. Thomas Tyler load_checkpoint.sh v2.0.5:
* Added support for handling case-insensitive and case
sensitive checkpoints. This works if $P4DBIN is
a symlink rather than a wrapper supplying '-C1', as
the need for '-C1' is dynamically detected. (This was
driven by a need to incorporate load_checkpoint.sh into
case conversion orchestration scripts using p4migrate).
* Added '-r' flag to load checkpoint into P4ROOT only,
skipping offline_db.
* Added '-b' flag to restart broker at the end.
* Normalized to template.sh structure, with typical
benefits (enhanced command line parsing, standard usage
message, better documentation).
* Added check for missing certificate.txt if using SSL.
* Made shellcheck v0.6.0 compliant.
#6 24602 C. Thomas Tyler Fixed a few obvious typos.
#5 24599 C. Thomas Tyler Fixed obvious typo.
#4 24482 C. Thomas Tyler Fixed obvious typo.

Removed redundant time as it clutters the output and introduces an
unnecessary dependency.

Change to new/unreleased script, bypassing review.
#3 24374 C. Thomas Tyler Added actual file content.
#2 24368 C. Thomas Tyler load_checkpoint.sh v1.0.6:
* Added safety warning and interactive confirmation.

* Supports systemd init mechanism rather than SysV init scripts if
systemd *.service files are configured.  Uses 'sudo systemctl
start/stop' calls for p4d and p4broker.  Use SysV init scripts on
other systems.

* Fixed bug with cleanup steps not executing.

TO DO:
Normalize into template.sh style with auto-logging and
standard command line flags.
#1 24198 C. Thomas Tyler Added utility script to load a specified checkpoint into both
/p4/N/root and /p4/N/offline_db, including logic to managing
p4d/p4broker processes, ensure no db.* files exist prior to
replaying the checkoint, managing the offline_db_usable.txt file,
etc.