#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ # This script is designed to rebuild an Edge server from a seed checkpoint from # the commit server WHILE KEEPING THE EXISTING EDGE SPECIFIC DATA. # # You have to first copy the seed checkpoint from the commit server, created with # edge_dump.sh, to the edge server before running this script. (Alternately, # a full checkpoint from the commit server can be used so long as the edge server # spec does not specify any filtering, e.g. does not make use of the # ArchiveDataFilter or RevisionDataFilter fields of the server spec.) # Then run this script on the edge server with the SDP instance name and full # path of the commit seed checkpoint as parameters. The checkpoint can be # a file or directory (for parallel checkpoints). # # Example 1: Recover for SDP instance 1, with a checkpoint file copied # from the commit server in the usual place: # ./recover_edge.sh 1 /p4/1/checkpoints/p4_1.edge_syd.seed.ckp.9188.gz # # # Example 2: Recover for SDP instance abc, with a checkpoint directory # (from a parallel checkpoint) stored in a non-SDP location: # ./recover_edge.sh abc /home/perforce/xfer_ckp/p4_abc.edge_syd.seed.ckp.9188 set -u declare ExcludedTables= declare CheckpointTables= declare Cmd= declare EdgeCheckpointsDir= declare EdgeDumpPrefix= declare NewEdgeDumpPrefix= declare -i DoEdgeSeedReplayParallel=0 # These two Create*Checkpoint values are set in the set_vars() function in # backup_functions.sh: # declare -i CreateParallelCheckpoint=0 # declare -i CreateMultifileParallelCheckpoint=0 declare -i Threads= declare Timestamp= declare MovedFile= declare LogLink= function usage () { echo -e "Usage:\\n\\t${0##*/} <SDP_Instance> <EdgeSeedCheckpoint>\\n" exit 1 } [[ $# -ne 2 || ${1:-Unset} == -h ]] && usage export SDP_INSTANCE=${SDP_INSTANCE:-Undefined} export SDP_INSTANCE=${1:-$SDP_INSTANCE} if [[ $SDP_INSTANCE == Undefined ]]; then echo -e "Usage Error: Instance parameter not supplied." usage fi declare EdgeSeedCheckpoint=${2:-Unset} if [[ "$EdgeSeedCheckpoint" == Unset ]]; then echo -e "Usage Error: EdgeSeedCheckpoint parameter not supplied. Usage:\n\t${0##*/} <SDP_Instance> <EdgeSeedCheckpoint>\n" echo "You must supply the Perforce instance as the second parameter to this script." exit 1 fi # shellcheck disable=SC1091 source /p4/common/bin/p4_vars "$SDP_INSTANCE" # shellcheck disable=SC1091 source /p4/common/bin/backup_functions.sh # shellcheck disable=SC1091 source /p4/common/bin/edge_vars # shellcheck disable=SC1091 source /p4/common/bin/log_functions.sh export LOGFILE= LOGFILE="$LOGS/recover_edge.$(date +'%Y%m%d-%H%M%S').log" # The LogLink symlink has no timestamp. It points to the most recent log file. LogLink="$LOGS/recover_edge.log" [[ -n "$ExcludedTables" && -n "$CheckpointTables" ]] ||\ die "Values for \$ExcludedTables and/or \$CheckpointTables not defined in $P4CBIN/edge_vars." ######### Start of Script ########## if [[ -e "$LogLink" ]]; then if [[ -L "$LogLink" ]]; then rm -f "$LogLink" else # If the name that should be a symlink is not a symlink, move it aside before # creating the symlink. OldLogTimestamp=$(get_old_log_timestamp "$LogLink") mv -f "$LogLink" "${LogLink%.log}.${OldLogTimestamp}.log" fi fi # Point LogLink symlink to current log. Use a subshell so the 'cd' doesn't persist. ( cd "$LOGS" && ln -s "${LOGFILE##*/}" "${LogLink##*/}"; ) EdgeCheckpointsDir="${CHECKPOINTS}.${SERVERID#p4d_}" EdgeDumpPrefix="$EdgeCheckpointsDir/${P4SERVER}.$(date +'%Y%m%d-%H%M%S').edge_dump" if [[ -r "$EdgeSeedCheckpoint" ]]; then # If the specified edge checkpoint is a directory, use paralell replay options. if [[ -d "$EdgeSeedCheckpoint" ]]; then DoEdgeSeedReplayParallel=1 # shellcheck disable=SC2072 [[ "$P4D_VERSION" > "2022.2" ]] ||\ die "The specifed edge seed checkpoint [$EdgeSeedCheckpoint] is a directory, but P4D version [$P4D_VERSION] is not new enough to handle parallel checkpoint directories. It must be 2023.1+. Aborting." # shellcheck disable=SC2072 # If we're doing parallel checkpoints (because we detected that the specified edge checkpoint is a directory), # determine the preferred number of threads from SDP configuration. Default to 4 otherwise. if [[ -n "${DO_PARALLEL_CHECKPOINTS:-}" && "$DO_PARALLEL_CHECKPOINTS" != "0" && "$DO_PARALLEL_CHECKPOINTS" =~ ^[1-9]{1}[0-9]*$ ]]; then Threads="$DO_PARALLEL_CHECKPOINTS" else Threads=4 fi else DoEdgeSeedReplayParallel=0 fi else die "The specifed edge seed checkpoint [$EdgeSeedCheckpoint] does not exist. Aborting." fi echo "Processing. This may take a while depending on checkpoint duration." echo "Log file is: $LOGFILE" check_vars set_vars ckp_running log "Remove offline db" rm -f "$OFFLINE_DB"/db.* > "$LOGFILE" 2>&1 # With -K filter out the various Edge-specific tables to be replaced with # current live versions. log "Phase 1: Recover edge seed from commit server into offline_db." if [[ "$DoEdgeSeedReplayParallel" -eq 1 ]]; then log "Recover checkpoint directory from commit server into offline_db, skipping tables not used on the edge." Cmd="$P4DBIN -r $OFFLINE_DB -K $ExcludedTables -N ${Threads:-4} -z -jrp $EdgeSeedCheckpoint" else log "Recover checkpoint file from commit server into offline_db, skipping tables not used on the edge." Cmd="$P4DBIN -r $OFFLINE_DB -K $ExcludedTables -z -jr $EdgeSeedCheckpoint" fi log "Running: $Cmd" $Cmd >> "$LOGFILE" 2>&1 || die "Failed to recover from $EdgeSeedCheckpoint." log "Phase 2: Shutdown the edge server." stop_p4d log "Phase 3: Create dump of local edge tables." # With -k (lowecase) we filter to include only edge-specific tables from the edge's live P4ROOT. Cmd="$P4DBIN -r $P4ROOT -k $CheckpointTables -z -jd $EdgeDumpPrefix" log "Creating a dump file of the edge specific data from P4ROOT." log "Running: $Cmd" $Cmd >> "$LOGFILE" 2>&1 ||\ die "Failed to dump with this command: $Cmd" log "Phase 4: Blend edge dump into offline_db (where seed was replayed earlier)." log "Phase 4: Recover the edge dump into offline_db." Cmd="$P4DBIN -r $OFFLINE_DB -jr ${EdgeDumpPrefix}.gz" log "Running: $Cmd" $Cmd >> "$LOGFILE" 2>&1 ||\ die "Failed to recover from edge dump with this command: $Cmd" log "Phase 5: Swap Tables and Restart Edge Replication" if [[ -r "$P4LOG" ]]; then Timestamp="$(date +'%Y-%m-%d-%H%M%S')" MovedFile="${P4LOG}.moved.${Timestamp}" log "Moving P4LOG [$P4LOG] aside to [$MovedFile]." mv -f "$P4LOG" "$MovedFile" else log "No P4LOG [$P4LOG] found. Skipping move of P4LOG." fi if [[ -r "$P4JOURNAL" ]]; then Timestamp="$(date +'%Y-%m-%d-%H%M%S')" MovedFile="${P4JOURNAL}.moved.${Timestamp}" log "Moving P4JOURNAL [$P4JOURNAL] aside to [$MovedFile]." mv -f "$P4JOURNAL" "$MovedFile" else log "No P4JOURNAL [$P4JOURNAL] found. Skipping move of P4JOURNAL." fi log "Reset replication state and clear the P4ROOT folder db files." # shellcheck disable=SC2129 rm -f "$P4ROOT"/db.* >> "$LOGFILE" 2>&1 rm -f "$P4ROOT"/state >> "$LOGFILE" 2>&1 rm -f "$P4ROOT"/rdb.lbr >> "$LOGFILE" 2>&1 rm -f "$P4JOURNAL" >> "$LOGFILE" 2>&1 log "Move the rebuilt database to P4ROOT" mv "$OFFLINE_DB"/db.* "$P4ROOT"/. >> "$LOGFILE" 2>&1 log "Start the edge server back up." start_p4d log "Phase 6: Recreate the offline_db." # With -K (uppercase), we filter to exclude edge-specific data from the commit. log "Phase 6A: Load seed from commit server into offline_db." if [[ "$DoEdgeSeedReplayParallel" -eq 1 ]]; then log "Recover checkpoint directory from commit server into offline_db, skipping tables not used on the edge." Cmd="$P4DBIN -r $OFFLINE_DB -K $ExcludedTables -N ${Threads:-4} -z -jrp $EdgeSeedCheckpoint" else log "Recover checkpoint file from commit server into offline_db, skipping tables not used on the edge." Cmd="$P4DBIN -r $OFFLINE_DB -K $ExcludedTables -jr $EdgeSeedCheckpoint" fi log "Running: $Cmd" $Cmd >> "$LOGFILE" 2>&1 ||\ die "Edge recovered OK, but could not replay edge seed into offline_db." log "Phase 6B: Load local edge dump into offline_db." Cmd="$P4DBIN -r $OFFLINE_DB -jr ${EdgeDumpPrefix}.gz" log "Running: $Cmd" $Cmd >> "$LOGFILE" 2>&1 ||\ die "Edge recovered OK, but could not replay edge tables into offline_db." echo "Offline db file restored successfully." > "${OFFLINE_DB}/offline_db_usable.txt" log "Phase 7: Create a new edge checkpoint from offline_db." get_offline_journal_num NewEdgeDumpPrefix="$EdgeCheckpointsDir/${P4SERVER}.${SERVERID#p4d_}.ckp.$((OFFLINEJNLNUM+1))" # CreateParallelCheckpoint is defined in set_vars in backup_functions.sh. # shellcheck disable=SC2154 if [[ "$CreateParallelCheckpoint" -eq 1 ]]; then # CreateMultifileParallelCheckpoint is defined in set_vars in backup_functions.sh. # shellcheck disable=SC2154 if [[ "$CreateMultifileParallelCheckpoint" -eq 1 ]]; then Cmd="$P4DBIN -r $OFFLINE_DB -z -N ${Threads:-4} -jdpm $NewEdgeDumpPrefix" log "Creating a dump directory of the edge specific data from offline_db." else Cmd="$P4DBIN -r $OFFLINE_DB -z -N ${Threads:-4} -jdp $NewEdgeDumpPrefix" log "Creating a dump directory of the edge specific data from offline_db." fi else Cmd="$P4DBIN -r $OFFLINE_DB -z -jd $NewEdgeDumpPrefix" log "Creating a dump file of the edge specific data from offline_db." fi log "Running: $Cmd" $Cmd >> "$LOGFILE" 2>&1 ||\ die "Edge recovered OK, but could not create new edge seed checkpoint from offline_db." ckp_complete log "End $P4SERVER Recover Edge" mail_log_file "$HOSTNAME $P4SERVER Recover Edge log."
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#16 | 30915 | C. Thomas Tyler |
Released SDP 2024.1.30913 (2024/11/20). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#15 | 28240 | C. Thomas Tyler |
Released SDP 2021.1.28238 (2021/11/12). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#14 | 27331 | C. Thomas Tyler |
Released SDP 2020.1.27325 (2021/01/29). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#13 | 26496 | C. Thomas Tyler |
Released SDP 2019.3.26494 (2020/04/23). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#12 | 25951 | C. Thomas Tyler |
Released SDP 2019.2.25949 (2019/08/12). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#11 | 25940 | C. Thomas Tyler |
Released SDP 2019.2.25938 (2019/08/05). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#10 | 25933 | C. Thomas Tyler |
Released SDP 2019.2.25923 (2019/08/05). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#9 | 23331 | C. Thomas Tyler |
Released SDP 2017.4.23329 (2017/12/05). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#8 | 23006 | C. Thomas Tyler |
Released SDP 2017.3.23003 (2017/10/19). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#7 | 21338 | C. Thomas Tyler |
Released SDP 2016.2.21328 (2016/12/16). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#6 | 19414 | C. Thomas Tyler | Released SDP/MultiArch/2016.1/19410 (2016/05/17). | ||
#5 | 17315 | C. Thomas Tyler |
Released SDP/MultiArch/2016.1/17297 (2016/02/03). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#4 | 16845 | C. Thomas Tyler | Fixed typo in script name in a comment; no functional change. | ||
#3 | 15856 | C. Thomas Tyler |
Replaced the big license comment block with a shortened form referencing the LICENSE file included with the SDP package, and also by the URL for the license file in The Workshop. |
||
#2 | 15777 | C. Thomas Tyler |
No functional changes. Style Policing only on bash scripts only. Normalized indentation and line breaks, removed offending tabs, and general whitespace usage. |
||
#1 | 15716 | Russell C. Jackson (Rusty) | Script for rebuilding an Edge server. |