#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ set -u # Global Variables. export P4DInitScript= export P4DRef= export P4DSystemdServiceFile= export P4BrokerInitScript= export P4BrokerRef= export P4BrokerSystemdServiceFile= export P4ProxyInitScript= export P4ProxyRef= export P4ProxySystemdServiceFile= export CKPTMPDIR= export OFFLINE_DB= export EDGESERVER= export STANDBYSERVER= # Common functions used in various SDP scripts. #------------------------------------------------------------------------------ # Verify key variables in the shell environment exist, or else abort. # # If checks in this function fail, this function an 'echo' and 'exit 1' # rather than calling 'log' or 'die', as this function is generally called # early in processing, before the log is initialized. #------------------------------------------------------------------------------ function check_vars () { local CheckVarsPreflightOK=1 CommonVars="SDP_INSTANCE P4HOME P4PORT P4ROOT P4JOURNAL P4BIN P4DBIN P4TICKETS P4TRUST KEEPCKPS KEEPJNLS KEEPLOGS CHECKPOINTS LOGS OSUSER" InstanceVars="P4MASTER_ID P4MASTERPORT" # First, check vars that should be set in /p4/common/bin/p4_vars. for var in $CommonVars; do # Detect unset variables, using ':-' to avoid 'unbound variable' errors. # shellcheck disable=SC1083 if [[ -z "$(eval echo \${"$var":-})" ]]; then echo "Error: Required variable \$$var is NOT set. It should be set in /p4/common/bin/p4_vars." CheckVarsPreflightOK=0 fi done # Next, check vars that should be set in /p4/common/config/p4_N.instance. # For some variables, provide additional details that help help users correct # the problem. for var in $InstanceVars; do # shellcheck disable=SC1083 if [[ -z "$(eval echo \${"$var":-})" ]]; then echo "Error: Required variable \$$var is NOT set. It should be set in /p4/common/config/p4_N.vars, where N is the SDP instance name." if [[ "$var" == "P4MASTER_ID" ]]; then echo "The value for P4MASTER_ID should be the name of the ServerID of the master server." fi CheckVarsPreflightOK=0 fi done if [[ "$CheckVarsPreflightOK" -eq 0 ]]; then echo "Use p4master_run or source p4_vars before calling this script." echo "Aborting to to errors in shell environment preflight checks." exit 1 fi } #------------------------------------------------------------------------------ # is_edge ($ServerID, $RootDir) # # Determine if a given ServerID is an edge server or not, checking a given # database root dir (e.g. $P4ROOT or $OFFLINE_DB). # # Input: # $1 - ServerID (required) # $2 - RootDir (optional, defaults to $P4ROOT) # # Output YES if an edge server, NO otherwise. # #------------------------------------------------------------------------------ function is_edge () { local ServerID=${1:-Unset} local RootDir=${2:-$P4ROOT} local ServicesData= local EdgeCheck= # Extract a slice of db.server referencing the given ServerID, # and then grab the field containing Services data. ServicesData=$("$P4DBIN" -r "$RootDir" -J off -L /dev/null -k db.server -jd - 2>&1 |\ $GREP "@db.server@ @${ServerID}@" | "$CUT" -d '@' -f 13) # Do a bitwise operation to determine if the ServicesData value indicates # this is an edge server. if [[ -n "$ServicesData" ]]; then EdgeCheck=$((ServicesData & 4096)) if [[ "$EdgeCheck" -gt 0 ]]; then echo YES else echo NO fi else echo NO fi } #------------------------------------------------------------------------------ # is_replica ($ServerID, $RootDir) # # Determine if a given ServerID is a replica server or not, checking a given # database root dir (e.g. $P4ROOT or $OFFLINE_DB). # # Input: # $1 - ServerID (required) # $2 - RootDir (optional, defaults to $P4ROOT) # # Output YES if an standby server, NO otherwise. # #------------------------------------------------------------------------------ is_replica () { local ServerID="${1:-Unset}" local RootDir="${2:-$P4ROOT}" local ServicesData= # Extract a slice of db.server referencing the given ServerID, # and then grab the field containing Services data. ServicesData=$("$P4DBIN" -r "$RootDir" -J off -L /dev/null -k db.server -jd - 2>&1 |\ "$GREP" "@db.server@ @${ServerID}@" | "$CUT" -d '@' -f 13 | tr -d ' ') # Do a check to see if the ServicesData value indicates # this is an standby server. if [[ -n "$ServicesData" ]]; then if [[ "$ServicesData" -eq '2533' ]]; then echo YES else echo NO fi else echo NO fi } #------------------------------------------------------------------------------ # is_standby ($ServerID, $RootDir) # # Determine if a given ServerID is an standby server or not, checking a given # database root dir (e.g. $P4ROOT or $OFFLINE_DB). # # Input: # $1 - ServerID (required) # $2 - RootDir (optional, defaults to $P4ROOT) # # Output YES if an standby server, NO otherwise. # #------------------------------------------------------------------------------ function is_standby () { local ServerID="${1:-Unset}" local RootDir="${2:-$P4ROOT}" local ServicesData= # Extract a slice of db.server referencing the given ServerID, # and then grab the field containing Services data. ServicesData=$("$P4DBIN" -r "$RootDir" -J off -L /dev/null -k db.server -jd - 2>&1 |\ "$GREP" "@db.server@ @${ServerID}@" | "$CUT" -d '@' -f 13 | tr -d ' ') # Do a check to see if the ServicesData value indicates # this is an standby server. if [[ -n "$ServicesData" ]]; then if [[ "$ServicesData" -eq '35141' || "$ServicesData" -eq '35301' ]]; then echo YES else echo NO fi else echo NO fi } #------------------------------------------------------------------------------ # Set variables for use in various scripts: # OFFLINE_DB=path to offline db directory # EDGESERVER=1 if this is an edge server, 0 otherwise. # STANDBYSERVER=1 if this is a standby server, 0 otherwise. # # This must be called after loading the standard shell environment by # doing: # source /p4/common/bin/p4_vars N # # This sets P4HOME, SERVERID, etc. needed by this function. #------------------------------------------------------------------------------ function set_vars () { P4DInitScript="$P4HOME/bin/p4d_${SDP_INSTANCE}_init" P4DRef="${P4DInitScript%_init}" P4DSystemdServiceFile="/etc/systemd/system/p4d_${SDP_INSTANCE}.service" P4BrokerInitScript="$P4HOME/bin/p4broker_${SDP_INSTANCE}_init" P4BrokerRef="${P4BrokerInitScript%_init}" P4BrokerSystemdServiceFile="/etc/systemd/system/p4broker_${SDP_INSTANCE}.service" P4ProxyInitScript="$P4HOME/bin/p4p_${SDP_INSTANCE}_init" P4ProxyRef="${P4ProxyInitScript%_init}" P4ProxySystemdServiceFile="/etc/systemd/system/p4p_${SDP_INSTANCE}.service" OFFLINE_DB="${P4HOME}/offline_db" CKPTMPDIR="${CHECKPOINTS}/ckp_tmp" # shellcheck disable=SC2153 if [[ -n "$SERVERID" ]]; then if [[ "$(is_edge "$SERVERID")" == YES ]]; then export EDGESERVER=1 else export EDGESERVER=0 fi else export EDGESERVER=0 fi if [[ -n "$SERVERID" ]]; then if [[ "$(is_replica "$SERVERID")" == YES ]]; then export REPLICASERVER=1 # Get commit server from P4TARGET setting in database else export REPLICASERVER=0 fi else export REPLICASERVER=0 fi if [[ -n "$SERVERID" ]]; then if [[ "$(is_standby "$SERVERID")" == YES ]]; then export STANDBYSERVER=1 # Get commit server from P4TARGET setting in database else export STANDBYSERVER=0 fi else export STANDBYSERVER=0 fi # Ensure that SDP_ADMIN_PASSWORD_FILE is set, using existing value if set (e.g. # in p4_vars), otherwise set it to the SDP standard value. export SDP_ADMIN_PASSWORD_FILE="${SDP_ADMIN_PASSWORD_FILE:-Unset}" if [[ "$SDP_ADMIN_PASSWORD_FILE" == Unset ]]; then export SDP_ADMIN_PASSWORD_FILE="$P4CCFG/.p4passwd.${P4SERVER}.admin" fi } #------------------------------------------------------------------------------ # Check if user is running as required OS user. #------------------------------------------------------------------------------ function check_uid () { user=$(id -un) if [[ "${user}" != "${OSUSER}" ]]; then die "Must be run by user: ${OSUSER}. Abort!" fi } #------------------------------------------------------------------------------ # Function log() - echo message to logfile or stdout. # # If $LOGFILE is defined, write message to the log file only; nothing goes to # stdout. Prepend a datestamp. # If $LOGFILE isn't defined, just echo to stdout, w/o timestamp or. # In all cases, support '-e' formatting. # Input: # $1 - message to log (must be quoted). #------------------------------------------------------------------------------ function log () { if [[ "${LOGFILE:-Unset}" != Unset ]]; then echo -n "$(date)" >> "$LOGFILE" 2>&1 echo -e " $0: $*" >> "$LOGFILE" 2>&1 else echo -e "$@" fi } #------------------------------------------------------------------------------ # Decide depending on our mail utility, how to specify sender (if we need to). # Mail on some platforms sets sender by default. # If the mail utility returns what looks like a version identifier # when given the '-V' flag, use a '-S' flag. If it does not return a # version identifier, don't set a mail sender option. # Allow GNU Mailutils alternative flag instead. #------------------------------------------------------------------------------ function get_mail_sender_opt () { local mail_sender_opt= local mail_ver= if [[ -n "$MAILFROM" ]]; then mail_ver=$($SDPMAIL -V 2>&1) # shellcheck disable=SC2076 if [[ "$mail_ver" =~ "GNU Mailutils" ]]; then mail_sender_opt="-aFrom:$MAILFROM" elif [[ "$mail_ver" =~ ^[0-9]+\.[0-9] ]]; then mail_sender_opt="-S from=$MAILFROM" fi fi echo "$mail_sender_opt" } #------------------------------------------------------------------------------ # Email the log file by $LOGFILE. #------------------------------------------------------------------------------ function mail_log_file () { local subject=$1 local mail_sender_opt mail_sender_opt=$(get_mail_sender_opt) $SDPMAIL -s "$subject" "$mail_sender_opt" "$MAILTO" < "$LOGFILE" } #------------------------------------------------------------------------------ # Deliver the $LOGFILE via AWS SNS. #------------------------------------------------------------------------------ function sns_log_file () { # AWS SNS has a 100 character limit for subject field local subject=$1 local short_subject="$(echo ${subject:0:100})" echo -e "Sending alert and log file contents to administrator via AWS SNS." >&2 aws --region "$AWS_DEFAULT_REGION" sns publish --topic-arn "$SNS_ALERT_TOPIC_ARN" --subject "$short_subject" --message "$(cat "$LOGFILE")" } #------------------------------------------------------------------------------ # Function die() - log message, send email/SNS, and exit. # If $LOGFILE is defined, write message to the log file, email/SNS log, # and exit. # If $LOGFILE is not defined, write message to the stdout, and skip # email/SNS. # If in terminal session, display message to stderr as well. #------------------------------------------------------------------------------ function die () { # mail the error (with more helpful subject line than cron) log "ERROR!!! - $HOSTNAME $P4SERVER $0: $*" if [[ "${LOGFILE:-Unset}" != Unset ]]; then if [[ "${SNS_ALERT_TOPIC_ARN:-Unset}" != Unset ]]; then log "Using SNS for log file delivery..." sns_log_file "ERROR!!! - $HOSTNAME $P4SERVER $0: $*" else log "Using email for log file delivery..." mail_log_file "ERROR!!! - $HOSTNAME $P4SERVER $0: $*" fi fi # if running from terminal, also send to stderr if tty >/dev/null; then echo -e "$@" >&2 fi rm -f "${LOGS}/ckp_running.txt" exit 1 } #------------------------------------------------------------------------------ # Convert various byte values (K,M,G,%) to bytes # Pass in values such as 1024K, 512M, 1G or 10% #------------------------------------------------------------------------------ function convert_to_bytes () { local value=$1 local totalsize=${2:-Undefined} local size= local unit= # Break up value into size (numeric) and unit (K,M,G) size=$("$GREP" -Eo '[[:alpha:]%]+|[0-9]+' <<< "$value" | head -1) unit=$("$GREP" -Eo '[[:alpha:]%]+|[0-9]+' <<< "$value" | tail -1) # Based on unit, convert to bytes case "$unit" in K) echo $((size * 1024)) ;; M) echo $((size * 1024**2)) ;; G) echo $((size * 1024**3)) ;; %) echo $((totalsize * size / 100)) ;; esac } #------------------------------------------------------------------------------ # Write a semaphore file, $LOGS/ckp_running.txt. This file is written at # the start of processing, and removed upon successful completion. It # prevents multiple concurrent operations from being launched accidentally # e.g. by multiple human admins, or a human inadvertently competing with a # cron job. # # It is also intended to get human admins to determine the root cause of # checkpoint failures. #------------------------------------------------------------------------------ function ckp_running() { if [[ -f "${LOGS}/ckp_running.txt" ]]; then die "Last checkpoint not complete. Check the backup process or contact support." fi echo "Checkpoint running." > "${LOGS}/ckp_running.txt" } #------------------------------------------------------------------------------ # Remove the ckp_running.txt semaphore file when checkpoint processing is # complete. #------------------------------------------------------------------------------ function ckp_complete() { rm -f "${LOGS}/ckp_running.txt" } #------------------------------------------------------------------------------ # Ensure key directories are writable. Abort if they are not. #------------------------------------------------------------------------------ function check_dirs () { # Check that key dirs are writable local -i dirsOK=1 dirList="$OFFLINE_DB $CHECKPOINTS $LOGS" [[ "$EDGESERVER" -eq 1 ]] && dirList+=" ${CHECKPOINTS}.${SERVERID#p4d_}" for dir in $dirList; do if [[ ! -d "$dir" || ! -w "$dir" ]]; then log "Error: Dir $dir does not exist or is not writable." dirsOK=0 fi done [[ "$dirsOK" -eq 1 ]] || die "Some expected dirs are missing or not writable. Aborting." } #------------------------------------------------------------------------------ # Add the results of df -h or df -m to the log file. #------------------------------------------------------------------------------ function check_disk_space () { log "Checking disk space..." $P4BIN diskspace >> "$LOGFILE" 2>&1 } #------------------------------------------------------------------------------ # Check value of journal; ensure it is an integer. #------------------------------------------------------------------------------ function check_journalnum () { local JNLNUM=${1:-Unset} re='^[0-9]+$' if ! [[ $JNLNUM =~ $re ]] ; then die "The journal counter value [$JNLNUM] is invalid. It must be numeric." fi } #------------------------------------------------------------------------------ # Check the checkpoints directory for the oldest checkpoint #------------------------------------------------------------------------------ function get_ckpnum () { if [[ "$EDGESERVER" -eq 0 ]]; then # shellcheck disable=SC2034 disable=SC2012 disable=SC2016 OLDESTCHECKPOINT=$(ls -1tr "${CHECKPOINTS}/" | "$GREP" ckp | "$GREP" -v md5 | head -n 1 | "$AWK" -F '.ckp.' '{ print $(2) }' | tr -d '.gz') else # shellcheck disable=SC2034 disable=SC2012 disable=SC2016 OLDESTCHECKPOINT=$(ls -1tr "${CHECKPOINTS}.${SERVERID#p4d_}/" | "$GREP" ckp | "$GREP" -v md5 | head -n 1 | "$AWK" -F '.ckp.' '{ print $(2) }' | tr -d '.gz') fi } #------------------------------------------------------------------------------ # Determine journal counter by checking counter in db.counters. #------------------------------------------------------------------------------ get_journalnum () { # get the current journal and checkpoint serial numbers. local nextCheckpointNum if [[ -r "$P4ROOT/db.counters" ]]; then nextCheckpointNum=$("$P4DBIN" -r "$P4ROOT" -k db.counters -jd - 2>&1 | grep @journal@ | cut -d '@' -f 8) if [[ -n "$nextCheckpointNum" ]]; then check_journalnum "$nextCheckpointNum" JOURNALNUM="$nextCheckpointNum" else # Special case: If db.counters is empty, then we have a new/empty data # set, so just set the value to 0. JOURNALNUM=0 fi else # Special case: If db.counters doesn't exist, then we have a new/empty # data set, so just set the value to 0. JOURNALNUM=0 fi # If we are on an edge server, the journal has already rotated, so we have to decrement the value # so that we replay the correct journal file and create the correct checkpoint number on the # edge server. # # In the case of a standby server, the journal rotation occurs on the master server, # so we don't need to increment the journal number again, so we decrement by 1. # Also, when replaying the journals to the offline db, we don't want to play to the live journal # because it is still being replicated. if [[ "$EDGESERVER" -eq 1 || "$REPLICASERVER" -eq 1 || "$STANDBYSERVER" -eq 1 ]]; then JOURNALNUM=$((JOURNALNUM - 1)) fi CHECKPOINTNUM=$((JOURNALNUM + 1)) } #------------------------------------------------------------------------------ # Determine journal space usage and minimum disk space requirement #------------------------------------------------------------------------------ get_journal_stats () { # Get minimum disk space required on server journal filesystem before server rejects commands # This will return the configured and default value, but grab the configured value which shows first # If a configured value is not present, it will use the default value # shellcheck disable=SC2034 disable=SC2016 P4JOURNALMIN=$("$P4BIN" configure show filesys.P4JOURNAL.min | "$AWK" '{ print $1 }' | $CUT -d'=' -f2 | head -1) # Get current journal free disk space # shellcheck disable=SC2034 P4JOURNALFREE=$("$P4BIN" -ztag -F "%freeBytes%" diskspace P4JOURNAL) # Get total available disk space for journal # shellcheck disable=SC2034 P4JOURNALTOTAL=$("$P4BIN" -ztag -F "%totalBytes%" diskspace P4JOURNAL) } #------------------------------------------------------------------------------ # Verify that the offline databases are usable by checking the existence # of a 'offline_db_usable.txt' file that is written only when databases # are in a known-good state, following successful recovery from a checkpoint. #------------------------------------------------------------------------------ check_offline_db_usable () { # Check it is OK if [[ ! -f "$OFFLINE_DB/offline_db_usable.txt" ]]; then die "Offline database not in a usable state. Check the backup process." fi if [[ ! -f "$OFFLINE_DB/db.counters" ]]; then die "Offline database not found. Consider creating it with live_checkpoint.sh. Be aware that live_checkpoint.sh locks the live system and may take a long time. Aborting." fi } #------------------------------------------------------------------------------ # Determine journal counter in offline databases. #------------------------------------------------------------------------------ get_offline_journal_num () { # Get the journal number of the offline database check_offline_db_usable OFFLINEJNLNUM=$("$P4DBIN" -r "$OFFLINE_DB" -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") || die "Cannot get the offline journal number. Abort!" check_journalnum "$OFFLINEJNLNUM" log "Offline journal number is: $OFFLINEJNLNUM" } #------------------------------------------------------------------------------ # Cleanup old checkpoint and numbered journal files. #------------------------------------------------------------------------------ remove_old_checkpoints_and_journals () { local CheckpointsDir= local StandbyReplicaJournalsDir= local FilePrefix= local JournalPrefix= local File= if [[ "$KEEPCKPS" -eq 0 ]]; then log "Skipping cleanup of old checkpoints because KEEPCKPS is set to 0." else log "Deleting obsolete checkpoints and journals. Keeping latest $KEEPCKPS per KEEPCKPS setting in p4_vars." # For the master server, we can safely rely on the SDP standard that the journalPrefix # is of the form '/p4/N/checkpoints/p4_N'. For replicas and edge servers, determine # the values dynamically based on the current journalPrefix value for the given ServerID. if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" else JournalPrefix="$("$P4DBIN" -r "$P4ROOT" -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)" if [[ -n "$JournalPrefix" ]]; then CheckpointsDir="${JournalPrefix%/*}" FilePrefix="${JournalPrefix##*/}" else log "Warning: Could not determine journalPrefix for ServerID $SERVERID." CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" fi fi if [[ -d "$CheckpointsDir" ]]; then # Remove selected checkpoint and journal files based on the KEEPCKPS # setting regardless of whether compressed or not. # We multiply KEEPCKP by 2 for the ckp files because of the md5 files. # shellcheck disable=SC2012 for File in $(ls -t "${CheckpointsDir}/${FilePrefix}".ckp.* 2>/dev/null | "$AWK" "NR > ($KEEPCKPS * 2)"); do log "rm -f $File" rm -f "$File" done # Use KEEPJNLS to allow for separate journal rotation at a higher # frequency. # shellcheck disable=SC2012 for File in $(ls -t "${CheckpointsDir}/${FilePrefix}".jnl.* 2>/dev/null | "$AWK" "NR > $KEEPJNLS"); do log "rm -f $File" rm -f "$File" done fi StandbyReplicaJournalsDir="${P4HOME}/journals.rep" if [[ -d "$StandbyReplicaJournalsDir" ]]; then # shellcheck disable=SC2012 for File in $(ls -t "$StandbyReplicaJournalsDir/${FilePrefix}".ckp.* 2>/dev/null | "$AWK" "NR > ($KEEPCKPS * 2)"); do log "rm -f $File" rm -f "$File" done # shellcheck disable=SC2012 for File in $(ls -t "${StandbyReplicaJournalsDir}/${FilePrefix}".jnl.* 2>/dev/null | "$AWK" "NR > $KEEPJNLS"); do log "rm -f $File" rm -f "$File" done fi # This is a workaround to cleanup $LOGS/journal.NNN files on standby replicas. # These files are normally removed by p4d during journal rotation on the standby # replica. Use only if standby journals are not removed due to a standby replica # sharing /hxdepots with its P4TARGET server. To use this workround, add this # line to the end of the /p4/common/config/p4_N.vars file: # # export SDP_REMOVE_STANDBY_JOURNALS=1 # if [[ "${SDP_REMOVE_STANDBY_JOURNALS:-0}" == 1 && "$(is_standby "$SERVERID")" == YES ]]; then log "Removing excess journal.NNN files due to SDP_REMOVE_STANDBY_JOURNALS=1." # shellcheck disable=SC2012 for File in $(ls -t "${LOGS}/journal."* 2>/dev/null | "$AWK" "NR > $KEEPJNLS"); do # Process only files named 'journal.NNN' in $LOGS. [[ "$File" =~ /journal.[0-9]+$ ]] || continue log "rm -f $File" rm -f "$File" done fi fi } #------------------------------------------------------------------------------ # Function: is_server_up ($server) # # Input: # $1 - server, one of 'p4d', 'p4p', or 'p4broker' # # Output: None # # Return Codes: # 0: Server is up. # 1: Server is down. # 2: Bad usage. # # Server up/down status is checked using the appropriate init script. #------------------------------------------------------------------------------ function is_server_up () { local server="${1:-Unset}" case "$server" in (p4d) "$P4DInitScript" status > /dev/null 2>&1 return $? ;; (p4broker) "$P4BrokerInitScript" status > /dev/null 2>&1 return $? ;; (p4p) "$P4ProxyInitScript" status > /dev/null 2>&1 return $? ;; (Unset) log "Internal Error: is_server_up(): No server type specified." return 2 ;; (*) log "Internal Error: is_server_up(): Unknown server specified: $server" return 2 ;; esac } #------------------------------------------------------------------------------ # Shutdown p4d using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the shutdown activity. # #------------------------------------------------------------------------------ stop_p4d () { log "Shutting down the ${P4DBIN##*/} server." local -i maxStopDelay=${SDP_MAX_STOP_DELAY_P4D:-43200} local -i stopVerified=0 local -i i=0 if [[ -r "$P4DSystemdServiceFile" ]]; then { sudo systemctl stop "${P4DBIN##*/}"; } >> "$LOGFILE" 2>&1 ||\ die "Failed to execute: sudo systemctl stop ${P4DBIN##*/}" # With systemd, we must independently confirm service stop, # waiting if needed. stopVerified=0 i=0; while [[ "$i" -lt "$maxStopDelay" ]]; do if is_server_up p4d; then sleep 1 else stopVerified=1 break fi i+=1 done else "$P4DInitScript" stop >> "$LOGFILE" 2>&1 stopVerified=1 fi if [[ "$stopVerified" -eq 1 ]]; then log "Stopped ${P4DBIN##*/} server." return 0 else log "Error: Server ${P4DBIN##*/} did not stop after $maxStopDelay seconds. Tailing $P4LOG:" tail "$P4LOG" >> "$LOGFILE" 2>&1 die "Aborting due to failed p4d stop." fi } #------------------------------------------------------------------------------ # Shutdown p4broker using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the shutdown activity. # #------------------------------------------------------------------------------ stop_p4broker () { log "Shutting down the ${P4BROKERBIN##*/} server." local -i maxStopDelay=${SDP_MAX_STOP_DELAY_P4BROKER:-600} local -i stopVerified=0 local -i i=0 if [[ -r "$P4BrokerSystemdServiceFile" ]]; then { sudo systemctl stop "${P4BROKERBIN##*/}"; } >> "$LOGFILE" 2>&1 ||\ die "Failed to execute: sudo systemctl stop ${P4BROKERBIN##*/}" # With systemd, we must independently confirm service stop, # waiting if needed. stopVerified=0 i=0; while [[ "$i" -lt "$maxStopDelay" ]]; do if is_server_up p4broker; then sleep 1 else stopVerified=1 break fi i+=1 done else "$P4BrokerInitScript" stop >> "$LOGFILE" 2>&1 stopVerified=1 fi if [[ "$stopVerified" -eq 1 ]]; then log "Stopped ${P4BROKERBIN##*/} server." return 0 else die "Server ${P4BROKERBIN##*/} did not stop after $maxStartDelay seconds." fi } #------------------------------------------------------------------------------ # Shutdown p4p using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the shutdown activity. # #------------------------------------------------------------------------------ stop_p4p () { log "Shutting down the ${P4PBIN##*/} server." local -i maxStopDelay=${SDP_MAX_STOP_DELAY_P4P:-600} local -i stopVerified=0 local -i i=0 if [[ -r "$P4ProxySystemdServiceFile" ]]; then { sudo systemctl stop "${P4PBIN##*/}"; } >> "$LOGFILE" 2>&1 ||\ die "Failed to execute: sudo systemctl stop ${P4PBIN##*/}" # With systemd, we must independently confirm service stop, # waiting if needed. stopVerified=0 i=0; while [[ "$i" -lt "$maxStopDelay" ]]; do if is_server_up p4p; then sleep 1 else stopVerified=1 break fi i+=1 done else "$P4ProxyInitScript" stop >> "$LOGFILE" 2>&1 stopVerified=1 fi if [[ "$stopVerified" -eq 1 ]]; then log "Stopped ${P4PBIN##*/} server." return 0 else die "Server ${P4PBIN##*/} did not stop after $maxStopDelay seconds." fi } #------------------------------------------------------------------------------ # Start p4d using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the startup activity. # # This is a do-or-die function. It returns success upon successful server # startup, or else dies. #------------------------------------------------------------------------------ function start_p4d () { log "Starting the ${P4DBIN##*/} server." local -i maxStartDelay=${SDP_MAX_START_DELAY_P4D:-120} local -i startVerified=0 local -i i=0 if [[ -r "$P4DSystemdServiceFile" ]]; then { sudo systemctl start "${P4DBIN##*/}"; } ||\ die "Failed to execute: sudo systemctl start ${P4DBIN##*/}" else "$P4DInitScript" start >> "$LOGFILE" 2>&1 fi # Confirm that p4d started, waiting if needed. startVerified=0 i=0; while [[ "$i" -lt "$maxStartDelay" ]]; do if is_server_up p4d; then startVerified=1 break else sleep 1 fi i+=1 done if [[ "$startVerified" -eq 1 ]]; then log "Server ${P4DBIN##*/} started successfully." return 0 else log "Error: Server ${P4DBIN##*/} did not start after $maxStartDelay seconds. Tailing $P4LOG:" tail "$P4LOG" >> "$LOGFILE" 2>&1 die "Aborting due to failed p4d start." fi } #------------------------------------------------------------------------------ # Start p4broker using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the startup activity. # # This is a do-or-die function. It returns success upon successful server # startup, or else dies. #------------------------------------------------------------------------------ function start_p4broker () { log "Starting the ${P4BROKERBIN##*/} server." local -i maxStartDelay=${SDP_MAX_START_DELAY_P4BROKER:-60} local -i startVerified=0 local -i i=0 if [[ -r "$P4BrokerSystemdServiceFile" ]]; then { sudo systemctl start "${P4BROKERBIN##*/}"; } ||\ die "Failed to execute: sudo systemctl start ${P4BROKERBIN##*/}" else "$P4BrokerInitScript" start >> "$LOGFILE" 2>&1 fi # Confirm that p4broker started, waiting if needed. startVerified=0 i=0; while [[ "$i" -lt "$maxStartDelay" ]]; do if is_server_up p4broker; then startVerified=1 break else sleep 1 fi i+=1 done if [[ "$startVerified" -eq 1 ]]; then log "Server ${P4BROKERBIN##*/} started successfully." return 0 else die "Server ${P4BROKERBIN##*/} did not start after $maxStartDelay seconds." fi } #------------------------------------------------------------------------------ # Start p4p using systemd if configured for systemd. Otherwise call the # underlying init script directly. # # Log the startup activity. # # This is a do-or-die function. It returns success upon successful server # startup, or else dies. #------------------------------------------------------------------------------ function start_p4p () { log "Starting the ${P4PBIN##*/} server." local -i maxStartDelay=${SDP_MAX_START_DELAY_P4P:-60} local -i startVerified=0 local -i i=0 if [[ -r "$P4ProxySystemdServiceFile" ]]; then { sudo systemctl start "${P4PBIN##*/}"; } ||\ die "Failed to execute: sudo systemctl start ${P4PBIN##*/}" else "$P4ProxyInitScript" start >> "$LOGFILE" 2>&1 fi # Confirm that p4p started, waiting if needed. startVerified=0 i=0; while [[ "$i" -lt "$maxStartDelay" ]]; do if is_server_up p4p; then startVerified=1 break else sleep 1 fi i+=1 done if [[ "$startVerified" -eq 1 ]]; then log "Server ${P4PBIN##*/} started successfully." return 0 else die "Server ${P4PBIN##*/} did not start after $maxStartDelay seconds." fi } #------------------------------------------------------------------------------ # Do a front-door 'p4d admin journal' command to rotate the current/active # journal file on the master server, starting a fresh new P4JOURNAL file. # # In a distributed topology with replicas/edge servers, this function must # be called only on the master/commit server. #------------------------------------------------------------------------------ function truncate_journal () { local CheckpointFile="${CHECKPOINTS}/${P4SERVER}.ckp.${CHECKPOINTNUM}.gz" local JournalFile="${CHECKPOINTS}/${P4SERVER}.jnl.${JOURNALNUM}" if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then [[ -f "$CheckpointFile" ]] && \ die "Checkpoint $CheckpointFile already exists, check the backup process." [[ -f "$JournalFile" ]] && \ die "Journal $JournalFile already exists, check the backup process." log "Truncating journal..." # During journal rotation, either by a front-door 'p4 admin journal' or a # back-door 'p4d -jj', p4d does a copy-then-delete rather than an mv at # the OS level. During rotation, the perforce server will pause # responses to clients (just as with a checkpoint), but this should be # for a short period of time even for large data sets, as the journal # typically represents a single day of metadata. # Curly braces capture output of 'time'. "$P4CBIN"/p4login -p "$P4MASTERPORT" { time "$P4BIN" -p "$P4MASTERPORT" admin journal; } >> "$LOGFILE" 2>&1 || { die "Journal rotation failed. Abort!"; } # The test below waits until the journal file exists in the checkpoints directory before proceeding. test=1 while [[ $test != 0 ]]; do sleep 5 if [[ -f "$JournalFile" ]]; then test=0 fi done "$P4CBIN"/p4login else log "Warning: The truncate_journal () function should only be called on the master server. It is ignored on edge and replica replica servers." fi } #------------------------------------------------------------------------------ # Call 'p4d -jj' to rotate the current/active journal file on the master # server from an edge server, starting a fresh new P4JOURNAL file. # # In a distributed topology with edge and standby servers, this function can be # used to trigger a journal rotation on master/commit server. It's not meant to # be used from the master server itself. #------------------------------------------------------------------------------ function truncate_journal_on_master () { # Increment Edge journal number since the journal will increment on the master after calling journal rotation local EdgeJournalNum=$((JOURNALNUM + 1)) local StandbyJournalNum=$((JOURNALNUM + 2)) # If using journalcopy, have to add 2 since live journal is in checkpoints folder local JournalFile= if [[ "$EDGESERVER" -eq 1 ]]; then # Refer to ckp/jnl files starting like (example ServerID=p4d_edge_nyc): # /p4/N/checkpoints.edge_nyc/p4_N.edge_nyc JournalFile="${CHECKPOINTS}.${SERVERID#p4d_}/${P4SERVER}.${SERVERID#p4d_}.jnl.${EdgeJournalNum}" elif [[ "$STANDBYSERVER" -eq 1 ]]; then JournalFile="${CHECKPOINTS}/${P4SERVER}.jnl.${StandbyJournalNum}" fi if [[ "$SERVERID" != "$P4MASTER_ID" ]]; then [[ -f "$JournalFile" ]] && \ die "Journal $JournalFile already exists, check the backup process." log "Truncating journal on ${P4MASTERPORT}." # 'p4d -jj' does a copy-then-delete, instead of a simple mv. # During 'p4d -jj' the perforce server will hang the responses to clients, # this should be for a very short period of time even for large data # sets, as the journal represents a single day of metadata. # Curly braces capture output of 'time'. "$P4CBIN"/p4login -p "$P4MASTERPORT" { time "$P4BIN" -p "$P4MASTERPORT" admin journal; } >> "$LOGFILE" 2>&1 || { die "Journal rotation failed. Abort!"; } # The test below waits until the journal file exists in the checkpoints directory before proceeding. test=1 while [[ $test != 0 ]]; do sleep 5 if [[ -f "$JournalFile" ]]; then test=0 fi done "$P4CBIN"/p4login -service else log "Warning: truncate_journal_on_master () function should not be called on a master server. Ignoring." fi } #------------------------------------------------------------------------------ # Similar to truncate_journal() above, p4d_truncate_journal() is intended to be # usable form the p4d_base init script, to allow journal rotation on p4d # start. As it may be called from the init script, it may be called on the # master, a replica, or the edge. However, it should will only do the journal # rotation if called on the master. #------------------------------------------------------------------------------ function p4d_truncate_journal () { local JournalFile="${CHECKPOINTS}/${P4SERVER}.jnl.${JOURNALNUM}" if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then [[ -f "$JournalFile" ]] && \ die "Journal $JournalFile already exists, check the backup process." log "Rotating journal prior to starting p4d." "$P4DBIN" -r "$P4ROOT" -J "$P4JOURNAL" -jj >> "$LOGFILE" 2>&1 ||\ die "Failed to rotate journal. Aborting p4d server start." else log "Warning: The p4d_truncate_journal() function has no effect if called on a server other than the master. Ignoring." fi } #------------------------------------------------------------------------------ # Replay any and all numbered journal files into the offline databases. #------------------------------------------------------------------------------ function replay_journals_to_offline_db () { local CheckpointsDir= local FilePrefix= local NumberedJournal= local JournalPrefix= log "Replay any unreplayed journals to the offline database." check_offline_db_usable # For the master server, we can safely rely on the SDP standard that the # journalPrefix is of the form '/p4/N/checkpoints/p4_N'. For replicas and # edge servers, determine the values dynamically based on the current journal # Prefix value for the given ServerID. if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" else JournalPrefix="$("$P4DBIN" -r "$P4ROOT" -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)" if [[ -n "$JournalPrefix" ]]; then CheckpointsDir="${JournalPrefix%/*}" FilePrefix="${JournalPrefix##*/}" else log "Warning: Could not determine journalPrefix for ServerID $SERVERID." CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" fi fi for (( j=OFFLINEJNLNUM; j <= JOURNALNUM; j++ )); do NumberedJournal="${CheckpointsDir}/${FilePrefix}.jnl.${j}" log "Replay journal $NumberedJournal to offline db." rm -f "${OFFLINE_DB}/offline_db_usable.txt" >> "$LOGFILE" 2>&1 # Curly braces capture output of 'time'. { time "$P4DBIN" -r "$OFFLINE_DB" -jr -f "$NumberedJournal"; } >> "$LOGFILE" 2>&1 || { die "Offline journal replay failed. Abort!"; } echo "Offline db file restored successfully." > "${OFFLINE_DB}/offline_db_usable.txt" done } #------------------------------------------------------------------------------ # Replay the live, active P4JOURNAL file into the offline database. #------------------------------------------------------------------------------ function replay_active_journal_to_offline_db () { log "Replay active journal to offline db." local ActiveJournal= # On a standby server, the current/active journal is named /p4/N/logs/journal.<jnlNum>. # On the master and other server types, the active journal is $P4JOURNAL. if [[ "$STANDBYSERVER" -eq 1 ]]; then local _JNLNUM _JNLNUM=$("$P4DBIN" -r "$P4ROOT" -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") || die "Cannot get $P4ROOT journal number. Abort!" ActiveJournal="$LOGS/journal.$_JNLNUM" else ActiveJournal="$P4JOURNAL" fi # Curly braces capture output of 'time'. { time "$P4DBIN" -r "$OFFLINE_DB" -jr -f "${ActiveJournal}"; } >> "$LOGFILE" 2>&1 || { die "Active Journal replay failed. Abort!"; } } #------------------------------------------------------------------------------ # Recreate offline databases from the latest checkpoint. #------------------------------------------------------------------------------ function recreate_offline_db_files () { local CheckpointsDir= local FilePrefix= local LastCheckpointMD5= local LastCheckpoint= # For the master server, we can safely rely on the SDP standard that the journalPrefix # is of the form '/p4/N/checkpoints/p4_N'. For replicas and edge servers, determine # the values dynamically based on the current journalPrefix value for the given ServerID. if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" else JournalPrefix="$("$P4DBIN" -r "$P4ROOT" -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)" if [[ -n "$JournalPrefix" ]]; then CheckpointsDir="${JournalPrefix%/*}" FilePrefix="${JournalPrefix##*/}" else log "Warning: Could not determine journalPrefix for ServerID $SERVERID." CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" fi fi if [[ -z "$(ls "${CheckpointsDir}/${FilePrefix}".ckp.*.md5 2>/dev/null)" ]]; then ckp_complete if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then die "No checkpoints found in $CheckpointsDir with prefix $FilePrefix. Consider running 'live_checkpoint.sh $SDP_INSTANCE'." else die "No checkpoints found in $CheckpointsDir with prefix $FilePrefix." fi fi # shellcheck disable=SC2012 LastCheckpointMD5=$(ls -t "${CheckpointsDir}/${FilePrefix}".ckp.*.md5 | head -1) [[ -n "$LastCheckpointMD5" ]] || \ die "Could not find *.md5 file for latest checkpoint. Abort!" # shellcheck disable=SC2129 rm -f "${OFFLINE_DB}"/offline_db_usable.txt >> "$LOGFILE" 2>&1 rm -f "${OFFLINE_DB}"/db.* >> "$LOGFILE" 2>&1 rm -f "${OFFLINE_DB}"/save/db.* >> "$LOGFILE" 2>&1 # Account for the idiosyncrasy that MD5 files for checkpoints may look # like p4_N.ckp.gz.md5 or p4_N.ckp.md5. if [[ "$LastCheckpointMD5" == *".gz.md5" ]]; then LastCheckpoint="${LastCheckpointMD5%.md5}" else LastCheckpoint="${LastCheckpointMD5%.md5}.gz" fi [[ -r "$LastCheckpoint" ]] || \ die "Missing last checkpoint file: $LastCheckpoint. Abort!" log "Recovering from last full checkpoint, $LastCheckpoint." # Curly braces capture output of 'time'. { time "$P4DBIN" -r "$OFFLINE_DB" -jr -z "${LastCheckpoint}"; } >> "$LOGFILE" 2>&1 || { die "Restore of checkpoint to $OFFLINE_DB failed!"; } echo "Offline db file restored successfully." > "${OFFLINE_DB}/offline_db_usable.txt" } #------------------------------------------------------------------------------ # Take a live checkpoint from db.* files in P4ROOT. #------------------------------------------------------------------------------ function checkpoint () { local CheckpointsDir= local FilePrefix= log "Create a new checkpoint from live db files in $P4ROOT." if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" else die "Live checkpoints can only be run on the master server." fi # Curly braces capture output of 'time'. { time "$P4DBIN" -r "$P4ROOT" -jc -Z "${CheckpointsDir}/${FilePrefix}"; } >> "$LOGFILE" 2>&1 || { die "ERROR - New live checkpoint failed!"; } } #------------------------------------------------------------------------------A # Take a checkpoint from the ROOTDIR, typically either /p4/N/root or # /p4/N/offline_db. #------------------------------------------------------------------------------ function dump_checkpoint () { local CheckpointsDir= local NewCheckpoint= local NewCheckpointMD5= local FilePrefix= local JournalPrefix= local -i DoSnapshot=0 local -i SnapshotOK=1 local -i CheckpointOK=1 # shellcheck disable=SC2153 log "Dump out new checkpoint from db files in $ROOTDIR." # For the master server, we can safely rely on the SDP standard that the journalPrefix # is of the form '/p4/N/checkpoints/p4_N'. For replicas and edge servers, determine # the values dynamically based on the current journalPrefix value for the given ServerID. if [[ "$SERVERID" == "$P4MASTER_ID" ]]; then # Refer to ckp/jnl files starting like: /p4/N/checkpoints/p4_N CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" else JournalPrefix="$("$P4DBIN" -r "$P4ROOT" -k db.config -jd - | grep "@${SERVERID}@ @journalPrefix@" | cut -d '@' -f 10)" if [[ -n "$JournalPrefix" ]]; then CheckpointsDir="${JournalPrefix%/*}" FilePrefix="${JournalPrefix##*/}" else log "Warning: Could not determine journalPrefix for ServerID $SERVERID." CheckpointsDir="${CHECKPOINTS}" FilePrefix="${P4SERVER}" fi fi NewCheckpoint="${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}.gz" NewCheckpointMD5="${NewCheckpoint}.md5" if [[ -r "$NewCheckpoint" && -r "$NewCheckpointMD5" ]]; then log "Warning: Skipping generation of existing checkpoint $NewCheckpoint.\\nVerified MD5 file exists: $NewCheckpointMD5." return fi # Curly braces capture output of 'time'. if { time "$P4DBIN" -r "$ROOTDIR" -jd -z "${CheckpointsDir}/${FilePrefix}.ckp.${CHECKPOINTNUM}.gz"; } >> "$LOGFILE" 2>&1; then CheckpointOK=1 else CheckpointOK=0 fi if [[ -n "${SNAPSHOT_SCRIPT:-}" ]]; then DoSnapshot=1 log "Calling site-specific snapshot script: $SNAPSHOT_SCRIPT" if "$SNAPSHOT_SCRIPT" >> "$LOGFILE" 2>&1; then SnapshotOK=1 else SnapshotOK=0 fi fi if [[ "$DoSnapshot" -eq 0 ]]; then if [[ "$CheckpointOK" -eq 1 ]]; then log "New checkpoint dump succeeded." else die "New checkpoint dump FAILED." fi else if [[ "$CheckpointOK" -eq 0 && "$SnapshotOK" -eq 0 ]]; then die "Both checkpoint dump and snapshot FAILED." elif [[ "$CheckpointOK" -eq 1 && "$SnapshotOK" -eq 0 ]]; then die "New checkpoint dump succeeded, but snapshot FAILED." elif [[ "$CheckpointOK" -eq 0 && "$SnapshotOK" -eq 1 ]]; then die "New checkpoint dump FAILED, but snapshot succeeded." else log "New checkpoint dump and snapshot succeeded." fi fi } #------------------------------------------------------------------------------ # Compare journal numbers between live and offline databases, to ensure # they can be safely swapped out. #------------------------------------------------------------------------------ function compare_journal_numbers () { local _OFFLINEJNLNUM _OFFLINEJNLNUM=$("$P4DBIN" -r "$OFFLINE_DB" -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") || die "Cannot get $OFFLINE_DB journal number. Abort!" check_journalnum "$_OFFLINEJNLNUM" # Get the journal number of the root database if [[ ! -f "$P4ROOT/db.counters" ]]; then die "$P4ROOT database not found. Something is seriously wrong since the server was just running a minute ago! Contact support-helix-core@perforce.com" fi local _JNLNUM _JNLNUM=$("$P4DBIN" -r "$P4ROOT" -jd - db.counters 2>&1 | grep '@journal@' | cut -d "@" -f 8 2>> "$LOGFILE") || die "Cannot get $P4ROOT journal number. Abort!" check_journalnum "$_JNLNUM" if [[ "$_JNLNUM" -gt "$_OFFLINEJNLNUM" ]]; then log "$P4ROOT journal number is: $_JNLNUM" log "$OFFLINE_DB journal number is: $_OFFLINEJNLNUM" die "$OFFLINE_DB journal number is less than $P4ROOT, cannot switch." fi } #------------------------------------------------------------------------------ # Swap out live db.* database files in P4ROOT with those in offline_db. #------------------------------------------------------------------------------ function switch_db_files () { local verifyCmd="$P4CBIN/verify_sdp.sh -skip crontab,excess,init,license,masterid,version -L off" log "Calling 'verify_sdp.sh' before swapping db.* files:\\n$verifyCmd" $verifyCmd >> "$LOGFILE" 2>&1 ||\ die "Error: Cannot confirm all is well with $P4CBIN/verify_sdp.sh. Aborting" # Compare the Offline and Master journal numbers before switching to make # sure they match. compare_journal_numbers log "Switching root and offline_db links." [[ -d "${P4ROOT}"/save ]] || mkdir -p "${P4ROOT}"/save >> "$LOGFILE" 2>&1 # shellcheck disable=SC2129 echo "P4ROOT is not available during switch_db_files() processing." > "$P4ROOT/P4ROOT_not_usable.txt" 2>> "$LOGFILE" echo "P4ROOT is not available during switch_db_files() processing." > "$OFFLINE_DB/P4ROOT_not_usable.txt" 2>> "$LOGFILE" # shellcheck disable=SC2129 rm -f "${P4ROOT}"/save/db.* >> "$LOGFILE" 2>&1 rm -rf "${P4ROOT}"/server.locks >> "$LOGFILE" 2>&1 mv "${P4ROOT}"/db.* "${P4ROOT}"/save/. >> "$LOGFILE" 2>&1 if [[ -r "$P4ROOT"/license ]]; then mv "${P4ROOT}"/license "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -n "$(ls "$P4ROOT"/license* 2>/dev/null)" ]]; then mv "${P4ROOT}"/license* "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -r "${P4ROOT}"/rdb.lbr ]]; then mv "${P4ROOT}"/rdb.lbr "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -n "$(ls "$P4ROOT"/state* 2>/dev/null)" ]]; then mv "${P4ROOT}"/state* "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -r "${P4ROOT}"/server.id ]]; then mv "${P4ROOT}"/server.id "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi if [[ -n "$(ls "$P4ROOT"/server.id* 2>/dev/null)" ]]; then mv "${P4ROOT}"/server.id* "${OFFLINE_DB}/." >> "$LOGFILE" 2>&1 fi rm -f "${OFFLINE_DB}/offline_db_usable.txt" >> "$LOGFILE" 2>&1 LinkOfflineDB="$(readlink "$OFFLINE_DB")" LinkP4ROOT="$(readlink "$P4ROOT")" unlink "$OFFLINE_DB" unlink "$P4ROOT" ln -s "$LinkOfflineDB" "$P4ROOT" >> "$LOGFILE" 2>&1 ||\ die "Link of $LinkOfflineDB to $P4ROOT failed." ln -s "$LinkP4ROOT" "$OFFLINE_DB" >> "$LOGFILE" 2>&1 ||\ die "Link of $LinkP4ROOT to $OFFLINE_DB failed." rm -f "$P4ROOT/P4ROOT_not_usable.txt" >> "$LOGFILE" 2>&1 rm -f "$OFFLINE_DB/P4ROOT_not_usable.txt" >> "$LOGFILE" 2>&1 } #------------------------------------------------------------------------------ # Function: trim_log_file ($LogToTrim, $MaxLines) # # For log files expected to be short, keep them at a max size. # # When they get too big, trim them from the top first, so the most recent # output is retained. # # Thus function treats its processing as non-essential; most errors are # silently ignored and output discarded. Only error output related to replacing # the original log is retained in LOGFILE or displayed. #------------------------------------------------------------------------------ function trim_log_file () { local LogToTrim="${1:-}" local MaxLines="${2:-5000}" local TmpFile= local Lines= [[ -w "$LogToTrim" ]] || return # Abort if MaxLines isn't numeric. [[ "$MaxLines" =~ ^[0-9]+$ ]] || return TmpFile="${LogToTrim}.trimming.$$.$RANDOM" Lines=$(wc -l "$LogToTrim") Lines=${Lines%% *} # Confirm Lines is a number, else just abort. [[ "$Lines" =~ ^[0-9]+$ ]] || return # If the file isn't big enough to need trimming, abort. [[ "$Lines" -gt "$MaxLines" ]] || return log "Trimming log $LogToTrim from $Lines to $MaxLines lines." # If the trimming fails, discard output and just return. if tail -"$MaxLines" "$LogToTrim" > "$TmpFile" 2>/dev/null; then if [[ -n "${LOGFILE:-}" ]]; then mv -f "$TmpFile" "$LogToTrim" >> "$LOGFILE" 2>&1 else mv -f "$TmpFile" "$LogToTrim" fi else return fi } #------------------------------------------------------------------------------ # Rotate specified log files, and compress with gzip. #------------------------------------------------------------------------------ function rotate_log_file () { local LogToRotate="${1:-}" local GzExt="${2:-}" local -i i=1 local Datestamp= local RotatedLog= local RotatedZippedLog= [[ -n "$LogToRotate" ]] || return if [[ -n "${LOGFILE:-}" ]]; then pushd "$LOGS" > /dev/null 2>> "$LOGFILE" || die "Could not cd to: $LOGS" else pushd "$LOGS" > /dev/null || die "Could not cd to: $LOGS" fi Datestamp=$(date +'%Y-%m-%d_%H-%M-%S') RotatedLog="${LogToRotate}.${Datestamp}" if [[ -f "${LogToRotate}" ]]; then if [[ -n "${LOGFILE:-}" ]]; then mv -f "${LogToRotate}" "${RotatedLog}" >> "$LOGFILE" 2>&1 if [[ -n "$GzExt" ]]; then RotatedZippedLog="${RotatedLog}${GzExt}" # If needed, move existing zipped log aside. if [[ -e "$RotatedZippedLog" ]]; then while [[ -e "${LogToRotate}.${Datestamp}.${i}${GzExt}" ]]; do i+=1 done log "Moving pre-existing $RotatedZippedLog aside to ${LogToRotate}.${Datestamp}.${i}${GzExt}" >> "$LOGFILE" 2>&1 mv -f "$RotatedZippedLog" "${LogToRotate}.${Datestamp}.${i}${GzExt}" >> "$LOGFILE" 2>&1 fi gzip "$RotatedLog" >> "$LOGFILE" 2>&1 fi else mv -f "${LogToRotate}" "${RotatedLog}" if [[ -n "$GzExt" ]]; then RotatedZippedLog="${RotatedLog}${GzExt}" # If needed, move existing zipped log aside. if [[ -e "$RotatedZippedLog" ]]; then while [[ -e "${LogToRotate}.${Datestamp}.${i}${GzExt}" ]]; do i+=1 done log "Moving pre-existing $RotatedZippedLog aside to ${LogToRotate}.${Datestamp}.${i}${GzExt}" mv -f "$RotatedZippedLog" "${LogToRotate}.${Datestamp}.${i}${GzExt}" fi gzip "$RotatedLog" fi fi fi if [[ -n "${LOGFILE:-}" ]]; then popd > /dev/null 2>> "$LOGFILE" || die "Could not cd to: $OLDPWD" else popd > /dev/null || die "Could not cd to: $OLDPWD" fi } #------------------------------------------------------------------------------ # At the start of each run for live_checkpoint.sh, daily_checkpoint.sh, and # recreate_db_checkpoint.sh, before *any* logging activity occurs, rotate the # logs from the most recent prior run, always named "checkpoint.log" or "log". #------------------------------------------------------------------------------ function rotate_last_run_logs () { # Rotate prior log file for the current script. rotate_log_file "$LOGFILE" # Rotate prior server log. rotate_log_file "log" ".gz" # Rotate prior broker log. rotate_log_file "p4broker.log" ".gz" # Rotate prior audit log. rotate_log_file "audit.log" ".gz" } #------------------------------------------------------------------------------ # Remove log files matching a specified name prefix, preserving a specified # number of the recent logs. #------------------------------------------------------------------------------ function remove_log_files () { REMOVE_LOGNAME=$1 KEEPNUM=$2 # shellcheck disable=SC2012 for I_LOGFILE in $(ls -t "${REMOVE_LOGNAME:?}"* 2>/dev/null | $AWK "NR > $KEEPNUM"); do log "rm -f $I_LOGFILE" rm -f "$I_LOGFILE" done } #------------------------------------------------------------------------------ # Remove old logs. #------------------------------------------------------------------------------ function remove_old_logs () { # Remove old Checkpoint Logs # Use KEEPJNLS rather than KEEPLOGS, so we keep the same number # of checkpoint logs as we keep checkpoints. pushd "$LOGS" > /dev/null 2>> "$LOGFILE" || die "Could not cd to: $LOGS" if [[ "$KEEPJNLS" -eq 0 ]]; then log "Skipping cleanup of old checkpoint logs because KEEPJNLS is set to 0." else log "Deleting old checkpoint logs. Keeping latest $KEEPJNLS, per KEEPJNLS setting in p4_vars." remove_log_files "checkpoint.log" "$KEEPJNLS" fi if [[ "$KEEPLOGS" -eq 0 ]]; then log "Skipping cleanup of old server logs because KEEPLOGS is set to 0." else log "Deleting old server logs. Keeping latest $KEEPLOGS, per KEEPLOGS setting in p4_vars." remove_log_files "log" "$KEEPLOGS" remove_log_files "p4broker.log" "$KEEPLOGS" remove_log_files "broker_rotate.log" "$KEEPLOGS" remove_log_files "audit.log" "$KEEPLOGS" remove_log_files "sync_replica.log" "$KEEPLOGS" remove_log_files "replica_status.log" "$KEEPLOGS" remove_log_files "replica_cleanup.log" "$KEEPLOGS" remove_log_files "request_checkpoint.log" "$KEEPLOGS" remove_log_files "recreate_offline_db.log" "$KEEPLOGS" remove_log_files "edge_shelf_replicate.log" "$KEEPLOGS" remove_log_files "upgrade.log" "$KEEPLOGS" remove_log_files "p4login" "$KEEPLOGS" remove_log_files "p4verify.log" "$KEEPLOGS" remove_log_files "journal_watch.log" "$KEEPLOGS" remove_log_files "refresh_P4ROOT_from_offline_db.log" "$KEEPLOGS" remove_log_files "purge_revisions.log" "$KEEPLOGS" fi popd > /dev/null 2>>"$LOGFILE" || die "Could not cd to: $OLDPWD" } #------------------------------------------------------------------------------ # Set the SDP Checkpoint counter to indicate last successful SDP checkpoint # operation. For standby servers, set the SDP Checkpoint counter on the master. #------------------------------------------------------------------------------ function set_counter() { "$P4CBIN/p4login" if [[ "$EDGESERVER" -eq 1 || "$STANDBYSERVER" -eq 1 ]]; then "$P4BIN" -u "$P4USER" -p "$P4MASTERPORT" counter "LastSDPCheckpoint.$SERVERID" "$(date +'%s (%Y/%m/%d %H:%M:%S %z %Z)')" > /dev/null else "$P4BIN" -u "$P4USER" -p "$P4PORT" counter "LastSDPCheckpoint.$SERVERID" "$(date +'%s (%Y/%m/%d %H:%M:%S %z %Z)')" > /dev/null fi }
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#77 | 30915 | C. Thomas Tyler |
Released SDP 2024.1.30913 (2024/11/20). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#76 | 30388 | C. Thomas Tyler |
Released SDP 2024.1.30385 (2024/06/11). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#75 | 30297 | C. Thomas Tyler |
Released SDP 2023.2.30295 (2024/05/08). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#74 | 30043 | C. Thomas Tyler |
Released SDP 2023.2.30041 (2023/12/22). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#73 | 29954 | C. Thomas Tyler |
Released SDP 2023.1.29949 (2023/12/01). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#72 | 29891 | C. Thomas Tyler |
Released SDP 2023.1.29699 (2023/07/11). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#71 | 29612 | C. Thomas Tyler |
Released SDP 2023.1.29610 (2023/05/25). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#70 | 29443 | C. Thomas Tyler |
Released SDP 2022.2.29441 (2023/02/27). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#69 | 29401 | C. Thomas Tyler |
Released SDP 2022.2.29399 (2023/02/06). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#68 | 29252 | C. Thomas Tyler |
Released SDP 2022.2.29250 (2022/12/08). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#67 | 29143 | C. Thomas Tyler |
Released SDP 2022.1.29141 (2022/10/29). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#66 | 28858 | C. Thomas Tyler |
Released SDP 2022.1.28855 (2022/05/27). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#65 | 28651 | C. Thomas Tyler |
Released SDP 2021.2.28649 (2022/03/03). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#64 | 28412 | C. Thomas Tyler |
Released SDP 2021.2.28410 (2021/11/24). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#63 | 28240 | C. Thomas Tyler |
Released SDP 2021.1.28238 (2021/11/12). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#62 | 27761 | C. Thomas Tyler |
Released SDP 2020.1.27759 (2021/05/07). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#61 | 27463 | C. Thomas Tyler |
Released SDP 2020.1.27457 (2021/02/17). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#60 | 27331 | C. Thomas Tyler |
Released SDP 2020.1.27325 (2021/01/29). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#59 | 26496 | C. Thomas Tyler |
Released SDP 2019.3.26494 (2020/04/23). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#58 | 26480 | C. Thomas Tyler |
Released SDP 2019.3.26478 (2020/04/12). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#57 | 26475 | C. Thomas Tyler |
Released SDP 2019.3.26472 (2020/04/10). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#56 | 26470 | C. Thomas Tyler |
Released SDP 2019.3.26468 (2020/04/10). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#55 | 26411 | C. Thomas Tyler |
Released SDP 2019.3.26407 (2020/03/28). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#54 | 26403 | C. Thomas Tyler |
Released SDP 2019.3.26400 (2020/03/28). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#53 | 26246 | C. Thomas Tyler |
Released SDP 2019.3.26239 (2020/01/08). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#52 | 26161 | C. Thomas Tyler |
Released SDP 2019.3.26159 (2019/11/06). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#51 | 25596 | C. Thomas Tyler |
Released SDP 2019.2.25594 (2019/05/02). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#50 | 25380 | C. Thomas Tyler |
Released SDP 2019.1.25374 (2019/03/21). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#49 | 25279 | C. Thomas Tyler |
Released SDP 2019.1.25276 (2019/03/06). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#48 | 25245 | C. Thomas Tyler |
Released SDP 2019.1.25238 (2019/03/02). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#47 | 23331 | C. Thomas Tyler |
Released SDP 2017.4.23329 (2017/12/05). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#46 | 23044 | C. Thomas Tyler |
Released SDP 2017.3.23041 (2017/10/24). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#45 | 23006 | C. Thomas Tyler |
Released SDP 2017.3.23003 (2017/10/19). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#44 | 22685 | Russell C. Jackson (Rusty) | Update main with current changes from dev. | ||
#43 | 22185 | C. Thomas Tyler |
Released SDP 2017.2.22177 (2017/05/17). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#42 | 21723 | C. Thomas Tyler |
Released SDP 2017.1.21720 (2017/02/17). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#41 | 21338 | C. Thomas Tyler |
Released SDP 2016.2.21328 (2016/12/16). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#40 | 21193 | Russell C. Jackson (Rusty) | Update main from dev. | ||
#39 | 20974 | C. Thomas Tyler |
Released SDP 2016.2.20972 (2016/11/01). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#38 | 20858 | C. Thomas Tyler |
Released SDP 2016.2.20856 (2016/10/04). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#37 | 20767 | C. Thomas Tyler |
Released SDP 2016.2.20755 (2016/09/29). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#36 | 20353 | C. Thomas Tyler |
Released SDP 2016.1.20348. Copy Up using 'p4 copy -r -b perforce_software-sdp-dev', with selective removal of changes related to work-in-progress changes. |
||
#35 | 19898 | C. Thomas Tyler |
Released SDP/MultiArch/2016.1/19888 (2016/07/07). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#34 | 19835 | C. Thomas Tyler |
Released Rev. SDP/MultiArch/2016.1/19768 (2016/06/24). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#33 | 19694 | C. Thomas Tyler |
Released SDP/MultiArch/2016.1/19661 (2016/06/08). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
#32 | 19414 | C. Thomas Tyler | Released SDP/MultiArch/2016.1/19410 (2016/05/17). | ||
#31 | 18961 | C. Thomas Tyler | Released: SDP/MultiArch/2016.1/18958 (2016/04/08). | ||
#30 | 18619 | Russell C. Jackson (Rusty) | Updating main with current changes. | ||
#29 | 18530 | Russell C. Jackson (Rusty) | Update main from dev. | ||
#28 | 16155 | Russell C. Jackson (Rusty) |
Removed check code that probably hasn't ever worked. Deleted mirror_ldap* since that functionality is built into the server now. |
||
#27 | 15856 | C. Thomas Tyler |
Replaced the big license comment block with a shortened form referencing the LICENSE file included with the SDP package, and also by the URL for the license file in The Workshop. |
||
#26 | 15784 | Russell C. Jackson (Rusty) | Added missing bracket. | ||
#25 | 15780 | C. Thomas Tyler |
Per Robert: Style police causing problems :) Fixed! |
||
#24 | 15777 | C. Thomas Tyler |
No functional changes. Style Policing only on bash scripts only. Normalized indentation and line breaks, removed offending tabs, and general whitespace usage. |
||
#23 | 15609 | C. Thomas Tyler | Pushing SDP 2015.1.15607 (2015/09/02). | ||
#22 | 15197 | Russell C. Jackson (Rusty) | Corrected versions from testing. | ||
#21 | 15193 | Russell C. Jackson (Rusty) |
Added semaphore file to indicate state of the offline database and added check into the backup process to fail if the state of the offline db is not good. |
||
#20 | 15190 | Russell C. Jackson (Rusty) |
Added a semaphore file to prevent the checkpoint process from running if another one hasn't finished. Added a check to make sure the journal number is numeric. |
||
#19 | 13928 | dsp |
Set lastSDPCounter after a successfull SDP checkpoint p4 admin checkpoint sets lastCheckpointAction, which is useful for monitoring, in particular when checkpoint age should be observed from the outside through p4. However the SDP is using p4d directly to create checkpoints and will not set checkpoints. In order to distinguish human actions from the SDP cronjobs, set a new counter lastSDPCounter in a similar format. |
||
#18 | 13908 | C. Thomas Tyler | Pushing SDP 2015.1.13906. | ||
#17 | 12171 | Russell C. Jackson (Rusty) | Merge in changes to remove the need for p4master_run. | ||
#16 | 11950 | Russell C. Jackson (Rusty) |
Made die function record ERROR!!! $HOSTNAME and $P4SERVER in subject. Cleaned up message passed to die command and corrected a typo. |
||
#15 | 11929 | Russell C. Jackson (Rusty) | Updated die function to just pass parameter to mail_log_file instead of echo. | ||
#14 | 11919 | Russell C. Jackson (Rusty) |
Added a SERVERID variable to p4_vars and updated backup_functions to use it. Changed the location and the names of the config files so that they could live in /p4/common/config (You're welcome Tom). The files names are: p4_$INSTANCE.vars p4_$INSTANCE.p4review.cfg p4_$INSTANCE.vars will now set P4REPLICA to FALSE if SERVERID matches MASTERNAME, otherwise it is TRUE. This change means that a user must change server.id now in order to change the role of the server rather than changing the instance vars file. This makes more sense to a user that is reading the admin guide about server.id rather than overwriting the file based on a setting that isn't in the admin guide. Change mkdirs to reflect all of the above changes. |
||
#13 | 11908 | adrian_waters | Use set -u to trap unbounded variables | ||
#12 | 11886 | Russell C. Jackson (Rusty) | Changed $prog to $0 so that we don't have to set prog in the calling functions. | ||
#11 | 11766 | Robert Cowham |
Missed a reference to $MAIL in @11758 Tweaked run_tests.sh to output more error messages on failure. Though this still doesn't show output of individual failed commands. |
||
#10 | 11758 | Russell C. Jackson (Rusty) |
Change MAIL variable to SDPMAIL to avoid conflicts with customer variables. Changed sdp_sync.sh to use get_mail_opts from backup_functions to avoid duplicate functions. |
||
#9 | 11730 | Russell C. Jackson (Rusty) |
Moved P4SERVER variable to p4_vars so that all scripts can use it properly. replica_status.sh referenced it, but it wasn't working since it was only in backup_functions.sh |
||
#8 | 11710 | Russell C. Jackson (Rusty) |
Changed die function to call new email function. Added su to OSUSER functionality to p4master_run to avoid problems with people running scripts manually as root by mistake. |
||
#7 | 11707 | Robert Cowham |
Refactored sending of mail to a common function. Make the setting of "MAILFROM" work for Ubuntu (GNU Mailutils) as well as CentOS |
||
#6 | 11570 | Russell C. Jackson (Rusty) |
Brought in changes from Mark Foundry to add -S $MAILFROM to mail commands. Changed sync_replica.sh and weekly_sync_replica.sh to use $LOGFILE for consistency. Added mail command to both files as well. |
||
#5 | 11540 | Russell C. Jackson (Rusty) | Converted to unix format. | ||
#4 | 11534 | Russell C. Jackson (Rusty) | Added -f to -jr to cover offline obliterates where the entries are already removed from the offline database. | ||
#3 | 11524 | Russell C. Jackson (Rusty) | Released updated version of the SDP from Dev. | ||
#2 | 11130 | Robert Cowham | Check for the existence of offline database and log error message if not found. | ||
#1 | 10148 | C. Thomas Tyler | Promoted the Perforce Server Deployment Package to The Workshop. |