failover_p4d_to_this_host.sh #2

  • //
  • guest/
  • perforce_software/
  • sdp/
  • main/
  • Server/
  • Unix/
  • p4/
  • common/
  • hms/
  • scripts/
  • failover_p4d_to_this_host.sh
  • View
  • Commits
  • Open Download .zip Download (4 KB)
#!/bin/bash
Version=1.0.4

# Usage: failover_p4d_from_this_host.sh SDP_INSTANCE Style Type NewServerID NO_OP
# This is intended to be called by 'hms', not directly by a human.

# Micro-functions.
function msg () { echo -e "$*"; }
function bail () { msg "\nError: ${1:-Unknown Error}\n"; exit ${2:-1}; }
function usage () { msg "Usage:\n\t$UsageSummary\n"; exit 2; }
function cmd () {
   if [[ $NO_OP -eq 0 ]]; then
      msg "Running: $*"
      $*
      return $?
   else
      msg "NO_OP: Would run: $*"
   fi
}

# Do nothing unless usage is correct.
export SDP_INSTANCE=${1:-Unset}
declare Style=${2:-Unset}
declare Type=${3:-Unset}
declare NewServerID=${4:-Unset}
declare NO_OP=${5:-Unset}
declare ThisScript=${0##*/}
declare UsageSummary="$ThisScript SDP_INSTANCE Style Type NewServerID NO_OP"
declare P4DInitScript=/p4/$SDP_INSTANCE/bin/p4d_${SDP_INSTANCE}_init
declare VerifyCmd=
declare -i ExitCode=

[[ $# -eq 5 ]] || bail "Bad Usage!  Usage:\n\t$UsageSummary\n"

msg "Starting $ThisScript v$Version on ${HOSTNAME%%.*} at $(date)."
msg "Args: I=$SDP_INSTANCE S=$Style T=$Type NS=$NewServerID N=$NO_OP"

# Load SDP controlled shell environment.
source /p4/common/bin/p4_vars "$SDP_INSTANCE" || \
   bail "Failed to load SDP environment for instance $SDP_INSTANCE."

[[ $(id -u -n) == $OSUSER ]] || bail "$0 can only be run by $OSUSER"

[[ $NO_OP -eq 1 ]] && msg "NO_OP: Preview Mode Enabled."

[[ -x $P4DInitScript ]] || bail "$P4DInitScript is not executable."

[[ -r $P4ROOT/server.id ]] || bail "Missing file $P4ROOT/server.id. Aborting."

[[ $Style == Scheduled && ! -r $P4JOURNAL.from_old_master && $NO_OP -eq 0 ]] && \
   bail "The journal file from the master [$P4JOURNAL.from_old_master] is expected with Scheduled Failover but is missing. Aborting."

[[ ${SERVERID^^} != *"MASTER"* ]] ||\
   bail "Instance $SDP_INSTANCE ServerID ($SERVERID) indicates it is already a master server! Aborting Failover."

cmd $P4DInitScript stop

if [[ $Style == Scheduled  ]]; then
   cmd /bin/mv -f $P4JOURNAL $P4JOURNAL.moved_by_hms.$(date +'%Y%m%d-%H%M%S') ||\
      bail "Failed to move $P4JOURNAL aside on ${HOSTNAME%%.%}."
   cmd /bin/mv -f $P4JOURNAL.from_old_master $P4JOURNAL ||\
      bail "Failed to move $P4JOURNAL.from_old_master to $P4JOURNAL."
fi

if [[ $Style == Scheduled ]]; then
   msg "Replaying journal from the master just in case this replica wasn't fully caught up."
   cmd $P4DBIN -r $P4ROOT -f -jr $P4JOURNAL || bail "Failed to replay journal from old master."
fi

cmd mv $P4ROOT/server.id $P4ROOT/server.id.moved_by_hms.$(date +'%Y%m%d-%H%M%S') ||\
   bail "Failed to move $P4ROOT/server.id file aside."

cmd $P4DBIN -xD $NewServerID || bail "Failed to set new ServerID to $NewServerID."

export SERVERID=$(cat ${P4ROOT}/server.id)
if [[ "$SERVERID" == "$NewServerID" ]]; then
   msg "Verified: New ServerID ($SERVERID) is set."
else
   if [[ $NO_OP -eq 0 ]]; then
      bail "Failed to confirm new ServerID was set. Aborting failover."
   else
      msg "Verified: New ServerID ($NewServerID) is set. (SIMULATED CHECK)."
   fi
fi

###
### Key Decisions:  If some archive files are lost, do we want to prevent a return
### to service?  For now, go live with information.
###

cmd $P4DInitScript start

if [[ $Type == Full ]]; then
   [[ $NO_OP -eq 0 ]] && sleep 2
   VerifyCmd="$P4CBIN/p4verify.sh $SDP_INSTANCE -recent"
   [[ $Style == Scheduled ]] && VerifyCmd+=" -recover"

   msg "Verifying recent changes to see if any where not replicated."
   cmd $VerifyCmd < /dev/null > /dev/null 2>&1
   cat $LOGS/p4verify.log

   if [[ $? -eq 0 ]]; then
      msg "\n$Style $Type Failover of p4d to ${HOSTNAME%%.*}, including verification of recent archive files, completed successfully at $(date)."
   else
      msg "\nError: Databases are OK, but couldn't verify recent archive files survived failover."
      msg "\nMetadata for $Style $Type Failover of p4d to ${HOSTNAME%%.*} completed at $(date)."
      ExitCode=2
   fi
else
   msg "\n$Style $Type Failover of p4d to ${HOSTNAME%%.*} completed successfully at $(date)."
fi

msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

exit $ExitCode
# Change User Description Committed
#4 25596 C. Thomas Tyler Released SDP 2019.2.25594 (2019/05/02).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#3 25245 C. Thomas Tyler Released SDP 2019.1.25238 (2019/03/02).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#2 22185 C. Thomas Tyler Released SDP 2017.2.22177 (2017/05/17).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
#1 20767 C. Thomas Tyler Released SDP 2016.2.20755 (2016/09/29).
Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'.
//guest/perforce_software/sdp/dev/Server/Unix/p4/common/hms/scripts/failover_p4d_to_this_host.sh
#1 20745 C. Thomas Tyler Approving as is since it isn't changing core SDP functionality, and reviewing it all line by line will take some time.
We can do that as we move forward with it.

First addition of HMS v1.0 files.  This change is a soft launch HMS
for initial deployment and testing.

Updates to HMS-related files are expected and will bypass pre-commit
code review until stabilized.