failover_p4d_to_this_host.sh #1

#!/usr/local/bin/bash
Version=1.0.3

# Usage: failover_p4d_from_this_host.sh SDP_INSTANCE Style Type NewServerID NO_OP
# This is intended to be called by 'hms', not directly by a human.

# Micro-functions.
function msg () { echo -e "$*"; }
function bail () { msg "\nError: ${1:-Unknown Error}\n"; exit ${2:-1}; }
function usage () { msg "Usage:\n\t$UsageSummary\n"; exit 2; }
function cmd () {
   if [[ $NO_OP -eq 0 ]]; then
      msg "Running: $*"
      $*
      return $?
   else
      msg "NO_OP: Would run: $*"
   fi
}

# Do nothing unless usage is correct.
export SDP_INSTANCE=${1:-Unset}
declare Style=${2:-Unset}
declare Type=${3:-Unset}
declare NewServerID=${4:-Unset}
declare NO_OP=${5:-Unset}
declare ThisScript=${0##*/}
declare UsageSummary="$ThisScript SDP_INSTANCE Style Type NewServerID NO_OP"
declare P4DInitScript=/p4/$SDP_INSTANCE/bin/p4d_${SDP_INSTANCE}_init
declare VerifyCmd=
declare -i ExitCode=

[[ $# -eq 5 ]] || bail "Bad Usage!  Usage:\n\t$UsageSummary\n"

msg "Starting $ThisScript v$Version on ${HOSTNAME%%.*} at $(date)."
msg "Args: I=$SDP_INSTANCE S=$Style T=$Type NS=$NewServerID N=$NO_OP"

# Load SDP controlled shell environment.
source /p4/common/bin/p4_vars "$SDP_INSTANCE" || \
   bail "Failed to load SDP environment for instance $SDP_INSTANCE."

[[ $(id -u -n) == $OSUSER ]] || bail "$0 can only be run by $OSUSER"

[[ $NO_OP -eq 1 ]] && msg "NO_OP: Preview Mode Enabled."

[[ -x $P4DInitScript ]] || bail "$P4DInitScript is not executable."

[[ -r $P4ROOT/server.id ]] || bail "Missing file $P4ROOT/server.id. Aborting."

[[ $Style == Scheduled && ! -r $P4JOURNAL.from_old_master && $NO_OP -eq 0 ]] && \
   bail "The journal file from the master [$P4JOURNAL.from_old_master] is expected with Scheduled Failover but is missing. Aborting."

[[ ${SERVERID^^} != *"MASTER"* ]] ||\
   bail "Instance $SDP_INSTANCE ServerID ($SERVERID) indicates it is already a master server! Aborting Failover."

cmd $P4DInitScript stop

if [[ $Style == Scheduled  ]]; then
   cmd /bin/mv -f $P4JOURNAL $P4JOURNAL.moved_by_hms.$(date +'%Y%m%d-%H%M%S') ||\
      bail "Failed to move $P4JOURNAL aside on ${HOSTNAME%%.%}."
   cmd /bin/mv -f $P4JOURNAL.from_old_master $P4JOURNAL ||\
      bail "Failed to move $P4JOURNAL.from_old_master to $P4JOURNAL."
fi

if [[ $Style == Scheduled ]]; then
   msg "Replaying journal from the master just in case this replica wasn't fully caught up."
   cmd $P4DBIN -r $P4ROOT -f -jr $P4JOURNAL || bail "Failed to replay journal from old master."
fi

cmd mv $P4ROOT/server.id $P4ROOT/server.id.moved_by_hms.$(date +'%Y%m%d-%H%M%S') ||\
   bail "Failed to move $P4ROOT/server.id file aside."

cmd $P4DBIN -xD $NewServerID || bail "Failed to set new ServerID to $NewServerID."

export SERVERID=$(cat ${P4ROOT}/server.id)
if [[ "$SERVERID" == "$NewServerID" ]]; then
   msg "Verified: New ServerID ($SERVERID) is set."
else
   if [[ $NO_OP -eq 0 ]]; then
      bail "Failed to confirm new ServerID was set. Aborting failover."
   else
      msg "Verified: New ServerID ($NewServerID) is set. (SIMULATED CHECK)."
   fi
fi

###
### Key Decisions:  If some archive files are lost, do we want to prevent a return
### to service?  For now, go live with information.
###

cmd $P4DInitScript start

if [[ $Type == Full ]]; then
   [[ $NO_OP -eq 0 ]] && sleep 2
   VerifyCmd="$P4CBIN/p4verify.sh $SDP_INSTANCE -recent"
   [[ $Style == Scheduled ]] && VerifyCmd+=" -recover"

   msg "Verifying recent changes to see if any where not replicated."
   cmd $VerifyCmd < /dev/null > /dev/null 2>&1
   cat $LOGS/p4verify.log

   if [[ $? -eq 0 ]]; then
      msg "\n$Style $Type Failover of p4d to ${HOSTNAME%%.*}, including verification of recent archive files, completed successfully at $(date)."
   else
      msg "\nError: Databases are OK, but couldn't verify recent archive files survived failover."
      msg "\nMetadata for $Style $Type Failover of p4d to ${HOSTNAME%%.*} completed at $(date)."
      ExitCode=2
   fi
else
   msg "\n$Style $Type Failover of p4d to ${HOSTNAME%%.*} completed successfully at $(date)."
fi

msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

exit $ExitCode