failover_p4d_local.sh #1

#!/usr/local/bin/bash
Version=1.0.6

# Usage: failover_p4d_local.sh SDP_INSTANCE Style NO_OP
# This is intended to be called by 'hms', not directly by a human.

# Micro-functions.
function msg () { echo -e "$*"; }
function bail () { msg "\nError: ${1:-Unknown Error}\n"; exit ${2:-1}; }
function usage () { msg "Usage:\n\t$UsageSummary\n"; exit 2; }
function cmd () {
   if [[ $NO_OP -eq 0 ]]; then
      msg "Running: $*"
      $*
      return $?
   else
      msg "NO_OP: Would run: $*"
   fi
}

# Do nothing unless usage is correct.
export SDP_INSTANCE=${1:-Unset}
declare Style=${2:-Unset}
declare NO_OP=${3:-Unset}
declare ThisScript=${0##*/}
declare UsageSummary="$ThisScript SDP_INSTANCE Style NO_OP"
declare DBCheckFile=
declare P4DInitScript=/p4/$SDP_INSTANCE/bin/p4d_${SDP_INSTANCE}_init

[[ $# -eq 3 ]] || bail "Bad Usage!  Usage:\n\t$UsageSummary\n"

msg "Starting $ThisScript v$Version on ${HOSTNAME%%.*} at $(date)."
msg "Args: I=$SDP_INSTANCE S=$Style NO_OP=$NO_OP"

# Load SDP controlled shell environment.
source /p4/common/bin/p4_vars "$SDP_INSTANCE" ||\
   bail "Failed to load SDP environment for instance $SDP_INSTANCE."

[[ $(id -u -n) == $OSUSER ]] || bail "$0 can only be run by $OSUSER"

[[ $NO_OP -eq 1 ]] && msg "NO_OP: Preview Mode Enabled."

[[ -x $P4DInitScript ]] || bail "$P4DInitScript is not executable."

msg "$0 v$Version starting Local Failover on host $(hostname) at $(date)."

DBCheckFile=$P4HOME/offline_db/offline_db_usable.txt

[[ -r "$DBCheckFile" ]] ||\
   bail "Could'nt verify offline DBs are good; missing check file [$DBCheckFile].\nAborting local failover."

msg "Verified: Offline DBs are usable."

[[ ${SERVERID^^} == *"MASTER"* ]] ||\
   bail "Local failover can only be done on a master server."

cmd $P4DInitScript stop

### TO DO: Add disk space avail safety checks.
### Thoughts:
### Get (and perhaps cahce) total size of db.* files in P4ROOT,
### e.g. with du -hc db.*| tail -n 1
### If there isn't enough space, what do we do? Just bail for now.
### We don't want to automate removal of live db.* files, even if
### they have some corruption.
msg "Warning: Disk space safety checks skipped."

if [[ -n "$(/bin/ls $P4ROOT/save/db.* 2>/dev/null)" ]]; then
   cmd /bin/rm -f $P4ROOT/save/db.* || bail "Failed to remove db.* files from $P4ROOT/save."
fi

# If the SDP was installed properly, the 'save' directory should exist, but let's
# be sure.
if [[ ! -d $P4ROOT/save ]]; then
   cmd mkdir $P4ROOT/save || bail "Failed to create $P4ROOT/save."
fi

# Move former live databases aside.
if [[ -n "$(/bin/ls $P4ROOT/db.*)" ]]; then
   cmd /bin/mv -f $P4ROOT/db.* $P4ROOT/save/. || bail "DB move root->save failed."
fi

cmd /bin/rm -rf $P4ROOT/server.locks || bail "Cleanup of server.locks directory failed."
cmd /bin/mv -f $P4HOME/offline_db/db.* $P4ROOT/. || bail "DB move offline->root failed."
cmd $P4DBIN -r $P4ROOT -f -jr $P4JOURNAL || bail "Failed to replay active journal to P4ROOT."

msg "Rotating journal to give a clean cut-off with the local failover."
cmd $P4DBIN -r $P4ROOT -jj || bail "Failed to rotate active journal."

msg "Starting server."
cmd $P4DInitScript start || bail "Failed to start $P4DBIN."

msg "Kicking off recreate_offline_db.sh, but not waiting for it."
if [[ $NO_OP -eq 0 ]]; then
   nohup $P4CBIN/recreate_offline_db.sh $SDP_INSTANCE < /dev/null > /dev/null 2>&1 &
else
   msg "NO_OP: Would run: nohup $P4CBIN/recreate_offline_db.sh $SDP_INSTANCE < /dev/null > /dev/null 2>&1 &"
fi

msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

exit 0