p4d_base #46

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

# Helix Server base init script

# Do nothing unless $SDP_INSTANCE is defined.
export SDP_INSTANCE="${1:-Unset}"
export OS_INIT_MECHANISM="${OS_INIT_MECHANISM:-Unset}"
declare -i StartDelay="${SDP_START_DELAY:-2}"
declare -i TailCmdUsable=0
declare -i UseSystemd=0
declare -i ExitCode=0
declare ThisScript="${0##*/}"
declare Version=5.0.3
declare CmdLine="$0 $*"
declare OpMode="${2:-usage}"
declare Log=
declare TmpDir=
declare TmpLog=
declare TmpJnl=
declare MustUseSystemdMsg=
declare ServiceName=

if [[ "$SDP_INSTANCE" == "Unset" ]]; then
   echo -e "\\nError: The SDP_INSTANCE is not defined.\\n."
   exit 1
fi

# Load SDP controlled shell environment.
# shellcheck disable=SC1091
source /p4/common/bin/p4_vars "$SDP_INSTANCE"
ExitCode=$?

if [[ "$ExitCode" -ne 0 ]]; then
   echo -e "\\nError: Failed to load SDP environment for instance $SDP_INSTANCE.\\n"
   exit 1
fi

# shellcheck disable=SC1090 disable=SC1091
 source "$P4CBIN/backup_functions.sh"
ExitCode=$?

if [[ "$ExitCode" -ne 0 ]]; then
   echo -e "\\nError: Failed to load SDP lib $P4CBIN/backup_functions.sh.\\n"
   exit 1
fi

if [[ "${P4D_FLAGS:-Unset}" == "Unset" ]]; then
   echo -e "\\nError: P4D_FLAGS is unset after loading SDP environment for instance $SDP_INSTANCE.\\n"
   exit 1
fi

if [[ $(id -u) -eq 0 ]]; then
   exec su - "$OSUSER" -c "$0 $*"
elif [[ $(id -u -n) != "$OSUSER" ]]; then
   echo "$0 can only be run by root or $OSUSER"
   exit 1
fi

if [[ ! -x "$P4DBIN" ]]; then
   echo -e "\\nError: $P4DBIN is not executable."
   exit 2;
fi

# Ensure that the '--pid-file' argument is provided, and add it if it is not.
# This is intended to prevent problems upgrading the SDP in case the
# /p4/common/config/p4_N.vars file isn't updated to use the new template.
if [[ "$P4D_FLAGS" != *"--pid-file"* ]]; then
   export P4D_FLAGS="$P4D_FLAGS --pid-file"
fi

# Determine if systemd is configured for this instance.
if [[ -n "$(command -v systemctl)" ]]; then
   ServiceName="${P4DBIN##*/}"
   if [[ -n "$(systemctl is-enabled "$ServiceName" 2>/dev/null)" ]]; then
      UseSystemd=1
      MustUseSystemdMsg="\\nError: It appears an attempt was made to start or stop the $ServiceName\\nservice without using systemd. When a systemd unit file is configured, only the\\nsystemd init mechanism can be used to start and stop the service, so that\\nsystemd maintains a more reliable indication of the status of the service. Run\\nas root or as $OSUSER with sudo. For example:\\n\\n\\tsudo systemctl start $ServiceName\\n\\tsudo systemctl stop $ServiceName\\n\\tsystemctl status $ServiceName\\n\\nNote that the 'status' check does not require sudo.\\n\\nThe systemd unit file is:\\n\\n<<<<<<<<\\n$(sudo systemctl cat "$ServiceName")\\n>>>>>>>\\n\\nIf you used 'systemctl' and still got this message, then please check if OS_INIT_MECHANISM is defined\\nin the above file. See /p4/sdp/Server/Unix/setup/systemd/p4d_1.service for an example."
   fi
fi

# For P4D 2017.1+, automatically replace '-d' with '--daemonsafe', unless
# we are running with systemd. If running with systemd, we drop the
# '-d'/'--daemonsafe' flags and let systemd handle the process forking.
# Disable shellcheck as we intend a string compare even though it looks like
# we're trying to compare a decimal value.
# shellcheck disable=SC2072
if [[ "$P4D_VERSION" > "2017.1" ]]; then
   if [[ "$UseSystemd" -eq 1 ]]; then
      export P4D_FLAGS=${P4D_FLAGS/ -d / }
      export P4D_FLAGS=${P4D_FLAGS/ --daemonsafe / }
   else
      export P4D_FLAGS=${P4D_FLAGS/ -d / --daemonsafe }
   fi
fi

Log="$LOGS/p4d_init.log"
export LOGFILE="$Log"

TmpDir=$(mktemp -d 2>/dev/null)
if [[ ! -d "$TmpDir" ]]; then
   TmpDir=$(mktemp -d -p "$P4TMP" -t 'tmp_p4d_base.XXXXXXXX')
fi

if [[ ! -d "$TmpDir" ]]; then
   echo -e "\\nError: Could not initialize TmpDir [$TmpDir]\\n" | tee -a "$Log"
   exit 1
fi

TmpLog="${TmpDir}/tmp.p4d_base.log"

echo -e "\\n$(date) Called $ThisScript v$Version with command line:\\n$CmdLine" >> "$Log"

set_vars

# If the 'tail' command responds to the '--version' flag, it is useful for our
# purposes in this script. If it does not recognize '--version' (e.g. on OSX),
# then it likely will not work with 'tail -<number>', and so not useful in this
# script.
if [[ -n "$(command -v tail)" ]]; then
   if tail --version > /dev/null 2>&1; then
      TailCmdUsable=1
   fi
fi

# See how we were called.
case "$OpMode" in
   (force_start)
      if [[ -e "$P4ROOT/P4ROOT_not_usable.txt" ]]; then
         echo -e "\\nWarning: The $P4ROOT/P4ROOT_not_usable.txt file exists, indicating\\ndatabases in P4ROOT are not safe to use. Contents:\\n$(cat "$P4ROOT/P4ROOT_not_usable.txt")\\nIgnoring this due to use of force_start option. Review the bottom of this log:\\n$Log\\n" | tee -a "$Log"
      fi

      if [[ "$UseSystemd" -eq 1 && "$OS_INIT_MECHANISM" != "systemd" ]]; then
         echo -e "\\nWarning: An attempt was made to start the $ServiceName service\\nwithout using systemd on a system configured to use systemd. Due to use of\\nforce_start, this will be allowed.\\n" | tee -a "$Log"
      fi

      echo "Starting $P4DBIN $P4D_FLAGS" | tee -a "$Log"
      # Delay start $StartDelay seconds, unless P4ROOT is empty.
      [[ -r "$P4ROOT/db.domain" ]] && sleep "$StartDelay"

      # shellcheck disable=SC2164
      cd "$P4ROOT" 2>&1 | tee -a "$Log"
      if [[ -n "$(command -v ulimit)" && "$(ulimit -c)" != "0" ]]; then
         ulimit -c unlimited
      fi

      # shellcheck disable=SC2086
      "$P4DBIN" $P4D_FLAGS > "$TmpLog" 2>&1
      ExitCode=$?
      cat "$TmpLog" >> "$Log"
      cat "$TmpLog"
   ;;

   (start)
      if [[ -e "$P4ROOT/P4ROOT_not_usable.txt" ]]; then
         echo -e "\\nError: The $P4ROOT/P4ROOT_not_usable.txt file exists, indicating\\ndatabases in P4ROOT are not safe to use. Contents:\\n$(cat "$P4ROOT/P4ROOT_not_usable.txt")\\nRefusing to start p4d.  Review the bottom of this log:\\n$Log\\n" | tee -a "$Log"
         exit 1
      fi

      if [[ "$UseSystemd" -eq 1 && "$OS_INIT_MECHANISM" != "systemd" ]]; then
         echo -e "$MustUseSystemdMsg" | tee -a "$Log"
         exit 1
      fi

      echo -e "\\nPreflight check: $P4CBIN/check_dir_ownership.sh $P4ROOT $OSUSER unset 1" | tee -a "$Log"
      # We call check_dir_ownership.sh checking for expected ownerhip, but
      # checking only for files in the P4ROOT directory itself (a 'find' with
      # '-maxdepth 1') to avoid any significant delay if anything under P4ROOT
      # is large (e.g. server.locks, partitioned client dirs, etc.). This is a
      # risk trade-off.
      if "$P4CBIN"/check_dir_ownership.sh "$P4ROOT" "$OSUSER" unset 1 > "$TmpLog" 2>&1; then
         # shellcheck disable=SC2002
         cat "$TmpLog" | tee -a "$Log"
      else
         # shellcheck disable=SC2002
         cat "$TmpLog" | tee -a "$Log"
         echo -e "\\nAborting due to failed ownership preflight check." | tee -a "$Log"
         exit 1
      fi

      if [[ -r "$P4ROOT/db.domain" ]]; then
         echo -e "\\nPreflight check: $P4DBIN -r $P4ROOT -xvU" > "$TmpLog"
         "$P4DBIN" -r "$P4ROOT" -xvU >> "$TmpLog" 2>&1
         ExitCode=$?
         echo "EXIT_CODE=$ExitCode" >> "$TmpLog"
         cat "$TmpLog" >> "$Log"

         if [[ "$ExitCode" -ne 0 ]]; then
            cat "$TmpLog"
            echo -e "\\nError: DB check with 'p4d -xvU' failed.  Database integrity is in question.\\nPlease Contact Perforce Support (support-helix-core@perforce.com).  The force_start option\\nis available, but not recommended.  Review the bottom of this log:\\n$Log\\n" | tee -a "$Log"
            exit 1
         fi

         if [[ -e "$P4JOURNAL" && "$TailCmdUsable" -eq 1 ]]; then
            echo "Preflight journal health check - validating journal in a temp directory" > "$TmpLog"
            TmpJnl="$TmpDir/jnl.test"
            tail -10000 "$P4JOURNAL" 2>/dev/null | grep -av "@vv@" > "$TmpJnl" 2>> "$TmpLog"

            # Check for first full record. Some records are multiline and we
            # may be part way through.
            FirstLine=$(grep -a -En "^@(ex|nx|pv|rv|dv)@" "$TmpJnl" 2>&1 | head -1 | cut -d: -f1)
            if [[ "$FirstLine" -gt "1" ]]; then
               # Skip lines if necessary to start with a full record.
               mv "$TmpJnl" "${TmpJnl}.1"
               tail -n "+$FirstLine" "${TmpJnl}.1" > "$TmpJnl" 2>> "$Log"
            fi

            "$P4DBIN" -r "$TmpDir" -f -jr "$TmpJnl" >> "$TmpLog" 2>&1
            ExitCode=$?
            echo "EXIT_CODE=$ExitCode" >> "$TmpLog"

            if [[ "$ExitCode" -ne 0 ]]; then
               cat "$TmpLog" >> "$Log"
               cat "$TmpLog"
               echo -e "\\nError: possible corruption at end of journal detected. Journal is being rotated so any corruption is at end of file. You may wish to contact Perforce Support (support-helix-core@perforce.com).  Server is still being started as normal. NOTE If there is any corruption then replicas will likely stop replicating until this is fixed!! If there are no problems with replication then this error can be ignored. Review this file:\\n$Log\\n" | tee -a "$Log"
               get_journalnum
               p4d_truncate_journal
               subject="ERROR!!! - $HOSTNAME $P4SERVER Possible journal corruption detected."
               mail_sender_opt=$(get_mail_sender_opt)
               echo "Sending mail: $SDPMAIL -s $subject $mail_sender_opt $MAILTO" | tee -a "$Log"
               "$SDPMAIL" -s "$subject" "$mail_sender_opt" "$MAILTO" < "$Log"
            fi
         fi
      fi

      echo "Starting $P4DBIN $P4D_FLAGS" | tee -a "$Log"

      # Delay start $StartDelay seconds, unless P4ROOT is empty.
      [[ -r "$P4ROOT/db.domain" ]] && sleep "$StartDelay"

      # shellcheck disable=SC2164
      cd "$P4ROOT" 2>&1 | tee -a "$Log"
      if [[ -n "$(command -v ulimit)" && "$(ulimit -c)" != "0" ]]; then
         ulimit -c unlimited
      fi

      # shellcheck disable=SC2086
      "$P4DBIN" $P4D_FLAGS > "$TmpLog" 2>&1
      ExitCode=$?

      cat "$TmpLog" >> "$Log"
      cat "$TmpLog"
   ;;

   (status)
      if [[ -r "$P4ROOT/server.pid" ]]; then
         pid=$(cat "$P4ROOT/server.pid")
         echo -e "\\nThe $P4ROOT/server.pid file contains pid $pid.  Pid info:"
         "$PS" -f -p "$pid" > "$TmpLog"
         ExitCode=$?

         cat "$TmpLog"

         if [[ "$ExitCode" -ne 0 ]]; then
            echo -e "\\nError: A server.pid file exists, but that process id is not running.  This could indicate abnormal process termination.\\n"
         fi
      fi

      if "$P4BIN" -p "$P4PORT" info -s > "$TmpLog" 2>&1; then
         ExitCode=0
      else
         # If we get an SSL trust error, then the server is online.
         if grep -lq 'The authenticity of' "$TmpLog"; then
            # Do a fire & forget attempt to fix the 'p4 trust' issue.
            "$P4BIN" -p "$P4PORT" trust -f -y > /dev/null 2>&1
            ExitCode=0
         else
            ExitCode=1
         fi
      fi

      cat "$TmpLog"
   ;;

   (admin_stop)
      if [[ "$UseSystemd" -eq 1 && "$OS_INIT_MECHANISM" != "systemd" ]]; then
         echo -e "$MustUseSystemdMsg"
         exit 1
      fi

      # If there is no server.pid file, shut down the old fashioned way.
      echo -n "Shutting down $P4DBIN: " | tee -a "$Log"
      if [[ "${P4REPLICA}" == "FALSE" ]]; then
         "$P4CBIN/p4login"
      fi
      echo "$P4BIN -p $P4PORT -u $P4USER admin stop" | tee -a "$Log"
      "$P4BIN" -p "$P4PORT" -u "$P4USER" admin stop 2>&1 | tee -a "$Log"
      sleep 5
      "$P4BIN" -p "$P4PORT" info > /dev/null 2>&1
      ExitCode=$?
      if [[ "$ExitCode" -eq 0 ]]; then
         echo -e "\\nError: Server shutdown failed." | tee -a "$Log"
         exit 1
      else
         exit 0
      fi
   ;;

   (stop)
      if [[ -r "$P4ROOT/server.pid" ]]; then
         pid=$(cat "$P4ROOT/server.pid")

         if "$PS" -p "$pid" > /dev/null 2>&1; then
            if [[ "$UseSystemd" -eq 1 && "$OS_INIT_MECHANISM" != "systemd" ]]; then
               echo -e "$MustUseSystemdMsg"
               exit 1
            fi

            echo -e "\\nSending SIGTERM signal to pid $pid in $P4ROOT/server.pid." | tee -a "$Log"
            kill "$pid" 2>&1 | tee -a "$Log"
            sleep 1

            "$PS" -p "$pid" > /dev/null 2>&1
            status=$?

            if [[ $status -eq 0 ]]; then
               echo -n "Waiting for p4d to shutdown ..." | tee -a "$Log"

               while [[ $status -eq 0 ]]; do
                  echo -n "."
                  sleep 5
                  "$PS" -p "$pid" > /dev/null 2>&1
                  status=$?
               done
            fi

            echo -e "\\nConfirmed shutdown of $P4DBIN." | tee -a "$Log"
         else
            echo -e "\\nError: A server.pid file exists, but that process id is not running.  This could indicate abnormal process termination.\\n" | tee -a "$Log"
            exit 1
         fi
      else
         # If there is no server.pid file, but we can still get a response from
	 # 'p4 info', try to shut down with the front-door method of calling
         # 'p4 admin stop'. If there is no server.pid file and no response from
         # 'p4 info', then p4d is down.
         if "$P4BIN" info -s > /dev/null 2>&1; then
            echo -e "\\nWarning: Missing $P4ROOT/server.pid. Attempting shutdown with 'p4 admin stop'.\\n" | tee -a "$Log"
            echo "$0" "$SDP_INSTANCE" admin_stop | tee -a "$Log"
            $0 "$SDP_INSTANCE" admin_stop
         fi
      fi
   ;;

   (restart)
      $0 "$SDP_INSTANCE" stop
      $0 "$SDP_INSTANCE" start
   ;;

   (*)
      echo -e "\\nUsage: $0 SDP_INSTANCE {start|stop|status|restart|force_start|admin_stop}\\n"
      exit 1
   ;;
esac

[[ -d "$TmpDir" ]] && rm -rf "$TmpLog"

exit "$ExitCode"