#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ # Helix Server base init script # Do nothing unless $SDP_INSTANCE is defined. export SDP_INSTANCE="${1:-Unset}" export OS_INIT_MECHANISM="${OS_INIT_MECHANISM:-Unset}" declare -i StartDelay="${SDP_START_DELAY:-2}" declare -i TailCmdUsable=0 declare -i UseSystemd=0 declare -i ExitCode=0 declare ThisScript="${0##*/}" declare Version=5.0.3 declare CmdLine="$0 $*" declare OpMode="${2:-usage}" declare Log= declare TmpDir= declare TmpLog= declare TmpJnl= declare MustUseSystemdMsg= declare ServiceName= if [[ "$SDP_INSTANCE" == "Unset" ]]; then echo -e "\\nError: The SDP_INSTANCE is not defined.\\n." exit 1 fi # Load SDP controlled shell environment. # shellcheck disable=SC1091 source /p4/common/bin/p4_vars "$SDP_INSTANCE" ExitCode=$? if [[ "$ExitCode" -ne 0 ]]; then echo -e "\\nError: Failed to load SDP environment for instance $SDP_INSTANCE.\\n" exit 1 fi # shellcheck disable=SC1090 disable=SC1091 source "$P4CBIN/backup_functions.sh" ExitCode=$? if [[ "$ExitCode" -ne 0 ]]; then echo -e "\\nError: Failed to load SDP lib $P4CBIN/backup_functions.sh.\\n" exit 1 fi if [[ "${P4D_FLAGS:-Unset}" == "Unset" ]]; then echo -e "\\nError: P4D_FLAGS is unset after loading SDP environment for instance $SDP_INSTANCE.\\n" exit 1 fi if [[ $(id -u) -eq 0 ]]; then exec su - "$OSUSER" -c "$0 $*" elif [[ $(id -u -n) != "$OSUSER" ]]; then echo "$0 can only be run by root or $OSUSER" exit 1 fi if [[ ! -x "$P4DBIN" ]]; then echo -e "\\nError: $P4DBIN is not executable." exit 2; fi # Ensure that the '--pid-file' argument is provided, and add it if it is not. # This is intended to prevent problems upgrading the SDP in case the # /p4/common/config/p4_N.vars file isn't updated to use the new template. if [[ "$P4D_FLAGS" != *"--pid-file"* ]]; then export P4D_FLAGS="$P4D_FLAGS --pid-file" fi # Determine if systemd is configured for this instance. if [[ -n "$(command -v systemctl)" ]]; then ServiceName="${P4DBIN##*/}" if [[ -n "$(systemctl is-enabled "$ServiceName" 2>/dev/null)" ]]; then UseSystemd=1 MustUseSystemdMsg="\\nError: It appears an attempt was made to start or stop the $ServiceName\\nservice without using systemd. When a systemd unit file is configured, only the\\nsystemd init mechanism can be used to start and stop the service, so that\\nsystemd maintains a more reliable indication of the status of the service. Run\\nas root or as $OSUSER with sudo. For example:\\n\\n\\tsudo systemctl start $ServiceName\\n\\tsudo systemctl stop $ServiceName\\n\\tsystemctl status $ServiceName\\n\\nNote that the 'status' check does not require sudo.\\n\\nThe systemd unit file is:\\n\\n<<<<<<<<\\n$(sudo systemctl cat "$ServiceName")\\n>>>>>>>\\n\\nIf you used 'systemctl' and still got this message, then please check if OS_INIT_MECHANISM is defined\\nin the above file. See /p4/sdp/Server/Unix/setup/systemd/p4d_1.service for an example." fi fi # For P4D 2017.1+, automatically replace '-d' with '--daemonsafe', unless # we are running with systemd. If running with systemd, we drop the # '-d'/'--daemonsafe' flags and let systemd handle the process forking. # Disable shellcheck as we intend a string compare even though it looks like # we're trying to compare a decimal value. # shellcheck disable=SC2072 if [[ "$P4D_VERSION" > "2017.1" ]]; then if [[ "$UseSystemd" -eq 1 ]]; then export P4D_FLAGS=${P4D_FLAGS/ -d / } export P4D_FLAGS=${P4D_FLAGS/ --daemonsafe / } else export P4D_FLAGS=${P4D_FLAGS/ -d / --daemonsafe } fi fi Log="$LOGS/p4d_init.log" export LOGFILE="$Log" TmpDir=$(mktemp -d 2>/dev/null) if [[ ! -d "$TmpDir" ]]; then TmpDir=$(mktemp -d -p "$P4TMP" -t 'tmp_p4d_base.XXXXXXXX') fi if [[ ! -d "$TmpDir" ]]; then echo -e "\\nError: Could not initialize TmpDir [$TmpDir]\\n" | tee -a "$Log" exit 1 fi TmpLog="${TmpDir}/tmp.p4d_base.log" echo -e "\\n$(date) Called $ThisScript v$Version with command line:\\n$CmdLine" >> "$Log" set_vars # If the 'tail' command responds to the '--version' flag, it is useful for our # purposes in this script. If it does not recognize '--version' (e.g. on OSX), # then it likely will not work with 'tail -', and so not useful in this # script. if [[ -n "$(command -v tail)" ]]; then if tail --version > /dev/null 2>&1; then TailCmdUsable=1 fi fi # See how we were called. case "$OpMode" in (force_start) if [[ -e "$P4ROOT/P4ROOT_not_usable.txt" ]]; then echo -e "\\nWarning: The $P4ROOT/P4ROOT_not_usable.txt file exists, indicating\\ndatabases in P4ROOT are not safe to use. Contents:\\n$(cat "$P4ROOT/P4ROOT_not_usable.txt")\\nIgnoring this due to use of force_start option. Review the bottom of this log:\\n$Log\\n" | tee -a "$Log" fi if [[ "$UseSystemd" -eq 1 && "$OS_INIT_MECHANISM" != "systemd" ]]; then echo -e "\\nWarning: An attempt was made to start the $ServiceName service\\nwithout using systemd on a system configured to use systemd. Due to use of\\nforce_start, this will be allowed.\\n" | tee -a "$Log" fi echo "Starting $P4DBIN $P4D_FLAGS" | tee -a "$Log" # Delay start $StartDelay seconds, unless P4ROOT is empty. [[ -r "$P4ROOT/db.domain" ]] && sleep "$StartDelay" # shellcheck disable=SC2164 cd "$P4ROOT" 2>&1 | tee -a "$Log" if [[ -n "$(command -v ulimit)" && "$(ulimit -c)" != "0" ]]; then ulimit -c unlimited fi # shellcheck disable=SC2086 "$P4DBIN" $P4D_FLAGS > "$TmpLog" 2>&1 ExitCode=$? cat "$TmpLog" >> "$Log" cat "$TmpLog" ;; (start) if [[ -e "$P4ROOT/P4ROOT_not_usable.txt" ]]; then echo -e "\\nError: The $P4ROOT/P4ROOT_not_usable.txt file exists, indicating\\ndatabases in P4ROOT are not safe to use. Contents:\\n$(cat "$P4ROOT/P4ROOT_not_usable.txt")\\nRefusing to start p4d. Review the bottom of this log:\\n$Log\\n" | tee -a "$Log" exit 1 fi if [[ "$UseSystemd" -eq 1 && "$OS_INIT_MECHANISM" != "systemd" ]]; then echo -e "$MustUseSystemdMsg" | tee -a "$Log" exit 1 fi echo -e "\\nPreflight check: $P4CBIN/check_dir_ownership.sh $P4ROOT $OSUSER unset 1" | tee -a "$Log" # We call check_dir_ownership.sh checking for expected ownerhip, but # checking only for files in the P4ROOT directory itself (a 'find' with # '-maxdepth 1') to avoid any significant delay if anything under P4ROOT # is large (e.g. server.locks, partitioned client dirs, etc.). This is a # risk trade-off. if "$P4CBIN"/check_dir_ownership.sh "$P4ROOT" "$OSUSER" unset 1 > "$TmpLog" 2>&1; then # shellcheck disable=SC2002 cat "$TmpLog" | tee -a "$Log" else # shellcheck disable=SC2002 cat "$TmpLog" | tee -a "$Log" echo -e "\\nAborting due to failed ownership preflight check." | tee -a "$Log" exit 1 fi if [[ -r "$P4ROOT/db.domain" ]]; then echo -e "\\nPreflight check: $P4DBIN -r $P4ROOT -xvU" > "$TmpLog" "$P4DBIN" -r "$P4ROOT" -xvU >> "$TmpLog" 2>&1 ExitCode=$? echo "EXIT_CODE=$ExitCode" >> "$TmpLog" cat "$TmpLog" >> "$Log" if [[ "$ExitCode" -ne 0 ]]; then cat "$TmpLog" echo -e "\\nError: DB check with 'p4d -xvU' failed. Database integrity is in question.\\nPlease Contact Perforce Support (support-helix-core@perforce.com). The force_start option\\nis available, but not recommended. Review the bottom of this log:\\n$Log\\n" | tee -a "$Log" exit 1 fi if [[ -e "$P4JOURNAL" && "$TailCmdUsable" -eq 1 ]]; then echo "Preflight journal health check - validating journal in a temp directory" > "$TmpLog" TmpJnl="$TmpDir/jnl.test" tail -10000 "$P4JOURNAL" 2>/dev/null | grep -av "@vv@" > "$TmpJnl" 2>> "$TmpLog" # Check for first full record. Some records are multiline and we # may be part way through. FirstLine=$(grep -a -En "^@(ex|nx|pv|rv|dv)@" "$TmpJnl" 2>&1 | head -1 | cut -d: -f1) if [[ "$FirstLine" -gt "1" ]]; then # Skip lines if necessary to start with a full record. mv "$TmpJnl" "${TmpJnl}.1" tail -n "+$FirstLine" "${TmpJnl}.1" > "$TmpJnl" 2>> "$Log" fi "$P4DBIN" -r "$TmpDir" -f -jr "$TmpJnl" >> "$TmpLog" 2>&1 ExitCode=$? echo "EXIT_CODE=$ExitCode" >> "$TmpLog" if [[ "$ExitCode" -ne 0 ]]; then cat "$TmpLog" >> "$Log" cat "$TmpLog" echo -e "\\nError: possible corruption at end of journal detected. Journal is being rotated so any corruption is at end of file. You may wish to contact Perforce Support (support-helix-core@perforce.com). Server is still being started as normal. NOTE If there is any corruption then replicas will likely stop replicating until this is fixed!! If there are no problems with replication then this error can be ignored. Review this file:\\n$Log\\n" | tee -a "$Log" get_journalnum p4d_truncate_journal subject="ERROR!!! - $HOSTNAME $P4SERVER Possible journal corruption detected." mail_sender_opt=$(get_mail_sender_opt) echo "Sending mail: $SDPMAIL -s $subject $mail_sender_opt $MAILTO" | tee -a "$Log" "$SDPMAIL" -s "$subject" "$mail_sender_opt" "$MAILTO" < "$Log" fi fi fi echo "Starting $P4DBIN $P4D_FLAGS" | tee -a "$Log" # Delay start $StartDelay seconds, unless P4ROOT is empty. [[ -r "$P4ROOT/db.domain" ]] && sleep "$StartDelay" # shellcheck disable=SC2164 cd "$P4ROOT" 2>&1 | tee -a "$Log" if [[ -n "$(command -v ulimit)" && "$(ulimit -c)" != "0" ]]; then ulimit -c unlimited fi # shellcheck disable=SC2086 "$P4DBIN" $P4D_FLAGS > "$TmpLog" 2>&1 ExitCode=$? cat "$TmpLog" >> "$Log" cat "$TmpLog" ;; (status) if [[ -r "$P4ROOT/server.pid" ]]; then pid=$(cat "$P4ROOT/server.pid") echo -e "\\nThe $P4ROOT/server.pid file contains pid $pid. Pid info:" "$PS" -f -p "$pid" > "$TmpLog" ExitCode=$? cat "$TmpLog" if [[ "$ExitCode" -ne 0 ]]; then echo -e "\\nError: A server.pid file exists, but that process id is not running. This could indicate abnormal process termination.\\n" fi fi if "$P4BIN" -p "$P4PORT" info -s > "$TmpLog" 2>&1; then ExitCode=0 else # If we get an SSL trust error, then the server is online. if grep -lq 'The authenticity of' "$TmpLog"; then # Do a fire & forget attempt to fix the 'p4 trust' issue. "$P4BIN" -p "$P4PORT" trust -f -y > /dev/null 2>&1 ExitCode=0 else ExitCode=1 fi fi cat "$TmpLog" ;; (admin_stop) if [[ "$UseSystemd" -eq 1 && "$OS_INIT_MECHANISM" != "systemd" ]]; then echo -e "$MustUseSystemdMsg" exit 1 fi # If there is no server.pid file, shut down the old fashioned way. echo -n "Shutting down $P4DBIN: " | tee -a "$Log" if [[ "${P4REPLICA}" == "FALSE" ]]; then "$P4CBIN/p4login" fi echo "$P4BIN -p $P4PORT -u $P4USER admin stop" | tee -a "$Log" "$P4BIN" -p "$P4PORT" -u "$P4USER" admin stop 2>&1 | tee -a "$Log" sleep 5 "$P4BIN" -p "$P4PORT" info > /dev/null 2>&1 ExitCode=$? if [[ "$ExitCode" -eq 0 ]]; then echo -e "\\nError: Server shutdown failed." | tee -a "$Log" exit 1 else exit 0 fi ;; (stop) if [[ -r "$P4ROOT/server.pid" ]]; then pid=$(cat "$P4ROOT/server.pid") if "$PS" -p "$pid" > /dev/null 2>&1; then if [[ "$UseSystemd" -eq 1 && "$OS_INIT_MECHANISM" != "systemd" ]]; then echo -e "$MustUseSystemdMsg" exit 1 fi echo -e "\\nSending SIGTERM signal to pid $pid in $P4ROOT/server.pid." | tee -a "$Log" kill "$pid" 2>&1 | tee -a "$Log" sleep 1 "$PS" -p "$pid" > /dev/null 2>&1 status=$? if [[ $status -eq 0 ]]; then echo -n "Waiting for p4d to shutdown ..." | tee -a "$Log" while [[ $status -eq 0 ]]; do echo -n "." sleep 5 "$PS" -p "$pid" > /dev/null 2>&1 status=$? done fi echo -e "\\nConfirmed shutdown of $P4DBIN." | tee -a "$Log" else echo -e "\\nError: A server.pid file exists, but that process id is not running. This could indicate abnormal process termination.\\n" | tee -a "$Log" exit 1 fi else # If there is no server.pid file, but we can still get a response from # 'p4 info', try to shut down with the front-door method of calling # 'p4 admin stop'. If there is no server.pid file and no response from # 'p4 info', then p4d is down. if "$P4BIN" info -s > /dev/null 2>&1; then echo -e "\\nWarning: Missing $P4ROOT/server.pid. Attempting shutdown with 'p4 admin stop'.\\n" | tee -a "$Log" echo "$0" "$SDP_INSTANCE" admin_stop | tee -a "$Log" $0 "$SDP_INSTANCE" admin_stop fi fi ;; (restart) $0 "$SDP_INSTANCE" stop $0 "$SDP_INSTANCE" start ;; (*) echo -e "\\nUsage: $0 SDP_INSTANCE {start|stop|status|restart|force_start|admin_stop}\\n" exit 1 ;; esac [[ -d "$TmpDir" ]] && rm -rf "$TmpLog" exit "$ExitCode"