p4dstate.sh #3

#!/bin/bash

#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

# The RCSCHANGE and RCSDATE values are copied verbatim from the originating
# Helix Server, and read-only elsewhere.  Do not change these and do not
# apply the '+k' file type modifer to this file.
RCSCHANGE="$Change: 1262283 $"
RCSDATE="$Date: 2015/11/05 $"

export SDP_INSTANCE=${SDP_INSTANCE:-Undefined}
export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ $SDP_INSTANCE == Undefined ]]; then
   echo "Instance parameter not supplied."
   echo "You must supply the Perforce instance as a parameter to this script."
   exit 1
fi

. /p4/common/bin/p4_vars $SDP_INSTANCE
. /p4/common/bin/backup_functions.sh

LOGFILE=$LOGS/p4dstate-`date +%Y%m%d%H%M%S`.out


TIMEOUT=120

SIGKILL=9
SIGTERM=15

plural()
{
	if [ $1 != "1" ]
	then
	    printf "%s %ss" $1 $2
	else
	    if [ $# -lt 3 ]
	    then
		printf "%s %s" $1 $2
	    else
		printf "%s" $3
	    fi
	fi
}

header()
{
	printf "Output from "
	if [ $# -gt 1 ]
	then
	    plural $2 second
	    printf " of "
	fi
	printf "\"%s\"" "$1"
	if [ $# -gt 2 ]
	then
	    printf " repeated every "
	    plural $3 second second
	fi
	printf ", starting at %s\n" "`date "+%Y/%m/%d %H:%M:%S"`"
}

initmask()
{
	printf "%.$1d" 0
}

setbit()
{
	masklen=`expr length $1`

	if [ $2 -gt 1 ]
	then
	    printf "%s" `expr substr $1 1 \`expr $2 - 1\``
	fi

	printf "1"

	if [ $2 -lt $masklen ]
	then
	    printf "%s" `expr substr $1 \`expr $2 + 1\` \`expr $masklen - $2\``
	fi
}

isset()
{
	test `expr substr $1 $2 1` = "1"
}

sleepkill()
{
	sleep $1 > /dev/null	# redirection required so as to not stall pipe
	kill -$2 $3 2> /dev/null
}

run()
{
	header "$1" $2 $3

	if [ $# -lt 3 ]
	then
	    $1 2>&1 &
	else
	    {
		while [ 1 ]
		do
		    date +%H:%M:%S
		    $1 2>&1
		    printf "=======\n"
		    sleep $3
		done
	    } &
	fi
	runpid=$!

	reportkills=`initmask 64`
	if [ $# -gt 1 ]
	then
	    reportkills=`setbit $reportkills $SIGKILL`

	    sleepkill $2 $SIGTERM $runpid

	    {
		sleepkill $TIMEOUT $SIGKILL $runpid
	    } &
	else
	    reportkills=`setbit $reportkills $SIGKILL`
	    reportkills=`setbit $reportkills $SIGTERM`

	    {
		sleepkill $TIMEOUT $SIGTERM $runpid
		sleepkill $TIMEOUT $SIGKILL $runpid
	    } &
	fi
	waitpid=$!
	wait $runpid 2> /dev/null
	signal=`expr $? - 128`
	test $signal -gt 0 && isset $reportkills $signal &&
	{
	    printf "Command timed out; terminated using \"kill %s\".\n" -$signal
	}
	kill $waitpid 2> /dev/null

	wait 2> /dev/null

	printf "\n"
}

lsofrun()
{
	run "lsof -r1 $P4ROOT/* $P4ROOT/server.locks/*/* $P4JOURNAL $P4LOG" 15 \
	    | grep -v '^tail'
}

getpids()
{
	printf "ps -e | grep '%s$'" `echo $P4DBIN | sed 's/.*\/\([^\/]*\)$/\1/'` \
	    | sh \
	    | sed 's/^[^0-9]*\([0-9]*\)[^0-9]*.*/\1/'
}

pidruns()
{
	getpids | while read pid
	do
	    run "lsof -p $pid"
	    run "strace -q -p $pid" 0.50
	done
}

collectfile()
{
	echo $LOGFILE.`printf "%.2d" $1`
}

collect()
{
	ncollects=`expr $ncollects + 1`

	"$@" > `collectfile $ncollects` &
}

ncollects=0

printf "Collecting p4d state..."

collect run "$P4DBIN -V"
collect run "$P4BIN -V"

collect run "uname -a"

collect run "tail -f -n10000 $P4LOG" 15
collect run "tail -f -n10000 $P4JOURNAL" 15

collect lsofrun

collect pidruns

collect run "ps -elfjH" 15 1
collect run "netstat -antp" 15 1

collect run "tail -f -n10000 /var/log/messages" 15

#
# Depending upon the state of the server, the following commands
# might not behave as expected.
#

collect run "$P4BIN monitor show -el" 15 1

#
# All commands have been started; wait for them to finish.
#
wait

icollect=0
while [ $icollect -lt $ncollects ]
do
	icollect=`expr $icollect + 1`

	collected=`collectfile $icollect`
	cat $collected
	rm -f $collected
done \
	> $LOGFILE

printf " done.\n"

exit 0