p4dstate.sh #4

#!/bin/bash

# Copyright (c) 2017, Perforce Software, Inc.  All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
# 
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

RCSCHANGE="$Change: 1488142 $"
RCSDATE="$Date: 2017/02/15 $"


export SDP_INSTANCE=${SDP_INSTANCE:-Undefined}
export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ $SDP_INSTANCE == Undefined ]]; then
   echo "Instance parameter not supplied."
   echo "You must supply the Perforce instance as a parameter to this script."
   exit 1
fi

. /p4/common/bin/p4_vars $SDP_INSTANCE
. /p4/common/bin/backup_functions.sh
 
outfile="${LOGS}/p4dstate-\`date +%Y%m%d%H%M%S\`.out"
LOGFILE=$outfile

p4d=$P4DBIN
p4=$P4BIN

NPARALLELPIDS=128	# number of concurrent per-PID data collectors

TIMEOUT=120

export P4PORT
export P4USER

SIGKILL=9
SIGTERM=15

plural()
{
	if [ $1 != "1" ]
	then
	    printf "%s %ss" $1 $2
	else
	    if [ $# -lt 3 ]
	    then
		printf "%s %s" $1 $2
	    else
		printf "%s" $3
	    fi
	fi
}

header()
{
	printf "Output from "
	if [ $# -gt 1 ]
	then
	    plural $2 second
	    printf " of "
	fi
	printf "\"%s\"" "$1"
	if [ $# -gt 2 ]
	then
	    printf " repeated every "
	    plural $3 second second
	fi
	printf ", starting at %s\n" "`date "+%Y/%m/%d %H:%M:%S"`"
}

initmask()
{
	printf "%.$1d" 0
}

setbit()
{
	masklen=`expr length $1`

	if [ $2 -gt 1 ]
	then
	    printf "%s" `expr substr $1 1 \`expr $2 - 1\``
	fi

	printf "1"

	if [ $2 -lt $masklen ]
	then
	    printf "%s" `expr substr $1 \`expr $2 + 1\` \`expr $masklen - $2\``
	fi
}

isset()
{
	test `expr substr $1 $2 1` = "1"
}

sleepkill()
{
	sleep $1 > /dev/null	# redirection required so as to not stall pipe
	kill -$2 $3 2> /dev/null
}

run()
{
	rcount=${2:-1} 
	scount=${3:-1} 
	header "$1" $rcount $scount 

	if [ $# -lt 3 ]
	then
	    $1 2>&1 &
	else
	    {
		while [ 1 ]
		do
		    date +%H:%M:%S
		    $1 2>&1
		    printf "=======\n"
		    sleep $3
		done
	    } &
	fi
	runpid=$!

	reportkills=`initmask 64`
	if [ $# -gt 1 ]
	then
	    reportkills=`setbit $reportkills $SIGKILL`

	    sleepkill $2 $SIGTERM $runpid

	    {
		sleepkill $TIMEOUT $SIGKILL $runpid
	    } &
	else
	    reportkills=`setbit $reportkills $SIGKILL`
	    reportkills=`setbit $reportkills $SIGTERM`

	    {
		sleepkill $TIMEOUT $SIGTERM $runpid
		sleepkill $TIMEOUT $SIGKILL $runpid
	    } &
	fi
	waitpid=$!
	wait $runpid 2> /dev/null
	signal=`expr $? - 128`
	test $signal -gt 0 && isset $reportkills $signal &&
	{
	    printf "Command timed out; terminated using \"kill %s\".\n" -$signal
	}
	kill $waitpid 2> /dev/null

	wait 2> /dev/null

	printf "\n"
}

collectfile()
{
	echo $1.`printf "%.5d" $2`
}

getrcsvalue()
{
	#
	# Return an RCS value from a string. The RCS value is the
	# substring between the first and last space characters.
	#
	echo $1 | sed 's/^[^ ]* \(.*\) [^ ]*$/\1/'
}

reportscript()
{
	printf "%s/%s (%s)\n" "`basename "$0" | tr [:lower:] [:upper:]`" \
	    "`getrcsvalue "$RCSCHANGE"`" "`getrcsvalue "$RCSDATE"`"
}

lsofrun()
{
	run "lsof -r1 $P4ROOT/* $P4ROOT/server.locks/*/* $P4JOURNAL $P4LOG" 15 \
	    | grep -v '^tail'
}

getpids()
{
	printf "ps -e | grep '%s$'" `echo $p4d | sed 's/.*\/\([^\/]*\)$/\1/'` \
	    | sh \
	    | sed 's/^[^0-9]*\([0-9]*\)[^0-9]*.*/\1/'
}

pidruns()
{
	collecting=`collectfile $outfile $ncollects`

	iparallelpid=0

	for pid in `getpids`
	do
	    iparallelpid=`expr $iparallelpid + 1`

	    {
		run "lsof -p $pid"
		run "strace -qT -p $pid" 0.50
	    } \
		> `collectfile $collecting $iparallelpid` &

	    if [ $iparallelpid -eq $NPARALLELPIDS ]
	    then
		wait

		package $collecting $NPARALLELPIDS

		iparallelpid=0
	    fi
	done

	wait

	package $collecting $iparallelpid
}

collect()
{
	ncollects=`expr $ncollects + 1`

	"$@" > `collectfile $outfile $ncollects` &
}

package()
{
	icollect=0
	while [ $icollect -lt $2 ]
	do
	    icollect=`expr $icollect + 1`

	    collected=`collectfile $1 $icollect`
	    cat $collected
	    rm -f $collected
	done
}

outfiletemplate="$outfile"

eval outfile=\"$outfiletemplate\"

ncollects=0

printf "Collecting p4d state into \"%s\"..." "$outfile"

collect run reportscript

collect run "$p4d -V"
collect run "$p4 -V"

collect run "uname -a"

collect run "tail -f -n10000 $P4LOG" 15
collect run "tail -f -n10000 $P4JOURNAL" 15

collect lsofrun

collect pidruns

collect run "ps -elfjH" 15 1
collect run "netstat -antp" 15 1

collect run "sysctl -a"

collect run "tail -f -n10000 /var/log/messages" 15

collect run "lslocks -o +BLOCKER"

#
# Depending upon the state of the server, the following commands
# might not behave as expected.
#

collect run "$p4 -z tag info"

collect run "$p4 configure show"
collect run "$p4d -r $P4ROOT -c show"

collect run "$p4 monitor show -el" 15 1

#
# All commands have been started; wait for them to finish.
#
wait

package $outfile $ncollects > $outfile

printf " done.\n"

exit 0