p4dstate.sh #5

#!/bin/bash

#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

# See original version of this file: //guest/perforce_software/admin_toolkit/p4dstate.sh
# This version is slightly customized for SDP - so assumes standard SDP directory locations/logfile etc
# It also requires SDP_INSTANCE as the single parameter.

# The RCSCHANGE and RCSDATE values are copied verbatim from the originating
# Helix Server, and read-only elsewhere.  Do not change these and do not
# apply the '+k' file type modifer to this file.
RCSCHANGE="$Change: 1488142 $"
RCSDATE="$Date: 2017/02/15 $"

export SDP_INSTANCE=${SDP_INSTANCE:-Undefined}
export SDP_INSTANCE=${1:-$SDP_INSTANCE}
if [[ $SDP_INSTANCE == Undefined ]]; then
   echo "Instance parameter not supplied."
   echo "You must supply the Perforce instance as a parameter to this script."
   exit 1
fi

# Source SDP variables and such as P4PORT and helper functions
. /p4/common/bin/p4_vars $SDP_INSTANCE
. /p4/common/bin/backup_functions.sh

LOGFILE=$LOGS/p4dstate-`date +%Y%m%d%H%M%S`.out

NPARALLELPIDS=128    # number of concurrent per-PID data collectors

TIMEOUT=120

SIGKILL=9
SIGTERM=15

plural()
{
    if [ $1 != "1" ]
    then
        printf "%s %ss" $1 $2
    else
        if [ $# -lt 3 ]
        then
            printf "%s %s" $1 $2
        else
            printf "%s" $3
        fi
    fi
}

header()
{
    printf "Output from "
    if [ $# -gt 1 ]
    then
        plural $2 second
        printf " of "
    fi
    printf "\"%s\"" "$1"
    if [ $# -gt 2 ]
    then
        printf " repeated every "
        plural $3 second second
    fi
    printf ", starting at %s\n" "`date "+%Y/%m/%d %H:%M:%S"`"
}

initmask()
{
    printf "%.$1d" 0
}

setbit()
{
    masklen=`expr length $1`

    if [ $2 -gt 1 ]
    then
        printf "%s" `expr substr $1 1 \`expr $2 - 1\``
    fi

    printf "1"

    if [ $2 -lt $masklen ]
    then
        printf "%s" `expr substr $1 \`expr $2 + 1\` \`expr $masklen - $2\``
    fi
}

isset()
{
    test `expr substr $1 $2 1` = "1"
}

sleepkill()
{
    sleep $1 > /dev/null    # redirection required so as to not stall pipe
    kill -$2 $3 2> /dev/null
}

run() 
{ 
    rcount=${2:-1} 
    scount=${3:-1} 
    header "$1" $rcount $scount 

    if [ $# -lt 3 ]
    then
        $1 2>&1 &
    else
        {
            while [ 1 ]
            do
                date +%H:%M:%S
                $1 2>&1
                printf "=======\n"
                sleep $3
            done
        } &
    fi
    runpid=$!

    reportkills=`initmask 64`
    if [ $# -gt 1 ]
    then
        reportkills=`setbit $reportkills $SIGKILL`

        sleepkill $2 $SIGTERM $runpid

        {
            sleepkill $TIMEOUT $SIGKILL $runpid
        } &
    else
        reportkills=`setbit $reportkills $SIGKILL`
        reportkills=`setbit $reportkills $SIGTERM`

        {
            sleepkill $TIMEOUT $SIGTERM $runpid
            sleepkill $TIMEOUT $SIGKILL $runpid
        } &
    fi
    waitpid=$!
    wait $runpid 2> /dev/null
    signal=`expr $? - 128`
    test $signal -gt 0 && isset $reportkills $signal &&
    {
        printf "Command timed out; terminated using \"kill %s\".\n" -$signal
    }
    kill $waitpid 2> /dev/null

    wait 2> /dev/null

    printf "\n"
}

collectfile()
{
    echo $1.`printf "%.5d" $2`
}

getrcsvalue()
{
    #
    # Return an RCS value from a string. The RCS value is the
    # substring between the first and last space characters.
    #
    echo $1 | sed 's/^[^ ]* \(.*\) [^ ]*$/\1/'
}

reportscript()
{
    printf "%s/%s (%s)\n" "`basename "$0" | tr [:lower:] [:upper:]`" \
        "`getrcsvalue "$RCSCHANGE"`" "`getrcsvalue "$RCSDATE"`"
}

lsofrun()
{
    run "lsof -r1 $P4ROOT/* $P4ROOT/server.locks/*/* $P4JOURNAL $P4LOG" 15 \
        | grep -v '^tail'
}

getpids()
{
    printf "ps -e | grep '%s$'" `echo $P4DBIN | sed 's/.*\/\([^\/]*\)$/\1/'` \
        | sh \
        | sed 's/^[^0-9]*\([0-9]*\)[^0-9]*.*/\1/'
}

pidruns()
{
    collecting=`collectfile $LOGFILE $ncollects`

    iparallelpid=0

    for pid in `getpids`
    do
        iparallelpid=`expr $iparallelpid + 1`

        {
            run "lsof -p $pid"
            run "strace -qT -p $pid" 0.50
        } \
            > `collectfile $collecting $iparallelpid` &

        if [ $iparallelpid -eq $NPARALLELPIDS ]
        then
            wait

            package $collecting $NPARALLELPIDS

            iparallelpid=0
        fi
    done

    wait

    package $collecting $iparallelpid
}

collect()
{
    ncollects=`expr $ncollects + 1`

    "$@" > `collectfile $LOGFILE $ncollects` &
}

package()
{
    icollect=0
    while [ $icollect -lt $2 ]
    do
        icollect=`expr $icollect + 1`

        collected=`collectfile $1 $icollect`
        cat $collected
        rm -f $collected
    done
}

ncollects=0

printf "Collecting p4d state into \"%s\"..." "$LOGFILE"

collect run reportscript

collect run "$P4DBIN -V"
collect run "$P4BIN -V"

collect run "uname -a"

collect run "tail -f -n10000 $P4LOG" 15
collect run "tail -f -n10000 $P4JOURNAL" 15

collect lsofrun

collect pidruns

collect run "ps -elfjH" 15 1
collect run "netstat -antp" 15 1

collect run "sysctl -a"

collect run "tail -f -n10000 /var/log/messages" 15

#
# Depending upon the state of the server, the following commands
# might not behave as expected.
#

collect run "$P4BIN -z tag info"

collect run "$P4BIN configure show"
collect run "$P4DBIN -r $P4ROOT -c show"

collect run "$P4BIN monitor show -el" 15 1

#
# All commands have been started; wait for them to finish.
#
wait

package $LOGFILE $ncollects > $LOGFILE

printf " done.\n"

exit 0