#!/bin/bash # Copyright (c) 2017, Perforce Software, Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. RCSCHANGE="$Change: 30147 $" RCSDATE="$Date: 2024/02/27 $" # === CONFIGURATION BEGIN === # # Change the "<absolute-path-to->" in the following "outfile" definition, but # leave the " and \` characters as they are. # export SDP_INSTANCE=${1:-$SDP_INSTANCE} if [[ $SDP_INSTANCE == Undefined ]]; then echo "Instance parameter not supplied." echo "You must supply the Perforce instance as a parameter to this script." exit 1 fi [[ $SDP_INSTANCE == $1 ]] && shift # Skip supplied param to avoid issues with later arg parsing # Setup SDP environment . /p4/common/bin/p4_vars $SDP_INSTANCE . /p4/common/bin/backup_functions.sh outfile=$LOGS/p4dstate-`date +%Y%m%d%H%M%S`.out # # Change the values of the following definitions as appropriate # for the installation. # p4d=$P4DBIN p4=$P4BIN # The following already set by p4_vars # P4ROOT=/<absolute-path-to-P4ROOT> # P4LOG=/<absolute-path-to->/log # P4JOURNAL=/<absolute-path-to->/journal # P4JOURNALPREFIX=/<absolute-path-to->/jnlPrefix # use if interested in rotated journals P4JOURNALPREFIX="" # P4PORT=<p4d-listener> # P4USER=<perforce-superuser> # === CONFIGURATION END === RED=$(tput setaf 1) GREEN=$(tput setaf 2) YELLOW=$(tput setaf 3) normal=$(tput sgr0) NPARALLELPIDS=128 # number of concurrent per-PID data collectors TIMEOUT=120 export P4PORT export P4USER SIGKILL=9 SIGTERM=15 # Initialise these for bash USAGE=0 NOP4=0 CONFIGONLY=0 SKIP_CONFIG_CHECK=0 while [ $# -gt 0 ]; do key="$1" case $key in # In case the server is nonresponse, use '--skipp4' --skipp4) NOP4=1 shift ;; # Config check only --check-config) CONFIGONLY=1 shift ;; -h) # Print usage USAGE=1 shift ;; --bypass-check-config) SKIP_CONFIG_CHECK=1 shift ;; esac done plural() { if [ $1 != "1" ] then printf "%s %ss" $1 $2 else if [ $# -lt 3 ] then printf "%s %s" $1 $2 else printf "%s" $3 fi fi } header() { printf "Output from " if [ $# -gt 1 ] then plural $2 second printf " of " fi printf "\"%s\"" "$1" if [ $# -gt 2 ] then printf " repeated every " plural $3 second second fi printf ", starting at %s\n" "`date "+%Y/%m/%d %H:%M:%S"`" } initmask() { printf "%.$1d" 0 } setbit() { masklen=`expr length $1` if [ $2 -gt 1 ] then printf "%s" `expr substr $1 1 \`expr $2 - 1\`` fi printf "1" if [ $2 -lt $masklen ] then printf "%s" `expr substr $1 \`expr $2 + 1\` \`expr $masklen - $2\`` fi } isset() { test `expr substr $1 $2 1` = "1" } sleepkill() { sleep $1 > /dev/null # redirection required so as to not stall pipe kill -$2 $3 2> /dev/null } run() { rcount=${2:-1} scount=${3:-1} header "$1" $rcount $scount if [ $# -lt 3 ] then $1 2>&1 & else { while [ 1 ] do date +%H:%M:%S $1 2>&1 printf "=======\n" sleep $3 done } & fi runpid=$! reportkills=`initmask 64` if [ $# -gt 1 ] then reportkills=`setbit $reportkills $SIGKILL` sleepkill $2 $SIGTERM $runpid { sleepkill $TIMEOUT $SIGKILL $runpid } & else reportkills=`setbit $reportkills $SIGKILL` reportkills=`setbit $reportkills $SIGTERM` { sleepkill $TIMEOUT $SIGTERM $runpid sleepkill $TIMEOUT $SIGKILL $runpid } & fi waitpid=$! wait $runpid 2> /dev/null signal=`expr $? - 128` test $signal -gt 0 && isset $reportkills $signal && { printf "Command timed out; terminated using \"kill %s\".\n" -$signal } kill $waitpid 2> /dev/null wait 2> /dev/null printf "\n" } collectfile() { echo $1.`printf "%.5d" $2` } getrcsvalue() { # # Return an RCS value from a string. The RCS value is the # substring between the first and last space characters. # echo $1 | sed 's/^[^ ]* \(.*\) [^ ]*$/\1/' } reportscript() { printf "%s/%s (%s)\n" "`basename "$0" | tr [:lower:] [:upper:]`" \ "`getrcsvalue "$RCSCHANGE"`" "`getrcsvalue "$RCSDATE"`" } lsofrun() { run "lsof -r1 $P4ROOT/* $P4ROOT/server.locks/*/* $P4JOURNAL $P4JOURNALPREFIX* $P4LOG" 15 \ | grep -v '^tail' } getpids() { printf "ps -e | grep '%s$'" `echo $p4d | sed 's/.*\/\([^\/]*\)$/\1/'` \ | sh \ | sed 's/^[^0-9]*\([0-9]*\)[^0-9]*.*/\1/' } pidruns() { collecting=`collectfile $outfile $ncollects` iparallelpid=0 for pid in `getpids` do iparallelpid=`expr $iparallelpid + 1` { run "lsof -p $pid" run "strace -qT -p $pid" 0.50 } \ > `collectfile $collecting $iparallelpid` & if [ $iparallelpid -eq $NPARALLELPIDS ] then wait package $collecting $NPARALLELPIDS iparallelpid=0 fi done wait package $collecting $iparallelpid } collect() { ncollects=`expr $ncollects + 1` "$@" > `collectfile $outfile $ncollects` & } package() { icollect=0 while [ $icollect -lt $2 ] do icollect=`expr $icollect + 1` collected=`collectfile $1 $icollect` cat $collected rm -f $collected done } check_binary() { printf ' Checking if %s installed and in PATH ' "$1" if ! type "$1" > /dev/null; then return 0; else return 1; fi } check_p4() { case $1 in p4d) printf ' Checking configured p4d binary (%s) ' "$p4d" eval $p4d -V > /dev/null 2>&1 ;; p4) printf ' Checking configured p4 binary (%s) ' "$p4" eval $p4 -V > /dev/null 2>&1 ;; P4LOG) printf ' Checking configured P4LOG (%s) ' "$P4LOG" eval tail -n1 "$P4LOG" > /dev/null 2>&1 ;; P4JOURNAL) printf ' Checking configured P4JOURNAL (%s) ' "$P4JOURNAL" eval tail -n1 "$P4JOURNAL" > /dev/null 2>&1 ;; esac if [ $? -gt 0 ] then return 0; else return 1; fi } check_config() { fatal_error=0; warnings=0 printf "Configuration Check\n" if [ $NOP4 -eq 0 ]; then printf " Checking 'p4 info' against configured P4PORT (%s) " "$P4PORT" eval $p4 -p $P4PORT -ztag info > /dev/null 2>&1 if [ $? -gt 0 ]; then printf '[%s]\n' "${YELLOW}warning${normal}" printf " -> Check connectivity to configured P4PORT or use the " printf "'--skipp4' option to p4dstate.sh.\n" warnings=1 else printf '[%s]\n' "${GREEN}pass${normal}" printf " Checking 'p4 configure show' against configured P4PORT " printf "as configured P4USER (%s,%s) " $P4PORT $P4USER eval $p4 -p $P4PORT -u $P4USER configure show > /dev/null 2>&1 if [ $? -gt 0 ]; then printf '[%s]\n' "${YELLOW}warning${normal}" printf " -> Check for valid login ticket for configured P4USER " printf "and P4PORT (%s,%s)\n" "$P4USER" "$P4PORT" warnings=1 else printf '[%s]\n' "${GREEN}pass${normal}" fi fi fi p4_binaries="p4d p4" for binary in ${p4_binaries} do if check_p4 "$binary"; then case $binary in p4d) printf '[%s]\n' "${RED}fail${normal}" fatal_error=1 ;; p4) printf '[%s]\n' "${YELLOW}warning${normal}" warnings=1 ;; esac printf ' -> No %s binary found, check p4dstate configuration.\n' "$binary" else printf '[%s]\n' "${GREEN}pass${normal}" fi done p4_components="P4LOG P4JOURNAL" for component in ${p4_components} do if check_p4 "$component"; then printf '[%s]\n' "${YELLOW}warning${normal}" printf ' -> No %s found, check p4dstate configuration.\n' "$component" warnings=1 else printf '[%s]\n' "${GREEN}pass${normal}" fi done; utilities="lsof strace netstat lslocks top" for utility in ${utilities} do if check_binary "$utility"; then if [ "$utility" != "lslocks" ] && [ "$utility" != "netstat" ]\ && [ "$utility" != "top" ];then printf '[%s]\n' "${RED}fail${normal}" fatal_error=1 else printf '[%s]\n' "${YELLOW}warning${normal}" warnings=1 fi else printf '[%s]\n' "${GREEN}pass${normal}" fi done; if [ $fatal_error -gt 0 ]; then if [ $warnings -gt 0 ]; then printf "Required components missing, fix failed and warning.\n" else printf "Required components missing, fix failed.\n" fi exit 1 fi if [ $warnings -gt 0 ]; then if [ $fatal_error -gt 0 ]; then printf "Required components missing, fix failed and warning.\n" exit 1 else printf "For a more complete capture of data, fix warnings.\n" fi fi if [ $CONFIGONLY -eq 1 ];then exit 1 fi } usage() { printf "Usage:\n" printf "\n\tp4dstate.sh [ options ]\n" printf "\n\toptions:" printf "\n\t\t${RED}-h ${normal}Display usage" printf "\n\t\t${RED}--skipp4 ${normal}Skip running p4 commands" printf "\n\t\t${RED}--check-config ${normal}Check configuration status only" printf "\n\n\tp4dstate.sh is used when a Linux Helix Core Server is exhibiting " printf "problematic behaviors like:\n" printf "\n\t * Slow or unresponsive Helix Core Server\n" printf "\t * Slow or unresponsive Helix Core Server command\n" printf "\t * Replication stalled or moving slowly on a Helix Core Replica Server\n" printf "\n\tp4dstate.sh collects data from multiple sources, during the time of an " printf "event, writing to its output \n\tfile a comprehensive set of data when " printf "paired with a Helix Core Server log file covering the time of \n\tthe event " printf "provides a more complete picture of events to aid in investigation of " printf "issues. The information \n\tgathered helps facilitate issue resolution and " printf "root cause investigations.\n\n" printf "\tPrerequisites for use:" printf "\n\n\t1. Configure the section near the top of the script to set the location of " printf "the output file and the \n\t relevant p4 settings for the Helix Core Server " printf "p4dstate.sh will be run against.\n" printf "\t2. Ensure the lsof, strace, lslocks, and netstat Linux utilities are installed and " printf "available in the\n\t \$PATH of the root Linux user running p4dstate.sh." printf "\n\t3. Ensure a valid Helix Core Server login ticket for the ${RED}<perforce-superuser> " printf "${normal}defined in the \n\t p4dstate.sh configuration." printf "\n\t4. Run p4dstate.sh as a Linux root user or under sudo." printf "\n\n\tUse 'p4dstate.sh --check-config' to check if prerequesites are met." printf "\n\n\tSee ${RED}https://portal.perforce.com/s/article/15261 ${normal}for further details.\n\n" } if [ $USAGE -gt 0 ] then usage exit 1 fi if [ $SKIP_CONFIG_CHECK -gt 0 ]; then check_config fi outfiletemplate="$outfile" eval outfile=\"$outfiletemplate\" ncollects=0 printf "Collecting p4d state from server at P4PORT=%s into \"%s\"..." "$P4PORT" "$outfile" collect run reportscript collect run "$p4d -r $P4ROOT -V" collect run "$p4d -r $P4ROOT -c show" collect run "$p4 -V" collect run "uname -a" collect run "tail -f -n10000 $P4LOG" 15 collect run "tail -f -n10000 $P4JOURNAL" 15 collect lsofrun collect pidruns collect run "ps -elfjH" 15 1 collect run "netstat -antp" 15 1 collect run "sysctl -a" collect run "tail -f -n10000 /var/log/messages" 15 collect run "tail -f -n10000 /var/log/syslog" 15 collect run "tail -f -n10000 /var/log/dmesg" 15 collect run "lslocks -J -o +BLOCKER" collect run "lslocks -o +BLOCKER" collect run "top -b -n 4" # # Depending upon the state of the server, the following commands # might not behave as expected. # if [ $NOP4 -eq 0 ] then collect run "$p4 -u $P4USER -p $P4PORT -ztag info" collect run "$p4 -u $P4USER -p $P4PORT configure show" collect run "$p4 -u $P4USER -p $P4PORT configure show allservers" collect run "$p4 -u $P4USER -p $P4PORT configure history" collect run "$p4 -u $P4USER -p $P4PORT -ztag servers -J" collect run "$p4 -u $P4USER -p $P4PORT journals" collect run "$p4 -u $P4USER -p $P4PORT monitor show -el" 15 1 fi # # All commands have been started; wait for them to finish. # wait package $outfile $ncollects > $outfile printf " done.\n" exit 0
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#14 | 30154 | Robert Cowham | Fix incorrect logic to call check_config, and fix getpids when SDP case-insensitive p4d is running. | ||
#13 | 30148 | Robert Cowham | Add usage comments for SDP version of script | ||
#12 | 30147 | Robert Cowham | Changed filetype to +k p4dstate.sh | ||
#11 | 29923 | C. Thomas Tyler |
Updated HTML hyperlinks to use 'portal.perforce.com'. This replaces currently broken links to 'answers.perforce.com' and currently redirected links to 'community.perforce.com'. #review-29924 |
||
#10 | 29709 | Robert Cowham | Customize lightly for SDP (sourcing variables for an instance) - requires use of /bin/bash rather than /bin/sh | ||
#9 | 28979 | Robert Cowham | Collect lslocks output even if older version of lslocks with no JSON option. | ||
#8 | 28731 | C. Thomas Tyler |
Reset p4dstate.sh to latest version from FTP server: http://ftp.perforce.com/perforce/tools/p4dstate/p4dstate.sh Also added p4pstate.sh and p4brokerstate.sh with URLs: http://ftp.perforce.com/perforce/tools/p4dstate/p4pstate.sh http://ftp.perforce.com/perforce/tools/p4dstate/p4brokerstate.sh These versions are not SDP-ified; these are the unmodified stock versions from the FTP server. TO DO Later: Enhance the stock SDP versions to be SDP-ready but not SDP-dependent. |
||
#7 | 27722 | C. Thomas Tyler |
Refinements to @27712: * Resolved one out-of-date file (verify_sdp.sh). * Added missing adoc file for which HTML file had a change (WorkflowEnforcementTriggers.adoc). * Updated revdate/revnumber in *.adoc files. * Additional content updates in Server/Unix/p4/common/etc/cron.d/ReadMe.md. * Bumped version numbers on scripts with Version= def'n. * Generated HTML, PDF, and doc/gen files: - Most HTML and all PDF are generated using Makefiles that call an AsciiDoc utility. - HTML for Perl scripts is generated with pod2html. - doc/gen/*.man.txt files are generated with .../tools/gen_script_man_pages.sh. #review-27712 |
||
#6 | 26368 | Robert Cowham | Shift lslocks earlier | ||
#5 | 23435 | Robert Cowham | Convert tabs to spaces - whitespace only no other changes | ||
#4 | 23434 | Robert Cowham | Require bash and fix problem with unspecified parameters to run(). | ||
#3 | 23424 | Robert Cowham |
Update with latest changes from the original script (merged in). //guest/perforce_software/admin_toolkit/p4dstate.sh |
||
#2 | 19059 | Russell C. Jackson (Rusty) | Updated to use SDP variables | ||
#1 | 16652 | C. Thomas Tyler |
Added p4dstate.sh to SDP. Changes from originating server (no functionality changes): * Adjusted Copyright message to SDP standard, which references the same basic content. * Added comment clarifying that RCS values are verbatim from the originating Helix Server, and read-only elsewhere. To Do: * Comments and usage instructions needed. * SDP-ifiy it; i.e. have it take an instance parameter, and which point all other settings (path to p4d and p4 executables, P4ROOT and other environment variables) are known. #review @michael_shields @robert_cowham |