#! /bin/bash
###############################################################################
# Copyright (c) Perforce Software, Inc., 2007-2016. All rights reserved
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
# 1  Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE
# SOFTWARE, INC. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 
# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
# DAMAGE.
###############################################################################
#
# Last Modified: $Date$
# Submitted by: $Author$
# Revision: $Revision$
#
# Usage:
#
USAGE=" Usage:
  check_helix_p4d_health -p <p4port> [options] [tests]

        options: [-u <p4-user>] [-T <p4-ticket-file-name>] [-P <p4-password>] 
                 [-t <p4trust-file>] [--notips] [--comment] [--version] [--help]

        tests: 
                 [--all]
                 [--remoteonly]
                 [--licensecheck]  [--licthreshold <expire-days-threshold>]
                 [--pidcheck] [--pidthreshold <running-p4-pid-threshold>] 
                 [--p4monitorcheck] [--monthreshold <p4-processes-threshold>]
                 [--p4diskcheck] [--diskthreshold <disk-used-threshold>]
                 [--p4repcheck]

    If no tests are specified then only the P4D server online test is run.
    Multiple test arguments can be supplied. Thresholds can be supplied
    with specific tests or when using '--all' or '--remoteonly'. 

    The '--all' flag runs all tests. 

    The '--remoteonly' flag runs only remote tests using the p4 client. These
    tests can be run from any machine with network access to the P4D server 
    (including the Nagios server).

    The '--message' flag allows you to prepend an alert with a custom message.

    The '-u' flag specifies the P4D user name. This user must be an 'operator'
    user or must have 'super' access to the P4D server

    The '-p' flag specifies the P4D hostname and port in the 
    format 'hostname:port'.

    The '-T' flag specifies the location of the tickets file.

    The '-P' flag explicitly sets the password to use. Note this may not be 
    secure and the use of 'p4 login' and a tickets file ('-T') on the server 
    provides better security.

    The '-t' flag specifies the location of the P4TRUST file for SSL 
    connections.
    
    The '--licensecheck' flag tests if the license file is nearing it's 
    expiry date.  By default it checks for expiry within '30' days but this 
    can be overriden with the '--licthreshold' flag.

    The '--p4diskcheck' flag checks for free disk space on the P4D drives using
    the Perforce command 'p4 diskspace' and warns if the disks are over 95% used.
    This value can be overriden by speciying a value between 0 and 99 using with 
    the '--diskthreshold' flag.
    
    The '--pidcheck' flag counts the number of connected p4d processes using 
    'netstat' and warns if the are over 500 processes running. This value can 
    be overriden with the '--pidthreshold' flag. This test must be run on the 
    P4D server machine.

    The '--p4monitorcheck' flag counts the number of commands in the 
    'p4 monitor' table and warns if there are over 500 running. This value can be 
    overriden with the '--monthreshold' flag.

    The '--p4repcheck' flags (REPLICA ONLY) checks that replication is running.
"
# Description:
#
DESCRIPTION="Helix P4D health checker - Example Nagios monitoring script"
#
# This plugin will run the following checks against your Helix P4D server:
#   - Online?
#   - Licensed and not expiring soon
#   - P4D process count in acceptable range
#   - P4D monitor count in acceptable range
#   - Disk space available on P4D volumes
#   - Replication OK?
#
# Output:
#
# Notes:
#
# Examples:
#
# Run all checks against server on localhost:1666
#
#  check_helix_p4d_health -p localhost:1666 --all
#
# Check if license will expire in next 45 days
#
#  check_helix_p4d_health -p localhost:1666 --licensecheck -licthreshold 45
#
# Multiple arguments can be combined to run multiple tests
#
#  check_helix_p4d_health -p localhost:1666 --licensecheck --p4diskcheck
#
###############################################################################

# Defaults - Can edit these or override at command prompt
LICEXPIRE=30
PIDCOUNT=100
MONCOUNT=100
LOWDISKSPACE=95
NOTIPS=false

# Define binary information
PROGNAME=$0
P4BIN=/usr/local/bin/p4

# Set return states values
STATE_OK=0
STATE_WARNING=1
STATE_CRITICAL=2
STATE_UNKNOWN=3
EXITSTATUS=${STATE_OK}

# Helper functions 

set_status() {
if [ ${1} -gt ${EXITSTATUS} ]; then EXITSTATUS=${1}; fi
}

echo_tip() {
if [ "${NOTIPS}" == "false" ]; then echo "TIP:";echo "${1}"; fi
}

print_usage() {
  echo "${USAGE}"
}

print_help() {
    echo ${PROGNAME} $Revision$
    echo ""
    echo " Description: ${DESCRIPTION}"
    echo ""
    print_usage
    echo ""
}

get_word(){ echo ${1} | cut -f${2} -d" "; }

run_p4()
{
    P4="${P4BIN} -p ${P4PORT} -u ${P4USER}"
    LOCALRESULT=$(${P4} "${@:2}" 2>&1)
    RETVALUE=$?
    case "${LOCALRESULT}" in
        *"Your session has expired, please login again."*)
             echo "WARNING: ${1} check P4D user ${P4USER} is not logged in." >&2
             set_status ${STATE_WARNING}
             ;;
        *"doesn't exist."*)
             echo "WARNING: ${1} check using invalid P4USER ${P4USER}." >&2
             set_status ${STATE_WARNING}
             ;;
        *"Perforce password (P4PASSWD) invalid or unset.")
             echo "WARNING: ${1} check P4D user ${P4USER} is not logged in." >&2
             set_status ${STATE_WARNING}
             ;;
        *"Unknown command."*)
             echo "WARNING: ${1} check ran unsupported command p4 ${2}." >&2
             set_status ${STATE_WARNING}
             ;;
        *"p4 is Perforce's client tool for the command line.  Try:"*)
             echo "WARNING: ${1} check used invalid command p4 ${@:2}." >&2
             set_status ${STATE_WARNING}
             ;;
        *"The authenticity of"*)
             echo "CRITICAL: User running ${1} check needs to trust this P4D server." >&2
             NOTIPS=true
             set_status ${STATE_CRITICAL}
             ;;
        *"You don't have permission for this operation."*)
             echo "WARNING: ${1} check P4D user ${P4USER} does not have permissions
to run p4 ${@:2}." >&2
             set_status ${STATE_WARNING}
             ;;
        *)
             echo "${LOCALRESULT}"
             ;;
    esac
    return ${RETVALUE}
}

# Tests start here

license_check() {
        LICENSE_EXPIRE_TIP="Please contact sales@perforce.com to request
a new license file.
"
        LICENSE_LINE=$(echo "${RESULT}" | grep 'Server license\:')
        if [ $(echo ${LICENSE_LINE} | grep ': none'|wc -l) -gt 0 ]
        then 
                echo "WARNING: P4D server on ${P4PORT} is unlicensed."
                set_status ${STATE_WARNING}
        else
                EXPIRES=$(echo ${LICENSE_LINE}| cut -f2 -d"(" | cut -f1 -d")"|rev|cut -f1 -d " "|rev)
                EXPIRES_SEC=$(date -ud ${EXPIRES} +'%s')

                TODAY=$(date +'%s')
                ((TIME_LEFT=(${EXPIRES_SEC} - ${TODAY})/60/60/24))
                if [ ${TIME_LEFT} -lt ${LICEXPIRE} ]
                then
                        echo "WARNING: License expires in ${TIME_LEFT} days."
                        echo_tip "${LICENSE_EXPIRE_TIP}"
                        set_status ${STATE_WARNING}
                fi
        fi
}

server_up() {
    SERVERDOWN_TIP="Check if the 'p4d' process is running on the box.
Check the P4D log file for errors if it unexpectedly
stopped.
"
    RESULT=$(run_p4 "P4D" info)
    SERVERUP=`echo ${RESULT} | grep "Server license" | wc -l`

    case $SERVERUP in
         [1])
           ;;
         *)
           echo "CRITICAL: P4D server not responding!"
           echo "${RESULT}"
           echo_tip "${SERVERDOWN_TIP}"
           set_status ${STATE_CRITICAL}
           ;;
    esac
}


pid_check(){
    PIDCHECK_TIP="This may be caused by a performance problem or by
a script that has gone wild. Use 'netstat -anp'
and 'p4 monitor show -ael' to find the patterns. 
Often the majority of connections will be from 
the same client address or user.
"

    PIDS=$(netstat -anp 2>&1 | grep ${P4PORT} | wc -l)
    if [ ${PIDS} -gt ${PIDCOUNT} ] 
    then 
        echo "WARNING: ${PIDS} running p4d pids exceeded threshold ${PIDCOUNT}."
        echo "${PIDCHECK_TIP}"
        set_status ${STATE_WARNING}
    fi
}

p4monitor_check() {
    P4MONCHECK_TIP="This may be caused by a performance problem or by
a script that has gone wild. Run 'p4 monitor -ael'
and look if most of the commands are run by the
same user or if there is one very long running
command. If you need assistance collect the
'p4 monitor -ael' output and the P4D log
file and send them to 'support@perforce.com'.
"

    RESULT=$(run_p4 "Monitor" monitor show -ael)
    P4DRUNNING=$(echo "${RESULT}" | wc -l)
    if [ ${P4DRUNNING} -gt ${MONCOUNT} ] 
    then 
        echo "WARNING: ${P4DRUNNING} running p4d commands exceeded threshold $MONCOUNT."
        echo_tip "${P4MONCHECK_TIP}"
        set_status ${STATE_WARNING}
    fi 
}

p4replication_check() {
# --replication-status is not worth using. Back to edge tests...
# Great example of error....
# $ p4 pull -l -j
# Perforce password (P4PASSWD) invalid or unset.
# Perforce password (P4PASSWD) invalid or unset.
# Current replica journal state is:     Journal 5,      Sequence 11400.
# Current master journal state is:      Journal 5,      Sequence -1.
# Current master journal state is:      Journal 0,      Sequence -1.
# The statefile was last modified at:   2016/03/21 11:40:03.
# The replica server time is currently: 2016/03/21 11:46:44 +0000 GMT
#
# On Commit:
#    Pull only allowed on replica servers.
#
    P4REPCHECK_TIP="There may be a problem with the journal
file on the master or the replica may
have been switched off. Check if the P4D
is runnning on the replica, that the 
replica has network connectivity to the 
master server and that there are no
errors in the replica log file.
"

   RESULT=$(run_p4 "Replication" pull -l -j)
   case "${RESULT}" in
       *"Pull only allowed on replica servers"*) 
            # It's a master/commit so no test needed.
            set_status ${STATE_OK}
            ;;
       *"Remote server refused request"*)
            echo "CRITICAL: Master server is not available."
            echo_tip "Check if P4D is running on the master."
            set_status ${STATE_CRITICAL}
            echo "${RESULT}"
            ;;
       *"Perforce password (P4PASSWD) invalid or unset."*)
            echo "CRITICAL: Replication user is no longer logged in."
            echo_tip "Log the replication user into the master server
from the replica."
            set_status ${STATE_CRITICAL}
            echo "${RESULT}"
            ;;
       *"Sequence -1"*)
            echo "CRITICAL: Replication failed. Replica is unable to access master journal."
            echo_tip "Check if journal on master has been moved or compressed"
            set_status ${STATE_CRITICAL}
            echo "${RESULT}"
            ;;
       *)
          if [ $( echo "${RESULT}" | grep Journal | wc -l ) -eq 3 ]
          then
             echo "CRITICAL: Replication has stalled."
             echo_tip "Check if journal on master has been damaged or removed."
             set_status ${STATE_CRITICAL}
             echo "${RESULT}"
          fi 
   esac
}

p4disk_check() {
    P4DISKCHECK_TIP="Failure to provide enough disk space for temporary
actions may cause corruption of the Helix system
or backups."
    WARNING_NEEDED=0
    RESULT=$(run_p4 "Disk" diskspace)
    if [ ${?} -eq 0 ]
    then
        while read LINE  
        do 
           VOLUME=$( get_word "${LINE}" 1 )
           USED=$( get_word "${LINE}" 11 | cut -f2 -d "("|cut -f1 -d "%")
           if [ ${USED} -ge ${LOWDISKSPACE} ]
           then
               echo "WARNING: ${USED}% disk space used on ${VOLUME} volume."
               set_status ${STATE_WARNING}
               WARNING_NEEDED=1
           fi
        done <<< "${RESULT}"
        if [ "${WARNING_NEEDED}" -eq 1 ]; then echo_tip "${P4DISKCHECK_TIP}"; fi
    fi 
}



# Main()
EXITSTATUS=${STATE_OK} #default

# Process arguments
while test -n "${1}"; do
    case "${1}" in
        --help|--version|-h|-V)
            print_help
            exit ${STATE_OK}
            ;;
        -u)
            P4USER=${2}
            shift
            ;;
        -p)
            P4PORT=${2}
            shift
            ;;
        -P)
            P4PASSWD=${2}
            shift
            ;;
        -T)
            export P4TICKETS=${2}
            shift
            ;;
        -t)
            export P4TRUST=${2}
            shift
            ;;
        --pidcheck)
            PIDCHECK=true
            ;;
        --pidthreshold)
            PIDCOUNT=${2}
            shift
            ;;
        --p4diskcheck)
            P4DISKCHECK=true
            ;;
        --diskthreshold)
            LOWDISKSPACE=${2}
            shift
            ;;
        --licensecheck)
            LICCHECK=true
            ;;
        --licthreshold)
            LICEXPIRE=${2}
            shift
            ;;
        --p4monitorcheck)
            P4MONCHECK=true
            ;;
        --monthreshold)
            MONCOUNT=${2}
            shift
            ;;
        --p4repcheck)
            P4REPCHECK=true
            ;;
        --all)
            LICCHECK=true
            PIDCHECK=true
            P4MONCHECK=true
            P4DISKCHECK=true
            P4REPCHECK=true
            ;;
        --remoteonly)
            LICCHECK=true
            P4MONCHECK=true
            P4DISKCHECK=true
            P4REPCHECK=true
            ;;
        --EXITSTATUS)
            EXITSTATUS=${2}
            exit ${EXITSTATUS}
            shift
            ;;
        --message)
            echo "${2}"
            shift
            ;;
        --notips)
            NOTIPS=true
            ;;
        --debug)
            set -x
            ;;
        *)
            echo "Unknown argument: $1"
            print_usage
            exit ${STATE_UNKNOWN}
            ;;
    esac
    shift
done

# Setup p4 binary
if [[ ! -f ${P4BIN} ]]; then echo "WARNING: ${P4BIN} is not executable. No checks run.";exit ${STATE_WARNING} ; fi

# If values not set try and pick them up from the environment
P4PORT=${P4PORT:-$(p4 set P4PORT | cut -f2 -d"=" | cut -f1 -d " ")}
P4USER=${P4USER:-$(p4 set P4USER | cut -f2 -d"=" | cut -f1 -d " ")}
P4USER=${P4USER:-$(whoami) }
P4TICKETS=${P4TICKETS:-$(p4 set P4TICKETS | cut -f2 -d"=" | cut -f1 -d " ")}
P4TRUST=${P4TRUST:-$(p4 set P4TRUST | cut -f2 -d"=" | cut -f1 -d " ")}
P4PASSWD=${P4PASSWD:-$(p4 set P4PASSWD | cut -f2 -d"=" | cut -f1 -d " ")}

P4="${P4BIN} -p ${P4PORT} -u ${P4USER}"

# Run tests
server_up
if [ ${EXITSTATUS} -lt ${STATE_CRITICAL} ]
then
  if [ "${LICCHECK}" == "true" ]; then license_check; fi
  if [ "${PIDCHECK}" == "true" ]; then pid_check; fi
  if [ "${P4MONCHECK}" == "true" ]; then p4monitor_check; fi
  if [ "${P4DISKCHECK}" == "true" ]; then p4disk_check; fi
  if [ "${P4REPCHECK}" == "true" ]; then p4replication_check; fi
fi

# If no problems need to return something to make Nagios happy.
if [ ${EXITSTATUS} -eq ${STATE_OK} ]; then echo "OK"; fi

exit ${EXITSTATUS}