#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ #============================================================================== # Declarations and Environment # Allow override of P4U_HOME, which is set only when testing P4U scripts. export P4U_HOME=${P4U_HOME:-/p4/common/bin} export P4U_LIB=${P4U_LIB:-/p4/common/lib} export P4U_ENV=$P4U_LIB/p4u_env.sh export P4U_LOG=Unset export VERBOSITY=${VERBOSITY:-3} # Environment isolation. For stability and security reasons, prepend # PATH to include dirs where known-good scripts exist. # known/tested PATH and, by implication, executables on the PATH. export PATH=$P4U_HOME:$PATH:~/bin:. export P4CONFIG=${P4CONFIG:-.p4config} [[ -r "$P4U_ENV" ]] || { echo -e "\nError: Cannot load environment from: $P4U_ENV\n\n" exit 1 } declare BASH_LIBS=$P4U_ENV BASH_LIBS+=" $P4U_LIB/libcore.sh" BASH_LIBS+=" $P4U_LIB/libp4u.sh" for bash_lib in $BASH_LIBS; do source $bash_lib done declare Version=1.1.3 declare HostList= declare CfgDir=${P4CCFG:-/p4/common/config} declare HostCfgFile=$CfgDir/sdp_hosts.cfg declare EnvFile=$P4U_HOME/p4_vars declare SDPInstance=${SDP_INSTANCE:-Unset} declare -i CheckFirewall=0 declare -i SilentMode=0 declare -i EmailAlways=0 declare -i EmailOnFailure=0 declare -i RemoveOldLogs=0 #============================================================================== # Local Functions #------------------------------------------------------------------------------ # Function: terminate function terminate { # Disable signal trapping. trap - EXIT SIGINT SIGTERM declare -i sendEmail=0 vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus" # Stop logging. if [[ "${P4U_LOG}" != off ]]; then stoplog # Email. [[ $EmailAlways -eq 1 ]] && sendEmail=1 [[ $EmailOnFailure -eq 1 && $OverallReturnStatus -ne 0 ]] && sendEmail=1 if [[ $sendEmail -eq 1 ]]; then emailStatus="OK" [[ $OverallReturnStatus -ne 0 ]] && emailStatus="Error" local mail_sender_opt=$(get_mail_sender_opt) run "$SDPMAIL -s \"${HOSTNAME%%\.*} p4_$SDPInstance global_replica_status [$emailStatus]\" \$mail_sender_opt \$MAILTO < $P4U_LOG" fi fi # Don't litter. cleanTrash # With the trap removed, exit. exit $OverallReturnStatus } #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). #------------------------------------------------------------------------------ function usage { declare style=${1:--h} echo "USAGE for $THISSCRIPT v$Version: $THISSCRIPT [-H <host1>[,<host2>,...]] [-i <tag>] [-f] [-r] [-e|-E] [-L <log>] [-si] [-v<n>] [-n|-N] [-D] or $THISSCRIPT [-h|-man|-V] " if [[ $style == -man ]]; then echo -e " DESCRIPTION: This script checks the statis of Helix replicas on all replica hosts, as defined in the SDP_REPLICA_HOSTS variable set in configuration file, $HostCfgFile. The underlying Perforce status command is a 'p4 pull -lj' command. This script is intended to be called by a cron job the master server. It then does ssh calls to the remaining SDP hosts. This depends on ssh keys being setup such that the Perforce login (as defined by OSUSER in $EnvFile) can ssh without a password to all SDP hosts. (To simplify failover, the backup servers should also be able to ssh to each other without a password; security implications should be considered here.) OPTIONS: -H <host1>[,<host2>,...] Specify a comma-delimited list of hosts to push to. By default, the SDP_REPLICA_HOSTS value defined in the config file $HostCfgFile determines the list of hosts to push to. -i <tag> Specify the SDP instance tag (e.g '-i 1' for /p4/1 or '-i fgs' for /p4/fgs) for the SDP instance to check replcas for. The default is to use the \$SDP_INSTANCE variable if defined, or else '1'. The default host config file $HostCfgFile is ignored in favor of an instance-specific file if one is found named $CfgDir/${HostCfgFile%.cfg}.<tag>.cfg. This can be used for cases where instances share the master/commit server, but have differing hosts on which replicas for each instance are run. -f Do a 'sudo service iptables status' to check firewalls. Currently only supports firewalls that work with that command (including RHEL 6/CentOS 6). This verifies that sudo access works, and displays information about the firewwall. This option is only suitable for sites where the Perforce admin account is allowed sudo access. -r Specify this option to remove old global_replica_status.*.log files. If this option is specified, log files named /p4/<n>/logs/global_replica_status.*.log (where '<n>' is the SDP instance name) that are older than the number of days indicated by the KEEPLOGS setting in $EnvFile are removed. The old log removal occurs only upon successful completion. -e Send email to MAILTO value defined in $EnvFile in event of failure only. -E Send email to MAILTO value defined in $EnvFile. -v<n> Set verbosity 1-5 (-v1 = quiet, -v5 = highest). -L <log> Specify the path to a log file, or the special value 'off' to disable logging. By default, all output (stdout and stderr) goes to a log file named global_replica_status.<datestame>.log in $LOGS. NOTE: This script is self-logging. That is, output displayed on the screen is simultaneously captured in the log file. Do not run this script with redirection operators like '> log' or '2>&1', and do not use 'tee.' -si Operate silently. All output (stdout and stderr) is redirected to the log only; no output appears on the terminal. This cannot be used with '-L off'. -n No-Op. Prints commands instead of running them. -N No-Op. Similar to '-n', but this command does execute the 'ssh' calls to get to the remote host, but then does 'p4 sync -n' rather than' 'p4 sync' on the remote host. -D Set extreme debugging verbosity. HELP OPTIONS: -h Display short help message -man Display man-style help message -V Dispay version info for this script and its libraries. FILES: The SDP environment file $EnvFile defines various SDP settings, and is used by several SDP scripts. The host config file $HostCfgFile defines the SDP_HOSTS and other values. EXAMPLES: Recommended crontab usage for SDP Instance 1: $P4CBIN/$THISSCRIPT -i 1 -si -r -E < /dev/null > /dev/null 2>&1 The redirect to /dev/null is to avoid any output that would generate a duplicate email from cron. Output is not lost; it is written to a timestampped log file: $P4HOME/logs/global_replica_status.<timestamp>.log SEE ALSO: See this related script: $P4CBIN/sdp_sync.sh " fi exit 1 } #============================================================================== # Command Line Processing declare -i shiftArgs=0 set +u while [[ $# -gt 0 ]]; do case $1 in (-i) SDPInstance=$2; shiftArgs=1;; (-H) HostList=$2; shiftArgs=1;; (-f) CheckFirewall=1;; (-r) RemoveOldLogs=1;; (-e) EmailOnFailure=1;; (-E) EmailAlways=1;; (-h) usage -h;; (-man) usage -man;; (-V) show_versions; exit 1;; (-v1) export VERBOSITY=1;; (-v2) export VERBOSITY=2;; (-v3) export VERBOSITY=3;; (-v4) export VERBOSITY=4;; (-v5) export VERBOSITY=5;; (-L) export P4U_LOG=$2; shiftArgs=1;; (-si) SilentMode=1;; (-n) export NO_OP=1;; (-D) set -x;; # Debug; use 'set -x' mode. (*) usageError "Unknown arg ($1).";; esac # Shift (modify $#) the appropriate number of times. shift; while [[ $shiftArgs -gt 0 ]]; do [[ $# -eq 0 ]] && usageError "Bad usage." shiftArgs=$shiftArgs-1 shift done done set -u #============================================================================== # Command Line Verification [[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \ usageError "Cannot use '-si' with '-L off'." [[ $EmailOnFailure -eq 1 && $P4U_LOG == off ]] && \ usageError "Cannot use '-e' or '-E' with '-L off'." [[ $EmailAlways -eq 1 && $EmailOnFailure -eq 1 ]] && \ usageError "The '-e' and '-E' flags are mutually exclusive." [[ $SDPInstance == Unset ]] && \ bail "SDP Environment is not set, and no instance parameter was provided. Aborting." source $EnvFile $SDPInstance ||\ bail "Failed to load environment from $EnvFile. Aborting." source $P4CBIN/backup_functions.sh ||\ warnmsg "Failed to load backup_functions.sh; email will not send." [[ $P4U_LOG == Unset ]] && export P4U_LOG="$LOGS/global_replica_status.$(date +'%Y%m%d-%H%M%S').log" #============================================================================== # Main Program declare -i OverallReturnStatus=0 trap terminate EXIT SIGINT SIGTERM if [[ "${P4U_LOG}" != off ]]; then touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]." # Redirect stdout and stderr to a log file. if [[ $SilentMode -eq 0 ]]; then exec > >(tee ${P4U_LOG}) exec 2>&1 else exec >${P4U_LOG} exec 2>&1 fi initlog fi msg "$THISSCRIPT v$Version reporting host status at $(date) for these hosts: $SDP_REPLICA_HOSTS." if [[ -r $CfgDir/sdp_hosts.${SDPInstance}.cfg ]]; then HostCfgFile=$CfgDir/sdp_hosts.${SDPInstance}.cfg msg "Using instance-specific host config file $HostCfgFile." fi source $HostCfgFile ||\ bail "Failed to load host configuration from $HostCfgFile. Aborting." # If '-H <host1>[,<host2>,...]' was specified as a comma-separated list # on the command line, overide the SDP_REPLICA_HOSTS value defined in # the host config file. [[ -n "$HostList" ]] && export SDP_REPLICA_HOSTS="$(echo $HostList|tr ',' ' ')" for host in $SDP_REPLICA_HOSTS; do msg "${H}\nChecking replication status on $host." if [[ $NO_OP -eq 0 ]]; then ssh -q $host "source $EnvFile $SDPInstance; $P4BIN serverid; $P4BIN pull -lj" ||\ OverallReturnStatus=1 else msg "NO_OP: Would run: ssh -q $host \"source $EnvFile $SDPInstance; $P4BIN serverid; $P4BIN pull -lj\"" fi done if [[ $CheckFirewall -eq 1 ]]; then for host in $SDP_REPLICA_HOSTS; do msg "${H}\nChecking sudo access and firewall on $host." if [[ $NO_OP -eq 0 ]]; then ssh -q -t $host "source $EnvFile $SDPInstance; $P4BIN serverid; sudo service iptables status" ||\ OverallReturnStatus=1 else msg "NO_OP: Would run: ssh -q -t $host \"source $EnvFile $SDPInstance; $P4BIN serverid; $P4BIN pull -lj\"" fi done fi if [[ $OverallReturnStatus -eq 0 ]]; then msg "${H}\nAll replicas reported back.\n"; else msg "${H}\nSome replicas did not report back. Review the output above carefully." fi # Illustrate using $SECONDS to display runtime of a script. msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n" # See the terminate() function, which is really where this script exits. exit $OverallReturnStatus
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#9 | 25744 | C. Thomas Tyler |
Removed global_replica_status.sh as this is being moved to HMS, due to dependencies on HMS configuration. |
||
#8 | 24467 | C. Thomas Tyler |
Implemented '-r' flag to clean up old logs in global_replica_status.sh. When this flag is used and upon successful completion of the script with exit code 0, old logs are cleaned up honoring the $KEEPLOGS setting in /p4/common/bin/p4_vars (as documented in the script). Also fixed issue where a proper error message was obscured if the sdp_hosts.cfg file was missing. |
||
#7 | 24338 | C. Thomas Tyler |
Minor refactoring in global_replica_status.sh to eliminate unintentional spam from 'cron' calls. Bypassing pre-commit review to enable testing. #review-24339 |
||
#6 | 22628 | C. Thomas Tyler |
Fixed minor order-of-processing bug resulting in a harmless error appearing at the end of script processing as cleanTrash() was called to clean garbage files. The run() function was called to clean garbage files/dirs just as a directory that function depended on got cleaned up. The fix was applied to scripts that used libcore.sh, including the template.sh template script. Also corrected comments in p4u_env.sh. Bypassing pre-commit review as this has been well tested. #review-22629 |
||
#5 | 21962 | C. Thomas Tyler |
Updated various scripts to use run() and rrun() functions in favor of predecessor runCmd() and runRemoteCmd(). The older functions won't be removed to avoid breaking scripts that rely on their behavior and have no issues with them. The newer fuctions are more scalable and avoid erroneous "Argument list too long" from bash due to buffer overruns when used with commands with large amounts of output. Enhanced runRemoteCmd() to clean up after itself, as it generated files in /tmp that didn't get automatically cleaned up. If used in scripts called very often (e.g. every 5 minutes in a crontab), this leads to significant issues with /tmp filling up with garbage files over a period of several weeks. Enhanced test_utils.sh to test new run() and rrun() calls. |
||
#4 | 18581 | C. Thomas Tyler |
global_replica_status.sh v1.1.0: * Moved sdp_hosts.cfg config file to /p4/common/config, with optional override by speciffying P4CCFG variable. * Changed recommended command line for crontab to suggest '-E' (always send emails) rather than '-e' (email on error only). * Enhanced docs. Added new P4CCFG setting to p4_vars.template, with value /p4/common/config. |
||
#3 | 17260 | C. Thomas Tyler |
global_replica_status.sh v1.0.5: * Added support for instance-specifc host config files. * Updated docs. |
||
#2 | 17219 | C. Thomas Tyler | Routine Merge Down to dev from main. | ||
#1 | 16563 | C. Thomas Tyler |
Routine Merge Down to dev from main using: p4 merge -b perforce_software-sdp-dev p4 resolve -as |
||
//guest/perforce_software/sdp/main/Server/Unix/p4/common/bin/global_replica_status.sh | |||||
#2 | 16535 | C. Thomas Tyler | Fixed bug sending emails; added missing lib. | ||
#1 | 16532 | C. Thomas Tyler |
Added new global_replica_status.sh script v1.0.1 to query status of replicas on all replica hosts configured with the SDP. Renamed sdp_sync.cfg to more generic sdp_hosts.cfg, as it is now used by both sdp_sync.sh and global_replica_status.sh. sdp_sync.sh v1.2.1: * Changed '-h <hosts>' flag to '-H <hosts> to avoid conflict with the '-h' (help) flag. * Fixed bug with default P4U_LOG setting. * Refined command line processing and environment handling. * Updated with latest changes from template.sh, so it now picks accepts the '-V' (show version info) flag. |