#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ # validate_sdp.sh # Validates current SDP environment according to current version of SDP #============================================================================== # Declarations and Environment export VS_SDP_P4CBIN=/p4/common/bin export VS_SDP_ENV=$VS_SDP_P4CBIN/p4_vars export SDP_INSTANCE=${SDP_INSTANCE:-UnsetSDPInstance} export SDP_INSTANCE=${1:-$SDP_INSTANCE} if [[ $SDP_INSTANCE == UnsetSDPInstance ]]; then echo "Instance parameter not supplied." echo "You must supply the SDP instance as a parameter to this script." exit 1 fi declare -i ServerOnline=0 declare -i ErrorCount=0 declare -i CheckCount=0 declare -i ExitCode=0 declare -i ShowLog=1 declare Version=5.2.0 declare H1="==============================================================================" declare H2="------------------------------------------------------------------------------" export LOGFILE=Unset #============================================================================== # Local Functions # Note: This script does not use SDP library files, as its purpose is to # verify the integrity of an SDP installation. Thus, it has its own # self-contained versions of some functions for which similar versions # would normally be sourced from files in /p4/common/lib, like libbcore.sh. # Micro-functions, one-liners used to avoid external dependencies. function msg () { if [[ "$LOGFILE" != Unset ]]; then echo -e "$*" >> "$LOGFILE"; else echo -e "$*"; fi; } function errmsg () { msg "\nError: ${1:-Unknown Error}\n"; ErrorCount+=1; } function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; } function run () { local cmd="${1:-echo}" local desc="${2:-}" local -i showOutput="${3:-1}" local log=$(mktemp) local -i exitCode= [[ -n "$desc" ]] && msg "$desc" msg "Executing: $cmd" $cmd > "$log" 2>&1 exitCode=$? if [[ "$showOutput" ]]; then echo "EXIT_CODE: $exitCode" >> "$log" cat "$log" >> $LOGFILE fi /bin/rm -f "$log" return "$exitCode" } #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). # The default is -h. # # $2 - error message (optional). Specify this if usage() is called due to # user error, in which case the given message displayed first, followed by the # standard usage message (short or long depending on $1). If displaying an # errror, usually $1 should be -h so that the longer usage message doesn't # obsure the error message. # # Sample Usage: # usage # usage -man # usage -h "Incorrect command line usage." # # This last example generates a usage error message followed by the short # '-h' usage summary. #------------------------------------------------------------------------------ function usage { declare style=${1:--h} declare errorMessage=${2:-Unset} if [[ $errorMessage != Unset ]]; then echo -e "\n\nUsage Error:\n\n$errorMessage\n\n" >&2 fi echo "USAGE for validate_sdp.sh v$Version: validate_sdp.sh [<instance>] or validate_sdp.sh -h|-man " if [[ $style == -man ]]; then echo -e "DESCRIPTION: This script validates the current SDP setup for the specified instance. Useful if you change anything, particularly after an SDP upgrade. OPTIONS: <instance> Specify the SDP instances. If not specified, the SDP_INSTANCE environment variable is used instead. If the instance is not defined by a parameter and SDP_INSTANCE is not defined, exits immediately with an error message. -online Online mode. Does additional checks that require P4D to be online. -si Silent mode. Does not display the generated log file to stdout at the end of processing. -L <log> Specify the log file to use. The default is /p4/N/logs/validate_SDP.log -D Set extreme debugging verbosity. HELP OPTIONS: -h Display short help message -man Display man-style help message EXAMPLES: This script is typically called after SDP update with only the instance name or number as an argument, e.g.: validate_SDP.sh 1 LOGGING: This script generates a log file and also displays it to stdout at the end of processing. By default, the log is: /p4/N/logs/validate_SDP.log. The exception is usage errors, which result an error being sent to stderr followed usage info on stdout, followed by an immediate exit. If the '-si' (silent) flag is used, the log is generated, but its contents are not displayed to stdout at the end of processing. EXIT CODES: An exit code of 0 indicates no errors were encounted attempting to perform validations, and that all checks verified cleanly. " fi exit 1 } #------------------------------------------------------------------------------ # Function: do_preflight_checks () # # Sample Usage: # do_preflght_checks || bail "Preflight checks failed. Aborting further checks." #------------------------------------------------------------------------------ function do_preflight_checks () { declare toolsList="which date ls grep awk id head tail" declare -i preflightOK=1 msg "$H2\nDoing preflight sanity checks." msg "Preflight Check 1: Ensuring these utils are in PATH: $toolsList" for tool in $toolsList; do CheckCount+=1 [[ -z "$(which "$tool" 2>/dev/null)" ]] && \ errmsg "Tool '$tool' not in PATH." done [[ $ErrorCount -eq 0 ]] || return 1 msg "Verified: Essential tools are in the PATH." msg "Preflight Check 2: cd $VS_SDP_P4CBIN" CheckCount+=1 cd "$VS_SDP_P4CBIN" >> $LOGFILE 2>&1 if [[ "$?" -ne 0 ]]; then errmsg "Could not cd to: $VS_SDP_P4CBIN" return 1 fi msg "Verified: cd works to: $VS_SDP_P4CBIN" msg "Preflight Check 3: Checking current user owns $VS_SDP_P4CBIN" # shellcheck disable=SC2012 export VS_SDP_OWNER=$(ls -ld . | awk '{print $3}') export ThisUser="$(id -n -u)" CheckCount+=1 if [[ "$ThisUser" == "$VS_SDP_OWNER" ]]; then msg "Verified: Current user [$ThisUser] owns $VS_SDP_P4CBIN" else errmsg "Current user [$ThisUser] does not own $VS_SDP_P4CBIN." return 1 fi return 0 } #------------------------------------------------------------------------------ # Function: check_file ($file, $errMsg) # # Checks for existance of a file, returns 0 if it exists, 1 otherwise. # Allows optional custom error message describing the file, to be displayed if # the file is missing. Default error message is "Missing file [FILE]." # # Inputs: function check_file () { local file=$1 local errMsg=${2:-Missing file} CheckCount+=1 msg "Checking existence of file $file" [[ -f $file ]] && return 0 errmsg "$errMsg: [$file]." return 1 } #------------------------------------------------------------------------------ # Function: check_configurable ($instance, $configurable, $scope, $expectedVal, $errMsg1, $errMsg2) # # Check that a configurable is set, and optionally check that it is set to # an expected value. # # Inputs: # $1 - SDP Instance. Required. # $2 - Configurable name. Required. # $3 - Configurable scope. Default is "any" # $4 - Expected value of variable. Optional. If defined, an additional check is # done, checking the current value against the expected value. # $5 - Optional error message to display if no value is defined. See code # below for the default message. # $6 - Optional error message to display if a value is defined but does not # match the expected value. See code below for the default message. # # Return Codes: # 1 - Verifications failed. # 0 - Verifications passed. # # Sample Usage: # check_configurable $SDP_INSTANCE journalPrefix # check_configurable $SDP_INSTANCE journalPrefix any "$CHECKPOINTS/$P4SERVER" # # check_configurable $SDP_INSTANCE journalPrefix any "$CHECKPOINTS/$P4SERVER" ||\ # bail "Yikes, journalPrefix is not set, all bets are off. Aborting." #------------------------------------------------------------------------------ function check_configurable () { local instance="$1" local configurable="$2" local scope="${3:-any}" local expectedValue="${4:-NoExpectedValue}" local errMsgMissing="${5:-No value defined}" local errMsgBadValue="${6:-Value does not match what is expected}" local value= CheckCount+=1 value=$($P4DBIN -r "$P4ROOT" -cshow | grep "^${scope}: ${configurable} = ") if [[ -n "$value" ]]; then value=${value##* = } msg "Verified: Configurable ${scope}:${configurable} is defined." else errmsg "$errMsgMissing for configurable [${scope}:${configurable}]." return 1 fi [[ "$expectedValue" == "NoExpectedValue" ]] && return 0 CheckCount+=1 if [[ "$value" == "$expectedValue" ]]; then msg "Verified: Configurable ${scope}:${configurable} has expected value [$value]." else errmsg "$errMsgBadValue for variable [${scope}:${configurable}]\n\tExpected value: [$expectedValue]\n\tActual value: [$value]" return 1 fi return 0 } #------------------------------------------------------------------------------ # Function: check_env_var ($instance, $var, $expectedval, $msg1, $msg2) # # Check that a shell environment variable is set when sourcing the SDP # environment. Optionally checks that variables are set to expected values. # # Inputs: # $1 - SDP Instance. Required. # $2 - Variable name. Required. # $3 - Expected value of variable. Optional. If defined, an additional check is # done, checking the current value against the expected value. # $4 - Optional error message to display if no value is defined. See code # below for the default message. # $5 - Optional error message to display if a value is defined but does not match # the expected value. See code below for the default message. # # Return Codes: # 1 - Verifications failed. # 0 - Verifications passed. # Sample Usage: # check_env_var $SDP_INSTANCE P4JOURNAL "/p4/$SDP_INSTANCE/logs/journal" # # check_env_var $SDP_INSTANCE P4JOURNAL "/p4/$SDP_INSTANCE/logs/journal" ||\ # bail "Yikes, P4JOURNAL is not set, all bets are off. Aborting." #------------------------------------------------------------------------------ function check_env_var () { local instance="$1" local var="$2" local expectedValue="${3:-NoExpectedValue}" local errMsgMissing="${4:-No value defined}" local errMsgBadValue="${5:-Value does not match what is expected}" local value= CheckCount+=1 eval unset "${var}" source "$VS_SDP_ENV" "$instance" set +u if [[ -n "$(eval echo \$"${var}")" ]]; then msg "Verified: Variable ${var} is defined." set -u else errmsg "$errMsgMissing for variable [$var]." set -u return 1 fi [[ "$expectedValue" == "NoExpectedValue" ]] && return 0 CheckCount+=1 value="$(eval echo \$"${var}")" if [[ "$value" == "$expectedValue" ]]; then msg "Verified: Variable ${var} has expected value [$value]." else errmsg "$errMsgBadValue for variable [$var]\n\tExpected value: [$expectedValue]\n\tActual value: [$value]" return 1 fi return 0 } #============================================================================== # Command Line Processing declare -i shiftArgs=0 set +u while [[ $# -gt 0 ]]; do case $1 in (-h) usage -h;; (-man) usage -man;; (-online) ServerOnline=1;; (-si) ShowLog=0;; (-L) LOGFILE="$2"; shiftArgs=1;; (-D) set -x;; # Debug; use 'set -x' mode. (-*) usage -h "Unknown command line option ($1).";; (*) export SDP_INSTANCE=$1;; esac # Shift (modify $#) the appropriate number of times. shift; while [[ $shiftArgs -gt 0 ]]; do [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments." shiftArgs=$shiftArgs-1 shift done done set -u #============================================================================== # Command Line Verification [[ $SDP_INSTANCE == Unset ]] && \ bail "The \$SDP_INSTANCE setting is not defined. It must be defined by doing:\n\n\tsource $VS_SDP_ENV <instance>\n\nor by passing in the instance name as a parameter to this script.\n" #============================================================================== # Main Program source "$VS_SDP_ENV" "$SDP_INSTANCE" ||\ bail "Failed to load SDP environment for instance $SDP_INSTANCE." source "$P4CBIN/backup_functions.sh" ||\ bail "Failed to load backup_functions.sh." [[ "$LOGFILE" == Unset ]] && export LOGFILE="$LOGS/validate_sdp.log" rotate_log_file "$LOGFILE" ".gz" msg "${0##*/} v$Version Starting validation at $(date +'%a %Y-%m-%d %H:%M:%S %Z')." msg "\nIf you have any questions about the output from this script, contact support@perforce.com." do_preflight_checks ||\ bail "Preflight checks failed. Aborting further checks." msg "${H2}\nChecking environment variables." check_env_var "$SDP_INSTANCE" SDP_INSTANCE check_env_var "$SDP_INSTANCE" P4ROOT "/p4/$SDP_INSTANCE/root" check_env_var "$SDP_INSTANCE" P4JOURNAL "/p4/$SDP_INSTANCE/logs/journal" check_env_var "$SDP_INSTANCE" SDP_ADMIN_PASSWORD_FILE "$P4CCFG/.p4passwd.$P4SERVER.admin" if [[ $? -eq 0 ]]; then check_file "$SDP_ADMIN_PASSWORD_FILE" "SDP admin password file doesn't exist" fi msg "${H2}\nRunning standard checks typically called within SDP scripts." check_vars set_vars check_dirs msg "${H2}\nChecking for a few database files." # Check db files exist file=db.counters for dir in $P4ROOT $OFFLINE_DB; do check_file "$dir/$file" "Expected database file doesn't exist" done msg "${H2}\nChecking for key files." check_file "$OFFLINE_DB/offline_db_usable.txt" "Offline database not in a usable state." check_file "$P4BIN" "The p4 binary (or symlink) doesn't exist" check_file "$RC" "The p4d init script doesn't exist" check_file "$P4TICKETS" "The P4TICKETS file doesn't exist" msg "${H2}\nChecking configurables values." check_configurable "$SDP_INSTANCE" journalPrefix any "$CHECKPOINTS/$P4SERVER" check_configurable "$SDP_INSTANCE" server.depot.root any "$DEPOTS" msg "${H2}\nChecking SDP structure." if [[ -L "$P4HOME" ]]; then errmsg "This is a symlink, should be a local directory: $P4HOME" fi if [[ "$ServerOnline" -eq 1 ]]; then msg "${H2}\nDoing online checks." CheckCount+=1 run "$P4CBIN/p4login" "Login check" 0 if [[ $? -eq 0 ]]; then msg "Verified: Login OK." else errmsg "Login as P4USER $P4USER to P4PORT $P4PORT could not be verified." fi fi ### P4="$P4BIN -p $P4PORT -u $P4USER" # TODO: # - links all present and correct for instance # - db files present (and quick validation OK) # - offline db directories named appropriately (e.g. db1/db2) # - crontab includes at least daily_checkpoint.sh # - checkpoints dir contains a checkpoint or two # - service password validated for a replica # - Add flag to check less-critical SDP configurables, and generate # warnings (rather than errors) if they are not set as expected. if [[ $ErrorCount -eq 0 ]]; then msg "\n${H1}\n\nALL CLEAN: $CheckCount verifications completed OK." else msg "\n${H1}\n\nVerifications completed, with $ErrorCount errors detected in $CheckCount checks." ExitCode=1 fi [[ "$ShowLog" -eq 1 && -s "$LOGFILE" ]] && cat "$LOGFILE" exit $ExitCode
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#4 | 24804 | C. Thomas Tyler |
Terminology tweak, 'validate' -> 'verify'. #review @robert_cowham |
||
#3 | 24534 | C. Thomas Tyler |
Various enhancements and internal refactoring for validate_sdp.sh. Added important check that /p4/N is a dir, not a symlink. #review-24532 @robert_cowham |
||
#2 | 24356 | C. Thomas Tyler |
Enhancements to validate_sdp.sh: * Added simple bold ALL CLEAN message to look for. * Added check_env_var() function to check shell environment variables, with some calls to it. * Added check_configurable() to check for configurables. This is implemented using back-door check methodology (using 'p4d_N -cshow') so values can be checked with P4D offline. This replaced stub function check_var(). * Removed stub function check_configurables(). It's easier to understand if all checks in the Main section of the code. * Changed so checks requiring p4d to be online are not done by default; added '-online' flag to run those tests. This is because I anticpate typical usage of the validator to be requiring it to be report ALL CLEAN before starting P4D after a server upgrade. * Added check for new $SDP_ADMIN_PASSWORD_FILE variable. * Added check admin password file pointed to by $SDP_ADMIN_PASSWORD_FILE. * Added errmsg() function, with corresponding tweak to bail(). * Consolidated Log an LOGIFLE to just LOGFILE. * Removed a few items from TOOD comments that got done. * Made a few tweaks for style normalization: - Functions are lowercase with undescore separators. - Functions vars are lowercase-initiated camelCase. - Indentation: 3 spaces for functions/loops/etc. * Added run() function replacing cmd() stub function. * Enhanced p4login check. * Added comment noting why this script uses self-contained copies of functions defined in other SDP files in /p4/common/lib. * And other things. Warning: In the short run, this may fail tests as the new SDP_ADMIN_PASSWORD_FILE variable is also pending review. #review @robert_cowham |
||
#1 | 23640 | Robert Cowham | Super basic validation - placeholder for many more tests to come! |