#!/bin/bash set -u #------------------------------------------------------------------------------ # Usage: To get usage info, run: ./aws_sync.sh - man #============================================================================== # Declarations and Environment declare ThisScript=${0##*/} declare ThisUser= declare DepotsDir= declare DefaultCfgFile="/p4/common/config/aws_s3_sync.cfg" declare SDPInstance="${SDP_INSTANCE:-}" declare SDPEnvFile="/p4/common/bin/p4_vars" declare SDPInstanceVarsFile= declare HxDepots= declare CfgFile= declare Version=1.1.5 declare -i NoOp=0 declare -i ErrorCount=0 declare -i SilentMode=0 declare -i CmdExitCode=0 declare Cmd= declare Log= declare H="==============================================================================" # Values for these variables are defined in the confgiration values. See the # CONFIGURATION FILES section of the manual pages. declare S3Bucket= declare StorageClass= #============================================================================== # Local Functions function msg () { echo -e "$*"; } function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; } function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; } #------------------------------------------------------------------------------ # Function: terminate function terminate { # Disable signal trapping. trap - EXIT SIGINT SIGTERM [[ "$Log" != off ]] && msg "Log is: $Log\\n${H}" # With the trap removed, exit. exit "$ErrorCount" } #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). # The default is -h. # # $2 - error message (optional). Specify this if usage() is called due to # user error, in which case the given message displayed first, followed by the # standard usage message (short or long depending on $1). If displaying an # error, usually $1 should be -h so that the longer usage message doesn't # obscure the error message. # # Sample Usage: # usage # usage -h # usage -man # usage -h "Incorrect command line usage." #------------------------------------------------------------------------------ function usage { declare style=${1:--h} declare errorMessage=${2:-Unset} if [[ "$errorMessage" != Unset ]]; then msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" fi msg "USAGE for $ThisScript v$Version: $ThisScript [-c <cfg_file>] [-L <log>] [-si] [-n] [-D] or $ThisScript [-h|-man|-V] " if [[ $style == -man ]]; then msg " DESCRIPTION: This script wraps the AWS command line, specifically the 'aws s3 sync' command, to sync the depots directory to AWS. CONFIGURATION FILE: This script requires a configuration file to define the following settings: * S3Bucket - Define the AWS S3 Bucket to copy to. * StorageClass Define the storage class to use. This impacts storage costs, accessibility, etc. For more information, see: https://aws.amazon.com/s3/storage-classes The default configuration file location is: $DefaultCfgFile" if [[ -r "$DefaultCfgFile" ]]; then msg "\\n${H}\\nBEGIN CONTENTS $DefaultCfgFile\\n$(cat "$DefaultCfgFile")\\n${H}\\nEND FILE CONTENTS" fi msg " EXAMPLES: This script can be called directly, though it is intended to be called via a crontab entry like this sample: INSTANCE=1 0 1 * * * [[ -e /hxdepots ]] && /p4/common/bin/run_if_master.sh \${INSTANCE} -si OPTIONS: -c <cfg_file> Specify the path to the configuration file. The default is: $DefaultCfgFile -L <log> Specify the path to a log file, or the special value 'off' to disable logging. By default, all output (stdout and stderr) goes to ${LOGS:-\$LOGS}/aws_s3_sync.<datestamp>.log NOTE: This script is self-logging. That is, output displayed on the screen is simultaneously captured in the log file. Do not run this script with redirection operators like '> log' or '2>&1', and do not use 'tee.' If an existing log file is specified, this script will append the the specified log file. -si Operate silently. All output (stdout and stderr) is redirected to the log only; no output appears on the terminal. This cannot be used with '-L off'. This is useful when running from cron, as it prevents automatic email from being sent by cron directly, as cron does when a script called from cron generates any output. This script is then responsible for email handling, if any is to be done. -n No-Op. Prints commands instead of running them. -D Set extreme debugging verbosity. HELP OPTIONS: -h Display short help message -man Display man-style help message " fi exit 1 } #============================================================================== # Command Line Processing declare -i shiftArgs=0 set +u while [[ $# -gt 0 ]]; do case $1 in (-i) SDPInstance="$2"; shiftArgs=1;; (-h) usage -h;; (-man) usage -man;; (-c) CfgFile="$2"; shiftArgs=1;; (-L) Log="$2"; shiftArgs=1;; (-si) SilentMode=1;; (-n) NoOp=1;; (-D) set -x;; # Debug; use 'set -x' mode. (*) usage -h "Unknown arg ($1).";; esac # Shift (modify $#) the appropriate number of times. shift; while [[ $shiftArgs -gt 0 ]]; do [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments." shiftArgs=$shiftArgs-1 shift done done set -u #============================================================================== # Command Line Verification [[ "$SilentMode" -eq 1 && "$Log" == off ]] && \ usage -h "Cannot use '-si' with '-L off'." [[ -n "$SDPInstance" ]] ||\ bail "The SDP '-i <SDP_Instance>' parameter is required unless \$SDP_INSTANCE is defined." [[ -z "$CfgFile" ]] && CfgFile="$DefaultCfgFile" [[ -r "$SDPEnvFile" ]] ||\ bail "The SDP Environment file is missing: $SDPEnvFile" SDPInstanceVarsFile="/p4/common/config/p4_${SDPInstance}.vars" [[ -r "$SDPInstanceVarsFile" ]] ||\ bail "The SDP Instances Vars file is missing: $SDPInstanceVarsFile" # shellcheck disable=SC1090 source "$SDPEnvFile" "$SDPInstance" ||\ bail "Failed to load SDP Environment." [[ -r "$CfgFile" ]] ||\ bail "The SDP Environment file is missing: $SDPEnvFile" # shellcheck disable=SC1090 source "$CfgFile" ||\ bail "Failed to load the configuration file: $CfgFile" [[ -z "$Log" ]] && Log="$LOGS/${ThisScript%.sh}.$(date +'%Y%m%d-%H%M%S').log" #============================================================================== # Main Program trap terminate EXIT SIGINT SIGTERM if [[ "$Log" != off ]]; then touch "$Log" || bail "Couldn't touch log file [$Log]." # Redirect stdout and stderr to a log file. if [[ "$SilentMode" -eq 0 ]]; then exec > >(tee -a "$Log") exec 2>&1 else exec >"$Log" exec 2>&1 fi msg "${H}\\nLog is: $Log" fi ThisUser=$(id -n -u) msg "Started $ThisScript v$Version as $ThisUser@${HOSTNAME%%.*} on $(date)." # shellcheck disable=SC2164 HxDepots=$(cd /p4/common; d=$(pwd -P); echo "${d%/p4/common}") [[ -d "$HxDepots" ]] || bail "Couldn't determine HxDepots with 'cd /p4/common; pwd -P'." DepotsDir="${HxDepots}${DEPOTS}" [[ -d "$DepotsDir" ]] || bail "The expected depots directory does not exist or is not accessible: $DepotsDir" Cmd="/usr/bin/aws s3 sync --storage-class $StorageClass $DepotsDir $S3Bucket" msg "Running:\\n$Cmd\\nLogging to: $Log" if [[ "$NoOp" -eq 0 ]]; then $Cmd CmdExitCode=$? msg "Exit code from 'aws s3 sync' command: $CmdExitCode" if [[ "$CmdExitCode" -eq 0 || "$CmdExitCode" -eq 2 ]]; then msg "Verified: The AWS Command returned with a happy zero exit code." else errmsg "AWS Command returned with a non-zero exit code." fi else msg "NO_OP: Would run: $Cmd" fi if [[ "$ErrorCount" -eq 0 ]]; then msg "\\nProcessing competed successfully." else errmsg "\\nErrors were encountered during processing." fi msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" exit "$ErrorCount"
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#1 | 29612 | C. Thomas Tyler |
Released SDP 2023.1.29610 (2023/05/25). Copy Up using 'p4 copy -r -b perforce_software-sdp-dev'. |
||
//guest/perforce_software/sdp/dev/Unsupported/Samples/cloud/aws/aws_s3_sync.sh | |||||
#1 | 29572 | C. Thomas Tyler | Added sample AWS s3 sync script and its config file. |