aws_s3_sync.sh #1

#!/bin/bash
set -u

#------------------------------------------------------------------------------
# Usage: To get usage info, run: ./aws_sync.sh - man

#==============================================================================
# Declarations and Environment

# Version ID Block. Relies on +k filetype modifier.
#------------------------------------------------------------------------------
# shellcheck disable=SC2016
declare VersionID='$Id: //p4-sdp/main/Unsupported/Samples/cloud/aws/aws_s3_sync.sh#1 $ $Change: 33433 $'
declare VersionStream=${VersionID#*//}; VersionStream=${VersionStream#*/}; VersionStream=${VersionStream%%/*};
declare VersionCL=${VersionID##*: }; VersionCL=${VersionCL%% *}
declare Version=${VersionStream}.${VersionCL}
[[ "$VersionStream" == r* ]] || Version="${Version^^}"

declare ThisScript=${0##*/}
declare ThisUser=
declare DepotsDir=
declare DefaultCfgFile="/p4/common/config/aws_s3_sync.cfg"
declare SDPInstance="${SDP_INSTANCE:-}"
declare SDPEnvFile="/p4/common/bin/p4_vars"
declare SDPInstanceVarsFile=
declare P4Depots=
declare CfgFile=
declare -i NoOp=0
declare -i ErrorCount=0
declare -i SilentMode=0
declare -i CmdExitCode=0
declare Cmd=
declare Log=
declare H="=============================================================================="

# Values for these variables are defined in the confgiration values.  See the
# CONFIGURATION FILES section of the manual pages.
declare S3Bucket=
declare StorageClass=

#==============================================================================
# Local Functions
function msg () { echo -e "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   [[ "$Log" != off ]] && msg "Log is: $Log\\n${H}"

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# error, usually $1 should be -h so that the longer usage message doesn't
# obscure the error message.
#
# Sample Usage:
# usage
# usage -h
# usage -man
# usage -h "Incorrect command line usage."
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ "$errorMessage" != Unset ]]; then
      msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n"
   fi

   msg "USAGE for $ThisScript version $Version:

$ThisScript [-c <cfg_file>] [-L <log>] [-si] [-n] [-D]

or

$ThisScript [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script wraps the AWS command line, specifically the 'aws s3 sync'
       	command, to sync the depots directory to AWS.

CONFIGURATION FILE:
	This script requires a configuration file to define the following
	settings:

	* S3Bucket - Define the AWS S3 Bucket to copy to.

	* StorageClass Define the storage class to use. This impacts storage
	costs, accessibility, etc. For more information, see:

	https://aws.amazon.com/s3/storage-classes

	The default configuration file location is: $DefaultCfgFile"

   if [[ -r "$DefaultCfgFile" ]]; then
      msg "\\n${H}\\nBEGIN CONTENTS $DefaultCfgFile\\n$(cat "$DefaultCfgFile")\\n${H}\\nEND FILE CONTENTS"
   fi
msg "
EXAMPLES:
	This script can be called directly, though it is intended to be called
	via a crontab entry like this sample:

	INSTANCE=1
	0 1 * * * [[ -e /mnt/p4depots ]] && /p4/common/bin/run_if_master.sh \${INSTANCE} -si

OPTIONS:
 -c <cfg_file>
 	Specify the path to the configuration file. The default is:
	$DefaultCfgFile

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to
	
        ${LOGS:-\$LOGS}/aws_s3_sync.<datestamp>.log

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

	If an existing log file is specified, this script will append the the
	specified log file.

 -si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
	This is useful when running from cron, as it prevents automatic
	email from being sent by cron directly, as cron does when a script called
	from cron generates any output.  This script is then responsible for
	email handling, if any is to be done.

 -n	No-Op.  Prints commands instead of running them.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-i) SDPInstance="$2"; shiftArgs=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-c) CfgFile="$2"; shiftArgs=1;;
      (-L) Log="$2"; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) NoOp=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usage -h "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ "$SilentMode" -eq 1 && "$Log" == off ]] && \
   usage -h "Cannot use '-si' with '-L off'."

[[ -n "$SDPInstance" ]] ||\
   bail "The SDP '-i <SDP_Instance>' parameter is required unless \$SDP_INSTANCE is defined."

[[ -z "$CfgFile" ]] && CfgFile="$DefaultCfgFile"

[[ -r "$SDPEnvFile" ]] ||\
   bail "The SDP Environment file is missing: $SDPEnvFile"

SDPInstanceVarsFile="/p4/common/config/p4_${SDPInstance}.vars"
[[ -r "$SDPInstanceVarsFile" ]] ||\
   bail "The SDP Instances Vars file is missing: $SDPInstanceVarsFile"

# shellcheck disable=SC1090
source "$SDPEnvFile" "$SDPInstance" ||\
   bail "Failed to load SDP Environment."

[[ -r "$CfgFile" ]] ||\
   bail "The SDP Environment file is missing: $SDPEnvFile"

# shellcheck disable=SC1090
source "$CfgFile" ||\
   bail "Failed to load the configuration file: $CfgFile"

[[ -z "$Log" ]] && Log="$LOGS/${ThisScript%.sh}.$(date +'%Y%m%d-%H%M%S').log"

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

if [[ "$Log" != off ]]; then
   touch "$Log" || bail "Couldn't touch log file [$Log]."

   # Redirect stdout and stderr to a log file.
   if [[ "$SilentMode" -eq 0 ]]; then
      exec > >(tee -a "$Log")
      exec 2>&1
   else
      exec >"$Log"
      exec 2>&1
   fi
   msg "${H}\\nLog is: $Log"
fi

ThisUser=$(id -n -u)
msg "Started $ThisScript version $Version as $ThisUser@${HOSTNAME%%.*} on $(date)."

# shellcheck disable=SC2164
P4Depots=$(cd /p4/common; d=$(pwd -P); echo "${d%/p4/common}")
[[ -d "$P4Depots" ]] || bail "Couldn't determine P4Depots with 'cd /p4/common; pwd -P'."

DepotsDir="${P4Depots}${DEPOTS}"

[[ -d "$DepotsDir" ]] || bail "The expected depots directory does not exist or is not accessible: $DepotsDir"

Cmd="/usr/bin/aws s3 sync --storage-class $StorageClass $DepotsDir $S3Bucket"

msg "Running:\\n$Cmd\\nLogging to: $Log"

if [[ "$NoOp" -eq 0 ]]; then
   $Cmd
   CmdExitCode=$?
   msg "Exit code from 'aws s3 sync' command: $CmdExitCode"

   if [[ "$CmdExitCode" -eq 0 || "$CmdExitCode" -eq 2 ]]; then
      msg "Verified: The AWS Command returned with a happy zero exit code."
   else
      errmsg "AWS Command returned with a non-zero exit code."
   fi
else
   msg "NO_OP: Would run: $Cmd"
fi

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\nProcessing competed successfully."
else
   errmsg "\\nErrors were encountered during processing."
fi

msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

exit "$ErrorCount"