get_helix_binaries.sh #1

  • //
  • guest/
  • perforce_software/
  • sdp/
  • dev/
  • helix_binaries/
  • get_helix_binaries.sh
  • View
  • Commits
  • Open Download .zip Download (8 KB)
#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------
set -u

# This script acquires Perforce Helix binaries from the Perforce FTP server.
# For documentation, run: get_helix_binaries.sh -man

#==============================================================================
# Declarations and Environment

declare DefaultPerforceHelixVersion=r20.1
declare DefaultBinList="p4 p4d p4broker p4p"

declare ThisScript=${0##*/}
declare Version=1.1.0
declare -i NoOp=0
declare -i ErrorCount=0
declare -i WarningCount=0
declare -i RetryCount=0
declare -i RetryMax=2
declare -i RetryDelay=2
declare -i RetryOK=0
declare PerforceHelixVersion=
declare BinList=
declare Platform=linux26x86_64
# declare PerforceFTPBaseURL="https://ftp.perforce.com/perforce"
declare PerforceFTPBaseURL="http://cdist2.perforce.com/perforce"
declare BinURL=
declare Cmd=

function msg () { echo -e "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unkonwn Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
# The default is -h.
#
# $2 - error message (optional).  Specify this if usage() is called due to
# user error, in which case the given message displayed first, followed by the
# standard usage message (short or long depending on $1).  If displaying an
# errror, usually $1 should be -h so that the longer usage message doesn't
# obsure the error message.
#
# Sample Usage:
# usage 
# usage -man
# usage -h "Incorrect command line usage."
#
# This last example generates a usage error message followed by the short
# '-h' usage summary.
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}
   declare errorMessage=${2:-Unset}

   if [[ $errorMessage != Unset ]]; then
      msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n"
   fi

msg "USAGE for $ThisScript v$Version:

$ThisScript [-r <HelixVersion>] [-b <Binary1>,<Binary2>,...]

   or

$ThisScript -h|-man"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script acquires Perforce Helix binaries from the Perforce FTP server.

	The four Helix binaries acquired are:

	* p4, the command line client
	* p4d, the Helix Core server
	* p4p, the Helix Proxy
	* p4broker, the Helix Broker

	This script gets the latest patch of binaries for the current server version.
	It is intended to acquire the latest patch for an existing install, or to
	get initial binaries for a fresh new install.

	When a newer major version of Helix binares is needed, this script should not
	be modified directly. Instead, the recommended approach is to upgrade the SDP
	to get the latest version of SDP first, which will included a newer version of
	this script. This helps ensure you have a version of the SDP that works with
	the version of p4d and other binaries.

EXAMPLES:
	All examples assume the SDP is in the standard location, /hxdepots/sdp.

	Example 1 - Typical Usage with no arguments:

	cd /hxdepots/sdp/helix_binaries
	./get_helix_binaries.sh

	This acquires the latest patch of all 4 binaries for the $DefaultPerforceHelixVersion
	release (aka 20${DefaultPerforceHelixVersion#r}).

	Example 2 - Specifying the major version:

	cd /hxdepots/sdp/helix_binaries
	./get_helix_binaries.sh -r r19.2

	This gets the latest patch of for the 2019.2 release of all 4 binaries.

	Note: Only supported Helix binares are guaranteed to be available from the
	Perforce FTP server.

	Note: Only the latest patch of any given binary is available from the Perforce
	FTP server.

	Example 3 - Sample getting r20.1 and skipping the proxy binary (p4p):

	cd /hxdepots/sdp/helix_binaries
	./get_helix_binaries.sh -r r20.1 -b p4,p4d,p4broker

DEPENDENCIES:
	This script requires outbound internet access. Depending on your environment,
	it may also require HTTPS_PROXY to be defined, or may not work at all.

	If this script doesn't work due to lack of outbound internet access, it is
	still useful illustrating the locations on the Perforce FTP server where
	Helix Core binaries can be found.


OPTIONS:
 -n	Specify the '-n' (No Operation) option to show the commands needed
	to fetch the Helix binaries from the Perforce FTP server without attempting
	to execute them.

 -D	Set extreme debugging verbosity using bash 'set -x' mode.

HELP OPTIONS:
 -h	Display short help message
 -man	Display this manual page

EXAMPLES:

EXIT CODES:
	An exit code of 0 indicates no errors were encounted. An
	non-zero exit code indicates errors were encounterd.
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-h) usage -h;;
      (-man) usage -man;;
      (-r) PerforceHelixVersion="${2:-}"; shiftArgs=1;;
      (-b) BinList="${2:-}"; shiftArgs=1;;
      (-n) NoOp=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown command line option ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

[[ -n "$PerforceHelixVersion" ]] || PerforceHelixVersion="$DefaultPerforceHelixVersion"
[[ -n "$BinList" ]] || BinList="$DefaultBinList"

#==============================================================================
# Command Line Verification

#==============================================================================
# Main Program

msg "\\nStarted $ThisScript v$Version as $USER@${HOSTNAME%%.*} at $(date)."

for bin in $(echo "$BinList"|tr ',' ' '); do
   msg "\\nGetting $bin ..."
   BinURL="${PerforceFTPBaseURL}/${PerforceHelixVersion}/bin.${Platform}/$bin"
   if [[ -f "$bin" ]]; then
      chmod +x "$bin"
      msg "Old version of $bin: $("./$bin" -V | grep Rev)"

      if [[ "$NoOp" -eq 0 ]]; then
         rm -f "$bin"
      fi
   fi

   Cmd="curl -s -k -O $BinURL"

   if [[ "$NoOp" -eq 1 ]]; then
      msg "NoOp: Would run: $Cmd"
      continue
   else
      msg "Running: $Cmd"
   fi

   if $Cmd; then
      chmod +x "$bin"
      msg "New version of $bin: $("./$bin" -V | grep Rev)"
   else
      # Replace the '-s' silent flag with '-v' after we have had an error, to
      # help with debugging.
      Cmd="curl -v -k -O $BinURL"
      warnmsg "Failed to download $bin with this URL: $BinURL\\nRetrying ..."
      RetryCount+=0

      while [[ "$RetryCount" -le "$RetryMax" ]]; do
         RetryCount+=1
         sleep "$RetryDelay"
         msg "Retry $RetryCount of $bin with command: $Cmd"
         if $Cmd; then
            chmod +x "$bin"
            msg "New version of $bin: $("./$bin" -V | grep Rev)"
            RetryOK=1
            break
         else
            warnmsg "Retry $RetryCount failed again to download $bin with this URL: $BinURL"
         fi
      done

      if [[ "$RetryOK" -eq 0 ]]; then
         errmsg "Failed to download $bin with this URL: $BinURL"
         rm -f "$bin"
      fi
   fi
done

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\nDownloading of Perforce Helix binaries completed OK."
else
   errmsg "\\There were $ErrorCount errors attempting to download Perforce Helix binaries."
fi

exit "$ErrorCount"
# Change User Description Committed
#20 31016 C. Thomas Tyler Changed default Helix Core version to r24.2.

Improved documentation and error messages in get_helix_binaries.sh.
#19 30625 C. Thomas Tyler Refined get_helix_binaries.sh to handle format of '-r rYY.N' that needs no conversion.
#18 30532 C. Thomas Tyler In get_helix_binaries.sh, enhanced error message for bad format of of '-r'.

#review-30533
#17 30402 C. Thomas Tyler Fixed issue if USER environment variable is not defined by the shell.
#16 30328 C. Thomas Tyler get_helix_binaries.sh: Added support for acquiring P4API files.

Use the new '-api' option to acquire whatever p4api.* files are
available for your Helix version and platform.

The '-b' option now allows '-b none', so you can donwload only
APIs with '-b none -api'

#review-30329
#15 30320 C. Thomas Tyler Added multi-platform support to include aarch64, and some modern OSX
variants.

Updated default Helix binary version to r24.1, which adds aarch64 builds.

Design Goals (achieved):
* No operational procedure changes requied, so users on existing platforms
don't need to learn new tricks.
* Changes to support multiple platforms/OS architectures are contained
entirely within the /p4/sdp/helix_binaries directory. The rest of the
SDP can remain blissfully unaware of platform specifics.

New capabilities:
* Uses 'uname' to detect current OS platform, mapping it to available Helix
builds for each binary.
* Uses 'jq' if available to parse P4*.json release list files to determine
if a build is available for the detected platform.
* Adds fallback logic.  For example, there is an aarch64 build for OSX 12+, but
not for x86_64. So if on OSX 12+ on x86_64, use the older-but-compatible
OSX 10.15 build for x86_64.
* Adds a new '-d' debug mode option, as the script gets complex.

New Files:
* Added P4*.json release list files to SDP package in /p4/sdp/helix_binaries.

These *.json files are updated as part of the Helix Core release process
for p4, p4d, p4broker, and p4p (and also P4V and others not relevant to
this script).

Bonus Content:
* Silently/harmlessly introduces as-yet-unused SDP_INSTALL_ROOT variable,
a prelude to future SDP refactoring.

Tested Platforms:
* linux26x86_64
* linux26aarch64
* macosx12arm64
* macosx12x86_64; exercises fallback to maxosx1015x86_64

#review-30313 @robert_cowham @mark_zinthefer @will_kreitzmann
#14 30221 C. Thomas Tyler In get_helix_binaries.sh, add '-sbd <StageBinDir>' option to get bins in any dir.

This supports alternate usages of this script outside documented workflows.

#review-30222
#13 29996 C. Thomas Tyler Changed DefaultHelixVersion from r23.1 -> r23.2, in preparation for
the coming SDP 2023.2 release.

#review-29997
#12 29558 C. Thomas Tyler Updated default Helix Core version from 22.2 -> r23.1.
#11 29218 C. Thomas Tyler Prep for SDP r22.2 release.

Changed default p4d version in get_helix_binaries.sh to prefer r22.2.
#10 28848 C. Thomas Tyler Adjusted get_helix_binaries.sh ftp.perforce.com rather than cdist2.perforce.com.

This fixes a hang issue when running get_helix_binaries.sh.

#review-28849
#9 28838 C. Thomas Tyler Updated default p4d version to r22.1.
#8 28376 C. Thomas Tyler In get_helix_binaries.sh, changed default version to r21.2.
#7 27892 C. Thomas Tyler Fixed typo and updated Version identifier.

Thanks, Adam!

#review-27893 @amo
#6 27890 C. Thomas Tyler Updated Release Notes and SDP Guide to clarify SDP r20.1 supports
Helix Core binaries up to r21.1, in advance of the coming SDP r21.1
release that will make it more obvious.

In get_helix_binaries.sh:
* Changed default Helix Core binary version to r21.1.
* Changed examples of getting a different version to reference r20.2.

#review-27891 @amo
#5 27722 C. Thomas Tyler Refinements to @27712:
* Resolved one out-of-date file (verify_sdp.sh).
* Added missing adoc file for which HTML file had a change (WorkflowEnforcementTriggers.adoc).
* Updated revdate/revnumber in *.adoc files.
* Additional content updates in Server/Unix/p4/common/etc/cron.d/ReadMe.md.
* Bumped version numbers on scripts with Version= def'n.
* Generated HTML, PDF, and doc/gen files:
  - Most HTML and all PDF are generated using Makefiles that call an AsciiDoc utility.
  - HTML for Perl scripts is generated with pod2html.
  - doc/gen/*.man.txt files are generated with .../tools/gen_script_man_pages.sh.

#review-27712
#4 27510 C. Thomas Tyler get_helix_binaries.sh: Added '-n' and '-D' options to usage summary.
Doc only; non-functional change.
#3 27030 C. Thomas Tyler Updated get_helix_binaries.sh v1.2.0:
* Completed documentation.
* Added command line verifications.
  - Must be run from */sdp/helix_binaries directory.
  - Format of '-r HelixVersion' value checked.
* Internal code improvements.
#2 27027 C. Thomas Tyler Fixed command line handling bug.
Invalid fragments result in usage error.
Changed variable name 'bin' -> 'binary' for clarity.
#1 27022 C. Thomas Tyler Changed 'exes' to 'helix_binaries' in folder and script name, per review.
//guest/perforce_software/sdp/dev/exes/get_latest_exes.sh
#5 27003 Robert Cowham Work around timing issues for tests.
#4 27000 C. Thomas Tyler Tweak so 'curl' command uses '-s' silent mode for normal
operation, and drops '-s' only on retry attempts in event of
an initial failure.
#3 26998 C. Thomas Tyler Fixed issue with existing files causing curl command to fail.
#2 26993 C. Thomas Tyler Added retry logic to get_latest_exes.sh.
#1 26982 C. Thomas Tyler mkdirs.sh v4.1.0:
* Accounted for directory structure change of Maintenance to Unsupported.
* Added standard command line processing with '-h' and '-man' doc flags,
and other flags (all documented).
* Added in-code docs and updated AsciiDoc.
* Enhanced '-test' mode to simulate /hx* mounts.
* Enhanced preflight testing, and fixed '-test' mode installs.
* Added support for installing to an alternate root directory.
* Added '-s <ServerID>' option to override REPLICA_ID.
* Added '-S <TargetServerID>' used for replicas of edge servers.
* Added '-t <server_type>' option to override SERVER_TYPE.
* Added '-M' option to override mount points.
* Added '-f' fast option to skip big chown/chmod commands, and
moved those commands near the end as well.

verify_sdp.sh v5.9.0:
* Added check for /p4/Version file, and checked that other legacy
SDP methods of checking version
* Added sanity check for crontab.
* Added 'test skip' mechanism to skip certain tests:
 - crontab: Skip crontab check. Use this if you do not expect crontab to be configured, perhaps if a different scheduler is used.
 - license: Skip license related checks.
 - version: Skip version checks.
 - excess: Skip checks for excess copies of p4d/p4p/p4broker in PATH.
* Added VERIFY_SDP_SKIP_TEST_LIST setting ton instance_vars.template,
to define a standard way to have verify_sdp.sh always skip certain
tests for a site.
* Extended '-online' checks to check for bogus P4MASTERPORT, a common
config error.

Update test_SDP.py:
* Adjusted test suite to account for various changes in mkdirs.sh.
* Added 'dir' parameter to run_cmd() and sudo_cmd(), to run a
command from a specified directory (as required to test new
mkdirs.sh)
* Added check_links() similar to existing check_dirs() function.

=== Upgrade Process Changes ===

Made /p4/common/bin/p4d/p4/p4broker/p4p shell script rather than binary.

This changes the way SDP new binaries are staged for upgrade.  For
safety, exes are now staged to a director outside the PATH, the
/p4/sdp/exes folder. A new 'get_latest_exes.sh' script simplifies
the task of pulling executables from the Perforce FTP server. This
can be used 'as is' for environments with outbound internet access,
and is useful in any case to describe now to acquire binaries.

This addresses an issue where a p4d binary staged for a future
upgrade might be called before the actual upgrade is performed.

upgrade.sh v4.0.0:
* All preflight checks are now done first. Added '-p' to abort after preflight.
* Added '-n' to show what would be done before anything is executed.
* Minimalist logic to start/stop only servers that are upgrade, and apply
upgrades only as needed.
* Staging of exes for upgrade is now separate from /p4/common/bin
* Improved in-code docs, added '-h' and '-man' options.
* Retained pre/post P4D 2019.1 upgrade logic.