#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment
set -u

# Version ID Block. Relies on +k filetype modifier.
#------------------------------------------------------------------------------
# shellcheck disable=SC2016
declare VersionID='$Id: //p4-sdp/dev_c2s/Server/Unix/setup/install_sdp.sh#2 $ $Change: 31472 $'
declare VersionStream=${VersionID#*//}; VersionStream=${VersionStream#*/}; VersionStream=${VersionStream%%/*};
declare VersionCL=${VersionID##*: }; VersionCL=${VersionCL%% *}
declare Version=${VersionStream}.${VersionCL}
[[ "$VersionStream" == r* ]] || Version="${Version^^}"

declare ThisScript="${0##*/}"
declare Log=
declare LogLink=
declare InheritedUmask=
declare OperatingUmask="0022"

# The latest SDP release tarball has a consistent name, sdp.Unix.tgz,
# alongside the version-stamped tarball (e.g. sdp.Unix.2024.1.30385.tgz).
# This is best when you want the latest officially released SDP.

declare SDPTar="sdp.Unix.tgz"

# See usage info for the '-d' flag in this script.
declare SDPLocalCopySource="/sdp"
declare WorkshopBaseURL="https://swarm.workshop.perforce.com"
declare SDPURL="$WorkshopBaseURL/download/guest/perforce_software/sdp/downloads/$SDPTar"
declare ThisScriptURL="$WorkshopBaseURL/download/guest/perforce_software/sdp/main/Server/Unix/setup/$ThisScript"
declare SDPInstallRoot="/p4"
declare SDPInstallMethod=FTP
declare PerforcePackageBase="/opt/perforce"
declare FTPURL="https://ftp.perforce.com/perforce"
declare HxMetadata1=
declare HxMetadata2=
declare HxLogs=
declare DirList=
declare Hostname=
declare Timezone=
declare UseSystemdOption=
declare -i UseSystemd=1
declare -i EnableSystemdServices=1
declare -i Debug=${SDP_DEBUG:-0}
declare -i ExtremeDebug=0
declare -i NoOp=1
declare -i CMDEXITCODE
declare H1="=============================================================================="

# The values for set here are for use in the usage() function if '-man' is
# used. They are set again further down in the code below, after settings
# are loaded from the config file which might change the values set here.
declare HxDepots="hxdepots"
declare HxCheckpoints="hxdepots"
declare DefaultP4BinRel=r24.2
declare RunUser="perforce"
declare SDPPackageBase="$PerforcePackageBase/helix-sdp"
declare ImmutableSDPDir="$SDPPackageBase/sdp"
declare WritableP4Dir="$SDPPackageBase/p4"
declare WritableSDPDir="$WritableP4Dir/sdp"
declare SDPUnixSetupDir="$WritableSDPDir/Server/Unix/setup"
declare SDPSetupDir="$WritableSDPDir/Server/setup"
declare BinDir="$WritableSDPDir/helix_binaries"
declare LocalInstallBinDir="$SDPPackageBase/helix_binaries"
declare DownloadsDir="$SDPPackageBase/downloads"
declare BinList=
declare ServerBin=
declare SiteBinDir="$SDPInstallRoot/common/site/bin"
declare MailSimulator="$SiteBinDir/mail"
declare RequiredUtils="date df grep head hostname id ls mkdir mv rm rsync sed sort stat su sudo tail tee touch uname whoami"
declare AllRequiredUtils="$RequiredUtils awk bc curl cut egrep wc"
declare TarVersion="1.00"
declare SELinuxMode=
declare BackupScript=/opt/perforce/helix-sdp/p4/sdp/Server/Unix/p4/common/bin/opt_perforce_sdp_backup.sh

declare SSLDir=
declare SSLConfig=
declare TmpFile=/tmp/tmp.install_sdp.$$.$RANDOM
declare CmdLine="${0##*/} $*"
declare -i WarningCount=0
declare -i ErrorCount=0
declare PackageManager=Unset
declare -A PackageList
declare -A ExtraPackageList
declare -A ExtraP4PackageList
declare -A Config ConfigDoc
declare -i InitializeEmptyServer=0
declare -i LoadSampleDepot=0
declare -i DemoInstall=0
declare -i SetDataHandling=0
declare -i PullFromWebAsNeeded=1
declare SSLPrefix=ssl:
declare -i GenDefaultConfig=0
declare -i DoSudo=1
declare -i LimitedSudoers=1
declare Cmd=
declare -i SetHostname=0
declare -i SetTimezone=0
declare -i SetSDPInstance=0
declare -i SetServerID=0
declare -i SetServerType=0
declare -i SetSimulateEmail=0
declare -i SetListenPort=0
declare -i SetTargetPort=0
declare -i SetTargetServerID=0
declare -i SimulateEmail=0
declare -i UseBroker=0
declare -i UseConfigFile=0
declare ConfigFile=Unset
declare RunUserNewHomeDir=
declare RunUserHomeDir=
declare RunGroup=Unset
declare UserAddCmd=
declare BackupTimestamp=
declare P4YumRepo="/etc/yum.repos.d/perforce.repo"
declare P4AptGetRepo="/etc/apt/sources.list.d/perforce.list"
declare PerforcePackageRepoURL="https://package.perforce.com"
declare PerforcePackagePubkeyURL="$PerforcePackageRepoURL/perforce.pubkey"
declare TmpPubKey=/tmp/perforce.pubkey
declare AptKeyRing=/usr/share/keyrings/perforce.gpg
declare -i AddPerforcePackageRepo=1
declare -i LoadActiveCrontab=1
declare -i InstallOSPackages=1
declare -i InstallExtraOSPackages=0
declare SampleDepotTar=sampledepot.tar.gz
declare ThisArch=
declare ThisHost=
declare ThisOS=
declare ThisOSName=
declare ThisOSDistro=
declare ThisOSMajorVersion=
declare ThisUser=
declare RunArchList="x86_64 aarch64"
declare CBIN="$SDPInstallRoot/common/bin"
declare CCFG="$SDPInstallRoot/common/config"
declare -i DoVerifySDP=0
declare -i DoFirewall=1
declare VerifySDPScript="$CBIN/verify_sdp.sh"
declare VerifySDPCmd=
declare VerifySDPOptions=
declare VerifySDPSkipTests=
declare SDPInstance="1"
declare SDPInstanceMkdirsCfg=
declare ServerID=
declare ServerType=
declare ListenPort=
declare TargetPort=
declare TargetServerID=
declare MkdirsCmd=

#==============================================================================
# Static Configuration - Package Lists

# The associative array 'PackageList' defines packages required for each
# package manager (yum, apt-get, or zypper).
PackageList['yum']="bc cronie curl mailx make openssl openssl-devel policycoreutils-python-utils rsync sos sysstat tar tuned wget which zlib zlib-devel"
ExtraPackageList['yum']="gcc gcc-c++"

PackageList['apt-get']="bc cron curl libbz2-dev libncurses5-dev libreadline-dev libsqlite3-dev libssl-dev llvm make rsync sysstat tuned wget zlib1g-dev"
ExtraPackageList['apt-get']="build-essential"

PackageList['zypper']="bc cronie curl make openssl openssl-devel rsync sos sysstat tuned wget which zlib zlib-devel"
ExtraPackageList['zypper']="gcc gcc-c++"

ExtraP4PackageList['yum']="perforce-p4python3"
ExtraP4PackageList['apt-get']="perforce-p4python3"
ExtraP4PackageList['zypper']=

#==============================================================================
# Static Configuration - User Config Data

# User modifiable data is defined in the 'Config' associative array, with
# corresponding user documentation in the 'ConfigDoc' array, corresponding
# to the sdp_install.cfg file the user modifies.

# To add a new setting, define values for both Config['YourNewValue'] and
# ConfigDoc['YourNewValue]' in this block. Also, ensure the values are written
# in the appropriate section of the sample config file generated in the
# function gen_default_config().

#------------------------------------------------------------------------------
# Settings Section 1: Localization
# Keep the order that settings are defined here in sync with the 'for c in'
# loop in gen_default_config() for Section 1 below. That defines the desired
# order of appearance in the generated file.

ConfigDoc['SMTPServer']="\\n# Specify email server for the p4review script. Ignore if Helix Swarm is used."
Config['SMTPServer']="smtp.p4demo.com"
ConfigDoc['P4AdminList']="\\n# Specify an email address to receive updates from admin scripts. This may be\\n# a distribution list or comma-separated list of addresses (with no spaces)."
Config['P4AdminList']="P4AdminList@p4demo.com"
ConfigDoc['MailFrom']="\\n# Specify an email address from which emails from admin scripts are sent.\\n# This must be a single email address."
Config['MailFrom']="P4Admin@p4demo.com"
ConfigDoc['DNS_name_of_master_server']="\\n# Specify the DNS alias to refer to the commit server, e.g. by end\\n# users. This might be something like 'perforce.example.com' or\\n# simply 'perforce', but probably not an actual host name like\\n# 'perforce-01', which would be known only to admins. The default value,\\n# localhost, is valid only for a single server topology."
Config['DNS_name_of_master_server']="localhost"
ConfigDoc['SiteTag']="\\n# Specify a geographic site tag for the master server location,\\n# e.g. 'bos' for Boston, MA, USA."
Config['SiteTag']="bos"
ConfigDoc['Hostname']="\\n# Specify the hostname.  This can be left blank. If set on a system that supports\\n# the 'hostnamectl' utility, that utility will be used to set the hostname.  If the\\n# command line parameter '-H <hostname>' is used, that will override this setting."
Config['Hostname']=""
ConfigDoc['Timezone']="\\n# Specify the timezone.  This can be left blank. If set on a system that supports\\n# the 'timedatectl' utility, that utility will be used to set the timezone.  If the\\n# command line parameter '-T <timezone>' is used, that will override this setting."
Config['Timezone']=""

#------------------------------------------------------------------------------
# Settings Section 2: Data Specific
# Keep the order that settings are defined here in sync with the 'for c in'
# loop in gen_default_config() for Section 2 below. That defines the desired
# order of appearance in the generated file.

ConfigDoc['P4_PORT']="\\n# Specify the TCP port for p4d to listen on. Typically this is 1999 if \\n# p4broker is used, or 1666 if only p4d is used."
Config['P4_PORT']="1999"
ConfigDoc['P4BROKER_PORT']="\\n# Specify the TCP port for p4broker to listen on. Must be different\\n# from the P4_PORT."
Config['P4BROKER_PORT']="1666"
ConfigDoc['Instance']="\\n# Specify SDP instance name, e.g. '1' for /p4/1."
Config['Instance']="1"
ConfigDoc['CaseSensitive']="\\n# Helix Core case sensitivity, '1' (sensitive) or '0' (insensitive). If\\n# data from a checkpoint is to be migrated into this instance, set this\\n# CaseSensitive value to match the case handling of the incoming data set\\n# (as shown with 'p4 info')."
Config['CaseSensitive']="1"
ConfigDoc['SSLPrefix']="\\n# If SSL (Secure Sockets Layer) encryption is to be used, specify the prefix,\\n# typically 'ssl:'. Leave blank if not using SSL."
Config['SSLPrefix']="ssl:"
ConfigDoc['P4USER']="\\n# Set the P4USER value for the Perforce super user."
Config['P4USER']="perforce"
ConfigDoc['Password']="\\n# Set the password for the super user (see P4USER). If using this Helix Installer to\\n# bootstrap a production installation, replace this default password with your own."
Config['Password']="F@stSCM!"
ConfigDoc['SimulateEmail']="\\n# Specify '1' to avoid sending email from admin scripts, or 0 to send\\n# email from admin scripts."
Config['SimulateEmail']="1"
ConfigDoc['ServerID']="\\n# Specify a ServerID value. If left blank for a master/commit server, a sensible\\n# default value will be assigned. See the Server Spec Naming Standard:\\n# https://swarm.workshop.perforce.com/view/guest/perforce_software/sdp/main/doc/SDP_Guide.Unix.html#_server_spec_naming_standard"
Config['ServerID']=""
ConfigDoc['ServerType']="\\n# Specify the type of server. Valid values are:\\n# * p4d_commit - A master/commit p4d server.\\n# * p4d_master - A synonym for p4d_commit.\\n# * p4d_replica - A p4d replica with all metadata from the master, not filtered in\\n#   any way. May or may not be forwarding.\\n# * p4d_filtered_replica - A filtered replica or filtered forwarding replica.\\n# * p4d_edge - An edge server.\\n# * p4d_edge_replica - Replica of an edge server. Also set TargetServerID.\\n# * p4broker - An SDP host running only a p4broker, e.g. as standalone broker\\n#   possibly deployed in a DMZ.\\n# * p4p - An SDP host running only a p4p.\\n# * p4proxy - A synonym for p4p.\\n#\\n# The ServerID must also be set if the ServerType is any p4d_* type other than\\n# 'p4d_commit' or 'p4d_master'."
Config['ServerType']="p4d_commit"
ConfigDoc['TargetServerID']="\\n# Set only if ServerType is p4d_edge_replica. The value is the ServerID of\\n# edge server that this server is a replica of, and must match the\\n# 'ReplicatingFrom:' field of the server spec."
Config['TargetServerID']=
ConfigDoc['TargetPort']="\\n# Specify the target port for a p4p or p4broker."
Config['TargetPort']=
ConfigDoc['ListenPort']="\\n# Specify the listening port for a p4p or p4broker."
Config['ListenPort']=

#------------------------------------------------------------------------------
# Settings Section 3: Deep Customization
# Keep the order that settings are defined here in sync with the code in
# gen_default_config() for Section 3 below. That defines the desired order of
# appearance in the generated file.

ConfigDoc['OSUSER']="\\n# Specify the Linux Operating System account under which p4d and other Helix\\n# services will run as. This user will be created if it does not exist. If\\n# created, the password will match that of the P4USER."
Config['OSUSER']="perforce"
ConfigDoc['OSGROUP']="\\n# Specify the primary group for the Linux Operating System account specified\\n# as OSUSER."
Config['OSGROUP']="perforce"
ConfigDoc['OSUSER_ADDITIONAL_GROUPS']="\\n#Specify a comma-delimited list of any additional groups the OSUSER to be\\n# created should be in.  This is passed to the 'useradd' command the '-G'\\n# flag. These groups must already exist."
Config['OSUSER_ADDITIONAL_GROUPS']=
ConfigDoc['OSUSER_HOME']="\\n# Specify home directory of the Linux account under which p4d and other Helix\\n# services will run as, and the group, in the form <user>:<group>.  This user\\n# and group will be created if they do not exist."
Config['OSUSER_HOME']="/home/perforce"
ConfigDoc['P4BinRel']="\\n# The version of Perforce Helix binaries to be downloaded: p4, p4d, p4broker, and p4p."
Config['P4BinRel']="$DefaultP4BinRel"
ConfigDoc['HxCheckpoints']="\\n# Define the directory that stores critical digital assets that must be\\n# backed up, including metadata checkpoints and numbered journal files. If set to the same value as HxDepots, all critical assets to be backed will be on a single volume."
Config['HxCheckpoints']="/hxdepots"
ConfigDoc['HxDepots']="\\n# Define the directory that stores critical digital assets that must be\\n# backed up, including contents of submitted and shelved versioned files."
Config['HxDepots']="/hxdepots"
ConfigDoc['HxLogs']="\\n# Define the directory used to store the active journal (P4JOURNAL) and\\n# various logs."
Config['HxLogs']="/hxlogs"
ConfigDoc['HxMetadata1']="\\n# The /HxMetadata1 and /HxMetadata1 settings define two interchangeable\\n# directories that store either active/live metadata databases (P4ROOT) or\\n# offline copies of the same (offline_db). These typically point to the same\\n# directory. Pointing them to the same directory simplifies infrastructure\\n# and enables the fastest recovery options. Using multiple metadata volumes\\n# is typically done when forced to due to capacity limitations for metadata\\n# on a single volume, or to provide operational survivability of the host in\\n# event of loss of a single metadata volume."
Config['HxMetadata1']="/hxmetadata"
ConfigDoc['HxMetadata2']=
Config['HxMetadata2']="/hxmetadata"

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style="${1:--h}"
   declare errorMessage="${2:-}"

   msg "USAGE for $ThisScript version $Version:

To Install a Helix Core P4D Server (with optional broker):

$ThisScript {-init|-empty|-sampledepot} [-demo] [-c <cfg>] [-no_cron] [-no_ppr] [-no_systemd|-no_enable] [-no_firewall] [-no_sudo|{-limited_sudo|-full_sudo}] [-v] [-no_pkgs|-extra_pkgs] [-s <ServerID>] [-si <SDPInstance>] [-ts <TargetServerID>] [-se] [-H <hostname>] [-T <timezone>] [-local] [-sdp_dir <sdp_dir>] [-d|-D]

To Install a standalone Helix Proxy:

$ThisScript -t p4p [-c <cfg>] [{-empty|-sampledepot}] [-no_cron] [-no_ppr] [-no_systemd|-no_enable] [-no_firewall] [-no_sudo|-limited_sudo|-full_sudo] [-v] [-no_pkgs|-extra_pkgs] [-s <ServerID>] [-si <SDPInstance>] [-ts <TargetServerID>] [-tp <TargetPort>] [-lp <ListenPort>] [-se] [-H <hostname>] [-T <timezone>] [-local] [-sdp_dir <sdp_dir>] [-d|-D]

To Install a standalone Helix Broker:

$ThisScript -t p4broker [-c <cfg>] [{-empty|-sampledepot}] [-no_cron] [-no_ppr] [-no_systemd|-no_enable] [-no_firewall] [-no_sudo|-limited_sudo|-full_sudo] [-v] [-no_pkgs|-extra_pkgs] [-s <ServerID>] [-si <SDPInstance>] [-ts <TargetServerID>] [-tp <TargetPort>] [-lp <ListenPort>] [-se] [-H <hostname>] [-T <timezone>] [-local] [-sdp_dir <sdp_dir>] [-d|-D]

or

$ThisScript -C > sdp_install.cfg

or

$ThisScript [-h|-man]
"
   if [[ -n "$errorMessage" ]]; then
      msg "\\nUSAGE ERROR: $errorMessage\\n"
      msg "See USAGE above for usage synopsis."

      # Set ErrorCount to 2. Depending on when this usage() function is called, the 'trap'
      # for the terminate() function may or may not have been set. If the trap is set, the
      # call to 'exit 2' below calls the terminate() function rather than exiting directly.
      # We set ErrorCount=2, the desired exit code, because the terminate() function exits
      # with the value of $ErrorCount. If the 'trap' has not yet been set, the 'exit 2'
      # exits this script directly.
      ErrorCount=2
      exit 2
   fi

   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script simplifies the process of installing Perforce Helix with the
	Server Deployment Package (SDP) on a fresh, new server machine.

	If you are adding a new SDP instance to a machine that already has SDP
	installed, use the mkdirs.sh script for that purpose.  See: mkdirs.sh -man

	If you are unsure if SDP is installed, see if there is a /p4 directory
	on the machine. If that directory exists, then SDP is already installed,
	and this install_sdp.sh script will refuse to operate on the machine.

	This script is intended only for initial installation on a new server
	machine. For safety, it will refuse to operate if it detects any existing
	SDP directory structures. If installed on a machine where Helix Core
	data exists in non-SDP structures, it will not interact with the existing
	data.  SDP structures include anything in or under the following
	directories:

	* /p4
	* $WritableSDPDir
	* /hx{checkpoints,depots,logs,metadata*}

	This script can be used to install any of the following:
	* A Helix Core Server (p4d commit, standby, edge, etc.) with an optional p4broker.
	* A standalone Helix Broker (p4broker).
	* A standalone Helix Proxy (p4p).

	If installing a Helix Core Server, there are three options, one of which
	must be specified:
	* Use '-init' to initialize a new data set using the configure_new_server.sh script to get
	started with various best practices, and create an initial checkpoint.
	* Use '-empty' to
	* Use '-sampledepot' to install the Sample Depot training data set.  This is helpful when
	  bootstrapping a training or demo server.

	The following SDP structure is initialized:
	$ImmutableSDPDir (root owned, immutable except by SDP upgrades).
	$WritableSDPDir (owned and writable by OSUSER).
	$DownloadsDir (owned and writable by OSUSER).
	$BinDir (owned and writable by OSUSER).

	This script handles many aspects of installation. It does the
	following:
	* Creates the operating system user (OSUSER) that Helix Core
	  processes (p4d, p4broker, and/p4 p4p) will run as.  The
	  default OSUSER is 'perforce'.  The 'useradd' command is
	  used to create the user as a local account on the machine,
	  and a password.  OSUSER creation and password setting is
	  skipped if that account already exists.  If a non-local
	  network account is to be used, that must be created
	  first before running this script.
	* Creates the home directory for the OSUSER user, if needed.

	Following installation, it also does the following to be more
	convenient for demos, and also give a more production-like feel:
	* Grants the perforce user sudo access (full or limited).
	* Creates default ~perforce/.bash_profile and .bashrc files.
	* Connects to the Perforce Package Repository (APT and YUM only).
	* Installs SDP crontab for the perforce OSUSER.

	This script calls mkdirs.sh for additional configuration:
	* Systemd service files are enabled (or SysV on older systems).
	* Firewall ports are opened for installed services.

PLATFORM SUPPORT:
	This script is intended to work on a variety of Linux distributions
	and Linux server machines.  The following are prioritized for support:

	* Ubuntu 24.04, 22.04, and 20.04. For Ubuntu, only even-numbered *.04
	releases are supported).
	* Rocky Linux 9, 8
	* Red Hat Enterprise Linux (RHEL) 9, 8
	* SuSE 15.

	This script recognizes SysV, Systemd.

	This script requires bash 4.x+ and works with bash 5.x.

	This script is not supported on Mac OSX. It recognizes the Launchd init
	mechanism on Mac but does not support it.

OS PACKAGES:
	The following OS packages are installed (unless '-no_pkgs' is used):

	* Yum: ${PackageList[yum]}

	* AptGet: ${PackageList[apt-get]}

	* Zypper: ${PackageList[zypper]}

	If '-extra_pkgs' is used, the following packages are installed in
	addition to those listed above:

	* Yum: ${ExtraPackageList[yum]}

	* AptGet: ${ExtraPackageList[apt-get]}

	* Zypper: ${ExtraPackageList[zypper]}

	Development utilities such as 'make', the 'gcc' compiler,
	and 'curl' will be available '-extra_pkgs' is used.

	In addition, if the Perforce Package Repository is added,
	these additional packages are installed:

	* Yum: ${ExtraP4PackageList[yum]}

	* AptGet: ${ExtraP4PackageList[apt-get]}

	* Zypper: None, as the Perforce Package Repository does
	not support the Zypper package management system (e.g.
	as used on SuSE Linux).

OPTIONS:
 -c <cfg>
	Specify a config file.  By default, values for various settings
	such as the email to send script logs to are configure with
	demo values, e.g. ${Config['P4AdminList']}.  Optionally, you can
	specify a config file to define your own values.

	For details on what settings you can define in this way, run:
	$ThisScript -C > sdp_install.cfg

	Then modify the generated config file sdp_install.cfg as desired.
	The generated config file contains documentation on settings and
	values.  If no changes are made to the generated file, running with
	'-c sdp_install.cfg' is the equivalent of running without using '-c' at
	all.

 -C	See '-c <cfg>' above.
 
 -init
	Specify '-init' to initialize a new Helix Core data set with the
	configure_new_server.sh script, which applies best practices for a
	production server installation.

	One of '-empty', '-init', or '-sampledepot' must be specified.

 -empty
	Specify '-empty' to avoid initialization of a Helix Core data set.  No
	db.* files will be created.

	This option should be used if you intend to load a checkpoint
	created elsewhere on this new server machine, as would be done
	if you are installing a replica or edge server.

	One of '-empty', '-init', or '-sampledepot' must be specified.

 -sampledepot
	Specify '-sampledepot' to load the Perforce Sample Depot training/demo
	data set.

	One of '-empty', '-init', or '-sampledepot' must be specified.

 -demo
 	By default, key SDP storage volumes are verified to not appear on
	on the OS root volume. If this is not the case, errors are given in
	pre-flight checks, and processing aborts.  Specify '-demo' to bypass
	these safety checks.

	This option should NOT be used for production installations.

 -no_cron
	Skip initialization of the crontab.  A crontab file is generated
	in the $SDPInstallRoot directory, but is not loaded as the active
	crontab.

 -no_ppr
	Skip addition of the Perforce Package Repository for YUM/APT
	repos.  By default, the Package Repository is added.

	Specifying '-local' implies '-no_ppr'.

 -no_sudo
	Specify that no updates to sudoers are to be made.

	WARNING: If systemd/systemctl is used to manage Perforce
	Helix services, the OSUSER that operates these services
	('perforce' by default) requires sufficient sudo access to
	start and stop services using systemctl. Using '-no_sudo'
	may result in a unusable service being created if used
	on a system where the systemctl command is available.

	If this option is used, consider also using '-no_systemd'
	to avoid the requiring systemd.  Using systemd is
	recommended where available.

	It is appropriate to use this option if the machine it
	operates on was based on a machine image that already
	grants the OSUSER sufficient sudo access.

	This option is mutually exclusive with '-limited_sudo' and '-full_sudo'.

 -limited_sudo
	Specify that limited sudo access for the OSUSER created
	is to be granted.  See 'gen_sudoers.sh -man' for details.

	This option is mutually exclusive with '-no_sudo' and '-full_sudo'.

	This option is recommended for optimal security.

 -full_sudo
 	Specify that full sudo access for the OSUSER created
	is to be granted.  See 'gen_sudoers.sh -man' for details.

	This option is mutually exclusive with '-no_sudo' and '-limited_sudo'.

 -v
 	Specify '-v' to run the verify_sdp.sh script after the SDP
	installation is complete. If '-v' is specified and the
	verify_sdp.sh script is available in the SDP, it is executed.

	If the Sample Depot is loaded with '-sampledepot' or a new server was
	initialized with '-init', the '-online' flag to the verify_sdp.sh
	script is added.  If '-no_cron' is specified, the corresponding
	'-skip cron' option is added verify_sdp.sh.  If '-empty' is
	specified, the '-skip' tests also exclude the offline_db and
	p4t_files checks in verify_sdp.sh.

 -no_pkgs
	Specify '-no_pkgs' to skip OS package installation using
	the package manager (yum, apt-get, or zypper).

	WARNING: Using this option may cause the initial install
	and/or subsequent operation to fail, and this using
	this is not advised.
 
-extra_pkgs
	Specify '-extra_pkgs' to install additional OS packages
	as may be needed for development of systems integrations,
	custom triggers, etc.  See package lists above for
	more detail.

 -local
	By default, various files and binaries are downloaded from
	the Perforce Workshop and the Perforce FTP server as needed.

	If the server machine on which this $ThisScript is to be
	run cannot reach the public internet or if using files from
	external sites is not desired, the '-local' flag can be used.

	With '-local', needed files must be acquired and put in place
	on the server machine on which this script is to be run.  Any
	missing files result in error messages and an aborted install.

	Specifying '-local' implies '-no_ppr'.

	For '-local' to work, the following must exist:

	1. Helix Binaries
	
	Helix binaries must exist in $LocalInstallBinDir:

	* $LocalInstallBinDir/p4
	* $LocalInstallBinDir/p4d
	* $LocalInstallBinDir/p4broker
	* $LocalInstallBinDir/p4p

	2. Server Deployment Package (SDP)

	The SDP tarball must be acquired an put in place here:

	* $DownloadsDir/$SDPTar

	It can be acquired on a machine that can reach the internet with this command:

	curl -L -O $SDPURL

	3. Sample Depot Tarball
	
	The Sample Depot appropriate to your platform must exist if the
	'-sampledepot' option is used:

	* $DownloadsDir/sampledepot.tar.gz (on UNIX/Linux or case-sensitive Mac)

	See EXAMPLES below for a sample of acquiring files for use with
	'-local' mode.

 -no_firewall
	Specify '-no_firewall' to skip updates to firewall.

	By default, if on a system for which the host-local firewall
	service (firewalld or ufw) is available and running when this
	script is called, then the firewall service is updated to open
	appropriate ports for the Perforce Helix services installed.

 -no_systemd
	Specify '-no_systemd' to avoid using systemd, even if it
	appears to be available. By default, systemd is used if it
	appears to be available.

	This is helpful in operating in containerized test environments
	where systemd is not available.

	This option is implied if the systemctl command is not available
	in the PATH of the root user.

	This option is mutually exclusive with '-no_enable'.

 -no_enable
	Specify '-no_enable' to avoid enabling systemd services that
	are installed and enabled by default. Specifically, this means
	that the call to 'systemctl enable' for installed services is
	skipped.

	This option is mutually exclusive with '-no_systemd'.

 -t <ServerType>
 
 	Specify the type of server. Valid values are:

	* p4d_master - A p4d master/commit server.
	* p4d_replica - A p4d replica with all metadata from the master (not
	  filtered in any way).
	* p4d_filtered_replica - A p4d filtered replica or filtered forwarding
	  replica.
	* p4d_edge - An p4d edge server.
	* p4d_edge_replica - A p4d replica of a p4d edge server. The TargetServerID
	  must also be set if ServerType is p4d_edge_replica.
	* p4broker - An SDP host running only a Helix Broker.
	* p4p - An SDP host running only a Helix Proxy.
	
 -s <ServerID>
 	Specify the ServerID.  A ServerID is required if the ServerType is
       	any p4d_* type other than p4d_master.

 -si <SDPInstance>
	Specify the SDP Instance name.  The SDP Instance name is incorporated
	into the folder structure of the installed service, with many files
	appearing under '/p4/<SDPInstance>', e.g. /p4/1.

	The default is '1'.

 -ts <TargetServerID>
 	Specify the Target ServerID. Set this only if ServerType is
	p4d_edge_replica. The value is the ServerID of edge server that
	this server is a replica of, and must match the ReplicatingFrom:
	field of the server spec.

 -tp <TargetPort>
 	Specify the target port.  For p4broker and p4p only.

 -lp <ListenPort>
 	Specify the port to listen on.  For p4broker and p4p only.

 -se
 	Specify -se to simulate email. This generates a mail simulator
	script: $MailSimulator

 -H <hostname>
 	Set the hostname.  This is only supported on systems that
	support the 'hostnamectl' command. The hostname is set by
	doing: hostnamectl set-hostname <hostname>

	If the corresponding 'Hostname' setting is defined in the
	configuration file and this '-H <hostname>' flag is used,
	the command line option will override the config file.

 -T <timezone>
 	Set the timezone.  This is only supported on systems that
	support the 'timedatectl' command. The timezone is set by
	doing: timedatectl set-timezone <timezone>

	If the corresponding 'Timezone' setting is defined in the
	configuration file and this '-T <timezone>' flag is used,
	the command line option will override the config file.

DEVELOPMENT OPTIONS:
 -sdp_dir <sdp_dir>
	Specify a directory on the local host containing the SDP to deploy.
	This should not be used for production installs.

	The directory specified by '-sdp_dir' is expected to contain either:
	* an SDP tarball ($SDPTar) file, or
	* an already-extracted SDP directory, which must include the SDP
	Version file.

	Use the special value '-sdp_dir default' to use the /sdp directory
	(as per the Docker-based SDP Test Suite environment).

DEBUGGING OPTIONS:
 -d     Enable debug message.
 -D     Enable extreme debugging with bash 'set -x'. Implies '-d'.

HELP OPTIONS:
 -h	Display short help message.
 -man	Display this full manual page.

 --help
	Alias for -man.

EXAMPLES:
	=== Demo Installation - Helix Core Server ===

	sudo su -
    	mkdir -p /root/sdp_install
	cd /root/sdp_install
	curl -L -O $ThisScriptURL
	chmod +x $ThisScript
	./$ThisScript -sampledepot -demo

	=== Typical Production Helix Core Server Installation - New Commit Server ===

	Following is a sample set of instructions for a typical new server setup.

	STEP 1: Configure storage.

	For a production install, storage must first be configured before this script
	can be run.

	See SDP documentation for guidance on storage configuration. There are a variety of
	options and methods for installing storage.  However accomplished, when storage is
	complete the following, the directories must exist and must have storage mounted
	that is NOT on the OS root volume:

	* /hxdepots
	* /hxmetadata
	* /hxlogs

	These paths are typical, but are configurable. More information is available in
	the install configuration file generated below.

	STEP 2: Install this script.

	Install this script in a directory under the root user's home directory with
	these commands:

	$ sudo su -
    	$ mkdir /root/sdp_install
	$ cd /root/sdp_install
	$ curl -L -O $ThisScriptURL
	$ chmod +x $ThisScript

 	STEP 3: Generate install configuration file.

	$ ./$ThisScript -C > sdp_install.cfg

	STEP 4: Modify install configuration file.

 	Edit the generated sdp_install.cfg using your preferred text editor, changing the values
	as desired. This file contains various settings with documentation for each setting.

	$ vi sdp_install.cfg

	Once settings are decided, save the file.

	STEP 5: Install SDP (Dry Run).

	Call this script and reference the configuration file, as a dry run/preview:

	$ ./$ThisScript -c sdp_install.cfg -init

	Review the generated log of the preview, and address any reported issues.

	STEP 6: Install SDP Live Run

	$ ./$ThisScript -c sdp_install.cfg -init -y

	This will install SDP per the per the command line and settings in the install
	configuration file.

	=== Typical Production Helix Core Server Installation - Edge/Replica ===

	When installing SDP on a machine intended to be a standby, replica, or edge
	server, the steps are exactly the same as for setting up a new commit server.

	The content of the generated and then edited sdp_install.cfg file will have different
	values for ServerType and ServerID settings. Ensure the CaseSensitity value matches
	the case of the commit server.  (If the commit server is a Windows server and this
	current server machine is to be a Linux replica of a Windows commit server, the
	Linux server must be setup as case-insensitive.)

	=== Standalone Proxy Installation ===

	STEP 1: Install this script.

	Install this script in a directory under the root user's home directory with
	these commands:

	$ sudo su -
    	$ mkdir /root/sdp_install
	$ cd /root/sdp_install
	$ curl -L -O $ThisScriptURL
	$ chmod +x $ThisScript

	STEP 2: Install the proxy.

	Install the proxy, specifying the listen port (ssl:1666 in this example)
	and the target port (ssl:p4d.myco.com:1666).

	$ ./$ThisScript -t p4p -lp ssl:1666 -tp ssl:p4d.myco.com:1666

	=== Standalone Broker Installation ===

	The instructions for installing the broker are identical to the instructions
	for installing the proxy, except that '-t p4broker' is used instead of
	'-t p4p'.

	=== Local Helix Core Server Install for Air Gap Networks ===

	The following sample commands illustrate how to acquire the dependencies for
	running with '-local' on a machine that can reach the public internet.  The
	resulting file structure, with paths as shown, must somehow be copied to the
	machine where this $ThisScript script is to be run.  This can be used to
	facilitate installation on a machine over an \"air gap\" network.

	$ sudo su -
	$ mkdir -p $LocalInstallBinDir
	$ cd $LocalInstallBinDir
	$ curl -L -O $WorkshopBaseURL/download/guest/perforce_software/sdp/main/helix_binaries/get_helix_binaries.sh
	$ chmod +x get_helix_binaries.sh

	If the latest major version available of Helix Core binaries are desired, do this:
	$ ./get_helix_binaries.sh -sbd .

	Or, if older Helix Core binaries are desired, append the version identifier with '-r rYY.N',
	as in this example to get Helix Core 2023.2 binaries:
	$ ./get_helix_binaries.sh -sbd . -r r23.2

	Next, get the SDP tarball and Sample Depot; the Sample Depot tarball:

	$ mkdir $DownloadsDir
	$ cd $DownloadsDir
	$ curl -O $FTPURL/tools/$SampleDepotTar
	$ curl -L -O $SDPURL

	Lastly, acquire this script:

	$ mkdir /root/install_sdp
	$ cd /root/install_sdp
	$ curl -L -O $ThisScriptURL
	$ chmod +x $ThisScript

	These acquired files must then be transferred to the machine where the install
	is to occur, and must appear in the same directory structure.  To recap, for
	a '-local' install, the following files in this structure must exist on the
	machine on which the install is to occur:

	/root/install_sdp/$ThisScript
	$DownloadsDir/$SDPTar
	$DownloadsDir/$SampleDepotTar (if the '-sampledepot' option is to be used).
	$LocalInstallBinDir/p4
	$LocalInstallBinDir/p4d
	$LocalInstallBinDir/p4broker
	$LocalInstallBinDir/p4p

	An install session would then look something like this:

	cd /root/install_sdp
	./$ThisScript -C > sdp_install.cfg         # Generate a config file
	vi sdp_install.cfg                            # Edit desired settings.
	./$ThisScript -c sdp_install.cfg -init     # Dry run. Review log to ensure success.
	./$ThisScript -c sdp_install.cfg -init -y  # Live run. Review log to ensure success.

	If the dry run is not successful, fix reported issues and try again.
"
   fi

   exit 2
}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   msg "\\n$ThisScript: EXITCODE: $ErrorCount\\n"
   [[ "$Log" != "off" ]] && msg "Log is: $Log"

   # With the trap removed, exit.
   exit "$ErrorCount"
}

#------------------------------------------------------------------------------
# Functions msg(), dbg(), and bail().
# Sample Usage:
#    bail "Missing something important. Aborting."
#    bail "Aborting with exit code 3." 3
function msg () { echo -e "$*"; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Function: get_old_log_timestamp ($log)
#
# Get the last modified timestamp of the old log in a cross-platform manner.
# If we don't get a correct value using 'stat' (which varies across the
# UNIX/Linux/Mac OSX spectrum), use the current time as a fallback. In that
# case, the timestamp will reflect the time the log was moved rather than when
# it was last modified, but that's still reasonable.  The file timestamp will
# still have the correct last-modified time.
#------------------------------------------------------------------------------
function get_old_log_timestamp () {
   local log=${1:-}
   local oldLogTimestamp=
   [[ -n "$log" ]] || return

   if [[ "$(uname -s)" == "Darwin" ]]; then
      oldLogTimestamp=$(stat -L -f %Sm -t '%Y-%m-%d-%H%M%S' "$log" 2>/dev/null)
   else
      oldLogTimestamp="$(stat -L -c '%10y' "$log" | sed -e 's@[.].*$@@g' -e 's@:@@g' -e 's@ @-@g')"
   fi

   [[ "$oldLogTimestamp" =~ ^[2-9]{1}[0-9]{3}- ]] ||
      oldLogTimestamp=$(date +'%Y-%m-%d-%H%M%S')

   echo "$oldLogTimestamp"
}

#------------------------------------------------------------------------------
# Functions run($cmdAndArgs, $desc, $ignoreNoOp)
#
# This function displays an optional description of a command to run. If the
# global NoOp (No Operation mode) is 0, then the command and arguments are
# displayed and then executed. If NoOp is 1, the command and arguments are
# displayed with  "NO_OP: Would have run" indication.
#
# Arguments:
# $1 - cmdAndArgs
# $2 - description
# $3 - ignoreNoOp. If 1, the command is always executed even if NoOp is 0.
function run {
   local cmdAndArgs="${1:-echo Testing run}"
   local desc="${2:-}"
   local -i ignoreNoOp=${3:-0}

   [[ -n "$desc" ]] && msg "$desc"
   if [[ "$NoOp" -eq 0 || "$ignoreNoOp" -eq 1 ]]; then
      msg "Running: $cmdAndArgs"
      eval "$cmdAndArgs"
      CMDEXITCODE=$?
   else
      msg "NO_OP: Would have run: $cmdAndArgs"
      CMDEXITCODE=0
   fi
   return $CMDEXITCODE
}

#------------------------------------------------------------------------------
# Function: gen_default_config()
#
# This generates a sample configuration settings file. Output is generated to
# stdout, making it easy for external automation to modify. By convention,
# output is generated to a file named sdp_install.cfg, e.g. with '-C sdp_install.cfg'
# flag with output redirected to that file.
#
# The sample file contains all required settings and reasonable sample values.
# The in-code documentation describes how the settings are used.  Settings
# are enumerated in the Config and ConfigDoc associative arrays, with the
# index of the arrays being the setting name.  For example, if the setting
# is ServerID, it can be referenced as ${Config['ServerID']}.
#
# The sections of the file are delineated by comments, with an description of
# what type of settings are in each section. The sections are Section 1:
# Localization, Section 2: Data Specific, and Section 3: Deep Customization.
# A hand-crafted 'for' loop in each section indicates what section any given
# setting belongs in.
#
# Generate a sample sdp_install.cfg file.
function gen_default_config {
   echo -e "\
#------------------------------------------------------------------------------
# Config file for $ThisScript version $Version.
#------------------------------------------------------------------------------
# This file is in bash shell script syntax.
# Note: Avoid spaces before and after the '=' sign.

# For demo and training installations, usually all defaults in this file
# are fine.

# For Proof of Concept (PoC) installation, Section 1 (Localization) settings
# should all be changed to local values. Some settings in Section 2 (Data
# Specific) might also be changed.

# Changing settings in Section 3 (Deep Customization) is generally
# discouraged unless necessary when bootstrapping a production installation or
# a high-realism PoC.

#------------------------------------------------------------------------------
# Section 1: Localization
#------------------------------------------------------------------------------
# Changing all these is typical and expected, even for PoC installations."

   for c in SMTPServer P4AdminList MailFrom DNS_name_of_master_server SiteTag Hostname Timezone; do
      echo -e "${ConfigDoc[$c]}"
      echo "$c=${Config[$c]}"
   done

   echo -e "
#------------------------------------------------------------------------------
# Section 2: Data Specific
#------------------------------------------------------------------------------
# These settings can be changed to desired values, though default values are
# preferred for demo installations."

   for c in P4_PORT P4BROKER_PORT Instance CaseSensitive SSLPrefix P4USER Password SimulateEmail ServerID ServerType TargetServerID TargetPort ListenPort; do
      echo -e "${ConfigDoc[$c]}"
      echo "$c=${Config[$c]}"
   done

   echo -e "
#------------------------------------------------------------------------------
# Section 3: Deep Customization
#------------------------------------------------------------------------------
# Changing these settings is gently discouraged, but may be necessary for
# bootstrapping some production environments with hard-to-change default values
# for settings such as OSUSER, OSGROUP, Hx*, etc.
#
# Changing these settings is gently discouraged because changing these values
# will cause the configuration to be out of alignment with documentation and
# sample instructions for settings that are typically left as defaults.
# However, there are no functional limitations to changing these settings."

   for c in OSUSER OSGROUP OSUSER_ADDITIONAL_GROUPS OSUSER_HOME P4BinRel; do
      echo -e "${ConfigDoc[$c]}"
      echo "$c=${Config[$c]}"
   done

   echo -e "
# The following Hx* settings reference directories that store Perforce
# Helix data.  If configuring for optimal performance and scalability,
# these folders can be mount points for storage volumes.  If so, they must
# be mounted prior to running the $ThisScript script (other than to generate
# this configuration file).
#
# See the Server Deployment Package (SDP) for information and guidance on
# provisioning these volumes."

   for c in HxDepots HxCheckpoints HxLogs; do
      echo -e "${ConfigDoc[$c]}"
      echo "$c=${Config[$c]}"
   done

   # Special case: The ConfigDoc for HxMetadata1 applies to both HxMetadata1
   # and HxMetadata2 settings, so display it only once.
   echo -e "${ConfigDoc['HxMetadata1']}"
   echo "HxMetadata1=${Config['HxMetadata1']}"
   echo "HxMetadata2=${Config['HxMetadata2']}"
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0
set +u

while [[ $# -gt 0 ]]; do
   case $1 in
      (-y) NoOp=0;;
      (-local) PullFromWebAsNeeded=0; AddPerforcePackageRepo=0;;
      (-init) InitializeEmptyServer=1; LoadSampleDepot=0; SetDataHandling=1;;
      (-empty) InitializeEmptyServer=0; LoadSampleDepot=0; SetDataHandling=1;;
      (-sampledepot) InitializeEmptyServer=0; LoadSampleDepot=1; SetDataHandling=1;;
      (-demo) DemoInstall=1;;
      (-no_cron) LoadActiveCrontab=0;;
      (-no_ppr) AddPerforcePackageRepo=0;;
      (-no_systemd) UseSystemd=0; UseSystemdOption="-no_systemd";;
      (-no_enable) EnableSystemdServices=0;;
      (-no_firewall) DoFirewall=0;;
      (-no_sudo) DoSudo=0;;
      (-limited_sudo) LimitedSudoers=1;;
      (-full_sudo) LimitedSudoers=0;;
      (-v) DoVerifySDP=1;;
      (-no_pkgs) InstallOSPackages=0;;
      (-extra_pkgs) InstallExtraOSPackages=1;;
      (-s) ServerID="$2"; SetServerID=1; shiftArgs=1;;
      (-si) SDPInstance="$2"; SetSDPInstance=1; shiftArgs=1;;
      (-t) ServerType="$2"; SetServerType=1; shiftArgs=1;;
      (-ts) TargetServerID="$2"; SetTargetServerID=1; shiftArgs=1;;
      (-tp) TargetPort="$2"; SetTargetPort=1; shiftArgs=1;;
      (-lp) ListenPort="$2"; SetListenPort=1; shiftArgs=1;;
      (-se) SimulateEmail=1; SetSimulateEmail=1;;
      (-H) Hostname="$2"; SetHostname=1; shiftArgs=1;;
      (-T) Timezone="$2"; SetTimezone=1; shiftArgs=1;;
      (-C) GenDefaultConfig=1;;
      (-c) ConfigFile="$2"; UseConfigFile=1; shiftArgs=1;;
      (-sdp_dir)
         SDPInstallMethod=Copy
         [[ "$2" == "default" ]] || SDPLocalCopySource="$2"
         shiftArgs=1
      ;;
      (-h) usage -h;;
      (-man|--help) usage -man;;
      (-V) msg "$ThisScript version $Version"; exit 2;;
      (-d) Debug=1;;
      (-D) Debug=1; ExtremeDebug=1; set -x;; # Debug; use bash 'set -x' mode.
      (-*) usage -h "Unknown flag/option ($1).";;
      (*) usage -h "Unknown parameter ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && bail "Usage Error: Wrong number of parameters to flags/options."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Validation
# See Additional Command Line Validation for validations that occur after
# settings are loaded from the config file.

[[ "$InitializeEmptyServer" -eq 1 && "$LoadSampleDepot" -eq 1 ]] && \
   usage -h "The '-init' and '-sampledepot' options are mutually exclusive."

[[ "$UseConfigFile" -eq 1 && "$GenDefaultConfig" -eq 1 ]] && \
   usage -h "The '-c <cfg>' and '-C' options are mutually exclusive."

[[ "$DoSudo" -eq 0 && "$LimitedSudoers" -eq 0 ]] && \
   usage -h "The '-no_sudo' and '-full_sudo' options are mutually exclusive."

[[ "$InstallOSPackages" -eq 0 && "$InstallExtraOSPackages" -eq 1 ]]  && \
   usage -h "The '-no_pkgs' and '-extra_pkgs' options are mutually exclusive."

#==============================================================================
# Main Program

ThisUser="$(whoami)"
ThisOS="$(uname -s)"
ThisArch="$(uname -m)"
ThisHost="$(hostname -s)"

#------------------------------------------------------------------------------
# Special Mode:  Generate Default Config File
# In this mode, generate a sample config file on stdout, and exit.

if [[ "$GenDefaultConfig" -eq 1 ]]; then
   gen_default_config
   exit 0
fi

#------------------------------------------------------------------------------
# Regular processing mode.

Log="$PWD/${ThisScript%.sh}.$(date +'%Y-%m-%d-%H%M%S').log"
LogLink="${ThisScript%.sh}.log"

touch "$Log" || bail "Could not initialize log with: touch \"$Log\""
trap terminate EXIT SIGINT SIGTERM

# Redirect stdout and stderr to the log file.
exec > >(tee "$Log")
exec 2>&1

if [[ -e "$LogLink" ]]; then
   if [[ -L "$LogLink" ]]; then
      rm -f "$LogLink"
   else
      # If the name that should be a symlink is not a symlink, move it aside before
      # creating the symlink.
      OldLogTimestamp=$(get_old_log_timestamp "$LogLink")
      mv -f "$LogLink" "${LogLink%.log}.${OldLogTimestamp}.log"
   fi
fi

# Point LogLink symlink to current log. Use a subshell so the 'cd' doesn't persist.
ln -s "$Log" "$LogLink"

msg "${H1}\\nLog is: $Log\\n"

msg "Started $ThisScript version $Version on host $ThisHost as user $ThisUser at $(date), called as:\\n\\t$CmdLine"

InheritedUmask=$(umask)

if [[ "$PullFromWebAsNeeded" -eq 1 ]]; then
   msg "\\nOnline mode: Files will be pulled from the web as needed using curl commands."
else
   msg "\\nLocal mode: No files will be pulled from the web. All assets must exist."
fi

#------------------------------------------------------------------------------
# Determine OS Package Manager, apt, yum, or zypper.
# TO DO: Ponder if/when to prefer dnf over yum.
if command -v yum > /dev/null; then
   PackageManager="yum"
elif command -v apt-get > /dev/null; then
   PackageManager="apt-get"
elif command -v zypper > /dev/null; then
   PackageManager="zypper"
else
   InstallOSPackages=0
fi

[[ "$ThisOS" == Darwin ]] && InstallOSPackages=0

#------------------------------------------------------------------------------
# Install Standard and Extra OS Packages.  Both apt-get and yum/dnf have nuances
# to working with that hinder reliable scripted operation. With apt-get, there
# are kernel upgrade warnings to be aware of. With yum/dnf, there are bugs to
# work around. Do whatever is needed to make the OS package installation
# "just work" for the user.
if [[ "$InstallOSPackages" -eq 1 ]]; then
   msg "Ensuring standard packages are installed."

   if [[ "$ThisOS" != "Darwin" ]]; then
      [[ "$PackageManager" == "Unset" ]] && \
         bail "Could not find one of these package managers: ${!PackageList[*]}"

      # For apt-get, do the update first.
      if [[ "$PackageManager" == apt-get ]]; then
         run "$PackageManager update" "Updating apt package repo list." ||\
            warnmsg "'apt-get update' returned non-zero exit code. Proceeding anyway."

         msg "Setting DEBIAN_FRONTEND=noninteractive for apt-get."
         export DEBIAN_FRONTEND=noninteractive
         msg "Accepting defaults for OS recommendations related to service restarts due to package installs doing:\\n
echo '* libraries/restart-without-asking boolean true' | debconf-set-selections
echo 'set needrestart/restart no' | debconf-set-selections"
         echo '* libraries/restart-without-asking boolean true' | debconf-set-selections
         echo 'set needrestart/restart no' | debconf-set-selections

         run "$PackageManager install -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' ${PackageList[$PackageManager]} < /dev/null" \
            "Installing these packages with $PackageManager: ${PackageList[$PackageManager]}" ||\
            warnmsg "Not all standard OS packages installed.  Proceeding anyway; this may or may not cause problems later."

         if [[ "$InstallExtraOSPackages" -eq 1 ]]; then
            run "$PackageManager install -y ${ExtraPackageList[$PackageManager]} < /dev/null" \
               "Installing these packages with $PackageManager: ${ExtraPackageList[$PackageManager]}" ||\
               warnmsg "Not all extra OS packages installed.  Proceeding anyway."
         fi
      elif [[ "$PackageManager" == yum || "$PackageManager" == zypper ]]; then
         # For RHEL/Rocky systems, we need to work around a bug where one missing
         # package in a list of packages causes silent failures to handle other
         # unrelated packages. For example, a missing 'mailx' package might cause
         # 'rsync' to not even be considered for install. The workaround is to just
         #  install each package separately.
         for pkg in ${PackageList[$PackageManager]}; do
            run "$PackageManager install -y $pkg < /dev/null" \
               "Installing standard OS Package '$pkg'." ||\
               warnmsg "Unable to install standard OS package '$pkg'. Proceeding anyway; this may or may not cause problems later."
         done

         if [[ "$InstallExtraOSPackages" -eq 1 ]]; then
            for pkg in ${ExtraPackageList[$PackageManager]}; do
               run "$PackageManager install -y $pkg < /dev/null" \
                  "Installing extra OS Package '$pkg'." ||\
                  warnmsg "Unable to install extra OS package '$pkg'. Proceeding anyway."
            done
         fi
      else
         InstallOSPackages=0
      fi
   else
      InstallOSPackages=0
   fi
fi

#------------------------------------------------------------------------------
# Install Extra OS Packages
if [[ "$InstallExtraOSPackages" -eq 1 ]]; then
   msg "Installing extra OS packages."

   if [[ "$ThisOS" != "Darwin" ]]; then
      run "$PackageManager install -y ${ExtraPackageList[$PackageManager]} < /dev/null" \
         "Installing these packages with $PackageManager: ${ExtraPackageList[$PackageManager]}" ||\
         warnmsg "Not all extra OS packages installed successfully.  Proceeding anyway."
   fi
fi

#------------------------------------------------------------------------------
# Check Umask
if [[ "$InheritedUmask" == "$OperatingUmask" ]]; then
   msg "Verified: Umask is as expected: $OperatingUmask"
else
   umask $OperatingUmask || bail "Could not do: umask OperatingUmask"
   msg "Umask as inherited from environment was $InheritedUmask; temporarily changed umask to $OperatingUmask for this script's operations."
fi

msg "\\nEnsuring required utilities to operate this script are available."
[[ "$PullFromWebAsNeeded" -eq 1 ]] && RequiredUtils+=" curl"
for u in $RequiredUtils; do
   if command -v "$u" > /dev/null; then
      dbg "Verified: Required utility '$u' found in PATH."
   else
      if [[ "$NoOp" -eq 0 ]]; then
         errmsg "Missing required utility '$u'.  You may need to install it or adjust the PATH for the root user to find it.\\n\\n"
      else
         warnmsg "Missing required utility '$u'.  You may need to install it or adjust the PATH for the root user to find it. Because this is DRY RUN mode and packages have not been installed, this is treated as a warning -- in live operation mode, this would cause an pre-flight failure resulting in an early abort (before any processing starts).\\n\\n"
      fi
   fi
done

[[ "$ErrorCount" -eq 0 ]] || bail "Aborting due to missing required utilities."

if [[ "$UseConfigFile" -eq 1 ]]; then
   [[ -r "$ConfigFile" ]] || \
      bail "Config file specified with '-c $ConfigFile' is not readable."

   msg "Loading configuration data from $ConfigFile."
   for c in "${!Config[@]}"; do
      value=$(grep "^$c"= "$ConfigFile")
      value="${value#*=}"
      Config[$c]="$value"
      dbg "  For config [$c] loaded value [$value]."
   done

   SDPInstance="${Config['Instance']}"
   RunGroup="${Config['OSGROUP']}"
   RunUserNewHomeDir="${Config['OSUSER_HOME']}"
else
   # We want to know whether RunGroup was set explicitly in the sdp_install.cfg
   # file or not.  If not, we set it to the value of 'Unset', enabling
   # OS-dependent logic to below to supply a platform-specific default.
   # We don't want platform-specific defaults to override values explicitly
   # defined in sdp_install.cfg if that is used.
   RunGroup=Unset

   # Similarly, we want to apply the home directory specified in account
   # creation if it was defined in sdp_install.cfg, but not otherwise.
   RunUserNewHomeDir=Unset
fi

# After the configuration data is loaded, set variables that depend on
# the loaded configuration.
# shellcheck disable=SC2086 disable=SC2116
SDPInstance=$(echo $SDPInstance)
RunUser="${Config['OSUSER']}"

HxDepots="${Config['HxDepots']}"
HxCheckpoints="${Config['HxCheckpoints']}"
[[ -n "$HxCheckpoints" ]] || HxCheckpoints="$HxDepots"
HxMetadata1="${Config['HxMetadata1']}"
HxMetadata2="${Config['HxMetadata2']}"
HxLogs="${Config['HxLogs']}"

# Trim the leading '/' from Hx* settings to be compatible with SDP mkdirs.cfg
HxDepots="${HxDepots#/}"
HxCheckpoints="${HxCheckpoints#/}"
HxMetadata1="${HxMetadata1#/}"
HxMetadata2="${HxMetadata2#/}"
HxLogs="${HxLogs#/}"

DownloadsDir="$SDPPackageBase/downloads"
SSLDir="$WritableSDPDir/Server/Unix/p4/ssl"
SSLConfig="$SSLDir/config.txt"
SSLPrefix="${Config['SSLPrefix']}"

#------------------------------------------------------------------------------
# Command Line Overrides
# Configuration settings in this block have corresponding command options.
# Settings from the config file will be overridden if their corresponding
# command line options is set. So if '-s <ServerID>' is given on the command
# line, the ServerID setting from the config file is ignored.
[[ "$SetHostname" -eq 0 ]] && Hostname="${Config['Hostname']}"
[[ "$SetTimezone" -eq 0 ]] && Timezone="${Config['Timezone']}"
[[ "$SetServerID" -eq 0 ]] && ServerID="${Config['ServerID']}"
[[ "$SetSDPInstance" -eq 0 ]] && SDPInstance="${Config['Instance']}"
[[ "$SetServerType" -eq 0 ]] && ServerType="${Config['ServerType']}"
[[ "$SetSimulateEmail" -eq 0 ]] && SimulateEmail="${Config['SimulateEmail']}"
[[ "$SetListenPort" -eq 0 ]] && ListenPort="${Config['ListenPort']}"
[[ "$SetTargetPort" -eq 0 ]] && TargetPort="${Config['TargetPort']}"
[[ "$SetTargetServerID" -eq 0 ]] && TargetServerID="${Config['TargetServerID']}"

#------------------------------------------------------------------------------
# Additional Command Line Validation
# See Command Line Validation for validations that occur earlier in processing.
[[ "$ServerType" == p4d_master && "$SetDataHandling" -eq 0 ]] && \
   usage -h "For installing a Helix Core server, one of '-init', '-local', or '-sampledepot' must be specified."
[[ "$ServerType" =~ ^(p4p|p4proxy)$ && -z "$ListenPort" ]] && \
   usage -h "ServerType is p4p, but ListenPort is undefined. Add '-lp <ListenPort>'."
[[ "$ServerType" =~ ^(p4p|p4proxy)$ && -z "$TargetPort" ]] && \
   usage -h "ServerType is p4p, but TargetPort is undefined. Add '-tp <TargetPort>'."
[[ "$ServerType" =~ ^p4broker$ && -z "$ListenPort" ]] && \
   usage -h "ServerType is p4broker, but ListenPort is undefined. Add '-lp <ListenPort>'."
[[ "$ServerType" =~ ^p4broker$ && -z "$TargetPort" ]] && \
   usage -h "ServerType is p4broker, but TargetPort is undefined. Add '-tp <TargetPort>'."

# Do data validations based on data loaded from the configuration file.
if [[ "$UseConfigFile" -eq 1 ]]; then
   msg "Doing sanity checks on data loaded from the config file."

   if [[ "$ServerType" != "p4d_commit" && "$ServerType" != "p4d_master" ]]; then
      if [[ -z "$ServerID" ]]; then
         if [[ "$ServerType" == "p4broker" || "$ServerType" = "p4p" || "$ServerType" = "p4proxy" ]]; then
            ServerID="$ServerType"
            msg "No ServerID defined for server of type $ServerType. Defaulting to $ServerType as the ServerID."
         else
            errmsg "ServerID is not set. ServerID must be set if ServerType is any p4d_* type other than p4d_commit or p4d_master. ServerType is [$ServerType]."
         fi
      fi
   fi

   if [[ -n "$TargetServerID" ]]; then
      [[ "$ServerType" != "p4d_edge_replica" ]] && \
         errmsg "TargetServerID is set (to $TargetServerID), but ServerType is not p4d_edge_replica. TargetServerID can only be set when ServerType is p4d_edge_replica. ServerType [$ServerType]."
   fi

   if [[ "${Config['CaseSensitive']}" == "0" && "$LoadSampleDepot" -eq 1 ]]; then
      errmsg "The 'CaseSensitive' setting in the config file was set to 0 (case-insensitive),\\nand the '-sampledepot' flag was specified, indicating intent to load the\\nSample Depot demo data set. Loading a case-insensitive Sample Depot is not\\nsupported.\\n"
   fi

   if [[ -n "$SSLPrefix" ]]; then
      if [[ "$SSLPrefix" =~ ^ssl[46]*:$ ]]; then
         dbg "Verified: SSL Prefix value [$SSLPrefix] is valid."
      else
         errmsg "The SSL prefix value specified, [$SSLPrefix] is not valid.  It should match this pattern: ^ssl[46]:\$"
      fi
   fi

   if [[ "$ErrorCount" -eq 0 ]]; then
      msg "Config file data passed sanity checks."
   else
      bail "Config file data failed sanity checks. Aborting."
   fi
fi

# Valid ServerType.  Based on ServerType, determine binary and services to
# install.
case "$ServerType" in
   (p4d_commit|p4d_master)
      BinList="p4d p4broker"
      UseBroker=1
   ;;
   (p4d_replica)
      BinList="p4d p4broker"
      UseBroker=1
   ;;
   (p4d_filtered_replica)
      BinList="p4d p4broker"
      UseBroker=1
   ;;
   (p4d_edge)
      BinList="p4d p4broker"
      UseBroker=1
   ;;
   (p4d_edge_replica)
      BinList="p4d p4broker"
      UseBroker=1
   ;;
   (p4broker)
      BinList="p4broker"
      LoadSampleDepot=0
      UseBroker=1
   ;;
   # Allow 'p4proxy' as an undocumented alias for 'p4p'.
   (p4p|p4proxy)
      BinList="p4p"
      LoadSampleDepot=0
      UseBroker=0
   ;;
   (*)
      bail "Invalid ServerType specified [$ServerType]. Run $ThisScript -man to see valid values."
   ;;
esac

# Set the ServerBin to the server binary that will be used to create SSL
# certificates. Use whichever server is available based on the server
# type to be installed; it can be p4d, p4p, or p4broker (but not the 'p4'
# client binary).
ServerBin="$SDPInstallRoot/${SDPInstance}/bin/${BinList%% *}_${SDPInstance}"

# Get just enough detailed OS info in order to fill in
# details in the Perforce package repository files.
if [[ "$ThisOS" == "Linux" ]]; then
   if [[ -r "/etc/redhat-release" ]]; then
      if grep -q ' 6\.' /etc/redhat-release; then
         ThisOSMajorVersion="6"
      elif grep -q ' 7\.' /etc/redhat-release; then
         ThisOSMajorVersion="7"
      elif grep -q ' 8\.' /etc/redhat-release; then
         ThisOSMajorVersion="8"
      elif grep -q ' 9\.' /etc/redhat-release; then
         ThisOSMajorVersion="9"
      fi
      [[ -n "$ThisOSMajorVersion" ]] || \
         warnmsg "Could not determine OS Major Version from contents of /etc/redhat-release."
   elif [[ -r "/etc/lsb-release" ]]; then
      ThisOSName=$(grep ^DISTRIB_ID= /etc/lsb-release)
      ThisOSName=${ThisOSName#*=}
      ThisOSName=${ThisOSName,,}
      ThisOSDistro=$(grep ^DISTRIB_CODENAME= /etc/lsb-release)
      ThisOSDistro=${ThisOSDistro#*=}

      [[ -n "$ThisOSName" && -n "$ThisOSDistro" ]] || \
         warnmsg "Could not determine OS Name and Distro from contents of /etc/lsb-release."
   fi
fi

if [[ "$ThisUser" != root ]]; then
   bail "Run as root, not $ThisUser."
else
   msg "Verified: Running as root user."
fi

#------------------------------------------------------------------------------
# Verify architecture support.
if [[ "$RunArchList" =~ $ThisArch ]]; then
   msg "Verified: Running on a supported architecture [$ThisArch]."
   case $ThisOS in
      (Darwin)
         [[ "$RunGroup" == Unset ]] && RunGroup=staff
         SampleDepotTar=sampledepot.mac.tar.gz
      ;;
      (Linux)
         # Set a platform-specific value for RunGroup if it wasn't defined
         # explicitly in a sdp_install.cfg file.
         if [[ "$RunGroup" == Unset ]]; then
            if [[ -r "/etc/SuSE-release" ]]; then
               RunGroup=users
            else
               # CentOS, RHEL, and Ubuntu default group is same as user name.
               RunGroup=perforce
            fi
         fi
         SampleDepotTar=sampledepot.tar.gz
      ;;
      (*) bail "Unsupported value returned by 'uname -s': $ThisOS. Aborting.";;
   esac
else
   bail "Running on architecture $ThisArch.  Run this only on hosts with one of these architectures: $RunArchList. Aborting."
fi

#------------------------------------------------------------------------------
# Extract the tar version number (e.g., 1.34).
TarVersion="$(tar --version | head -n 1 | awk '{print $4}')"
dbg "TarVersion is: $TarVersion"

#------------------------------------------------------------------------------
# Verify key storage paths.
if [[ "$ServerType" == p4d_* ]]; then
   msg "\\nVerifying storage mounts."

   if [[ "$DemoInstall" -eq 0 ]]; then
      DirList="/$HxDepots"
      [[ "$HxDepots" == "$HxCheckpoints" ]] || DirList+=" /$HxCheckpoints"
      [[ "$HxDepots" == "$HxCheckpoints" ]] || DirList+=" /$HxCheckpoints"
      DirList+=" /$HxLogs"
      DirList+=" /$HxMetadata1"
      [[ "$HxMetadata1" == "$HxMetadata2" ]] || DirList+=" /$HxMetadata2"

      for d in $DirList; do
         if [[ -d "$d" ]]; then
            device=$(df --output=target "$d/" | tail -1)
            if [[ "$device" != / ]]; then
               dbg "Verified: $d is on a mounted volume."
            else
               errmsg "Directory $d is on the OS root volume, NOT on a mounted volume as expected."
            fi
         else
            errmsg "Directory $d should be a mounted volume, but is not a directory."
         fi
      done

      if [[ "$ErrorCount" -eq 0 ]]; then
         msg "\\nStorage mounts verified for all checked directories."
      else
         bail "Aborting due to failed storage configuration checks. If this install is not for production, you can bypass this check with '-demo'."
      fi
   else
      msg "Skipping volume mount checks due to '-demo'."
   fi
fi

#------------------------------------------------------------------------------
# For '-local' mode, do extra pre-flight checks.
if [[ "$PullFromWebAsNeeded" -eq 0 ]]; then
   msg "\\nFor -local mode, scanning to ensure certain SDP directories exist prior to install."

   # If in '-local' mode only, the /opt/perforce/helix-sdp directory is allowed to exist
   # prior to starting, and must contain the helix-sdp and downloads directories. The
   # other directories to be installed must not yet exist.
   if [[ -d "$DownloadsDir" ]]; then
      msg "Verified: The downloads directory [$DownloadsDir] exists as required when '-local' is specified.."
   else
      errmsg "The downloads directory [$DownloadsDir] must exist prior to install when '-local' is specified. Aborting."
   fi

   if [[ -d "$LocalInstallBinDir" ]]; then
      msg "Verified: The helix_binaries directory [$LocalInstallBinDir] exists as required when '-local' is specified.."
   else
      errmsg "The helix_binaries directory [$LocalInstallBinDir] must exist prior to install when '-local' is specified. Aborting."
   fi

   if [[ "$ErrorCount" -eq 0 ]]; then
      msg "\\nVerified: For '-local' mode, found all directories required."
   else
      bail "Aborting because one or more required directories for '-local' do not exist."
   fi
fi

#------------------------------------------------------------------------------
# Safety check: Ensure no SDP data exists prior to install.
msg "\\nScanning to ensure no SDP data or package structure directories exist before install."

DirList="$SDPInstallRoot /$HxDepots/p4"

if [[ "$PullFromWebAsNeeded" -eq 1 ]]; then
   # For a new install where we can pull files from the web, the /opt/perforce/helix-sdp
   # directory must not exist prior to starting.
   DirList+=" $SDPPackageBase"
else
   DirList+=" $ImmutableSDPDir $WritableSDPDir"
fi

[[ "$HxDepots" == "$HxCheckpoints" ]] || DirList+=" /$HxCheckpoints/p4"
DirList+=" /$HxMetadata1/p4"
[[ "$HxMetadata1" == "$HxMetadata2" ]] || DirList+=" /$HxMetadata2/p4"
DirList+=" /$HxLogs/p4"

for d in $DirList; do
   if [[ -d "$d" ]]; then
      errmsg "SDP directory unexpectedly exists prior to SDP install: $d"
   else
      dbg "Verified: SDP directory does not exist prior to install [$d]."
   fi
done

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\nVerified: Scans for pre-existing SDP data and structure came back clean."
else
   bail "Aborting because one or more directories exist that must not prior to install."
fi

#------------------------------------------------------------------------------
# Set hostname and timezone.
if [[ -n "$Hostname" ]]; then
   if command -v hostnamectl > /dev/null; then
      Cmd="hostnamectl set-hostname $Hostname"
      if run "$Cmd" "Setting hostname to: $Hostname"; then
         msg "Hostname set to $(hostname); short hostname is $(hostname -s)."
      else
         errmsg "Failed to set hostname with: hostnamectl set-hostname $Hostname";
      fi
   else
      errmsg "Not setting hostname due to lack of 'hostnamectl' utility."
   fi
else
   dbg "Not configured to set hostname, so skipping call to hostnamectl."
fi

if [[ -n "$Timezone" ]]; then
   if command -v timedatectl > /dev/null; then
      Cmd="timedatectl set-timezone $Timezone"
      if run "$Cmd" "Setting timezone to: $Timezone."; then
         msg "Timezone is set. Date is: $(date)"
      else
         errmsg "Failed to set timezone with: $Cmd";
      fi
   else
      errmsg "Not setting timezone due to lack of 'timedatectl' utility."
   fi
else
   dbg "Not configured to set time zone, so skipping call to timedatectl."
fi

#------------------------------------------------------------------------------
# Add 'perforce' or similar OS group.

if command -v getent > /dev/null; then
   if getent group "$RunGroup" > /dev/null 2>&1; then
      msg "Verified: Group $RunGroup exists."
   else
      run "groupadd $RunGroup" "Creating group $RunGroup." ||\
         bail "Failed to create group $RunGroup."
   fi
fi

#------------------------------------------------------------------------------
# Create local OSUSER (default 'perforce') account.

if getent passwd "$RunUser" > /dev/null; then
   msg "Verified: User $RunUser exists."
else
   if command -v useradd > /dev/null; then
      # If additional groups are defined, do a pre-flight to ensure those
      # users exist.
      if [[ -n "${Config['OSUSER_ADDITIONAL_GROUPS']}" ]]; then
         for g in ${Config['OSUSER_ADDITIONAL_GROUPS']}; do
            if getent group "$g" > /dev/null 2>&1; then
               msg "Verified: OSUSER_ADDITIONAL_GROUPS group $g exists."
            else
               bail "OSUSER_ADDITIONAL_GROUPS group $g does not exist. Groups defined in OSUSER_ADDITIONAL_GROUPS must already exist before calling $ThisScript."
            fi
         done
      fi

      UserAddCmd="useradd -s /bin/bash -g $RunGroup"

      # Specify the home dir only if explicitly defined in sdp_install.cfg;
      # otherwise defer to the useradd default.
      [[ "$RunUserNewHomeDir" != Unset ]] && \
         UserAddCmd+=" -d $RunUserNewHomeDir"

      # Specify the -G value to useradd if and only if values for additional
      # groups were defined in sdp_install.cfg.
      [[ -n "${Config['OSUSER_ADDITIONAL_GROUPS']}" ]] && \
         UserAddCmd+=" -G ${Config['OSUSER_ADDITIONAL_GROUPS']/ /,}"

      UserAddCmd+=" $RunUser"
      run "$UserAddCmd" "Creating user $RunUser with command: $UserAddCmd" ||\
         bail "Failed to create user $RunUser."

      msg "Setting OS password for user $RunUser."
      echo "${Config['Password']}" > "$TmpFile"
      echo "${Config['Password']}" >> "$TmpFile"

      msg "Running: passwd $RunUser"
      if [[ "$NoOp" -eq 0 ]]; then
         if passwd "$RunUser" < "$TmpFile"; then
            msg "Verified: Password for user $RunUser set successfully."
         else
            warnmsg "Failed to set password for user $RunUser."
         fi
      else
         msg "NO_OP: Password would have been set for user $RunUser."
      fi
      run "rm -f $TmpFile"

      RunUserHomeDir="$(eval echo ~"$RunUser" 2>/dev/null)"
      if [[ -d "$RunUserHomeDir" ]]; then
         msg "Verified: Home directory exists for user $RunUser."
      else
         if [[ -n "$RunUserHomeDir" ]]; then
            run "mkdir -p $RunUserHomeDir" \
               "Creating home dir for $RunUser." ||\
               bail "Failed to create home directory $RunUserHomeDir for OS user $RunUser."
            run "chown -R $RunUser:$RunGroup $RunUserHomeDir" \
               "Ensuring $RunUser owns home dir $RunUserHomeDir." ||\
               errmsg "Failed to change ownership of home directory $RunUserHomeDir for OS user $RunUser."
         else
            errmsg "Could not determine home dir for user $RunUser with: eval echo ~\"$RunUser\""
         fi
      fi
   else
      bail "User $RunUser does not exist, and the 'useradd' utility was not found. Aborting."
   fi
fi

#------------------------------------------------------------------------------
# Create /opt/perforce/helix-sdp

if [[ "$PullFromWebAsNeeded" -eq 1 ]]; then
   if [[ -d "$PerforcePackageBase" ]]; then
      dbg "Verified: The Perforce Package Base directory exists: $PerforcePackageBase"
   else
      run "mkdir -p $PerforcePackageBase" "Ensuring Perforce Package Base directory exists: $PerforcePackageBase" ||\
         bail "Could not do: mkdir -p $SDPPackageBase"
   fi

   if [[ -d "$SDPPackageBase" ]]; then
      dbg "Verified: The SDP Package Base directory exists: $PerforcePackageBase"
   else
      run "mkdir -p $SDPPackageBase" "Ensuring SDP Package Base directory exists: $SDPPackageBase" ||\
         bail "Could not do: mkdir -p $SDPPackageBase"
   fi
fi

run "chmod 755 $PerforcePackageBase" \
   "Ensuring 755 perms for $PerforcePackageBase." ||\
   bail" Could not do: chmod 755 $PerforcePackageBase"

run "chmod 755 $SDPPackageBase" \
   "Ensuring 755 perms for $SDPPackageBase." ||\
   bail" Could not do: chmod 755 $SDPPackageBase"

#------------------------------------------------------------------------------
# Ensure proper ownership and permissions.
run "chown root $SDPPackageBase" "Setting owner for SDP Package Base directory to root." ||\
   bail "Could not do: chown root $SDPPackageBase"
run "chgrp $RunGroup $SDPPackageBase" "Setting group for SDP Package Base directory to $RunGroup." ||\
   bail "Could not do: chgrp $RunGroup $SDPPackageBase"
run "chmod 775 $SDPPackageBase" "Setting 775 perms for SDP Package Base directory." ||\
   bail "Could not do: chmod 775 $SDPPackageBase"

#------------------------------------------------------------------------------
# Create /opt/perforce/helix-sdp/downloads
if [[ "$PullFromWebAsNeeded" -eq 1 ]]; then
   run "mkdir -p $DownloadsDir" "Making SDP Downloads directory: $DownloadsDir" ||\
      bail "Could not do: mkdir -p $DownloadsDir"
fi

if [[ "$NoOp" -eq 0 ]]; then
   cd "$DownloadsDir" || bail "Could not cd to downloads dir: $DownloadsDir"

   msg "Operating in: $PWD"
else
   msg "NO_OP: Would be operating in: $DownloadsDir"
fi

if [[ "$SDPInstallMethod" == FTP ]]; then
   if [[ "$PullFromWebAsNeeded" -eq 1 ]]; then
      run "curl -L -s -O $SDPURL" ||\
         bail "Could not get SDP tar file from [$SDPURL]."
   else
      if [[ -r "$DownloadsDir/$SDPTar" ]]; then
         msg "In '-local' mode, using SDP tarball: $DownloadsDir/$SDPTar"
      elif [[ -r "$DownloadsDir/sdp/Version" ]]; then
         msg "In '-local' mode, using SDP directory: $DownloadsDir/sdp"
         SDPInstallMethod=Copy
         SDPLocalCopySource="$DownloadsDir/sdp"
      else
         bail "In '-local' mode, no SDP install tarball or directory found. Checked $DownloadsDir/$SDPTar and $DownloadsDir/sdp."
      fi
   fi
fi

if [[ "$LoadSampleDepot" -eq 1 ]]; then
   if [[ "$PullFromWebAsNeeded" -eq 1 ]]; then
      run "curl -s -O $FTPURL/tools/$SampleDepotTar" ||\
         bail "Could not get file [$SampleDepotTar]. Aborting."
   else
      if [[ -r "$DownloadsDir/$SampleDepotTar" ]]; then
         msg "In '-local' mode, using Sample Depot: $DownloadsDir/$SampleDepotTar"
      fi
   fi

   if [[ ! -d PerforceSample ]]; then
      # shellcheck disable=SC2072
      if [[ "$TarVersion" > "1.29" ]]; then
         run "tar --warning=no-file-removed -xzpf $SampleDepotTar" \
            "Unpacking $SampleDepotTar in $PWD." ||\
            errmsg "Error unpacking tarball $SampleDepotTar"
      else
         run "tar -xzpf $SampleDepotTar" "Unpacking $SampleDepotTar in $PWD." ||\
            errmsg "Error unpacking tarball $SampleDepotTar"
      fi
   else
      msg "Using existing extracted Sample Depot dir $PWD/PerforceSample."
   fi
else
   dbg "Skipping download of Sample Depot; '-sampledepot' was not specified."
fi

run "chown -R $RunUser:$RunGroup $DownloadsDir" \
   "Setting ownership on downloads dir." ||\
   bail "Failed to set ownership on downloads dir [$DownloadsDir]. Aborting."

#------------------------------------------------------------------------------
# SDP Package Directory Structure Setup
if [[ "$NoOp" -eq 0 ]]; then
   cd "$SDPPackageBase" || bail "Could not do: cd $SDPPackageBase"
   msg "Operating in: $PWD"
else
   msg "NO_OP: Would be operating in: $SDPPackageBase"
fi

#------------------------------------------------------------------------------
# Extract tarball to create root-owned /opt/perforce/helix-sdp/sdp
if [[ "$SDPInstallMethod" == FTP ]]; then
   # If tar version is 1.30 or later, use --warning to selectively suppress warnings.
   # shellcheck disable=SC2072
   if [[ "$TarVersion" > "1.29" ]]; then
      run "tar --warning=no-file-removed -xzpf $DownloadsDir/$SDPTar" \
         "Extracting from downloaded SDP tarball $DownloadsDir/$SDPTar in $SDPPackageBase." ||\
         bail "Failed to untar SDP tarball."
   else
      run "tar -xzpf $DownloadsDir/$SDPTar" \
         "Extracting from downloaded SDP tarball $DownloadsDir/$SDPTar in $SDPPackageBase." ||\
         bail "Failed to untar SDP tarball."
   fi
elif [[ "$SDPInstallMethod" == Copy ]]; then
   if [[ -r "$SDPLocalCopySource/$SDPTar" ]]; then
      # shellcheck disable=SC2072
      if [[ "$TarVersion" > "1.29" ]]; then
         run "tar --warning=no-file-removed -xzf $SDPLocalCopySource/$SDPTar" \
            "Extracting from local SDP tarball: $SDPLocalCopySource/$SDPTar" ||\
             bail "Failed to extract SDP tarball."
      else
         run "tar -xzf $SDPLocalCopySource/$SDPTar" \
            "Extracting from local SDP tarball: $SDPLocalCopySource/$SDPTar" ||\
             bail "Failed to extract SDP tarball."
      fi
   elif [[ -r "$SDPLocalCopySource/Version" ]]; then
      run "rsync -a $SDPLocalCopySource/ $ImmutableSDPDir" "Deploying SDP via local copy from: $SDPLocalCopySource" ||\
         bail "Failed to rsync SDP from $SDPLocalCopySource."
   else
      bail "The SDP directory [$SDPLocalCopySource] contains neither an SDP tarball file ($SDPTar) nor a Version file to indicate a pre-extracted SDP tarball. Aborting."
   fi
else
   bail "Unknown SDPInstallMethod value [$SDPInstallMethod]."
fi

run "chown -R root:root $ImmutableSDPDir" "Setting owner:group for Immutable SDP directory to root:root." ||\
   bail "Could not do: chown root:root $ImmutableSDPDir"
run "chmod 755 $ImmutableSDPDir" "Setting 755 perms for Immutable SDP directory." ||\
   bail "Could not do: chmod 755 $ImmutableSDPDir"

#------------------------------------------------------------------------------
# Copy immutable root-owned immutable SDP dir to as-deployed structure, i.e. in
# /opt/perforce/helix-sdp from 'sdp' -> 'p4/sdp'.
run "mkdir $SDPPackageBase/p4" || bail "Could not do: mkdir $SDPPackageBase/p4"
run "rsync -a $ImmutableSDPDir/ $WritableSDPDir"

if [[ "$NoOp" -eq 0 ]]; then
   if [[ -r "$WritableSDPDir/Version" ]]; then
      msg "SDP from $WritableSDPDir/Version is: $(cat "$WritableSDPDir/Version")"
   else
      bail "Cannot determine SDP Version; file $WritableSDPDir/Version is missing."
   fi
else
   msg "NO_OP: Would have determined SDP version from SDP version file: $WritableSDPDir/Version."
fi

#------------------------------------------------------------------------------
# Permissions adjustments.

DirList="$DownloadsDir $BinDir $WritableP4Dir"

for d in $DirList; do
   if [[ -d "$d" ]]; then
      run "chown -R $RunUser:$RunGroup $d" \
         "Adjusting ownership of $d to $RunUser:$RunGroup." ||\
          bail "Failed to adjust ownership of $d."
   fi
done

#------------------------------------------------------------------------------
# Call mkdirs.sh after basic setup.
if [[ "$NoOp" -eq 0 ]]; then
   cd "$SDPUnixSetupDir" || bail "Could not do: cd $SDPUnixSetupDir"
   msg "Operating in: $PWD"
else
   msg "NO_OP: Would be operating in: $SDPUnixSetupDir"
fi

SDPInstanceMkdirsCfg="mkdirs.${SDPInstance}.cfg"
msg "\\nGenerating $SDPInstanceMkdirsCfg."

if [[ "$NoOp" -eq 0 ]]; then
   if sed \
      -e "s|P4MASTERHOST=localhost|P4MASTERHOST=${Config['DNS_name_of_master_server']}|g" \
      -e "s|=DNS_name_of_master_server|=${Config['DNS_name_of_master_server']}|g" \
      -e "s|^MAILTO=.*|MAILTO=${Config['P4AdminList']}|g" \
      -e "s|^MAILFROM=.*|MAILFROM=${Config['MailFrom']}|g" \
      -e "s|mail.example.com|${Config['SMTPServer']}|g" \
      -e "s|^CASE_SENSITIVE=.*|CASE_SENSITIVE=${Config['CaseSensitive']}|g" \
      -e "s|^DB1=.*|DB1=${HxMetadata1}|g" \
      -e "s|^DB2=.*|DB2=${HxMetadata2}|g" \
      -e "s|^DD=.*|DD=${HxDepots}|g" \
      -e "s|^CD=.*|CD=${HxCheckpoints}|g" \
      -e "s|^LG=.*|LG=${HxLogs}|g" \
      -e "s|^P4_PORT=.*|P4_PORT=SeeBelow|g" \
      -e "s|^P4BROKER_PORT=.*|P4BROKER_PORT=SeeBelow|g" \
      -e "s|^P4P_TARGET_PORT=.*|P4P_TARGET_PORT=$TargetPort|g" \
      -e "s|# P4_PORT=1666|P4_PORT=${Config['P4_PORT']}|g" \
      -e "s|# P4BROKER_PORT=1667|P4BROKER_PORT=${Config['P4BROKER_PORT']}|g" \
      -e "s|=adminpass|=${Config['Password']}|g" \
      -e "s|=GudL0ngAdminP@sswerd|=${Config['Password']}|g" \
      -e "s|=servicepass|=${Config['Password']}|g" \
      -e "s|ADMINUSER=perforce|ADMINUSER=${Config['P4USER']}|g" \
      -e "s|OSUSER=perforce|OSUSER=$RunUser|g" \
      -e "s|OSGROUP=perforce|OSGROUP=$RunGroup|g" \
      -e "s|REPLICA_ID=replica|REPLICA_ID=p4d_ha_${Config['SiteTag']}|g" \
      -e "s|SVCUSER=service|SVCUSER=svc_p4d_ha_${Config['SiteTag']}|g" \
      -e "s|^SSL_PREFIX=.*|SSL_PREFIX=$SSLPrefix|g" \
      mkdirs.cfg > "$SDPInstanceMkdirsCfg"; then

      # Make additional tweaks in the generated file.
      # Change SERVER_TYPE setting in mkdirs.N.cfg.
      if [[ -n "$ServerType" ]]; then
         sed -i -E -e "s|^SERVER_TYPE=.*|SERVER_TYPE=$ServerType|g" "$SDPInstanceMkdirsCfg" ||\
            errmsg "Could not update SERVER_TYPE in $SDPInstanceMkdirsCfg."

         # Change either MASTER_ID or REPLICA_ID depending on the server type we are creating.
         if [[ -n "$ServerID" ]]; then
            if [[ "$ServerType" == p4d_commit || "$ServerType" == p4d_master ]]; then
               sed -i -E -e "s|^MASTER_ID=.*|MASTER_ID=$ServerID|g" "$SDPInstanceMkdirsCfg" ||\
                  errmsg "Could not update MASTER_ID in $SDPInstanceMkdirsCfg."
            else
               sed -i -E -e "s|^REPLICA_ID=.*|REPLICA_ID=$ServerID|g" "$SDPInstanceMkdirsCfg" ||\
                  errmsg "Could not update REPLICA_ID in $SDPInstanceMkdirsCfg."
            fi
         fi
      fi

      msg "Generated $SDPInstanceMkdirsCfg."
   else
      bail "Failed to generate $SDPInstanceMkdirsCfg"
   fi
else
   msg "NO_OP: Would have generated $SDPInstanceMkdirsCfg"
fi

msg "\\nSDP Localizations in mkdirs.cfg:"
run "diff mkdirs.cfg $SDPInstanceMkdirsCfg" \
   "Diffs between mkdirs.cfg (sample/template) and $SDPInstanceMkdirsCfg"

# Build the command to mkdirs.sh
MkdirsCmd="./mkdirs.sh $SDPInstance -f -r ${Config['P4BinRel']}"

[[ "$Debug" -eq 0 ]] || MkdirsCmd+=" -d"
[[ "$ExtremeDebug" -eq 0 ]] || MkdirsCmd+=" -D"
[[ -n "$TargetServerID" ]] && MkdirsCmd+=" -S $TargetServerID"

# The mkdir.sh doesn't do sudo by default. We pass in '-limited_sudo' for limited
# sudoers, and '-full_sudo' for full sudoers.
if [[ "$DoSudo" -eq 1 ]]; then
   if [[ "$LimitedSudoers" -eq 1 ]]; then
      MkdirsCmd+=" -ls"
   else
      MkdirsCmd+=" -fs"
   fi
fi

# If this install_sdp.sh script was called with '-no_systemd', pass that option
# thru to the mkdirs.sh script.
if [[ "$UseSystemd" -eq 0 ]]; then
   MkdirsCmd+=" -no_systemd"
fi

# If this install_sdp.sh script was called with '-no_enable', pass that option
# thru to the mkdirs.sh script.
if [[ "$EnableSystemdServices" -eq 0 ]]; then
   MkdirsCmd+=" -no_enable"
fi

# If this install_sdp.sh script was called with '-no_cron', pass that option
# thru to the mkdirs.sh script.
if [[ "$LoadActiveCrontab" -eq 0 ]]; then
   MkdirsCmd+=" -no_cron"
fi

# If this install_sdp.sh script was called with '-no_firewall', pass that option
# thru to the mkdirs.sh script.
if [[ "$DoFirewall" -eq 0 ]]; then
   MkdirsCmd+=" -no_firewall"
fi

log="$PWD/mkdirs.${SDPInstance}.log"
msg "Initializing SDP instance [$SDPInstance], writing log [$log]."
if [[ "$NoOp" -eq 0 ]]; then
   msg "Calling: $MkdirsCmd"
   if $MkdirsCmd > "$log" 2>&1; then
      cat "$log"
      msg "\\nVerified: mkdirs.sh was successful."
   else
      cat "$log"
      bail "Aborting because the called script mkdirs.sh was NOT successful. Review the output above."
   fi
else
   msg "NO_OP: Would have run: $MkdirsCmd"
fi

msg "Apply custom proxy/broker-only host rules."
iCfg="$SDPInstallRoot/common/config/p4_${SDPInstance}.vars"
if [[ -r "$iCfg" ]]; then
   iCfgTmp="$(mktemp)"
   if [[ -n "$TargetPort" && "$ServerType" == "p4p"* ]]; then
      sed -e "s|^export PROXY_TARGET=.*|export PROXY_TARGET=$TargetPort|g" "$iCfg" > "$iCfgTmp"
      run "mv -f $iCfgTmp $iCfg" "Updating $iCfg." || errmsg "Could not deploy $iCfg."
   fi
   if [[ -n "$TargetPort" && "$ServerType" == "p4broker" ]]; then
      sed -e "s|^export P4PORT=.*|export P4PORT=$TargetPort|g" "$iCfg" > "$iCfgTmp"
      run "mv -f $iCfgTmp $iCfg" "Updating $iCfg." || errmsg "Could not deploy $iCfg."
   fi
   if [[ -n "$ListenPort" && "$ServerType" == "p4p"*  ]]; then
      sed -e "s|^export PROXY_PORT=.*|export PROXY_PORT=$ListenPort|g" \
         -e "s|^export P4PORT=.*|export P4PORT=$ListenPort|g" \
         "$iCfg" > "$iCfgTmp"
      run "mv -f $iCfgTmp $iCfg" "Updating $iCfg." || errmsg "Could not deploy $iCfg."
   fi
   if [[ -n "$ListenPort" && "$ServerType" == "p4broker" ]]; then
      sed -e "s|^export P4BROKERPORT=.*|export P4BROKERPORT=$ListenPort|g" \
         "$iCfg" > "$iCfgTmp"
      run "mv -f $iCfgTmp $iCfg" "Updating $iCfg." || errmsg "Could not deploy $iCfg."
   fi
   run "chown $RunUser:$RunGroup $iCfg" "Adjusting ownership of $iCfg."
fi

if [[ "$UseBroker" -eq 1 ]]; then
   msg "\\nGenerating broker config for instance $SDPInstance.\\n"
   if [[ "$NoOp" -eq 0 ]]; then
      su -l "$RunUser" -c "$CBIN/gen_default_broker_cfg.sh ${SDPInstance} > $CCFG/p4_${SDPInstance}.broker.cfg"
   else
      msg "NO_OP: Would have generated broker with: $CBIN/gen_default_broker_cfg.sh ${SDPInstance} > $CCFG/p4_${SDPInstance}.broker.cfg"
   fi
fi

if [[ -n "$SSLPrefix" ]]; then
   msg "Generating SSL config file for self-signed certificate."
   if [[ "$NoOp" -eq 0 ]]; then
      sed -e "s|REPL_DNSNAME|helix|g" "$SSLConfig" > "$TmpFile" ||\
         bail "Failed to substitute content in $SSLConfig."
   else
      msg "NO_OP: Would have generated SSL config from $SSLConfig."
   fi
   run "mv -f $TmpFile $SSLConfig"

   if [[ "$NoOp" -eq 0 ]]; then
      msg "Contents of $SSLConfig:\\n$(cat "$SSLConfig")\\n"
   else
      msg "NO_OP: Would display contents of generated SSL config file: $SSLConfig\\n"
   fi

   if [[ -x "$ServerBin" ]]; then
      msg "Generating SSL certificates."
      # If there are multiple SDP Instances, we only generate SSL certificates
      # for one instance.  Any one will do because SSL certs are not
      # instance-specific.
      if [[ "$NoOp" -eq 0 ]]; then
         su -l "$RunUser" -c "$SDPInstallRoot/common/bin/p4master_run $SDPInstance $ServerBin -Gc" ||\
            warnmsg "Failed to generate SSL Certificates."
      else
         msg "NO_OP: Would have generate SSL certs with: $SDPInstallRoot/common/bin/p4master_run $SDPInstance $ServerBin -Gc"
      fi
   fi
fi

#------------------------------------------------------------------------------
# To simulate email, generate a faux mail utility in the PATH.
if [[ "$SimulateEmail" -eq 1 ]]; then
   msg "Generating mail simulator."
   if [[ ! -d "$SiteBinDir" ]]; then
      run "mkdir -p $SiteBinDir" "Creating SiteBin dir: $SiteBinDir" ||\
         warnmsg "Failed to generate SiteBin dir: $SiteBinDir"
   fi

   if [[ "$NoOp" -eq 0 ]]; then
      echo -e "#!/bin/bash\\necho Simulated Email: \$*\\n" > "$MailSimulator"
      chmod +x "$MailSimulator"
      [[ -x "$MailSimulator" ]] || \
         warnmsg "Failed to generate mail simulator script: $MailSimulator"
      run "chown $RunUser:$RunGroup $MailSimulator" "Adjusting ownership of $MailSimulator" || \
         warnmsg "Failed to do: chown \"$RunUser:$RunGroup\" \"$MailSimulator\""
   else
      msg "NO_OP: Would have generated mail simulator script: $MailSimulator"
   fi
fi

#------------------------------------------------------------------------------
# Add Perforce Package Repository to repo list (YUM and APT only).

if [[ "$AddPerforcePackageRepo" -eq 1 ]]; then
   if [[ -d "${P4YumRepo%/*}" ]]; then
      if [[ -n "$ThisOSMajorVersion" ]]; then
         run "rpm --import $PerforcePackagePubkeyURL" \
            "Adding Perforce's packaging key to RPM keyring." ||\
            warnmsg "Failed to add Perforce packaging key to RPM keyring."

         msg "Generating $P4YumRepo."
         if ! echo -e "[perforce]\\nname=Perforce\\nbaseurl=$PerforcePackageRepoURL/yum/rhel/$ThisOSMajorVersion/x86_64\\nenabled=1\\ngpgcheck=1\\n" > "$P4YumRepo"; then
            warnmsg "Unable to generate $P4YumRepo."
         fi
      else
         warnmsg "Skipping generation of $P4YumRepo due to lack of OS Major Version info."
      fi
   elif [[ -d "${P4AptGetRepo%/*}" ]]; then # /etc/apt/sources.list.d
      if [[ -n "$ThisOSName" && -n "$ThisOSDistro" ]]; then
         msg "Acquiring Perforce's package repository public key."
         if [[ "$NoOp" -eq 0 ]]; then
            if wget -qO - "$PerforcePackagePubkeyURL" > $TmpPubKey; then
               msg "Using gpg --dearmor on key for Perforce package repo acquired as $TmpPubKey."
               if gpg --dearmor < "$TmpPubKey" > "$AptKeyRing"; then
                  msg "Adding Perforce's packaging key to APT keyring."
                  if apt-key add < "$TmpPubKey"; then
                     msg "APT keyring added successfully."
                  else
                     warnmsg "Failed to add Perforce packaging key to APT keyring."
                  fi
               else
                  warnmsg "Failed to dearmour apt public key."
               fi

               msg "Doing apt-get update after adding the new perforce.list repo."
               if [[ -n $(command -v sync) ]]; then
                  run "sync" \
                     "Flushing buffers before calling apt-get update." ||\
                     warnmsg "The call to 'sync' to flush buffers returned a non-zero exit code. Ignoring."
               fi

               run "apt-get update --allow-releaseinfo-change" \
                  "Forcing apt to refresh properly." ||\
                  warnmsg "Call to 'apt-get update --allow-releaseinfo-change' returned a non-zero exit code. Ignoring."

               run "sleep 3" \
                  "Waiting a few seconds before calling 'apt-get update' to enhance reliability."

               if run "apt-get update" "Updating apt repository list."; then
                  msg "Update completed."
               else
                  warnmsg "An apt-get update did not return a zero exit code."
               fi
            else
               warnmsg "Failed to acquire Perforce package repo public key."
               InstallOSPackages=0
            fi
         else
            msg "NO_OP: Would have run: wget -qO - $PerforcePackagePubkeyURL > $TmpPubKey"
            msg "Using gpg --dearmor on key for Perforce package repo acquired as $TmpPubKey."
            msg "NO_OP: Would have run: gpg --dearmor < $TmpPubKey > $AptKeyRing"
            msg "Adding Perforce's packaging key to APT keyring."
            msg "NO_OP: Would have run: apt-key add < $TmpPubKey"
            msg "APT keyring added successfully."
         fi

         msg "Generating $P4AptGetRepo."
         if [[ "$NoOp" -eq 0 ]]; then
            if ! echo "deb [signed-by=$AptKeyRing] $PerforcePackageRepoURL/apt/$ThisOSName $ThisOSDistro release" > "$P4AptGetRepo"; then
               warnmsg "Unable to generate $P4AptGetRepo."
               InstallOSPackages=0
            fi
         else
            msg "NO_OP: Would have run: echo \"deb [signed-by=$AptKeyRing] $PerforcePackageRepoURL/apt/$ThisOSName $ThisOSDistro release\" > $P4AptGetRepo"
         fi
      else
         warnmsg "Skipping generation of $P4AptGetRepo due to lack of OS Name and Distro info."
         InstallOSPackages=0
      fi
   else
      warnmsg "No Perforce supported package repository, RPM or APT, found to add. Skipping."
      InstallOSPackages=0
   fi

   if [[ "$InstallOSPackages" -eq 1 && "$PackageManager" != zypper ]]; then
      msg "Adding packages from the Perforce Package Repository."

      run "$PackageManager install -y ${ExtraP4PackageList[$PackageManager]} < /dev/null" \
         "Installing these packages with $PackageManager: ${PackageList[$PackageManager]}" ||\
         warnmsg "Not all Perforce packages installed successfully.  Proceeding."
   fi
else
   msg "Skipping addition of Perforce Package repository due to '-no_ppr' and/or '-local'."
fi

#------------------------------------------------------------------------------
# Install the standard ~perforce/.bash_profile and ~perforce/.bashrc files.

RunUserHomeDir="$(eval echo ~"$RunUser" 2>/dev/null)"
if [[ -d "$RunUserHomeDir" ]]; then
   BackupTimestamp="$(date +'%Y-%m-%d-%H%M%S')"
   if [[ -e "$RunUserHomeDir/.bash_profile" ]]; then
      run "mv -f $RunUserHomeDir/.bash_profile $RunUserHomeDir/.bash_profile.bak.$BackupTimestamp" \
         "Backing up $RunUserHomeDir/.bash_profile to $RunUserHomeDir/.bash_profile.bak.$BackupTimestamp" ||\
         errmsg "Failed to backup $RunUserHomeDir/.bash_profile to $RunUserHomeDir/.bash_profile.bak.$BackupTimestamp"
   fi

   run "cp $SDPUnixSetupDir/bash/perforce_bash_profile $RunUserHomeDir/.bash_profile" \
      "Creating $RunUserHomeDir/.bash_profile." ||\
      errmsg "Failed to copy to $RunUserHomeDir/.bash_profile."

   if [[ -e "$RunUserHomeDir/.bashrc" ]]; then
      run "mv -f $RunUserHomeDir/.bashrc $RunUserHomeDir/.bashrc.bak.$BackupTimestamp" \
         "Backing up $RunUserHomeDir/.bashrc to $RunUserHomeDir/.bashrc.bak.$BackupTimestamp" ||\
         errmsg "Failed to backup $RunUserHomeDir/.bashrc to $RunUserHomeDir/.bashrc.bak.$BackupTimestamp"
   fi

   msg "Creating $RunUserHomeDir/.bashrc."
   if [[ "$NoOp" -eq 0 ]]; then
      sed "s|EDITME_SDP_INSTANCE|$SDPInstance|g" \
         "$SDPUnixSetupDir/bash/perforce_bashrc" > "$RunUserHomeDir/.bashrc" ||\
         errmsg "Failed to copy to $RunUserHomeDir/.bashrc."
   else
      msg "NO_OP: Would have generated $RunUserHomeDir/.bashrc with: sed \"s|EDITME_SDP_INSTANCE|$SDPInstance|g\" $SDPUnixSetupDir/bash/perforce_bashrc > $RunUserHomeDir/.bashrc"
   fi

   if [[ -e "$RunUserHomeDir/.p4aliases" ]]; then
      run "mv -f $RunUserHomeDir/.p4aliases $RunUserHomeDir/.p4aliases.bak.$BackupTimestamp" \
         "Backing up $RunUserHomeDir/.p4aliases to $RunUserHomeDir/.p4aliases.bak.$BackupTimestamp" ||\
         errmsg "Failed to backup $RunUserHomeDir/.p4aliases to $RunUserHomeDir/.p4aliases.bak.$BackupTimestamp"
   fi

   run "cp $SDPUnixSetupDir/bash/perforce_p4aliases $RunUserHomeDir/.p4aliases" \
      "Creating $RunUserHomeDir/.p4aliases." ||\
      errmsg "Failed to copy to $RunUserHomeDir/.p4aliases."

   run "chown $RunUser:$RunGroup $(eval echo ~"$RunUser")/.bash_profile $(eval echo ~"$RunUser")/.bashrc $(eval echo ~"$RunUser")/.p4aliases" "Adjusting ownership of ~$RunUser/.{bash_profile,.bashrc,.p4aliases}." ||\
      errmsg "Adjusting ownership of ~$RunUser/{.bash_profile,.bashrc,.p4aliases} failed."
fi

#------------------------------------------------------------------------------
# Handle '-init', '-empty', and '-sampledepot' options.
if [[ "$ServerType" == p4d_commit || "$ServerType" == p4d_master ]]; then
   #------------------------------------------------------------------------------
   # Initialize an empty data set.
   if [[ "$InitializeEmptyServer" -eq 1 ]]; then
      msg "Initializing empty data set with SDP best practices for instance $SDPInstance."
      if [[ "$NoOp" -eq 0 ]]; then
         if su -l "$RunUser" -c "$SDPSetupDir/configure_new_server.sh $SDPInstance -checkpoint"; then
            msg "\\nNew server p4d_$SDPInstance initialized.\\n"
         else
            bail "Failed to initialize empty depot."
         fi
      else
         msg "NO_OP: Would have run: su -l $RunUser -c \"$SDPSetupDir/configure_new_server.sh $SDPInstance\""
         msg "\\nNew server p4d_$SDPInstance initialized.\\n"
      fi
   elif [[ "$LoadSampleDepot" -eq 1 ]]; then
   #------------------------------------------------------------------------------
   # Load the Sample Depot training data set, useful for eval/demo installs,
   # PoCs, training, etc.
      msg "Configuring Sample Depot for SDP on Instance $SDPInstance."
      if [[ "$NoOp" -eq 0 ]]; then
         if su -l "$RunUser" -c "$SDPUnixSetupDir/configure_sample_depot_for_sdp.sh -i $SDPInstance -d $SDPUnixSetupDir -u $RunUser $UseSystemdOption"; then
            msg "\\nSample Depot configured successfully for instance $SDPInstance.\\n"
         else
            bail "Failed to load the Sample Depot."
         fi
      else
         msg "NO_OP: Would have run: su -l $RunUser -c \"$SDPUnixSetupDir/configure_sample_depot_for_sdp.sh -i $SDPInstance -d $SDPUnixSetupDir -u $RunUser $UseSystemdOption\""
         msg "\\nSample Depot configured successfully for instance $SDPInstance.\\n"
      fi
   else
      msg "No data loaded because '-empty' was specified."
   fi
else
   dbg "Skipping data initializing for server type $ServerType."
fi

#------------------------------------------------------------------------------
msg "\\nInstall Backup Service for SDP OS Package Dirs"
run "rsync /p4/common/etc/systemd/system/opt_perforce_sdp_backup.service /etc/systemd/system/opt_perforce_sdp_backup.service" \
   "Updating opt_perforce_sdp_backup.service" ||\
   errmsg "Failed to update opt_perforce_sdp_backup.service."
run "rsync /p4/common/etc/systemd/system/opt_perforce_sdp_backup.timer /etc/systemd/system/opt_perforce_sdp_backup.timer" \
   "Updating opt_perforce_sdp_backup.timer" ||\
   errmsg "Failed to update opt_perforce_sdp_backup.timer."

SELinuxMode="$(getenforce 2>/dev/null || sestatus 2>/dev/null | grep -i 'Current mode' | awk '{print $NF}')"
if [[ "$SELinuxMode" =~ ^(Enforcing|Permissive)$ ]]; then
   TmpFile=$(mktemp)
   if [[ -n "$(command -v semanage)" ]]; then
      if run "semanage fcontext -a -t bin_t $BackupScript" \
         "Configuring SELinux for script: $BackupScript" > "$TmpFile" 2>&1; then
         cat "$TmpFile"
         msg "SELinux configured OK for $BackupScript"
      else
         # If we get an error indicating semanage is already defined, suppress that.
         if grep -q -E "ValueError: File context for .* already defined" "$TmpFile"; then
            msg "SELinux configured OK for $BackupScript (it was already defined)."
         else
            cat "$TmpFile"
            errmsg "Failed SELinux addition of backup script: $BackupScript"
         fi
      fi
   else
       warnmsg "SELinux is available but semanage not in PATH. Skipping semanage setup"
   fi

   rm -f "$TmpFile"

   if [[ -n "$(command -v restorecon)" ]]; then
      run "restorecon -vF $BackupScript" ||\
         errmsg "Failed SELinux restorecon of backup script: $BackupScript"
   else
       warnmsg "SELinux is available but restorecon not in PATH. Skipping restorecon setup for: $BackupScript"
   fi
fi

run "systemctl daemon-reload" "Reloading systemd"
run "systemctl enable opt_perforce_sdp_backup.timer" \
   "Enabling opt_perforce_sdp_backup.timer" ||\
   errmsg "Failed to enable opt_perforce_sdp_backup.timer."
run "systemctl start opt_perforce_sdp_backup.timer" \
   "Starting opt_perforce_sdp_backup.timer" ||\
   errmsg "Failed to start opt_perforce_sdp_backup.timer"
run "systemctl start opt_perforce_sdp_backup.service" \
   "Starting opt_perforce_sdp_backup.service" ||\
   errmsg "opt_perforce_sdp_backup.service"
run "systemctl list-timers --all | grep opt_perforce_sdp_backup" \
  "Listing timer opt_perforce_sdp_backup."

#------------------------------------------------------------------------------
# Call verify_sdp.sh if '-v' was specified.
if [[ "$DoVerifySDP" -eq 1 && -x "$VerifySDPScript" ]]; then
   VerifySDPOptions+=" -L off"
   if [[ "$LoadSampleDepot" -eq 1 ]]; then
      VerifySDPOptions+=" -online"
   else
      VerifySDPSkipTests="offline_db,p4root,p4t_files"
   fi

   [[ "$LoadActiveCrontab" -eq 0 ]] && VerifySDPSkipTests+=",cron"
   [[ -n "$VerifySDPSkipTests" ]] && VerifySDPOptions+=" -skip $VerifySDPSkipTests"

   VerifySDPCmd="$VerifySDPScript $SDPInstance $VerifySDPOptions"
   msg "\\nDoing SDP verification for instance $SDPInstance."
   if run "$VerifySDPCmd"; then
      msg "SDP verification for instance $SDPInstance was OK."
   else
      errmsg "SDP verification for instance $SDPInstance reported errors."
   fi
elif [[ "$DoVerifySDP" -eq 1 ]]; then
   errmsg "No $VerifySDPScript script found to execute. Skipping it."
fi

#------------------------------------------------------------------------------
# Now that SDP is installed, display warnings about any utilities that will
# be needed by other SDP scripts.
msg "\\nEnsuring required utilities to operate SDP scripts are available."
for u in $AllRequiredUtils; do
   if command -v "$u" > /dev/null; then
      dbg "Verified: Required utility '$u' found in PATH."
   else
      warnmsg "Missing required utility '$u'.  You may need to install it or adjust the PATH for the root user to find it.\\n\\n"
   fi
done

if [[ "$ErrorCount" -eq 0 ]]; then
   if [[ "$WarningCount" -eq 0 ]]; then
      if [[ "$NoOp" -eq 0 ]]; then
         msg "\\nSUCCESS:  SDP Installation complete with no errors or warnings."
      else
         msg "\\nSUCCESS:  SDP Installation complete in DRY RUN (preview) mode with no errors or warnings.  Run with '-y' to proceed with the real installation."
      fi
   else
      if [[ "$NoOp" -eq 0 ]]; then
         msg "\\nSUCCESS:  SDP Installation complete with $WarningCount warnings."
      else
         msg "\\nSUCCESS:  SDP Installation complete with $WarningCount warnings in DRY RUN (preview) mode.  Run with '-y' to proceed with the real installation."
      fi
   fi
else
   if [[ "$NoOp" -eq 0 ]]; then
      msg "\\nSDP Installation completed, but with $ErrorCount errors and $WarningCount warnings."
   else
      msg "\\nSDP Installation completed, but with $ErrorCount errors and $WarningCount warnings in DRY RUN (preview) mode."
   fi
fi

exit "$ErrorCount"
