reset_sdp.sh #56

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# this package, and also available online:
# https://swarm.workshop.perforce.com/view/guest/perforce_software/helix-installer/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations
set -u
declare Version=3.1.2

# The tar file specified here is used if the '-t' flag is used to
# specify installing from a tar file, which is good for getting a known,
# fixed version of the SDP to work with.  The default install method
# uses Helix DVCS features to get the very latest code from a branch
# ('main' by default, but '-b dev' could be specified as well) by doing
# a 'p4 clone' command.
declare SDPTar="sdp.Unix.2019.1.25238.tgz"
declare SDPURL="https://swarm.workshop.perforce.com/projects/perforce-software-sdp/download/downloads/$SDPTar"
declare SDPInstallMethod=DVCS
declare SDPInstallBranch=Unset
declare WorkshopPort="public.perforce.com:1666"
declare WorkshopUser=sdp_reader
declare WorkshopUserPasswordFile="/tmp/.p4passwd.$WorkshopUser"
declare WorkshopRemote=
declare HelixInstallerBaseURL="https://swarm.workshop.perforce.com/download/guest/perforce_software/helix-installer"
declare HelixInstallerBranch=main
declare HelixInstallerURL=
declare HxDepots=/hxdepots
declare ResetHome="$HxDepots/reset"
declare HxMetadata=/hxmetadata
declare HxLogs=/hxlogs
declare SDPHome="$HxDepots/sdp"
declare TmpFile=/tmp/tmp.reset_sdp.$$.$RANDOM
declare TmpDir=/tmp/tmp.dir.reset_sdp.$$.$RANDOM
declare ShelvedChange=Unset
declare LocalShelvedChange=Unset
declare CmdLine="${0##*/} $*"
declare -i WarningCount=0
declare -i ErrorCount=0
declare InitMechanism=Unset
declare PackageManager=Unset
declare -A PackageList
PackageList['yum']="curl gcc gcc-c++ make openssl openssl-devel rsync wget zlib zlib-devel"
PackageList['apt-get']="build-essential libssl-dev make zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev"
PackageList['zypper']="curl gcc gcc-c++ make openssl openssl-devel wget zlib zlib-devel"

# If using this Helix Installer to bootsrap a production installation,
# please do NOT be tempted to keep this password.  Replace it with your
# own.  And after bootsrapping, be sure to remove the Helix Installer,
# as this software has no business being on a production machine.
declare Password="F@stSCM!"

declare ExesDir=$HxDepots/exes
declare DownloadsDir=$HxDepots/downloads
declare -i BlastDownloadsAndExes=0
declare -i ExtremeCleanup=0
declare -i FastMode=0
declare -i LoadSampleDepot=1
declare -i PullFromWebAsNeeded=1
declare -i UseSSL=1
declare PreserveDirList=Unset
declare P4ExeRel=r18.2
declare P4DExeRel=$P4ExeRel
declare P4PExeRel=$P4ExeRel
declare P4BrokerExeRel=$P4ExeRel
declare P4SwarmRel=r18.3
declare P4Rel=r18.2
declare RunUser=perforce
declare RunUserHomeDir=
declare RunGroup=
declare SampleDepotTar=
declare ApiArch=
declare ThisArch=
declare ThisHost=
declare ThisOS=
declare ThisUser=
declare RunArch="x86_64"
declare CBIN="/p4/common/bin"
declare CCFG="/p4/common/config"
declare SDPSetupDir="$SDPHome/Server/Unix/setup"
declare SDPConfigDir="$SDPHome/Server/Unix/p4/common/config"
declare ThisScript="${0##*/}"
declare SDPInstances="1"

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   msg "USAGE for $ThisScript v$Version:

$ThisScript [-B|-local] [-fast] [-no_ssl] [-no_sd] [-b <branch>[,@cl] |-t <tarfile>] [-p <dir1>[,<dir2>,...]>] [-i <helix_installer_branch>] [-D] [-X]

or

$ThisScript [-h|-man]
"
   if [[ $style == -man ]]; then
      msg "
SAFETY NOTICE:
	This script SHOULD NEVER EXIST on a Production Perforce server.

DESCRIPTION:
	This script simplifies the process of testing an SDP installation,
	repetitively blasting all process by the 'perforce' user and resetting
	the SDP from the ground up, blasting typical SDP folders each time.

	It is helpful when bootstrapping a demo server with a sample data
	set, complete with broker, and optionally Perl/P4Perl and
	Python/P4Python.

PLATFORM SUPPORT:
	This works on Red Hat Enterprise Linux, CentOS, and Mac OSX
	10.10+ thru Mojave platforms.  It works on RHEL/CentOS
	6.4-7.6, SuSE Linux 12, and likely on Ubuntu 18 and other Linux
	distros with little or no modification.

	This script currently supports the bin.linux26x86_64 (Linux) and
	bin.maxosx1010x86_64 (Mac OSX/Darwin) architectures.

	This script recognizes SysV, Systemd, and Launchd init mechanisms,
	though does not currently support Launchd on OSX.

	For Mac OSX, note that this requires bash 4.x, and the default
	bash on Mac OSX remains 3.x as of OSX Mojave.  For operating on
	Mac, the /bin/bash shebang line needs to be adjusted to reference
	a bash 4 version, e.g. /usr/local/bin/bash if installed with
	Homebrew.

REQUIREMENTS:
	Development utilities such as 'make', the 'gcc' compiler,
	and 'curl' must be installed and available in the PATH (unless
	running with '-fast'), or accessible from the OS package manager.

	The following OS packages are installed (unless '-fast' is
	used):

	* Yum: ${PackageList[yum]}

	* AptGet: ${PackageList[apt-get]}

	* Zypper: ${PackageList[zypper]}

OPTIONS:
 -B	Specify '-B' to blast base SDP dirs, for a clean start.

	Otherwise without '-B', downloaded components from earlier
	runs will be used (which should be fine if they were run
	recently).

	The '-B' flag also replaces files in the $ResetHome
	directory, where this script lives, with those downloaded
	from The Workshop (the versions of which are affected
	by the '-i <helix_installer_branch>' flag, described
	below).

	The '-B' flag also blasts the /tmp/downloads and /tmp/p4perl
	directories, used by install_sdp_python.sh and
	install_sdp_perl.sh, if they exist.

	The '-B' and '-local' arguments are mutually exclusive.

 -local
	By default, various files and executables are downloaded from
	the Perforce Workshop or the Perforce FTP server as needed.
	With '-local', any missing files are treated as an error
	condition.

	The '-B' and '-local' arguments are mutually exclusive.

 -fast	Specify '-fast' to skip installation SDP Perl and
	SDP Python, to include P4Perl and P4Python.

	The '-fast' argument changes a check for GCC/G++
	from a fatal error to a warning message.

	The '-fast' option typically takes just a few minutes,
	as compared to 20+ minutes without due to the time
	needed to compile and test the Perl and Python bits.

	The '-fast' flag should not be used if you plan to
	deploy or develop triggers that use P4Python or P4Perl,
	such as the SDP CheckCaseTrigger.py.  Alternately, you
	can run with '-fast', and then afterward run the
	following as the OS user $RunUser:

	cd $SDPSetupDir
	./install_sdp_python.sh
	./install_sdp_perl.sh

 -no_ssl
	By default, the Perforce server is setup SSL-enabled.  Specify
	'-no_ssl' to avoid using SSL feature.

 -no_sd
	By default, the Perforce Sample Depot data set is loaded.
	Specify '-no_sd' to skip loading the Sample Depot.

 -p <dir1>[,<dir2>,...]>]
	Specify a comma-delimited list of directories under /p4/common
	to preserve that would otherwise be removed.  Directories must
	be specified as paths relative to /p4/common, and cannot contain
	spaces.

	For example, the value '-p config,bin/triggers' would preserve the
	/p4/common/config and /p4/common/bin/triggers directories.

	Directories specified are moved aside to a temporary working area
	before the SDP folders are removed.  After installation, they are
	moved back via an 'rsync' command with no '--delete' option.  This
	means any files that overlap with the stock install are replaced
	by ones that originally existed, but non-overlapping files are not
	removed.

	This is intended to be useful for developing test suites that
	install server under /p4/common/bin, e.g. Component Based Development
	scripts which install under /p4/common/bin/cbd would use '-p bin/cbd'.

 -b <branch>[,@cl]
	The default SDP install method is to clone the SDP from the main
	branch in The Workshop ($WorkshopPort).  Specify '-b' to use
	a different branch, e.g. 'dev'.

	If '-b' is specified with the optional @cl syntax, where @cl
	specifies a changelist with files shelved on the given branch,
	a form of unshelving is done, enabling a workflow for testing
	shelved changes with the Helix Installer.  So for example,
	specify '-b dev,@23123' to clone from the dev branch, and then
	followed by a fetch of shelved changelist @23123, which is
	expected to have files shelved in the dev branch.

 -t <tarfile>
	Specify a tarfile to use, e.g. '-t sdp.Unix.2017.3.23041.tgz', or
	use the special value '-t default' to use the default SDP tarfile,
	$SDPTar

	The specified tar file must exist here:
	https://swarm.workshop.perforce.com/projects/perforce-software-sdp/files/downloads

	If '-t <tarfile>' is not specified, default install method is
	used instead, which is to clone the SDP main branch from The
	Workshop ($WorkshopPort).

	Use '-t' to get a known version of the SDP for testing.  Avoid '-t'
	to test with the very latest SDP from a given branch (see '-b' above
	for more info), even if the latest has not yet been packaged into a
	distribution tar file.

DEBUGGING OPTIONS:
 -i	<helix_installer_branch>

	Specify the branch of the Helix Installer to use.  This affects the
	URL from which Helix Installer files in $ResetHome are pulled from
	The Workshop.  The default is main; an alternative is '-i dev'.

 -D     Set extreme debugging verbosity.

 -X	Extreme reset. This removes the user accout for perforce and
	blast all SDP-related directories at the start of script operations.

	Using '-X' does not blast the Helix Installer downloads or exes
	directories, and thus is compatible with either the'-B' or '-local'
	options.


HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	COMPREHENSIVE INSTALLATION:

	su -
    	mkdir -p /hxdepots/reset
	cd /hxdepots/reset
	curl -k -s -O $HelixInstallerURL/src/reset_sdp.sh
	./reset_sdp.sh 2>&1 | tee reset_sdp.log

	FAST INSTALLATION (skipping Perl, Python):

	su -
    	mkdir -p /hxdepots/reset
	cd /hxdepots/reset
	curl -k -s -O $HelixInstallerURL/src/reset_sdp.sh
	curl -k -s -O $HelixInstallerURL/src/r
	./r

	Note that the 'r' wrapper script calls the reset_sdp.sh script with
	a pre-defined of flags optimized for fast opreration.  The 'r' wapper
	also handles log capture, writing to the file 'reset.log'.

	SDP DEV BRANCH TESTING:
	The Helix Installer can be used to test SDP changes shelved to the SDP
	dev branch in The Workshop.  The following example illustrates testing
	a change in shelved changelist 23123:

	su -
    	mkdir -p /hxdepots/reset
	cd /hxdepots/reset
	curl -k -s -O $HelixInstallerURL/src/reset_sdp.sh

	./reset_sdp.sh -b dev,@23123 2>&1 | tee reset_sdp.CL23123.log

	After the first test, an iterative test cycle may follow on the same
	shelved changelist. For each test iteration, the shelved changelist
	is first updated in the workspace from which the change was originally
	shelved, e.g. with a command like 'p4 shelve -f -c 23123'.

	Then a new test can be done by calling reset_sdp.sh with the same
	arguments. The script will re-install the SDP cleanly, and then
	re-apply the updated shelved changelist.

"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Functions msg(), dbg(), and bail().
# Sample Usage:
#    bail "Missing something important. Aborting."
#    bail "Aborting with exit code 3." 3
function msg () { echo -e "$*"; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function dbg () { msg "DEBUG: $*" >&2; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Functions run($cmd, $desc)
#
# This function is similar to functions defined in SDP core libraries, but we
# need to duplicate them here since this script runs before the SDP is
# available on the machine (and we require dependencies for this
# script).
function run {
   cmd="${1:-echo Testing run}"
   desc="${2:-}"
   [[ -n "$desc" ]] && msg "$desc"
   msg "Running: $cmd"
   $cmd
   return $?
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0
set +u

while [[ $# -gt 0 ]]; do
   case $1 in
      (-B) BlastDownloadsAndExes=1;;
      (-local) PullFromWebAsNeeded=0;;
      (-fast) FastMode=1;;
      (-no_ssl) UseSSL=0;;
      (-no_sd) LoadSampleDepot=0;;
      (-t)
         if [[ "$2" != "default" ]]; then
            SDPTar=$2
         fi
         SDPInstallMethod=Tarfile
         shiftArgs=1
      ;;
      (-b)
         if [[ "$2" == *",@"* ]]; then
            SDPInstallBranch=${2%%,@*};
            ShelvedChange=${2##*,@}
         else
            SDPInstallBranch=$2;
         fi
         shiftArgs=1
      ;;
      (-p) PreserveDirList=$2; shiftArgs=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (-X) ExtremeCleanup=1;;
      (-i) HelixInstallerBranch="$2"; shiftArgs=1;;
      (*) bail "Usage Error: Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && bail "Usage Error: Wrong number of args or flags to args."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#------------------------------------------------------------------------------
# Command Line Validation

ThisUser="$(whoami)"
ThisOS="$(uname -s)"
ThisArch="$(uname -m)"
ThisHost="$(hostname -s)"

if [[ $ThisUser != root ]]; then
   bail "Run as root, not $ThisUser."
else
   msg "Verified: Running as root user."
fi

[[ "$BlastDownloadsAndExes" -eq 1 && "$PullFromWebAsNeeded" -eq 0 ]] && \
   bail "The '-B' and '-local' arguments are mutually exclusive."

[[ "$SDPInstallBranch" != "Unset" && "$SDPInstallMethod" == "Tarfile" ]] && \
   bail "The '-b <branch>' and '-t <tarfile>' arguments are mutually exclusive."

[[ $SDPInstallBranch == Unset ]] && SDPInstallBranch=main

HelixInstallerURL="$HelixInstallerBaseURL/$HelixInstallerBranch"

#------------------------------------------------------------------------------
msg "Started $ThisScript v$Version on host $ThisHost at $(date), called as:\\n\\t$CmdLine"

if [[ "$ExtremeCleanup" -eq 1 ]]; then
   msg "\\nStarted Extreme Cleanup due to -X."
   if id -u "$RunUser" > /dev/null 2>&1; then
      RunUserHomeDir="$(eval echo ~$RunUser)"
      run "pkill -9 -u $RunUser" "Blasting processes by OS user $RunUser." ||\
         warnmsg "Failed to blast all processes by OS user $RunUser."
      sleep 1
      run "userdel $RunUser" "Removing OS user $RunUser." ||\
         warnmsg "Failed to remove OS user $RunUser."

      run "/bin/rm -rf $RunUserHomeDir" "Removing home dir $RunUserHomeDir" ||\
         warnmsg "Failed to remove $RunUserHomeDir."
   else
      msg "Extreme Cleanup: OS User $RunUser does not exist."
   fi

   run "/bin/rm -rf /p4 $HxDepots/p4 $HxMetadata/p4 $HxLogs/p4" \
      "Extreme Cleanup: Blasting several SDP dirs." ||\
      warnmsg "Failed to blast some SDP dirs."

   msg "Extreme Cleanup complete.\\n"
fi

# Digital asset acquisition and availability checks.
[[ ! -d "$HxDepots" ]] && run "/bin/mkdir -p $HxDepots"

cd "$HxDepots" || bail "Could not cd to [$HxDepots]."

if command -v yum > /dev/null; then
   PackageManager="yum"
elif command -v apt-get > /dev/null; then
   PackageManager="apt-get"
elif command -v zypper > /dev/null; then
   PackageManager="zypper"
fi

if [[ -d "/etc/systemd/system" ]]; then
   InitMechanism="Systemd"
elif [[ -x "/sbin/launchd" ]]; then
   InitMechanism="Launchd"
elif [[ -d "/etc/init.d" ]]; then
   InitMechanism="SysV"
fi

if [[ "$FastMode" -eq 0 ]]; then
   msg "Ensuring needed packages are installed."

   [[ "$PackageManager" == "Unset" ]] && \
      bail "Could not find one of these package managers: ${!PackageList[*]}"

   run "$PackageManager install -y ${PackageList[$PackageManager]}" \
      "Installing these pacakges with $PackageManager: ${PackageList[$PackageManager]}" ||\
      warnmsg "Not all packages installed successfully.  Proceeding."
else
   msg "Skipping package dependency checks in -fast mode."
   if ! command -v gcc > /dev/null || ! command -v g++ > /dev/null; then
      msg "Warning: No gcc found in the path.  You may need to install it.  Please\\n check that the gcc and gcc-c++ packages are\\n installed, e.g. with:\\n\\t$PackageManager install -y gcc gcc-c++\\nIgnoring missing gcc/g++ due to '-fast'.\\n"
   else
      msg "Verified: gcc and g++ are available and in the PATH."
   fi
fi

if [[ "$PullFromWebAsNeeded" -eq 1 ]]; then
   if ! command -v curl > /dev/null; then
      bail "No 'curl' found in the path.  You may need to install it or adjust the PATH for the root user to find it.\\n\\n"
   fi
fi

if ! command -v su > /dev/null; then
   bail "No 'su' found in the path.  You may need to install it or adjust the PATH for the root user to find it.\\n\\n"
fi

if [[ "$ThisArch" == "$RunArch" ]]; then
   msg "Verified:  Running on a supported architecture [$ThisArch]."
   ApiArch=UNDEFINED_API_ARCH
   case $ThisOS in
      (Darwin)
         ApiArch="macosx1010x86_64"
         RunGroup=staff
         SampleDepotTar=sampledepot.mac.tar.gz
      ;;
      (Linux)
         ApiArch="linux26x86_64"
         if [[ -r "/etc/SuSE-release" ]]; then
            RunGroup=users
         else
            # CentOS, RHEL, and Ubuntu default group is same as user name.
            RunGroup=perforce
         fi
         SampleDepotTar=sampledepot.tar.gz
      ;;
      (*) bail "Unsupported value returned by 'uname -m': $ThisOS. Aborting.";;
   esac
else
   bail "Running on architecture $ThisArch.  Run this only on hosts with '$RunArch' architecture. Aborting."
fi

# In this block, we just check that directories specified to be preserved
# with the '-p' flag actually exist, in which case we abort before further
# processing.
if [[ "$PreserveDirList" != Unset ]]; then
   for d in $(echo "$PreserveDirList"| tr ',' ' '); do
      preserveDir="$HxDepots/p4/common/$d"
      if [[ -d "$preserveDir" ]]; then
         parentDir=$(dirname "$TmpDir/$d")
         if [[ ! -d "$parentDir" ]]; then
            run "/bin/mkdir -p $parentDir" "Creating parent temp dir [$parentDir]." ||\
               bail "Failed to create parent temp dir [$parentDir]."
         fi
      fi
   done
fi

if [[ "$BlastDownloadsAndExes" -eq 1 ]]; then
   if [[ -d "$ExesDir" ]]; then
      run "/bin/rm -rf $ExesDir" \
         "Blasting exes dir [$ExesDir] due to '-B'." ||\
          warnmsg "Failed to blast exes dir."

      for d in /tmp/downloads /tmp/p4perl; do
         if [[ -d "$d" ]]; then
            run "/bin/rm -rf $d" \
               "Blasting $d dir due to '-B'." ||\
                warnmsg "Failed to blast dir $d."
         fi
      done
   fi
fi

if [[ ! -d "$ExesDir" ]]; then
   [[ "$PullFromWebAsNeeded" -eq 0 ]] && bail "ExesDir [$ExesDir] is missing and '-local' specified. Aborting."
   run "/bin/mkdir -p $ExesDir" ||\
      bail "Could not create dir [$ExesDir]."

   cd "$ExesDir" || bail "Could not cd to $ExesDir."
   msg "Working in [$PWD]."
   run "curl -s -O http://ftp.perforce.com/perforce/$P4ExeRel/bin.$ApiArch/p4" ||\
      bail "Could not get 'p4' executable."
   run "curl -s -O http://ftp.perforce.com/perforce/$P4DExeRel/bin.$ApiArch/p4d" ||\
      bail "Could not get 'p4d' executable."
   run "curl -s -O http://ftp.perforce.com/perforce/$P4PExeRel/bin.$ApiArch/p4p" ||\
      bail "Could not get 'p4p' executable."
   run "curl -s -O http://ftp.perforce.com/perforce/$P4BrokerExeRel/bin.$ApiArch/p4broker" ||\
      bail "Could not get 'p4broker' executable."

   run "chmod +x p4 p4d p4p p4broker" \
      "Doing chmod +x for downloaded executables."
else
   msg "Using existing exes dir [$ExesDir]."
fi

#------------------------------------------------------------------------------
# Services Shutdown and Cleanup.

if id -u "$RunUser" > /dev/null 2>&1; then
   msg "Verified: User $RunUser exists."
else
   if command -v useradd > /dev/null; then
      run "useradd $RunUser" "Creating user $RunUser." ||\
         bail "Failed to create user $RunUser."

      RunUserHomeDir="$(eval echo ~$RunUser)"
      if [[ -d "$RunUserHomeDir" ]]; then
         msg "Verified: Home directory for user $RunUser exists."
      else
         run "mkdir -p $RunUserHomeDir" \
            "Creating home dir for $RunUser" ||\
            bail "Failed to create home directory $RunUserHomeDir for OS user $RunUser."
      fi

      run "cp $ResetHome/perforce_bash_profile $RunUserHomeDir/.bash_profile" \
         "Creating $RunUserHomeDir/.bash_profile." ||\
         warnmsg "Failed to copy to $RunUserHomeDir/.bash_profile."

      run "cp $ResetHome/perforce_bashrc $RunUserHomeDir/.bashrc" \
         "Creating $RunUserHomeDir/.bashrc." ||\
         warnmsg "Failed to copy to $RunUserHomeDir/.bashrc."

      run "chown $RunUser:$RunGroup $(eval echo ~$RunUser)/.bash_profile $(eval echo ~$RunUser)/.bashrc" "Adjusting perms of .bash_profile and .bashrc." ||\
         warnmsg "Adjusting ownership failed."
   else
      bail "User $RunUser does not exist, and the 'useradd' utility was not found."
   fi
fi

for i in $SDPInstances; do
   if [[ "$ThisOS" == Linux || "$ThisOS" == Darwin ]]; then
      msg "Stopping Perforce-related servcies for Instance $i."
      for svc in p4d p4broker p4p p4web p4dtg; do
         processCmd="${svc}_${i}"

         # This 'ps' command should work on Linux and Mac (Yosemite+ at least).
         # shellcheck disable=SC2009
         Pids=$(ps -u $RunUser -f | grep -v grep | grep "/$processCmd "| awk '{print $2}')

         if [[ -z "$Pids" && $svc == p4d ]]; then
            msg "$processCmd not found for p4d service; looking for _bin variant instead."
            # For the p4d service, the process command may look like 'p4d_1_bin' or just 'p4d_1', so
            # we check for both.
            processCmd="${svc}_${i}_bin"
            # shellcheck disable=SC2009
            Pids=$(ps -u $RunUser -f | grep -v grep | grep "/$processCmd "| awk '{print $2}')
         fi

         if [[ -n "$Pids" ]]; then
            run "kill -9 $Pids" \
               "Killing user $RunUser processes for command $processCmd."
            sleep 1
         else
            msg "Verified: No processes by user $RunUser for command [$processCmd] are running."
         fi
      done
   fi

   if [[ "$InitMechanism" == "SysV" ]]; then
      cd /etc/init.d || bail "Could not cd to /etc/init.d."

      msg "Removing Perforce-related SysV services in $PWD."

      for svc in p4*_init; do
         run "chkconfig --del $svc"
         run "rm -f $svc"
      done
   elif [[ "$InitMechanism" == "Systemd" ]]; then
      cd /etc/systemd/system || bail "Could not cd to /etc/systemd/system."

      msg "Disabling and removing Perforce-related Systemd services in $PWD."
      for svcFile in p4*.service; do
         run "systemctl disable ${svcFile%.service}"
         run "rm -f $svcFile"
      done
   fi
done

#------------------------------------------------------------------------------
if [[ -d $DownloadsDir && $BlastDownloadsAndExes -eq 1 ]]; then
   run "/bin/rm -r -f $DownloadsDir" \
      "Blasting downloads dir [$DownloadsDir] due to '-B'."
fi

if [[ ! -d $DownloadsDir ]]; then
   [[ $PullFromWebAsNeeded -eq 0 ]] && bail "DownloadsDir [$DownloadsDir] is missing and '-local' specified. Aborting."
   run "/bin/mkdir -p $DownloadsDir"

   cd $DownloadsDir || bail "Could not cd to [$DownloadsDir]."

   msg "Working in [$PWD]."
   run "curl -s -O http://ftp.perforce.com/perforce/$P4Rel/bin.$ApiArch/p4api.tgz" ||\
      bail "Could not get file 'p4api.tgz'"
   run "curl -s -O http://ftp.perforce.com/perforce/$P4SwarmRel/bin.multiarch/swarm.tgz" ||\
      bail "Could not get file 'swarm.tgz'"

   if [[ $SDPInstallMethod == Tarfile ]]; then
      run "curl -k -s -O $SDPURL" ||\
         bail "Could not get SDP tar file from [$SDPURL]."
   fi

   run "curl -s -O http://ftp.perforce.com/perforce/tools/$SampleDepotTar" ||\
      bail "Could not get file [$SampleDepotTar]. Aborting."
   if [[ ! -d PerforceSample ]]; then
      run "tar -xzpf $SampleDepotTar" "Unpacking $SampleDepotTar in $PWD."
   fi
   run "chown -R $RunUser:$RunGroup $DownloadsDir" \
      "Setting ownership on downloads dir." ||\
      bail "Failed to set ownership on downloads dir [$DownloadsDir]. Aborting."
else
   msg "Using existing downloads dir [$DownloadsDir]."

   cd "$DownloadsDir" || bail "Could not cd to downloads dir: $DownloadsDir"

   if [[ $SDPInstallMethod == Tarfile ]]; then
      if [[ -r "$SDPTar" ]]; then
         msg "Using existing SDP tarfile [$SDPTar]."
      else
         run "curl -k -s -O $SDPURL" ||\
            bail "Could not get SDP tar file from [$SDPURL]."
      fi
   fi

fi

#------------------------------------------------------------------------------
# Cleanup
cd "$HxDepots" || bail "Could not cd to [$HxDepots]. Aborting."

msg "Working in [$PWD]."

for d in $HxMetadata $HxLogs; do
   if [[ ! -d "$d" ]]; then
      run "/bin/mkdir -p $d" "Initialized empty dir [$d]." ||\
         bail "Failed to create dir [$d]."
   else
      msg "Verified: Dir [$d] exists."
   fi
done

if [[ $PreserveDirList != Unset ]]; then
   run "/bin/mkdir -p $TmpDir" "Creating temp dir [$TmpDir]." ||\
      bail "Failed to create temp dir [$TmpDir]."

   for d in $(echo "$PreserveDirList" | tr ',' ' '); do
      preserveDir="$HxDepots/p4/common/$d"
      if [[ -d "$preserveDir" ]]; then
         parentDir=$(dirname "$TmpDir/$d")
         if [[ ! -d "$parentDir" ]]; then
            run "/bin/mkdir -p $parentDir" "Creating parent temp dir [$parentDir]." ||\
               bail "Failed to create parent temp dir [$parentDir]."
         fi

         run "/bin/mv $preserveDir $TmpDir/$d" \
            "Moving preserved dir $preserveDir aside for safe keeping."
      else
         bail "Missing expected preserve dir [$preserveDir]. Check that paths specified with '-p' are relative to $HxDepots/p4/common."
      fi
   done
fi

for i in $SDPInstances; do
   run "/bin/rm -rf $HxDepots/p4/$i $HxMetadata/p4/$i $HxLogs/p4/$i" \
      "Blasting any existing SDP structures." ||\
      bail "Failed to blast existing SDP structures."
   if [[ -L "/p4/$i" ]]; then
      run "/bin/rm -f /p4/$i" "Blasting symlink /p4/$i." ||\
         bail "Failed to remove SDP symlink."
   elif [[ -d "/p4/$i" ]]; then
      run "/bin/rm -rf /p4/$i" "Blasting directory /p4/$i." ||\
         bail "Failed to remove SDP instance directory /p4/$i."
   fi
done

run "/bin/rm -rf $HxDepots/p4/common" \
   "Blasting SDP common folder." ||\
   bail "Failed to remove SDP common folder."

if [[ -L /p4/common ]]; then
   run "/bin/rm -f /p4/common" "Removing /p4/common symlink." ||\
      bail "Failed to remove /p4/common symlink."
elif [[ -d /p4/common ]]; then
   run "/bin/rm -rf /p4/common" "Removing /p4/common directory." ||\
      bail "Failed to remove /p4/common local directory."
else
   msg "Warning: /p4/common did not exist as a directory or symlink."
fi

run "/bin/rm -rf $SDPHome /p4/ssl" \
   "Removing old SDP Home and /p4/ssl." ||\
   bail "Failed to remove old $SDPHome and /p4/ssl."

#------------------------------------------------------------------------------
# SDP Setup
if [[ $SDPInstallMethod == Tarfile ]]; then
   run "tar -xzpf $DownloadsDir/$SDPTar" "Unpacking $DownloadsDir/$SDPTar in $PWD." ||\
      bail "Failed to untar SDP tarfile."
else
   export PATH="$ExesDir:$PATH"
   export P4ENVIRO=/dev/null/.p4enviro
   export P4CONFIG=.p4config.local
   run "/bin/mkdir -p $SDPHome" "Creating dir $SDPHome" ||\
      bail "Failed to create dir $SDPHome. Aborting."
   cd "$SDPHome" || bail "Failed to cd to [$SDPHome]."
   WorkshopRemote=perforce_software-sdp_${SDPInstallBranch}
   touch "$WorkshopUserPasswordFile"
   chmod 600 "$WorkshopUserPasswordFile"
   for c in R 0 c k \' n H 3 l i x R U  l 3 z; do
      echo -n "$c" >> "$WorkshopUserPasswordFile"
   done

   $ExesDir/p4 -s -u "$WorkshopUser" -p "$WorkshopPort" login < "$WorkshopUserPasswordFile" > /tmp/login.txt 2>&1
   /bin/rm -f "$WorkshopUserPasswordFile"

   run "$ExesDir/p4 -s -u $WorkshopUser clone -p $WorkshopPort -r $WorkshopRemote" \
      "Cloning SDP $SDPInstallBranch branch from The Workshop." ||\
      bail "Failed to clone SDP from The Workshop."

   run "$ExesDir/p4 -s sync -f .p4ignore" \
      "Force-sync .p4ignore file."

   if [[ "$ShelvedChange" != Unset ]]; then
      run "$ExesDir/p4 -s fetch -s $ShelvedChange" \
         "Fetching shelved change @$ShelvedChange from The Workshop." ||\
      bail "Failed to fetch shelved change @$ShelvedChange from The Workshop."

      LocalShelvedChange=$($ExesDir/p4 -ztag -F %change% changes -s shelved -m 1)

      [[ -n "$LocalShelvedChange" ]] || \
         bail "Could not determine local shelved change fetched for shelved change @$ShelvedChange."

      run "$ExesDir/p4 -s unshelve -s $LocalShelvedChange" \
         "Unshelving local shelved change @$LocalShelvedChange." ||\
         bail "Failed to unshelve local shelved change @$LocalShelvedChange."
   fi

   unset P4ENVIRO
   unset P4CONFIG
fi

cd $SDPConfigDir ||\
   bail "Could not cd to [$SDPConfigDir]."

msg "Generating custom p4_vars.template in [$PWD]."

run "/bin/mv -f p4_vars.template p4_vars.template.orig"
run "chmod 444 p4_vars.template.orig"

sed -e "s:SDPMAIL=mailx:SDPMAIL='/bin/echo Simulated mail':g" \
   -e "s:SDPMAIL=mail:SDPMAIL='/bin/echo Simulated mail':g" \
   p4_vars.template.orig > p4_vars.template

msg "Changes in p4_vars.template:"
diff p4_vars.template.orig p4_vars.template

cd $SDPSetupDir ||\
   bail "Could not cd to [$SDPSetupDir]."

msg "Generating custom mkdirs.sh in $PWD."

run "/bin/mv -f mkdirs.sh mkdirs.sh.orig"
run "chmod 444 mkdirs.sh.orig"

sed -e "s:export MAIL=mailx:export MAIL='/bin/echo Simulated mail':g" \
   -e "s:export MAIL=mail:export MAIL='/bin/echo Simulated mail':g" \
   mkdirs.sh.orig > mkdirs.sh

run "/bin/mv -f mkdirs.cfg mkdirs.cfg.orig"

sed -e "s:=DNS_name_of_master_server:=helix:g" \
   -e "s:admin@example.com:P4AdminList@p4demo.com:g" \
   -e "s:mail.example.com:smtp.p4demo.com:g" \
   -e "s:^DB1=.*:DB1=$HxMetadata:g" \
   -e "s:^DB2=.*:DB2=$HxMetadata:g" \
   -e "s:^P4_PORT=.*:P4_PORT=SeeBelow:g" \
   -e "s:^P4BROKER_PORT=.*:P4BROKER_PORT=SeeBelow:g" \
   -e "s:# P4_PORT=1666:P4_PORT=1999:g" \
   -e "s:# P4BROKER_PORT=1667:P4BROKER_PORT=1666:g" \
   -e "s:=adminpass:=$Password:g" \
   -e "s:=servicepass:=$Password:g" \
   -e "s:OSGROUP=perforce:OSGROUP=$RunGroup:g" \
   -e "s:REPLICA_ID=replica:REPLICA_ID=p4d_ha_bos:g" \
   -e "s:SVCUSER=service:SVCUSER=svc_p4d_ha_bos:g" \
   mkdirs.cfg.orig > mkdirs.cfg

if [[ $UseSSL -eq 0 ]]; then
   msg "Not using SSL feature due to '-no_ssl'."
   sed "s/SSL_PREFIX=ssl:/SSL_PREFIX=/g" mkdirs.cfg > $TmpFile
   run "mv -f $TmpFile mkdirs.cfg"
fi

chmod +x mkdirs.sh

msg "SDP Localizations in mkdirs.sh:"
diff mkdirs.sh.orig mkdirs.sh

msg "\nSDP Localizations in mkdirs.cfg:"
diff mkdirs.cfg.orig mkdirs.cfg

run "cp -p $ExesDir/p4* $SDPHome/Server/Unix/p4/common/bin/." \
   "Copying perforce executables."

msg "Initializing SDP instances and configuring $InitMechanism services."

for i in $SDPInstances; do
   cd "$SDPSetupDir" || bail "Could not cd to [$SDPSetupDir]."
   log="$PWD/mkdirs.${i}.log"
   msg "Initializing SDP instance [$i], writing log [$log]."
   "$PWD/mkdirs.sh" "$i" > "$log" 2>&1
   cat "$log"

   if [[ "$InitMechanism" == "SysV" ]]; then
      msg "\\nConfiguring $InitMechanism services.\\n"
      cd /etc/init.d || bail "Could not cd to [/etc/init.d]."
      for svc in p4broker p4d; do
         initScript=${svc}_${i}_init
         if [[ -x /p4/${i}/bin/$initScript ]]; then
            run "ln -s /p4/${i}/bin/$initScript"
            run "chkconfig --add $initScript"
            run "chkconfig $initScript on"
         fi
      done
   elif [[ "$InitMechanism" == "Systemd" ]]; then
      msg "\\nConfiguring $InitMechanism services.\\n"
      cd /etc/systemd/system || bail "Could not cd to /etc/systemd/system."
      for exe in p4broker p4d; do
         svcName=${exe}_${i}
         svcFile=${svcName}.service
         sed "s:1:${i}:g" "$SDPSetupDir/systemd/$svcFile" > $svcFile ||\
            bail "Failed to generated $PWD/svcFile."
         run "systemctl enable $svcName"
      done
   fi

   run "chown -R $RunUser:$RunGroup $HxDepots" \
      "Adjusting ownership of $HxDepots to $RunUser:$RunGroup." || bail "Failed to adjust ownership of $HxDepots."

   msg "\nGenerating broker config for instance $i.\n"
   su -l $RunUser -c "$CBIN/gen_default_broker_cfg.sh ${i} > $CCFG/p4_${i}.broker.cfg"

done

if [[ $UseSSL -eq 1 ]]; then
   msg "Generating /p4/ssl/config.txt SSL config file for autogen cert."
   sed -e "s/REPL_DNSNAME/helix/g" /p4/ssl/config.txt > $TmpFile ||\
      bail "Failed to substitute content in /p4/ssl/config.txt."
   run "mv -f $TmpFile /p4/ssl/config.txt"
   msg "Contents of /p4/ssl/config.txt:\n$(cat /p4/ssl/config.txt)\n"
fi

if [[ $PreserveDirList != Unset ]]; then
   for d in $(echo "$PreserveDirList" | tr ',' ' '); do
      preserveDir=$HxDepots/p4/common/$d
      tempCopyDir=$TmpDir/$d
      run "rsync -av --exclude=.p4root --exclude=.p4ignore --exclude=.p4config $tempCopyDir/ $preserveDir" \
         "Restoring $preserveDir" ||\
         bail "Failed to restore $preserveDir."
   done

   run "/bin/rm -rf $TmpDir" "Cleanup: Removing temp dir [$TmpDir]." ||\
      bail "Failed to remove temp dir [$TmpDir]."
fi

#------------------------------------------------------------------------------
# Install P4Perl and P4Python.
if [[ "$FastMode" -eq 0 ]]; then
   msg "\\nInstalling P4Python for SDP."
   su -l $RunUser -c '/hxdepots/sdp/Server/Unix/setup/install_sdp_python.sh' ||\
      warnmsg "Failed to install P4Python"

   msg "\\nInstalling P4Perl for SDP."
   su -l $RunUser -c '/hxdepots/sdp/Server/Unix/setup/install_sdp_perl.sh' ||\
      warnmsg "Failed to install P4Perl."
fi

msg "Preparing to run Sample Depot configuration script."
if [[ ! -d "$ResetHome" ]]; then
   run "/bin/mkdir -p $ResetHome" "Creating reset home dir [$ResetHome]." ||\
      bail "Could not create reset home dir [$ResetHome]. Aborting."
fi

cd "$ResetHome" || bail "Could not cd to $HxDepots/ResetHome. Aborting."

if [[ ! -x configure_sample_depot_for_sdp.sh ]]; then
   run "curl -k -s -O $HelixInstallerURL/src/configure_sample_depot_for_sdp.sh" \
      "Getting script configure_sample_depot_for_sdp.sh."
fi

for f in NoTicketExpiration.group.p4s admin.user.p4s configure_sample_depot_for_sdp.sh protect.p4s perforce_bash_profile perforce_bashrc r; do
   if [[ ! -f $f ]]; then
      [[ "$PullFromWebAsNeeded" -eq 0 ]] && bail "Missing file [$f] and '-local' specified. Aborting."
      run "curl -k -s -O $HelixInstallerURL/src/$f" "Getting file $f." ||\
         bail "Failed to download file [$PWD/$f]. Aborting."
   else
      if [[ "$BlastDownloadsAndExes" -eq 1 ]]; then
         run "curl -k -s -O $HelixInstallerURL/src/$f" \
            "Replacing file $f due to '-B'." ||\
            bail "Failed to download file [$PWD/$f]. Aborting."
      else
         msg "Using existing file $PWD/$f."
      fi
   fi

   if [[ ! -x $f ]]; then
      if [[ $f == *".sh" || $f == r ]]; then
         run "chmod +x $f" "chmod +x" || bail "Failed to do: chmod +x $f. Aborting."
      fi
   fi
done

run "chown -R $RunUser:$RunGroup $ResetHome" "Fixing ownership of $ResetHome" ||:

if [[ $LoadSampleDepot -eq 1 ]]; then
   msg "Configuring Sample Depot for SDP on Instance 1 only."
   if su -l $RunUser -c "$ResetHome/configure_sample_depot_for_sdp.sh -i 1 -d $ResetHome"; then
      msg "\\nSample Depot configured successfully.\\n"
   else
      bail "Failed to load the Sample Depot."
   fi
else
   msg "\\nSkipping configuration of Sample Depot due to '-no_sd'.\\n"
fi

if [[ "$ErrorCount" -eq 0 ]]; then
   if [[ "$WarningCount" -eq 0 ]]; then
      msg "\\nSUCCESS:  SDP Configuration complete."
   else
      msg "\\nSUCCESS:  SDP Configuration complete with $WarningCount warnings."
   fi
else
   msg "\\nSDP Configuration completed, but with $ErrorCount errors and $WarningCount warnings."
fi

exit 0