configure_sample_depot_for_sdp.sh #24

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# this package, and also available online:
# https://swarm.workshop.perforce.com/view/guest/perforce_software/helix-installer/main/LICENSE
#------------------------------------------------------------------------------
set -u

#------------------------------------------------------------------------------
# Declarations
declare Version=1.11.0
declare HxDepots=
declare DownloadsDir=
declare BackupDir=Unset
declare BackupFile=
declare TmpFile="/tmp/tmp.csd4sdp.$$.$RANDOM"
declare RunUser=perforce
declare ThisUser=
declare ThisHost=
declare CfgScriptBak=
declare CBIN=/p4/common/bin
declare ThisScript="${0##*/}"
declare CmdLine="$0 $*"
declare SDPInstance=Unset
declare PasswordFile=
declare P4DInitScript=
declare P4DSystemdServiceFile=
declare P4BrokerInitScript=
declare P4BrokerSystemdServiceFile=
declare -i MaxStartDelay=120
declare -i P4DStartVerified=0
declare -i P4BrokerStartVerified=0
declare -i UseSystemd=1
declare -i i=0

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage {
   declare style=${1:--h}

   msg "USAGE for $ThisScript v$Version:

$ThisScript -i <sdp_instance> [-d <data_dir>] [-u <osuser>] [-no_systemd]

or

$ThisScript [-h|-man]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script transforms a stock Sample Depot instance into
	one that works with the SDP.

REQUIREMENTS:
	A P4D process must be live and running with the stock
	Sample Depot data set, on a sport 

ARGUMENTS:
 -i <sdp_instance>
	Specify the SDP Instance in which the Sample Depot data set is
	running.  This argument is required.

 -d <data_dir>
	Specify the data directory where supporting files exist, such as the
	*.p4s data files used by this script.

 -u <osuser>
	Specify the Linux operating system user account under which p4d runs.
	If omitted, the default is 'perforce'.

 -no_systemd
	Specify '-no_systemd' to avoid using systemd, even if it
	appears to be available. By default, systemd is used if it
	appears to be available.

	This is helpful in operating in containerized test environments
	where systemd is not available.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message

EXAMPLES:
	Usage to configure Instance 1:
	cd /where/this/script/is
	$ThisScript 1 2>&1 | tee log.${ThisScript%.sh}.1

	Usage to configure Instance abc:
	cd /where/this/script/is
	$ThisScript abc 2>&1 | tee log.${ThisScript%.sh}.abc
"
   fi

   exit 1
}

#------------------------------------------------------------------------------
# Function bail().
# Sample Usage:
#    bail "Missing something important. Aborting."
#    bail "Aborting with exit code 3." 3
function msg () { echo -e "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Functions.  The runCmd() function is similar to functions defined in SDP core
# libraries, but we need to duplicate them here since this script runs before
# the SDP is available on the machine (and we want no dependencies for this
# script.
function runCmd {
   declare cmd=${1:-echo Testing runCmd}
   declare desc=${2:-""}

   declare cmdToShow=$cmd

   [[ "$cmdToShow" == *"<"* ]] && cmdToShow=${cmdToShow%%<*}
   [[ "$cmdToShow" == *">"* ]] && cmdToShow=${cmdToShow%%>*}

   [[ -n "$desc" ]] && msg "$desc"
   msg "Running: $cmdToShow"
   $cmd
   return $?
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0
declare -i ErrorCount=0
declare -i WarningCount=0
declare DataDir="$PWD"

set +u

while [[ $# -gt 0 ]]; do
   case $1 in
      (-no_systemd) UseSystemd=0;;
      (-i) SDPInstance=$2; shiftArgs=1;;
      (-d) DataDir="$2"; HxDepots=${DataDir%/*};shiftArgs=1;;
      (-u) RunUser="$2"; shiftArgs=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-D) set -x;; # Debug; use 'set -x' mode.
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && bail "Usage Error: Wrong numbers of args or flags to args."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#------------------------------------------------------------------------------
# Usage Validation

[[ $SDPInstance == Unset ]] && \
   bail "Bad Usage: The '<sdp_instance>' argument is required."

[[ ! -r $CBIN/p4_vars ]] && \
   bail "Missing SDP Environment File [$CBIN/p4_vars]. Aborting."

#------------------------------------------------------------------------------
# Main Program

ThisUser=$(whoami)
ThisHost=${HOSTNAME%%.*}

msg "Started $ThisScript v$Version on host $ThisHost as user $ThisUser at $(date), called as:\\n\\t$CmdLine"

if [[ "$ThisUser" != "$RunUser" ]]; then
   bail "Run as $RunUser, not $ThisUser."
else
   msg "Verified: Running as user $RunUser."
fi

# Load SDP environment and variable definitions.
# shellcheck disable=SC1090 disable=SC1091
source "$CBIN/p4_vars" "$SDPInstance" ||\
   bail "Failed to load SDP environment. Aborting."

export P4ENVIRO=/dev/null/.p4enviro
export P4CONFIG=.p4config

PasswordFile="$SDP_ADMIN_PASSWORD_FILE"
P4DInitScript="${P4DBIN}_init"
P4BrokerInitScript="${P4BROKERBIN}_init"

if [[ "$UseSystemd" -eq 1 ]]; then
   P4DSystemdServiceFile="/etc/systemd/system/${P4DBIN##*/}.service"
   P4BrokerSystemdServiceFile="/etc/systemd/system/${P4BROKERBIN##*/}.service"
else
   P4DSystemdServiceFile=
   P4BrokerSystemdServiceFile=
fi

[[ -n "$HxDepots" ]] || HxDepots="/hxdepots"
DownloadsDir="$HxDepots/downloads"

cd "$HxDepots/sdp/Server/setup" ||\
   bail "Failed to cd to [$HxDepots/sdp/Server/setup]."

msg "Operating in SDP server setup area [$PWD]."

runCmd "$P4BIN -u bruno -s info -s" "Verifying server is offline." &&\
   bail "Perforce server is unexpectedly online. Aborting."

runCmd "/p4/${SDPInstance}/bin/p4d_${SDPInstance} -jr $DownloadsDir/PerforceSample/checkpoint" \
   "Loading the Sample Depot metadata in instance ${SDPInstance}." ||\
   bail "Failed to load Sample Depot checkpoint."

runCmd "/p4/${SDPInstance}/bin/p4d_${SDPInstance} -xu" \
   "Upgrading databases (p4d -xu) for instance ${SDPInstance}." ||\
   bail "Failed to upgrade databases."

if [[ $P4PORT == "ssl:"* ]]; then
   runCmd "/p4/${SDPInstance}/bin/p4d_${SDPInstance} -Gc" \
      "Generating OpenSSL Certificates for instance $SDPInstance." ||\
      bail "Failed to generate OpenSSL Certs for Instance $SDPInstance."
fi

if [[ -x "$P4DInitScript" ]]; then
   if [[ -f "$P4DSystemdServiceFile" ]]; then
      msg "Starting p4d as user $ThisUser with: sudo systemctl start p4d_${SDPInstance}"
      if sudo systemctl start "p4d_${SDPInstance}"; then
         msg "Service p4d_${SDPInstance} started with systemd."
      else
         warnmsg "Non-zero exit code $? starting p4d service with systemd."
         if [[ -r "$LOGS/p4d_init.log" ]]; then
            msg "Contents of $LOGS/p4d_init.log:\\n$(cat "$LOGS/p4d_init.log")"
         else
            warnmsg "No $LOGS/p4d_init.log file found."
         fi
      fi
   elif [[ -f "/etc/init.d/p4d_${SDPInstance}_init" ]]; then
      msg "Starting p4d as user $ThisUser with: sudo service p4d_${SDPInstance}_init start"
      if sudo service "p4d_${SDPInstance}_init" start; then
         msg "Service p4d_${SDPInstance}_init started with SysV init."
      else
         warnmsg "Non-zero exit code $? starting p4d service with SysV init."
         if [[ -r "$LOGS/p4d_init.log" ]]; then
            msg "Contents of $LOGS/p4d_init.log:\\n$(cat "$LOGS/p4d_init.log")"
         else
            warnmsg "No $LOGS/p4d_init.log file found."
         fi
      fi
   else
      msg "Starting p4d as user $ThisUser with: $P4DInitScript start"
      if "$P4DInitScript" start; then
         msg "Service p4d_${SDPInstance}_init started with no init mechanism."
      else
         warnmsg "Non-zero exit code $? starting p4d service with no init mechanism."
         if [[ -r "$LOGS/p4d_init.log" ]]; then
            msg "Contents of $LOGS/p4d_init.log:\\n$(cat "$LOGS/p4d_init.log")"
         else
            warnmsg "No $LOGS/p4d_init.log file found."
         fi
      fi
   fi
else
   bail "Init script is missing or not executable: $P4DInitScript"
fi

P4DStartVerified=0
i=0; while [[ "$i" -lt "$MaxStartDelay" ]]; do
   if "$P4BIN" -p "$P4PORT" info -s > "$TmpFile" 2>&1; then
      msg "Verified: p4d has started."
      P4DStartVerified=1
      rm -f "$TmpFile"
      break
   else
      # If we get an SSL trust error, then the server is indeed online.
      if grep -lq 'The authenticity of' "$TmpFile"; then
         msg "Verified: p4d has started."
         P4DStartVerified=1
         "$P4BIN" -p "$P4PORT" trust -f -y > /dev/null 2>&1
         rm -f "$TmpFile"
         break
      else
         sleep 1
      fi
   fi
   i+=1
done

[[ "$P4DStartVerified" -eq 1 ]] || \
   bail "p4d start attempted but could not be verified after $MaxStartDelay seconds.\\nTail of server log $P4LOG:\\n$(tail "$P4LOG")\\n\\n"

if [[ -x "$P4BrokerInitScript" ]]; then
   if [[ -f "$P4BrokerSystemdServiceFile" ]]; then
      msg "Starting p4broker as user $ThisUser with: sudo systemctl start p4broker_${SDPInstance}"
      if sudo systemctl start "p4broker_${SDPInstance}"; then
         msg "Service p4broker_${SDPInstance} started with systemd."
      else
         warnmsg "Non-zero exit code $? starting broker service with systemd."
      fi
   elif [[ -f "/etc/init.d/p4broker_${SDPInstance}_init" ]]; then
      msg "Starting p4broker as user $ThisUser with: sudo service p4broker_${SDPInstance}_init start"
      if sudo service "p4broker_${SDPInstance}_init" start; then
         msg "Service p4broker_${SDPInstance}_init started with SysV init."
      else
         warnmsg "Non-zero exit code $? starting broker service with SysV init."
      fi
   else
      msg "Starting p4broker as user $ThisUser with: $P4BrokerInitScript start"
      if "$P4BrokerInitScript" start; then
         msg "Service p4broker_${SDPInstance}_init started with no init mechanism."
      else
         warnmsg "Non-zero exit code $? starting broker service with no init mechanism."
      fi
   fi
else
   bail "Init script is missing or not executable: $P4BrokerInitScript"
fi

P4BrokerStartVerified=0
i=0; while [[ "$i" -lt "$MaxStartDelay" ]]; do
   if "$P4BIN" -p "$P4BROKERPORT" info -s > "$TmpFile" 2>&1; then
      msg "Verified: p4broker has started."
      P4BrokerStartVerified=1
      rm -f "$TmpFile"
      break
   else
      if grep -lq 'The authenticity of' "$TmpFile"; then
         msg "Verified: p4d has started."
         P4BrokerStartVerified=1
         "$P4BIN" -p "$P4BROKERPORT" trust -f -y > /dev/null 2>&1
         rm -f "$TmpFile"
         break
      else
         sleep 1
      fi
   fi
   i+=1
done

[[ "$P4BrokerStartVerified" -eq 1 ]] || \
   bail "p4broker start attempted but could not be verified after $MaxStartDelay seconds.\\n\\nTail of broker log $LOGS/p4broker.log:\\n$(tail "$LOGS/p4broker.log")\\n\\n"

if [[ $P4PORT == "ssl:"* ]]; then
   # Note: Automating a 'p4 trust -y' (especially with '-f') may not be appropriate
   # in a production environment, as it defeats the purpose of the Open SSL trust
   # mechanism.  But for our purposes here, where scripts spin up throw-away data
   # sets for testing or training purposes, it's just dandy.
   runCmd "/p4/${SDPInstance}/bin/p4_${SDPInstance} -p $P4PORT trust -y -f" \
      "Trusting the OpenSSL Cert of the server." ||\
      bail "Failed to trust the server."
   runCmd "/p4/${SDPInstance}/bin/p4_${SDPInstance} -p $P4BROKERPORT trust -y -f" \
      "Trusting the OpenSSL Cert of the broker." ||\
      bail "Failed to trust the broker."
fi

runCmd "$P4BIN -s info -s" "Verifying direct connection to Perforce server." ||\
   bail "Could not connect to Perforce server."

runCmd "$P4BIN -u bruno -s -p $P4BROKERPORT info -s" "Verifying via-broker connection to Perforce server." ||\
   bail "Could not connect to Perforce server via broker."

[[ "$($P4BIN -u bruno protects -m)" == super ]] ||\
   bail "Could not verify super user access for $P4USER on port $P4PORT.  Is this the Sample depot? Aborting."

msg "Super user access for bruno verified."

msg "Creating user $P4USER."
sed "s:__EDITME_ADMIN_P4USER__:$P4USER:g" "$DataDir/admin.user.p4s" > "$TmpFile"
"$P4BIN" -u bruno user -f -i < "$TmpFile"

msg "Adding user to NoTicketExpiration group."
sed "s:__EDITME_ADMIN_P4USER__:$P4USER:g" "$DataDir/NoTicketExpiration.group.p4s" > "$TmpFile"
"$P4BIN" -u bruno group -i < "$TmpFile"

msg "Promoting user $P4USER to super user."
"$P4BIN" -u bruno protect -o > "$TmpFile"
msg "\\tsuper user $P4USER * //...\\n" >> "$TmpFile"
"$P4BIN" -u bruno protect -i < "$TmpFile"

cat "$PasswordFile" > "$TmpFile"
cat "$PasswordFile" >> "$TmpFile"

"$P4BIN" -u bruno passwd "$P4USER" < "$TmpFile"

runCmd "/p4/common/bin/p4login" "Logging in $P4USER super user." ||\
   bail "Failed to login super user $P4USER. Aborting."

# Variable     Format                              Sample Values
# P4PORT       [ssl:]<P4DPortNum>                  ssl:1999, 1999
# P4BROKERPORT [ssl:]<BrokerPort>                  ssl:1666, 1666
for p in $P4PORT $P4BROKERPORT; do
   if [[ $p == "ssl:"* ]]; then
      runCmd "$P4BIN -p $p trust -y" "Trusting P4PORT=$p." ||\
         bail "Failed to trust P4PORT=$p."
   fi
   cmd="$P4BIN -u $P4USER -p $p login -a"
   msg "Running: $cmd < $PasswordFile"
   $cmd < "$PasswordFile" ||\
      bail "Login as perforce using P4PORT=$p failed.  Aborting."
done

runCmd "cat $P4TICKETS" "Showing P4TICKETS:"

CfgScriptBak="configure_new_server.sh.$(date +'%Y%m%d-%H%M%S').bak"
runCmd "mv -f configure_new_server.sh $CfgScriptBak" \
   "Tweaking configure_new_server.sh settings to values more appropriate for a demo-grade installation, e.g. reducing 5G storage limits." ||\
   bail "Failed to move configure_new_server.sh to $CfgScriptBak."

# Warning: If the values in configure_new_server.sh are changed from 5G, this
# will need to be updated.
sed -e 's/filesys.P4ROOT.min=5G/filesys.P4ROOT.min=10M/g' \
   -e 's/filesys.depot.min=5G/filesys.depot.min=10M/g' \
   -e 's/filesys.P4JOURNAL.min=5G/filesys.P4JOURNAL.min=10M/g' \
   "$CfgScriptBak" >\
   configure_new_server.sh ||\
   bail "Failed to do sed substitutions in $HxDepots/sdp/Server/setup/$CfgScriptBak"

runCmd "chmod -x $CfgScriptBak"
runCmd "chmod +x configure_new_server.sh"

msg "Changes made to configure_new_server.sh:"
diff "$CfgScriptBak" configure_new_server.sh

runCmd "./configure_new_server.sh $SDPInstance" \
   "Applying SDP configurables." ||\
   bail "Failed to set SDP configurables. Aborting."

for depot in $(/bin/ls -d "$HxDepots"/downloads/PerforceSample/*); do
   [[ $depot == *"checkpoint"* ]] && continue
   [[ $depot == *"README"* ]] && continue
   [[ $depot == *"readme"* ]] && continue
   if [[ $depot == *"spec"* ]]; then
      runCmd "/usr/bin/rsync -a $depot/ /p4/$SDPInstance/depots/${depot##*/}" \
         "Copying Sample Depot archive files for spec depot [${depot##*/}]." ||\
         warnmsg "Non-zero exit code $? from rsync for depot ${depot##*/}."
   else
      runCmd "/usr/bin/rsync -a --delete $depot/ /p4/$SDPInstance/depots/${depot##*/}" \
         "Copying Sample Depot archive files for depot [${depot##*/}]." ||\
         warnmsg "Non-zero exit code $? from rsync for depot ${depot##*/}."
   fi
done

runCmd "$P4BIN admin updatespecdepot -a" \
   "Updating spec depot." || bail "Failed to udpate spec depot. Aborting."

runCmd "/usr/bin/rsync -a /p4/$SDPInstance/root/spec/ /p4/$SDPInstance/depots/spec" \
   "Copying a few spec depot files." ||\
   warnmsg "Non-zero exit code $? from rsync for spec depot."

runCmd "/bin/rm -rf /p4/$SDPInstance/root/spec" \
   "Cleanup redundant copy of spec depot files." ||:

runCmd "/p4/common/bin/live_checkpoint.sh $SDPInstance" \
   "Taking Live Checkpoint." || bail "Live checkpoint failed. Aborting."

[[ $BackupDir == Unset ]] && BackupDir=/p4/$SDPInstance/backup

if [[ -d $BackupDir ]]; then
    runCmd "/bin/rm -rf $BackupDir" \
       "Removing old backup dir [$BackupDir]."
fi

if [[ ! -d $BackupDir ]]; then
   runCmd "/bin/mkdir -p $BackupDir" \
      "Creating new empty backups directory: $BackupDir." ||\
      bail "Failed to create backups dir [$BackupDir]. Aborting."
fi

BackupFile="$BackupDir/p4_$SDPInstance.backup.$(date +'%Y-%m-%d-%H%M').tgz"
LastCheckpoint="$(ls -1 -t /p4/"$SDPInstance"/checkpoints/p4_"${SDPInstance}".ckp.*.gz 2>/dev/null)"
BackupPaths="/p4/${SDPInstance}/depots"
[[ -n "$LastCheckpoint" ]] && BackupPaths="$BackupPaths $LastCheckpoint"

runCmd "tar -czf $BackupFile $BackupPaths" \
   "Creating backup $BackupFile." ||\
   bail "Failed to backup instance $SDPInstance. Aborting."

msg "\\nSUCCESS:  SDP Instance $SDPInstance loaded with sample depot data, live checkpoint done, and backup created.  Good to go!\\n"

exit 0