#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Helix Management System (HMS), and also available online: # https://swarm.workshop.perforce.com/projects/perforce_software-hms/view/main/LICENSE #------------------------------------------------------------------------------ #============================================================================== # Declarations and Environment declare -i ErrorCount=0 declare -i WarningCount=0 declare -i Debug=0 declare -i NoOp=0 declare ThisScript=${0##*/} declare ThisUser= declare Version=3.0.1 declare CmdLine="$0 $*" declare Log= declare H1="==============================================================================" declare H2="------------------------------------------------------------------------------" declare RequiredUtils="date egrep grep id ls rsync tail wc" declare HxDepots= declare SDPInstallDirDefault= declare SDPSetupDir= declare SDPInstallDir=Unset declare SDPMgmtWs= declare SDPMgmtP4Cfg= declare SDPConfigScript= declare SDPHostsFile= declare SDPHostsSample= declare HMSVarsFile="/p4/common/config/p4_hms.vars" declare HMSSetupDir="$PWD" declare HMSInstallRoot="${HMSSetupDir%/p4/common/site/hms/setup}" declare CustomSetupScript="${HMS_CUSTOM_SETUP_SCRIPT:-}" declare -i UseSudo=0 declare -i Interactive=1 declare ActiveInstances= declare ThisHost="${HOSTNAME%%.*}" declare DirList="${HOSTNAME%%.*}" declare TmpFile1= declare TmpFile2= declare TmpFile3= declare TmpFile4= #============================================================================== # Local Functions function msg () { echo -e "$*"; } function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; } function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; } function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; } function bail () { errmsg "${1:-Unknown Error}"; exit "$ErrorCount"; } #------------------------------------------------------------------------------ # Function: run # # Short: Run a command with optional description, honoring $Debug # and $NoOp settings. # # Input: # $1 - cmd. The command to run. The command is displayed first # if $Debug is not 0. # $2 - desc. Text description of command to run. # This parameter is optional. # $3 - honorNoOpFlag. Pass in 1 to mean "Yes, honor the $NoOp setting # and display (but don't run) commands if $NoOp is 1." Otherwise # $NoOp is ignored, and the command is always run. # This parameter is optional; the default value is 1. # $4 - alwaysShowOutputFlag. If set to 1, show output regardless of $Debug # value. Otherwise, only show output if $Debug is not 0. # This parameter is optional; the default value is 0. # $5 - grepString. If set, this changes the exit code behavior. # If the specified string exists in the output, a 0 is returned, else 1. # Strings suitable for use with 'egrep' are allowed. # # Description: # Display an optional description of a command, and then run the # command. This is affected by $NoOp and $Debug. If $NoOp is 1, # the command is shown, but not run, provided $honorNoOpFlag is 1. # The description is not shown if $Debug is not set. # # Output is shown if either $alwaysShowOutputFlag is 1 or $Debug is not 0. #------------------------------------------------------------------------------ function run () { dbg "CALL: run ($*)" local cmd=${1:-} local desc=${2:-} local -i honorNoOpFlag=${3:-1} local -i alwaysShowOutputFlag=${4:-0} local grepString=${5:-} local cmdScript= local cmdOut= local -i grepExit local -i cmdExitCode=0 cmdScript=$(mktemp XXXXXXXX_cmd.sh) cmdOut="${cmdScript%.sh}.out" [[ -n "$desc" ]] && msg "$desc" if [[ "$honorNoOpFlag" -eq 1 && "$NoOp" -eq 1 ]]; then msg "NO_OP: Would run: \"$cmd\"" else msg "Running: \"$cmd\"." echo -e "#!/bin/bash\n$cmd\n" > "$cmdScript" chmod +x "$cmdScript" [[ "$Debug" -ne 0 ]] && dbg "BEGIN Contents of cmdScript $cmdScript:\\n$(cat "$cmdScript")\\nEND Contents of cmdScript" $cmdScript > "$cmdOut" 2>&1 cmdExitCode=$? if [[ -n "$grepString" ]]; then # shellcheck disable=SC2196 egrep "$grepString" "$cmdOut" > /dev/null 2>&1 grepExit=$? cmdExitCode="$grepExit" fi if [[ "$alwaysShowOutputFlag" -eq 1 || "$Debug" -ne 0 ]]; then cat "$cmdOut" fi # Be clean and tidy. rm -f "$cmdScript" "$cmdOut" # If a grep was requested, return the exit code from the egrep, # otherwise return the exit code of the command executed. In # any case, $cmdExitCode contains the exit code of the command. if [[ -n "$grepString" ]]; then return "$grepExit" else return "$cmdExitCode" fi fi return 0 } #------------------------------------------------------------------------------ # Function: terminate function terminate { # Disable signal trapping. trap - EXIT SIGINT SIGTERM # Stop logging. [[ "$Log" == off ]] || msg "Log is: $Log\\n${H1}" # With the trap removed, exit. exit "$ErrorCount" } #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). # The default is -h. # # $2 - error message (optional). Specify this if usage() is called due to # user error, in which case the given message displayed first, followed by the # standard usage message (short or long depending on $1). If displaying an # errror, usually $1 should be -h so that the longer usage message doesn't # obsure the error message. # # Sample Usage: # usage # usage -h # usage -man # usage -h "Incorrect command line usage." #------------------------------------------------------------------------------ function usage { declare style=${1:--h} declare errorMessage=${2:-Unset} if [[ "$errorMessage" != Unset ]]; then echo -e "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" fi echo "USAGE for $ThisScript v$Version: $ThisScript [-y] [-d <sdp_dir>] [-s] [-L <log>] [-n] [-debug|-D] or $ThisScript [-h|-man|-V] " if [[ "$style" == -man ]]; then echo -e " DESCRIPTION: The HMS \"Tight Ship\" setup script installs HMS server, or adds an 'hms' SDP instance to an existing SDP installation. into a tight-ship style installation on the master server machine. A tight ship installation is one in which: * The key SDP folders, e.g. /p4/common and /p4/N/bin (for each instance) are versioned locally using SDP.<hostname> workspaces in a local HMS instance, /p4/hms, which this script creates and initializes. * The crontab for the Perforce user is versioned. * Any local configuration and/or customizations of the SDP are versioned. * The HMS instance is aware of The Workshop, a public Helix server hosted by Perforce Software. It fetches SDP updates from The Workshop into the local hms instance using Helix DVCS features. This simplifies future updates of the SDP, allowing them to be done with a 'p4 fetch' followed by a 'p4 merge' on your local server (if the server can reach The Workshop directly; if not an alternative approach using an interim personal repo can be used to bridge 'air gap' networks). PRE-REQUISITES: The following are pre-requisites to running $ThisScript: 1. The Helix Installer must have been run. The SDP is expected to exist in /p4/sdp (or whatever diretory is specified with '-d <sdp_dir>'). This may have been populated via a tar file or workspace pulling files from The Workshop (P4PORT=public.perforce.com:1666), pulling files from //guest/perforce_software/sdp/main on that server. 2. No hms instance. There should not (yet) be an hms instance with data. It is OK if only /p4/hms structure exists due to being initialized with the Helix Installer, but there should be no existing data in the /p4/hms SDP instance. 3. Determine if sudo access is available. If The OSUSER (typically 'perforce') has full sudo privs, then HMS can go further in the management of things; e.g. it can manage Swarm (which requires root access). Specify the '-s' option on the command line if the OSUSER is be able to execute 'sudo su' commands without a password. This can be accomplished on modern Linux systems by creating /etc/sudoers.d/perforce, with this one line of contents: perforce ALL=(ALL:ALL) NOPASSWD: ALL If configured properly, the following command should execute without error: sudo /bin/ls /etc/sudoers.d ADDITIONAL SETUP: The following setup is necessary before operating in a tight ship environment, but are not necessary before running this script. 1. The SSH keys must be configured enabling the 'perforce' user to ssh without a password to all machines listed in sdp_hosts.cfg configured above, including the current host (i.e. it must be able to ssh to itself using the current hostname). OPTIONS: -y Continue past the preflight without an interactive prompt. -d <sdp_dir> Specify the root of the directory tree where the SDP package is available. The default is: $SDPInstallDirDefault The version of the SDP installed must be recent enough to contain the 'hms' setup directory: $SDPInstallDirDefault/Server/Unix/setup/hms. -s Specify '-s' if sudo access is available. If sudo access is available, HMS can manage more things than it can without sudo access, e.g. Swarm. Adding '-s' adds a preflight check to verify sudo access. -d Set debug verbosity. -L <log> Specify the path to a log file, or the special value 'off' to disable logging. By default, all output (stdout and stderr) goes to: ${HOME}/${ThisScript%.sh}.<timestamp>.log NOTE: This script is self-logging. That is, output displayed on the screen is simultaneously captured in the log file. Do not run this script with redirection operators like '> log' or '2>&1', and do not use 'tee.' -n No-Op. Prints commands instead of running them. -D Use bash 'set -x' extreme debugging verbosity. Implies -d. HELP OPTIONS: -h Display short help message -man Display man-style help message -V Dispay version info for this script and its libraries. FILES: EXAMPLES: This is best done immediately after a running the Helix Installer to initialize the hms instance on a dedicated p4hms machine with no data. " fi exit 1 } #============================================================================== # Command Line Processing declare -i shiftArgs=0 set +u while [[ $# -gt 0 ]]; do case $1 in (-d) SDPInstallDir="$2"; shiftArgs=1;; (-s) UseSudo=1;; (-y) Interactive=0;; (-h) usage -h;; (-man) usage -man;; (-V) msg "$ThisScript v$Version"; exit 2;; (-L) Log="$2"; shiftArgs=1;; (-n) NoOp=1;; (-debug) Debug=1;; (-D) Debug=1; set -x;; # Debug; use bash 'set -x' extreme debug mode. (*) usage -h "Unknown arg ($1).";; esac # Shift (modify $#) the appropriate number of times. shift; while [[ "$shiftArgs" -gt 0 ]]; do [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments." shiftArgs=$shiftArgs-1 shift done done set -u #============================================================================== # Command Line Verification HxDepots=$(cd /p4/common 2>/dev/null || exit; d=$(pwd -P); echo "${d%/p4/common}") SDPInstallDirDefault="$HxDepots/sdp" [[ -n "$Log" ]] || \ Log="${HOME}/${ThisScript/.sh/}.$(date +'%Y%m%d-%H%M%S').log" [[ "$SDPInstallDir" == Unset ]] && SDPInstallDir="$SDPInstallDirDefault" [[ -d "$SDPInstallDir" ]] || \ bail "The SDP install dir [$SDPInstallDir] does not exist. Specify the correct value with '-d <sdp_dir>'." [[ -d "$HMSSetupDir" ]] || \ bail "The HMS setup dir [$HMSSetupDir] does not exist." #============================================================================== # Main Program trap terminate EXIT SIGINT SIGTERM for u in $RequiredUtils; do [[ -n "$(command -v "$u")" ]] || errmsg "Missing required utility: $u" done [[ "$ErrorCount" -eq 0 ]] || bail "Aborted early due to missing required utilitiies." if [[ "$Log" != off ]]; then touch "$Log" || bail "Couldn't touch log file: $Log" # Redirect stdout and stderr to a log file. exec > >(tee "$Log") exec 2>&1 msg "${H1}\\nLog is: $Log" fi ThisUser=$(id -n -u) msg "Started $ThisScript v$Version as $ThisUser@${HOSTNAME%%.*} on $(date) as:\\n\\t$CmdLine" SDPSetupDir="$HxDepots/sdp/Server/Unix/setup" TmpFile1=$(mktemp) TmpFile2=$(mktemp) TmpFile3=$(mktemp) #------------------------------------------------------------------------------ msg "Starting Preflight Checklist." SDPConfigScript="$SDPInstallDir/Server/Unix/setup/mkdirs.sh" SDPConfigFile="$SDPInstallDir/Server/Unix/setup/mkdirs.cfg" if [[ -x "$SDPConfigScript" ]]; then msg "Verified: SDP config script is executable: [$SDPConfigScript]." else errmsg "SDP config script not executable: [$SDPConfigScript]." fi if [[ -r "$SDPConfigFile" ]]; then msg "Verified: SDP config file exists: [$SDPConfigFile]." else errmsg "SDP config file not found: [$SDPConfigFile]." fi SDPHostsFile="/p4/common/site/config/sdp_hosts.cfg" SDPHostsSample="$HMSInstallRoot/p4/common/site/config/sdp_hosts.cfg.sample" if [[ -f "$SDPHostsFile" ]]; then msg "Verified: SDP hosts file exists: [$SDPHostsFile]." else if [[ -e "$SDPHostsSample" ]]; then msg "Generate SDP Hosts file [$SDPHostsFile] from sample [$SDPHostsSample]." TmpFile4=$(mktemp) sed -E \ -e "s|ALL_SDP_HOSTS=.*|ALL_SDP_HOSTS=\"${HOSTNAME%%.*}\"|g" \ -e "s|SDP_REPLICA_HOSTS=.*|SDP_REPLICA_HOSTS=|g" \ -e "s|SDP_REMOTE_HOSTS=.*|SDP_REMOTE_HOSTS=|g" \ -e "s|SDP_SYNC_HOSTS=.*|SDP_SYNC_HOSTS=\"\$ALL_SDP_HOSTS\"|g" \ -e "s|SDP_STATUS_DIRS=.*|SDP_STATUS_DIRS=\"/p4/common/bin /p4/common/cloud /p4/common/config /p4/common/etc /p4/common/lib /p4/common/test\"|g" \ "$SDPHostsSample" > "$TmpFile4" ||\ errmsg "Could not generate SDP Hosts temp file [$TmpFile4] from sample [$SDPHostsSample]." dbg "Generated SDP hosts file:\\n$(cat "$TmpFile4")" if [[ "$NoOp" -eq 0 ]]; then # The /p4/common/site should always exist, but /p4/common/site/config might not. # Create that directory if needed; use 'mkdir' but not 'mkdir -p' since we should # only need to create one directory level. [[ -d /p4/common/site/config ]] || mkdir /p4/common/site/config mv -f "$TmpFile4" "$SDPHostsFile" ||\ errmsg "Failed to move generated SDP hosts temp file [$TmpFile4] to [$SDPHostsFile]." else msg "NO_OP: Would update SDP hosts file [$SDPHostsFile] from temp file [$TmpFile4]." fi else errmsg "SDP hosts file missing: [$SDPHostsFile], and also the sample from which to generate one is missing: $SDPHostsSample\\n" fi fi # Extact values already defined in mkdirs.cfg. DD=$(grep "^DD=" "$SDPConfigFile"|tail -1|cut -d '=' -f 2) DB1=$(grep "^DB1=" "$SDPConfigFile"|tail -1|cut -d '=' -f 2) DB2=$(grep "^DB2=" "$SDPConfigFile"|tail -1|cut -d '=' -f 2) if [[ -z "$DB1" ]]; then # Backward compatibility for older form of mkdirs.cfg, which # used MD= for /metadata rather than DB1 for /hxmetadata. DB1=$(grep "^MD=" "$SDPConfigFile"|tail -1|cut -d '=' -f 2) DB2="$DB1" fi LG=$(grep "^LG=" "$SDPConfigFile"|tail -1|cut -d '=' -f 2) OSUSER=$(grep "^OSUSER=" "$SDPConfigFile"|tail -1|cut -d '=' -f 2) MAILFROM=$(grep "^MAILFROM=" "$SDPConfigFile"|tail -1|cut -d '=' -f 2) ADMINPASS=$(grep "^ADMINPASS=" "$SDPConfigFile"|tail -1|cut -d '=' -f 2) ADMINUSER=$(grep "^ADMINUSER=" "$SDPConfigFile"|tail -1|cut -d '=' -f 2) # Removed excess whitespace. # shellcheck disable=SC2116 DB1=$(echo "$DB1") # shellcheck disable=SC2116 DD=$(echo "$DD") # shellcheck disable=SC2116 LG=$(echo "$LG") # shellcheck disable=SC2116 OSUSER=$(echo "$OSUSER") # Find all active SDP Instances. ActiveInstances= for i in /p4/*; do [[ -d "$i" ]] || continue [[ -f "$i/root/db.domain" ]] || continue ActiveInstances+=" ${i#/p4/}" done # Trim excess whitespace. # shellcheck disable=SC2116 ActiveInstances=$(echo "$ActiveInstances") if [[ -n "$ActiveInstances" ]]; then msg "The following active SDP instances were detected: $ActiveInstances." else msg "No Active SDP Instances Detected." fi DirList="/p4 /$DD/p4/common/bin /$LG/p4 /$DB1/p4" [[ "$DB1" == "$DB2" ]] || DirList="$DirList /$DB2/p4" for dir in $DirList; do if [[ -d "$dir" ]]; then msg "Verified: SDP directory exists: [$dir]." else errmsg "Missing SDP directory: [$dir]." fi done if [[ -n "$ActiveInstances" ]]; then msg "Checking directories for active instances: $ActiveInstances" for i in $ActiveInstances; do DirList="/p4/$i/bin /$DD/p4/$i/checkpoints /$DD/p4/$i/depots /$LG/p4/$i/logs /$DB1/p4/$i/db1 /$DB2/p4/$i/db2" for dir in $DirList; do if [[ -d "$dir" ]]; then msg "Verified: SDP directory for instance $i exists: [$dir]." else errmsg "Missing SDP directory for instance $i: [$dir]." fi done done fi msg "Checking whether any directories for hms instance already exist." DirList="/$DD/p4/hms/checkpoints /$DD/p4/hms/depots /$LG/p4/hms/logs /$LG/p4/hms/tmp" if [[ "$DB1" == "$DB2" ]]; then DirList="$DirList /$DB1/p4/hms/db1 /$DB1/p4/hms/db2" else DirList="$DirList /$DB1/p4/hms/db1 /$DB2/p4/hms/db2" fi for dir in $DirList; do if [[ -d "$dir" ]]; then if [[ -z "$(cd "$dir"; ls|grep -Ev '(server.id|save)')" ]]; then msg "Verified: HMS SDP directory is empty: [$dir]." else errmsg "HMS SDP directory exists but it should not exist or should be empty: [$dir]." fi else msg "Verified: HMS SDP directory doesn't (yet) exist: [$dir]." fi done if [[ $(id -u -n) != "$OSUSER" ]]; then errmsg "Must run as $OSUSER, not $(id -u -n)." else msg "Verified: Running os OSUSER ($OSUSER)." fi if [[ "$UseSudo" -eq 1 ]]; then # Test sudo access. Ignore the NoOp setting since we want # to try this preflight check even in no-op mode if run "sudo /bin/ls /etc/sudoers.d > /dev/null" "Testing sudo access. If prompted for a password, do Ctrl-C. Then enable sudo access for the perforce user and try again." 0 1; then msg "Verified: OSUSER ($OSUSER) has sudo privs." else errmsg "Could not verify sudo access for OSUSER $OSUSER. Sudo access required." fi else msg "Not verifying sudo access." fi if [[ "$ErrorCount" -eq 0 ]]; then msg "Preflight Checklist Completed OK." else bail "Aborting due to failed preflight checks." fi if [[ "$Interactive" -ne 0 ]]; then input="" while [[ -z "$input" ]]; do echo -e -n "\\nEnter Y to proceed, N to stop [y/Y/n/N]: " read -r -e input if [[ "${input}" == "Y" || $input == "y" ]]; then continue elif [[ "${input}" == "N" || $input == "n" ]]; then msg "Confirmation to proceed not received. Halting." exit 1 else input= fi done fi #------------------------------------------------------------------------------ msg "${H2}\\nStep 1: Initialize HMS SDP instance as /p4/hms." if [[ ! -e /p4/hms ]]; then cd "$SDPSetupDir" ||\ bail "Failed to cd to SDP setup dir: $SDPSetupDir" msg "Operating in: $PWD" if [[ "$UseSudo" -eq 1 ]]; then run "sudo $SDPConfigScript hms > mkdirs.hms.log 2>&1" \ "Configuring new HMS instance of SDP in /p4/hms as root." 1 1 ||\ bail "Failed to initialize HMS instance as root." else run "$SDPConfigScript hms > mkdirs.hms.log 2>&1" \ "Configuring new HMS instance of SDP in /p4/hms as $OSUSER." 1 1 ||\ bail "Failed to initialize HMS instance as $OSUSER." fi else msg "Skipping call to mkdirs.sh because /p4/hms already exists." fi [[ -f "$HMSVarsFile" ]] ||\ bail "After HMS initialization, expected file is missing: $HMSVarsFile." run "chmod +w $HMSVarsFile" "Making HMS vars file writable." if [[ "$NoOp" -eq 0 ]]; then sed -e "s:export P4PORTNUM=.*:export P4PORTNUM=7468:g" \ -e "s:export P4BROKERPORTNUM=.*:export P4BROKERPORTNUM=7467:g" \ $HMSVarsFile > "$TmpFile1" ||\ bail "Failed to export." mv -f "$TmpFile1" "$HMSVarsFile" || bail "Failed to update $HMSVarsFile." else msg "NO_OP: Would have updated $HMSVarsFile." fi #------------------------------------------------------------------------------ msg "${H2}\\nStep 2: Populate HMS instance with static load data." cd "$HMSSetupDir" ||\ bail "Failed to cd to HMS setup dir: $HMSSetupDir" msg "Operating in: $PWD" if [[ "$NoOp" -eq 0 ]]; then msg "Loading HMS environment." # shellcheck disable=SC1091 source /p4/common/bin/p4_vars hms ||\ bail "Failed to load HMS environment." else msg "NO_OP: Would load HMS environment." fi # Environment safety check: if [[ "$P4ROOT" == "/p4/hms/root" ]]; then msg "Verified: HMS Environment looks OK." else # In NO_OP mode, P4ROOT won't be set correctly, so just skip this test. if [[ "$NoOp" -eq 0 ]]; then bail "HMS environment is not as expected, P4ROOT is $P4ROOT." fi fi if [[ "$P4PORT" =~ ^ssl[46]*: && ! -f /p4/ssl/certificate.txt ]]; then run "/p4/hms/bin/p4d_hms -Gc" "Generating SSL Certificate" 1 1 ||\ bail "Failed to generate SSL certificate." fi # Lower SDP default 5G diskspace safey limits by hacking configure_new_server.sh before # running it. sed -Ei -e 's|5G|20M|g' /p4/sdp/Server/setup/configure_new_server.sh run "/p4/sdp/Server/setup/configure_new_server.sh hms" \ "Starting and configuring new HMS instance of p4d." 1 1 ||\ bail "Failed to start new HMS instance." sed -e "s:REPL_MAILFROM:$MAILFROM:g" \ swarm.user.p4t > swarm.user.p4s run "p4 user -f -i < swarm.user.p4s" "Creating user swarm." ||\ bail "Failed to create user swarm." for g in *.group.p4t; do _gtmp=$(echo "$g" | cut -f1 -d.) rm -f "$_gtmp.group.p4s" sed -e "s:REPL_ADMINUSER:$ADMINUSER:g" "$g" > "$_gtmp.group.p4s" done for g in *.group.p4s; do run "p4 group -i < $g" "Loading group spec file $g." ||\ bail "Failed to load group spec file $g with contents:\\n$(cat "$g")" done run "p4 triggers -i < triggers.p4s" "Loading triggers table." ||\ bail "Failed to load triggers table with contents:\\n$(cat triggers.p4s)" for d in *.depot.p4t; do _dtmp=$(echo "$d" | cut -f1 -d.) rm -f "$_dtmp.depot.p4s" sed -e "s:REPL_ADMINUSER:$ADMINUSER:g" "$d" > "$_dtmp.depot.p4s" done for d in *.depot.p4s; do run "p4 depot -i < $d" "Loading depot spec file $d." ||\ bail "Failed to load depot spec file $d with contents:\\n$(cat "$d")" done for s in *.stream.p4t; do _stmp=$(echo "$s" | cut -f1 -d.) rm -f "$_stmp.stream.p4s" sed -e "s:REPL_ADMINUSER:$ADMINUSER:g" "$s" > "$_stmp.stream.p4s" # shellcheck disable=SC2072 if [[ "$P4D_VERSION" > "2020.2" ]]; then echo -e "\\nParentView: inherit\\n" >> "$_stmp.stream.p4s" fi done # Stream specs need to be created in a certain order, respecting the # parenting relationships; e.g. 'main' must be created before a child # of 'main' can be created. Rather than work out the parenting # relationships to elegantly create streams in the correct order, here # we take a simple brute-force approach of making several passes at # creating the entire list of streams. Streams that can't be created on # the first pass because their parent does't yet exist will succeed on a # subsequent pass. This simple approach results in harmless errors # creating streams. We arbitrarily select 5 passes, which will create up # to 5 levels of streams. # shellcheck disable=SC2034 for pass in {1..5}; do for s in *.stream.p4s; do run "p4 stream -i < $s" "Loading stream spec file $s." ||\ msg "Info: Ignore this known-harmless error creating a stream spec." done done # Do one final pass at stream spec creation. On this final passs, the stream # specs should already exist, so problems loading stream specs at this point # are treated as hard errors. for s in *.stream.p4s; do run "p4 stream -i < $s" "Loading stream spec file $s." ||\ bail "Failed to load stream spec file $s with contents:\\n$(cat "$s")" done msg "Create the deployment virtual stream for host $ThisHost." echo -e "Stream: //sdp/deploy_${ThisHost}\\n Owner: $ADMINUSER\\n Name: deploy_${ThisHost}\\n Parent: //sdp/main\\n Type: virtual\\n Description: \\tThis stream stream manages the deployment of SDP files \\ton $ThisHost.\\n Options: allsubmit unlocked notoparent nofromparent mergedown\\n Paths: \\tshare Server/Unix/.p4ignore \\tshare Server/Unix/p4/common/..." > "$TmpFile3" for i in hms $ActiveInstances; do echo -e "\\tshare host/$ThisHost/p4/$i/bin/..." >> "$TmpFile3" done echo -e "\\nRemapped: \\tServer/Unix/.p4ignore .p4ignore \\tServer/Unix/p4/common/... common/..." >> "$TmpFile3" for i in hms $ActiveInstances; do echo -e "\\thost/$ThisHost/p4/$i/bin/... $i/bin/..." >> "$TmpFile3" done # shellcheck disable=SC2072 if [[ "$P4D_VERSION" > "2020.2" ]]; then echo -e "\\nParentView: inherit" >> "$TmpFile3" fi run "p4 stream -i < $TmpFile3" "Loading generated deployment stream spec." ||\ bail "Failed to load this generated stream spec:\\n$(cat "$TmpFile3")\\n" rm -f "$TmpFile3" for r in *.remote.p4s; do run "p4 remote -i < $r" "Loading remote spec file $r." ||\ bail "Failed to load remote spec file $r." done for b in *.branch.p4t; do # Remove the generated Perforce spec file with a *.p4s extension, to be # generated from a template with a .p4t extension. rm -f "${b/p4t/p4s}" sed -e "s:REPL_ADMINUSER:$ADMINUSER:g" \ "$b" > "${b/p4t/p4s}" run "p4 branch -i < ${b/p4t/p4s}" "Loading branch spec file ${b/p4t/p4s}." ||\ bail "Failed to load branch spec file ${b/p4t/p4s}." done rm -f sdp.hostname.client.p4s sed -e "s:REPL_ADMINUSER:$ADMINUSER:g" \ -e "s:REPL_SHORT_HOSTNAME:$ThisHost:g" \ -e "s:REPL_HOSTNAME:$(hostname):g" sdp.hostname.client.p4t > sdp.hostname.client.p4s ||\ warnmsg "Error generating client spec for deployment workspace." rm -f superuser.sdp_main.client.p4s sed -e "s:REPL_P4USER:$P4USER:g" \ -e "s:REPL_ADMINUSER:$ADMINUSER:g" \ -e "s:REPL_SHORT_HOSTNAME:$ThisHost:g" \ -e "s:REPL_HOSTNAME:$(hostname):g" \ superuser.sdp_main.client.p4t > superuser.sdp_main.client.p4s ||\ warnmsg "Error generating client spec for //sdp_main." for c in *.client.p4s; do run "p4 client -i < $c" "Loading client spec file $c." ||\ bail "Failed to load client spec file $c with contents:\\n$(cat "$c")" done run "p4master_run hms p4 configure unset filesys.checklinks" ||\ warnmsg "Failed to unset filesys.checklinks configurable." msg "Enabling DVCS features." run "p4master_run hms p4 configure set server.allowpush=3" ||\ bail "Failed to set server.allowpush configurable." run "p4master_run hms p4 configure set server.allowfetch=3" ||\ bail "Failed to set server.allowfetch configurable." #------------------------------------------------------------------------------ msg "${H2}\\nStep 3: Fetch SDP from The Workshop." msg "Fetching the SDP." run "p4master_run hms p4 fetch" ||\ bail "Failed to fetch SDP from The Workshop." #------------------------------------------------------------------------------ msg "${H2}\\nStep 4: Populate local mainline." msg "Branching files from Workshop SDP mainline //sdp/workshop_main to local mainline //sdp/main." run "p4 populate -S //sdp/workshop_main" ||\ bail "Failed to populate //sdp/main from //sdp/workshop_main." msg "Generating P4CONFIG file /p4/.p4config.SDP." run "p4master_run hms echo P4PORT=\$P4PORT > /p4/.p4config.SDP" run "p4master_run hms echo P4USER=\$P4USER >> /p4/.p4config.SDP" run "echo P4CLIENT=sdp.$ThisHost >> /p4/.p4config.SDP" run "p4master_run hms echo P4TICKETS=\$P4TICKETS >> /p4/.p4config.SDP" run "p4master_run hms echo P4TRUST=\$P4TRUST >> /p4/.p4config.SDP" run "echo P4IGNORE=.p4ignore >> /p4/.p4config.SDP" run "echo net.autotune=1 >> /p4/.p4config.SDP" msg "Setting P4CONFIG=/p4/.p4config.SDP." export P4CONFIG=/p4/.p4config.SDP run "p4 set" run "p4 flush" ||\ bail "Failed to flush." #------------------------------------------------------------------------------ msg "${H2}\\nStep 5: Overlay HMS on the SDP." run "rsync -av $HMSInstallRoot/p4/common/ /p4/common" \ "Overlaying files in /p4/common" ||\ bail "Failed to overlayfiles into: /p4/common" #------------------------------------------------------------------------------ msg "${H2}\\nStep 6: Version key files in /p4/common and /p4/N/bin dirs." run "p4 -s sync -f /p4/.p4ignore" "Deploy /p4/.p4ignore." ||\ bail "Failed to force-sync the /p4/.p4ignore file." DirList="/p4/common/bin /p4/common/etc /p4/common/config /p4/common/site" for i in $ActiveInstances; do DirList+=" /p4/$i/bin" done msg "Reconciling to detect local differences." for d in $DirList; do run "p4 -d $d rec" ||\ bail "Failed to reconcile in $d." done #------------------------------------------------------------------------------ msg "${H2}\\nStep 7: Sync SDP management workspace." SDPMgmtWs="$P4USER.${HOSTNAME%%.*}.sdp_main" SDPMgmtP4Cfg=/p4/hms/tmp/sdp_main/.p4config run "p4master_run hms p4 -c $SDPMgmtWs -s sync" ||\ bail "Failed to sync SDP management workspace [$SDPMgmtWs]." msg "Generating P4CONFIG file $SDPMgmtP4Cfg" run "p4master_run hms echo P4PORT=\$P4PORT > $SDPMgmtP4Cfg" run "p4master_run hms echo P4USER=\$P4USER >> $SDPMgmtP4Cfg" run "echo P4CLIENT=$SDPMgmtWs >> $SDPMgmtP4Cfg" run "p4master_run hms echo P4TICKETS=\$P4TICKETS >> $SDPMgmtP4Cfg" run "p4master_run hms echo P4TRUST=\$P4TRUST >> $SDPMgmtP4Cfg" run "echo P4IGNORE=.p4ignore >> $SDPMgmtP4Cfg" run "echo net.autotune=1 >> $SDPMgmtP4Cfg" if [[ "$NoOp" -eq 0 ]]; then [[ -r "$SDPMgmtP4Cfg" ]] || \ bail "Failed to generate SDP management P4CONFIG file [$SDPMgmtP4Cfg]." fi #------------------------------------------------------------------------------ msg "${H2}\\nStep 8: Do initial live checkpoint for new hms instance." msg "Taking a live checkpoint." run "p4master_run hms -c $P4CBIN/live_checkpoint.sh" ||\ bail "Failed to initialize SDP offline checkpoint process with a live checkpoint." #------------------------------------------------------------------------------ msg "${H2}\\nStep 9: Custom site-local setup." if [[ -x "$CustomSetupScript" ]]; then run "$CustomSetupScript" \ "Executing custom setup script specified with \$HMS_CUSTOM_SETUP_SCRIPT." ||\ bail "Custom setup script returned non-zero exit code." else msg "No custom setup script found." fi #------------------------------------------------------------------------------ if [[ "$ErrorCount" -eq 0 && "$WarningCount" -eq 0 ]]; then msg "${H2}\\nAll processing completed successfully.\\n" elif [[ "$ErrorCount" -eq 0 ]]; then msg "${H2}\\nProcessing completed with no errors but with $WarningCount warnings. Scan above output carefully.\\n" else msg "${H2}\\nProcessing completed, but with $ErrorCount errors and $WarningCount warnings. Scan above output carefully.\\n" fi # Illustrate using $SECONDS to display runtime of a script. msg "That took $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n" # See the terminate() function, which is really where this script exits. exit "$ErrorCount"
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#8 | 30242 | C. Thomas Tyler | Updated package reference in LICENSE to refer to HMS rather than SDP license. | ||
#7 | 30162 | C. Thomas Tyler | Hack 5G disk space safety limits down to 20M for demo environments. | ||
#6 | 30124 | C. Thomas Tyler |
hms_ts_setup.sh v3.0.0: * Now works with now-preferred model of a stand-alone dedicated p4hms server machine. * Adapted to SDP 2023.2. |
||
#5 | 29190 | C. Thomas Tyler | Minor revisions; functional version. | ||
#4 | 29189 | C. Thomas Tyler | Tweaked order of ops. | ||
#3 | 29188 | C. Thomas Tyler | Enhanced environment isolation. | ||
#2 | 29185 | C. Thomas Tyler |
Added ParentView field to generated stream specs if P4D_VERSION is 2020.2+. Fixed bugs in logic to cleanup *.p4s (p4 spec) files before regenerationg from *.p4t (p4 spec template) files. |
||
#1 | 29182 | C. Thomas Tyler |
Moved HMS files from /p4/common/bin -> /p4/common/site/bin. Moved HMS files from /p4/common/lib -> /p4/common/site/lib. Removed dependency on SDP libs so that HMS can be deployed with a wider variety of SDP versions. |
||
//guest/perforce_software/hms/dev/p4/common/hms/setup/hms_ts_setup.sh | |||||
#11 | 27694 | C. Thomas Tyler | Copyright Update; non-functional change. | ||
#10 | 27693 | C. Thomas Tyler | Fixed issue in generation of P4CONFIG file for SDP Management workspace. | ||
#9 | 27692 | C. Thomas Tyler | Honor /p4/.p4ignore file when versioning files under /p4/common, /p4/N/bin. | ||
#8 | 26857 | C. Thomas Tyler | Fixed typo in password file. | ||
#7 | 25948 | C. Thomas Tyler |
hms_ts_setup.sh v2.4.0: * Adjusted so lack of any active SDP instances is no longer an error condition, and results in an info message (not even a warning). Previously, the typical scenario was to run this after installing a live data p4d instance on the same host. Now it is becoming more common to install HMS right after running the Helix Installer, with no live data instances, on a Linux bastion host where the p4_hms instance is the only one intended to be on the machine. * Minor style consistency changes with usage of quotes. |
||
#6 | 25877 | C. Thomas Tyler | Correct substituion issue. | ||
#5 | 25875 | C. Thomas Tyler |
Added logic to generate and load branch spec file for the HMS_to_SDP branch spec. |
||
#4 | 25873 | C. Thomas Tyler |
Added new phase to generate SDP management workspace on //sdp/main stream. |
||
#3 | 25712 | C. Thomas Tyler |
Made hms_ts_setup.sh shellcheck compliant. Removed custom hacks for Battle School demo from hms_ts_setup.sh. Added generic mechanism for calling custom scripts by defining environment variables referencing them (undoc'd) to both hms_ts_setup.sh and reset_hms.sh. |
||
#2 | 25653 | C. Thomas Tyler |
Updated the HMS "Tight Ship" setup to work with HMS now being a stand-lone product, layered on the SDP but separate from it. This version has a few 'HACK' comments needed to do a demo; these will be removed in a subsequent version. |
||
#1 | 25533 | C. Thomas Tyler |
Copied updated and new files from SDP into the new HMS "overlay" structure. A 'p4 copy' was done in all cases, so files in this change match what they did in the SDP. Corresponding files in the SDP are to be deleted. Some files will need modification to adapt to the new HMS structure, e.g. the 'setup' tree. |
||
//guest/perforce_software/sdp/dev/Server/Unix/setup/hms/hms_ts_setup.sh | |||||
#16 | 25501 | C. Thomas Tyler | Hardening fix for hms_ts_setup.sh | ||
#15 | 25478 | C. Thomas Tyler |
Adapted HMS setup to new SDP structure. Bypassing pre-commit review for obvious fixes and to enable testing. #review-25479 |
||
#14 | 25114 | C. Thomas Tyler |
Fixed bug where an extra user named REPL_OSER was created on the 'hms' server, and the 'perforce' user was created wrongly. |
||
#13 | 24474 | C. Thomas Tyler |
In hms_ts_setup.sh, removed assumption that /p4/N is a symlink. This account for new standard structure where /p4/N is a regular directory on the root '/' volume containing symlinks, but is not itself a symlink. |
||
#12 | 24219 | C. Thomas Tyler |
Changed hms remote spec for the Public Depot to be the default 'origin'. This simplifies the update even further, so the fetch can be done using simply 'p4 fetch' without requiring the '-r' flag to specify a remote spec name. |
||
#11 | 23455 | cgeen |
Updates to allow an alternate P4USER and not OSUSER. This is because in a hardened setup OSUSER should have no permissions into perforce |
||
#10 | 22401 | C. Thomas Tyler |
hms_ts_setup.sh v1.1.1: * Adjusted to not require 'sudo' by default, and to only check for sudo access if '-s' flag is provided. Since the 'mkdirs.sh' script (SDP directory structure initialization script) must have already been run once, we can assume we don't need to run it with root when this script runs it a second time. The '-s' flag is there to verify that sudo access is setup correctly if desired, as having sudo access is preferred for HMS, since it is necessary to manage Swarm. * Adjusted to preferred HMS ports, 7467 for broker (PHMS on a phone keypad, for Perforce Helix Management System), and 7468 for p4d. |
||
#9 | 22268 | C. Thomas Tyler |
Updated HMS tight ship setup script to adapt to changes in mkdirs.sh and introduction of mkidrs.cfg. Also fixed minor bugs in output indicating certain assumptions were verifed after an error messsage indicated they weren't. |
||
#8 | 22015 | C. Thomas Tyler |
Corrected format of sudo file for perforce in HMS docs. Doc correction only; no functional change. |
||
#7 | 21526 | C. Thomas Tyler |
Updated hms_ts_setup.sh: * Adapted to recent changes in SDP structure. * Fixed issue with doing a shallow clone with '-m1'; it must now do a full clone to avoid a funky error. * Updated so that failure to unset filesys.checklinks configurable is merely a warning, not a hard error. |
||
#6 | 21404 | C. Thomas Tyler |
Added settings for P4TICKETS and P4TRUST to generated SDP P4CONFIG file, /p4/.p4config.SDP. |
||
#5 | 21403 | C. Thomas Tyler |
Added support for SSL. Fixed issue with '-n' mode always failing environment safety check. Fixed issue with confirmation to proceed (Yes was Yes, No was No, but random text was treated as a Yes. Now it's just treated as random text). Corrected log file name. |
||
#4 | 21398 | C. Thomas Tyler |
Added missing .p4ignore file to virtural stream spec used for SDP host management workspaces. Unset filesys.checklinks configurable. This is set in configure_new_server.sh. It's a reasonable configurable for most environments, but doesn't work well for versioning the SDP, since it's all about versioning a structure with symlinks in directory paths. |
||
#3 | 21298 | C. Thomas Tyler |
Updated to new default /hxdepots structure. Completed basic implementation; now does live checkpoint. Bypassing pre-commit review; update to new/unreleased feature. |
||
#2 | 21294 | C. Thomas Tyler | Merged down hms dev work. | ||
#1 | 21103 | C. Thomas Tyler | Merge down to dev from main. | ||
//guest/perforce_software/sdp/main/Server/Unix/setup/hms/hms_ts_setup.sh | |||||
#2 | 21091 | C. Thomas Tyler |
Enhancements to hms_ts_setup.sh and auxiliary files. Enahnced preflight checks to include sudo access check, etc. Enhanced cosmetics. Enhanced doc, adding Additional Setup section. This is still a doc-only verison. Bypassing pre-commit review. #review-21092 |
||
#1 | 21081 | C. Thomas Tyler |
Added preliminary files in support of automation of the HMS Tight Ship installation. The key file here hs hms_ts_setup.sh, which is currently a 'doc only' script that documents what it is intended to do, although it doesn't currently do anything that affects data. The '-h' and '-man' flags work to convey their intent. This also contains various spec files to be used in the actual implementation. Bypassing pre-commit review since this is only adding new 'doc only' code, not active software. #review-21082 |