#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # this package, and also available online: # https://swarm.workshop.perforce.com/view/guest/perforce_software/helix-installer/main/LICENSE #------------------------------------------------------------------------------ set -u #------------------------------------------------------------------------------ # Declarations declare Version=1.8.2 declare ResetTarget=/hxdepots declare DownloadsDir="$ResetTarget/downloads" declare BackupDir=Unset declare BackupFile= declare TmpFile="/tmp/tmp.csd4sdp.$$.$RANDOM" declare RunUser=perforce declare ThisUser= declare ThisHost= declare CBIN=/p4/common/bin declare ThisScript="${0##*/}" declare CmdLine="$0 $*" declare SDPInstance=Unset declare PasswordFile= declare P4DInitScript= declare P4DSystemdServiceFile= declare P4BrokerInitScript= declare P4BrokerSystemdServiceFile= declare -i MaxStartDelay=120 declare -i P4DStartVerified=0 declare -i P4BrokerStartVerified=0 declare -i i=0 #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). #------------------------------------------------------------------------------ function usage { declare style=${1:--h} msg "USAGE for $ThisScript v$Version: $ThisScript -i <sdp_instance> [-d <data_dir>] [-u <osuser>] or $ThisScript [-h|-man] " if [[ $style == -man ]]; then msg " DESCRIPTION: This script transforms a stock Sample Depot instance into one that works with the SDP. REQUIREMENTS: A P4D process must be live and running with the stock Sample Depot data set, on a sport ARGUMENTS: -i <sdp_instance> Specify the SDP Instance in which the Sample Depot data set is running. This argument is required. -d <data_dir> Specify the data directory where supporting files exist, such as the *.p4s data files used by this script. -u <osuser> Specify the Linux operating system user account under which p4d runs. If omitted, the default is 'perforce'. -D Set extreme debugging verbosity. HELP OPTIONS: -h Display short help message -man Display man-style help message EXAMPLES: Usage to configure Instance 1: cd /where/this/script/is $ThisScript 1 2>&1 | tee log.${ThisScript%.sh}.1 Usage to configure Instance abc: cd /where/this/script/is $ThisScript abc 2>&1 | tee log.${ThisScript%.sh}.abc " fi exit 1 } #------------------------------------------------------------------------------ # Function bail(). # Sample Usage: # bail "Missing something important. Aborting." # bail "Aborting with exit code 3." 3 function msg () { echo -e "$*"; } function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; } function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; } function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; } #------------------------------------------------------------------------------ # Functions. The runCmd() function is similar to functions defined in SDP core # libraries, but we need to duplicate them here since this script runs before # the SDP is available on the machine (and we want no dependencies for this # script. function runCmd { declare cmd=${1:-echo Testing runCmd} declare desc=${2:-""} declare cmdToShow=$cmd [[ "$cmdToShow" == *"<"* ]] && cmdToShow=${cmdToShow%%<*} [[ "$cmdToShow" == *">"* ]] && cmdToShow=${cmdToShow%%>*} [[ -n "$desc" ]] && msg "$desc" msg "Running: $cmdToShow" $cmd return $? } #============================================================================== # Command Line Processing declare -i shiftArgs=0 declare -i ErrorCount=0 declare -i WarningCount=0 declare DataDir="$PWD" set +u while [[ $# -gt 0 ]]; do case $1 in (-i) SDPInstance=$2; shiftArgs=1;; (-d) DataDir="$2"; shiftArgs=1;; (-u) RunUser="$2"; shiftArgs=1;; (-h) usage -h;; (-man) usage -man;; (-D) set -x;; # Debug; use 'set -x' mode. esac # Shift (modify $#) the appropriate number of times. shift; while [[ $shiftArgs -gt 0 ]]; do [[ $# -eq 0 ]] && bail "Usage Error: Wrong numbers of args or flags to args." shiftArgs=$shiftArgs-1 shift done done set -u #------------------------------------------------------------------------------ # Usage Validation [[ $SDPInstance == Unset ]] && \ bail "Bad Usage: The '<sdp_instance>' argument is required." [[ ! -r $CBIN/p4_vars ]] && \ bail "Missing SDP Environment File [$CBIN/p4_vars]. Aborting." #------------------------------------------------------------------------------ # Main Program ThisUser=$(whoami) ThisHost=${HOSTNAME%%.*} msg "Started $ThisScript v$Version on host $ThisHost as user $ThisUser at $(date), called as:\\n\\t$CmdLine" if [[ "$ThisUser" != "$RunUser" ]]; then bail "Run as $RunUser, not $ThisUser." else msg "Verified: Running as user $RunUser." fi # Load SDP environment and variable definitions. # shellcheck disable=SC1090 source "$CBIN/p4_vars" "$SDPInstance" ||\ bail "Failed to load SDP environment. Aborting." export P4ENVIRO=/dev/null/.p4enviro export P4CONFIG=.p4config PasswordFile="$SDP_ADMIN_PASSWORD_FILE" P4DInitScript="${P4DBIN}_init" P4DSystemdServiceFile="/etc/systemd/system/${P4DBIN##*/}.service" P4BrokerInitScript="${P4BROKERBIN}_init" P4BrokerSystemdServiceFile="/etc/systemd/system/${P4BROKERBIN##*/}.service" cd "$ResetTarget/sdp/Server/setup" ||\ bail "Failed to cd to [$ResetTarget/sdp/Server/setup]." msg "Operating in SDP server setup area [$PWD]." runCmd "$P4BIN -u bruno -s info -s" "Verifying server is offline." &&\ bail "Perforce server is unexpectedly online. Aborting." runCmd "/p4/${SDPInstance}/bin/p4d_${SDPInstance} -jr $DownloadsDir/PerforceSample/checkpoint" \ "Loading the Sample Depot metadata in instance ${SDPInstance}." ||\ bail "Failed to load Sample Depot checkpoint." runCmd "/p4/${SDPInstance}/bin/p4d_${SDPInstance} -xu" \ "Upgrading databases (p4d -xu) for instance ${SDPInstance}." ||\ bail "Failed to upgrade databases." if [[ $P4PORT == "ssl:"* ]]; then runCmd "/p4/${SDPInstance}/bin/p4d_${SDPInstance} -Gc" \ "Generating OpenSSL Certificates for instance $SDPInstance." ||\ bail "Failed to generate OpenSSL Certs for Instance $SDPInstance." fi if [[ -x "$P4DInitScript" ]]; then if [[ -f "$P4DSystemdServiceFile" ]]; then msg "Starting p4d as user $ThisUser with: sudo systemctl start p4d_${SDPInstance}" if sudo systemctl start "p4d_${SDPInstance}"; then msg "Service p4d_${SDPInstance} started with systemd." else warnmsg "Non-zero exit code $? starting p4d service with systemd." if [[ -r "$LOGS/p4d_init.log" ]]; then msg "Contents of $LOGS/p4d_init.log:\\n$(cat "$LOGS/p4d_init.log")" else warnmsg "No $LOGS/p4d_init.log file found." fi fi else msg "Starting p4d as user $ThisUser with: sudo service p4d_${SDPInstance}_init start" if sudo service "p4d_${SDPInstance}_init" start; then msg "Service p4d_${SDPInstance}_init started with SysV init." else warnmsg "Non-zero exit code $? starting p4d service with SysV init." if [[ -r "$LOGS/p4d_init.log" ]]; then msg "Contents of $LOGS/p4d_init.log:\\n$(cat "$LOGS/p4d_init.log")" else warnmsg "No $LOGS/p4d_init.log file found." fi fi fi else bail "Init script is missing or not executable: $P4DInitScript" fi P4DStartVerified=0 i=0; while [[ "$i" -lt "$MaxStartDelay" ]]; do if "$P4BIN" -p "$P4PORT" info -s > "$TmpFile" 2>&1; then msg "Verified: p4d has started." P4DStartVerified=1 rm -f "$TmpFile" break else # If we get an SSL trust error, then the server is indeed online. if grep -lq 'The authenticity of' "$TmpFile"; then msg "Verified: p4d has started." P4DStartVerified=1 "$P4BIN" -p "$P4PORT" trust -f -y > /dev/null 2>&1 rm -f "$TmpFile" break else sleep 1 fi fi i+=1 done [[ "$P4DStartVerified" -eq 1 ]] || \ bail "p4d start attempted but could not be verified after $MaxStartDelay seconds.\\nTail of server log $P4LOG:\\n$(tail "$P4LOG")\\n\\n" if [[ -x "$P4BrokerInitScript" ]]; then if [[ -f "$P4BrokerSystemdServiceFile" ]]; then msg "Starting p4broker as user $ThisUser with: sudo systemctl start p4broker_${SDPInstance}" if sudo systemctl start "p4broker_${SDPInstance}"; then msg "Service p4broker_${SDPInstance} started with systemd." else warnmsg "Non-zero exit code $? starting broker service with systemd." fi else msg "Starting p4broker as user $ThisUser with: sudo service p4broker_${SDPInstance}_init start" if sudo service "p4broker_${SDPInstance}_init" start; then msg "Service p4broker_${SDPInstance}_init started with SysV init." else warnmsg "Non-zero exit code $? starting broker service with SysV init." fi fi else bail "Init script is missing or not executable: $P4BrokerInitScript" fi P4BrokerStartVerified=0 i=0; while [[ "$i" -lt "$MaxStartDelay" ]]; do if "$P4BIN" -p "$P4BROKERPORT" info -s > "$TmpFile" 2>&1; then msg "Verified: p4broker has started." P4BrokerStartVerified=1 rm -f "$TmpFile" break else if grep -lq 'The authenticity of' "$TmpFile"; then msg "Verified: p4d has started." P4BrokerStartVerified=1 "$P4BIN" -p "$P4BROKERPORT" trust -f -y > /dev/null 2>&1 rm -f "$TmpFile" break else sleep 1 fi fi i+=1 done [[ "$P4BrokerStartVerified" -eq 1 ]] || \ bail "p4broker start attempted but could not be verified after $MaxStartDelay seconds.\\n\\nTail of broker log $LOGS/p4broker.log:\\n$(tail "$LOGS/p4broker.log")\\n\\n" if [[ $P4PORT == "ssl:"* ]]; then # Note: Automating a 'p4 trust -y' (especially with '-f') may not be appropriate # in a production environment, as it defeats the purpose of the Open SSL trust # mechanism. But for our purposes here, where scripts spin up throw-away data # sets for testing or training purposes, it's just dandy. runCmd "/p4/${SDPInstance}/bin/p4_${SDPInstance} -p $P4PORT trust -y -f" \ "Trusting the OpenSSL Cert of the server." ||\ bail "Failed to trust the server." runCmd "/p4/${SDPInstance}/bin/p4_${SDPInstance} -p $P4BROKERPORT trust -y -f" \ "Trusting the OpenSSL Cert of the broker." ||\ bail "Failed to trust the broker." fi runCmd "$P4BIN -s info -s" "Verifying direct connection to Perforce server." ||\ bail "Could not connect to Perforce server." runCmd "$P4BIN -u bruno -s -p $P4BROKERPORT info -s" "Verifying via-broker connection to Perforce server." ||\ bail "Could not connect to Perforce server via broker." [[ "$($P4BIN -u bruno protects -m)" == super ]] ||\ bail "Could not verify super user access for $P4USER on port $P4PORT. Is this the Sample depot? Aborting." msg "Super user access for bruno verified." msg "Creating user $P4USER." sed "s:__EDITME_ADMIN_P4USER__:$P4USER:g" "$DataDir/admin.user.p4s" > "$TmpFile" "$P4BIN" -u bruno user -f -i < "$TmpFile" msg "Adding user to NoTicketExpiration group." sed "s:__EDITME_ADMIN_P4USER__:$P4USER:g" "$DataDir/NoTicketExpiration.group.p4s" > "$TmpFile" "$P4BIN" -u bruno group -i < "$TmpFile" msg "Promoting user $P4USER to super user." "$P4BIN" -u bruno protect -o > "$TmpFile" msg "\\tsuper user $P4USER * //...\\n" >> "$TmpFile" "$P4BIN" -u bruno protect -i < "$TmpFile" cat "$PasswordFile" > "$TmpFile" cat "$PasswordFile" >> "$TmpFile" "$P4BIN" -u bruno passwd "$P4USER" < "$TmpFile" runCmd "/p4/common/bin/p4login" "Logging in $P4USER super user." ||\ bail "Failed to login super user $P4USER. Aborting." # Variable Format Sample Values # P4PORT [ssl:]<P4DPortNum> ssl:1999, 1999 # P4BROKERPORT [ssl:]<BrokerPort> ssl:1666, 1666 for p in $P4PORT $P4BROKERPORT; do if [[ $p == "ssl:"* ]]; then runCmd "$P4BIN -p $p trust -y" "Trusting P4PORT=$p." ||\ bail "Failed to trust P4PORT=$p." fi cmd="$P4BIN -u $P4USER -p $p login -a" msg "Running: $cmd < $PasswordFile" $cmd < "$PasswordFile" ||\ bail "Login as perforce using P4PORT=$p failed. Aborting." done runCmd "cat $P4TICKETS" "Showing P4TICKETS:" runCmd "mv configure_new_server.sh configure_new_server.sh.orig" \ "Tweaking configure_new_server.sh settings to values more appropriate for a demo-grade installation, e.g. reducing 5G storage limits." ||\ bail "Failed to move configure_new_server.sh to configure_new_server.sh.orig." # Warning: If the values in configure_new_server.sh are changed from 5G, this # will need to be updated. sed -e 's/filesys.P4ROOT.min=5G/filesys.P4ROOT.min=10M/g' \ -e 's/filesys.depot.min=5G/filesys.depot.min=10M/g' \ -e 's/filesys.P4JOURNAL.min=5G/filesys.P4JOURNAL.min=10M/g' \ configure_new_server.sh.orig >\ configure_new_server.sh ||\ bail "Failed to do sed substitutions in $ResetTarget/sdp/Server/setup/configure_new_server.sh.orig." runCmd "chmod +x configure_new_server.sh" msg "Changes made to configure_new_server.sh:" diff configure_new_server.sh.orig configure_new_server.sh runCmd "./configure_new_server.sh $SDPInstance" \ "Applying SDP configurables." ||\ bail "Failed to set SDP configurables. Aborting." for depot in $(/bin/ls -d $ResetTarget/downloads/PerforceSample/*); do [[ $depot == *"checkpoint"* ]] && continue [[ $depot == *"README"* ]] && continue [[ $depot == *"readme"* ]] && continue if [[ $depot == *"spec"* ]]; then runCmd "/usr/bin/rsync -a $depot/ /p4/$SDPInstance/depots/${depot##*/}" \ "Copying Sample Depot archive files for spec depot [${depot##*/}]." ||\ warnmsg "Non-zero exit code $? from rsync for depot ${depot##*/}." else runCmd "/usr/bin/rsync -a --delete $depot/ /p4/$SDPInstance/depots/${depot##*/}" \ "Copying Sample Depot archive files for depot [${depot##*/}]." ||\ warnmsg "Non-zero exit code $? from rsync for depot ${depot##*/}." fi done runCmd "$P4BIN admin updatespecdepot -a" \ "Updating spec depot." || bail "Failed to udpate spec depot. Aborting." runCmd "/usr/bin/rsync -a /p4/$SDPInstance/root/spec/ /p4/$SDPInstance/depots/spec" \ "Copying a few spec depot files." ||\ warnmsg "Non-zero exit code $? from rsync for spec depot." runCmd "/bin/rm -rf /p4/$SDPInstance/root/spec" \ "Cleanup redundant copy of spec depot files." ||: runCmd "/p4/common/bin/live_checkpoint.sh $SDPInstance" \ "Taking Live Checkpoint." || bail "Live checkpoint failed. Aborting." [[ $BackupDir == Unset ]] && BackupDir=/p4/$SDPInstance/backup if [[ -d $BackupDir ]]; then runCmd "/bin/rm -rf $BackupDir" \ "Removing old backup dir [$BackupDir]." fi if [[ ! -d $BackupDir ]]; then runCmd "/bin/mkdir -p $BackupDir" \ "Creating new empty backups directory: $BackupDir." ||\ bail "Failed to create backups dir [$BackupDir]. Aborting." fi BackupFile="$BackupDir/p4_$SDPInstance.backup.$(date +'%Y-%m-%d-%H%M').tgz" LastCheckpoint="$(ls -1 -t /p4/"$SDPInstance"/checkpoints/p4_"${SDPInstance}".ckp.*.gz 2>/dev/null)" BackupPaths="/p4/${SDPInstance}/depots" [[ -n "$LastCheckpoint" ]] && BackupPaths="$BackupPaths $LastCheckpoint" runCmd "tar -czf $BackupFile $BackupPaths" \ "Creating backup $BackupFile." ||\ bail "Failed to backup instance $SDPInstance. Aborting." msg "\\nSUCCESS: SDP Instance $SDPInstance loaded with sample depot data, live checkpoint done, and backup created. Good to go!\\n" exit 0
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#21 | 28415 | C. Thomas Tyler |
Released SDP 2021.4.28409 (2021/11/24). Copy Up using 'p4 copy -r -b perforce_software-helix-installer-dev'. |
||
#20 | 28129 | C. Thomas Tyler |
Released SDP 2021.3.28126 (2021/10/24). Copy Up using 'p4 copy -r -b perforce_software-helix-installer-dev'. |
||
#19 | 27427 | C. Thomas Tyler |
Released SDP 2021.2.27425 (2021/02/09). Copy Up using 'p4 copy -r -b perforce_software-helix-installer-dev'. |
||
#18 | 27229 | C. Thomas Tyler | Released Helix Installer 2021.1.27227 (2021/01/20). | ||
#17 | 27165 | C. Thomas Tyler | Released HelixInstaller 2021.1.27163 (2021/01/11). | ||
#16 | 27139 | C. Thomas Tyler | Released HelixInstaller 2021.1.27137 (2021/01/03). | ||
#15 | 26793 | C. Thomas Tyler | Released HelixInstaller/MultiArch/2020.1/26784 (2020/09/25). | ||
#14 | 26578 | C. Thomas Tyler | Released Helix Installer 2020.1.26576 (2020/06/17). | ||
#13 | 26031 | C. Thomas Tyler | Released Helix Installer 2019.4.26026 (2019/08/22). | ||
#12 | 25822 | C. Thomas Tyler | Released HelixInstaller/MultiArch/2019.3/25820 (2019/07/11). | ||
#11 | 22292 | C. Thomas Tyler |
Tweaked to use P4CONFIG=.p4config.local for 'p4 clone' operation only, and then it is unset to defer to the SDP standard. This fixes an issue where the local repo cloned from The Workshop with 'p4 clone' (using DVCS features) was getting confused with the 'p4d' instance we were trying to initialize. Also updated to download 2017.1 p4/p4d/p4broker executables plus Swarm 2017.1 package. |
||
#10 | 22279 | C. Thomas Tyler |
configure_sample_depot_for_sdp.sh v1.4.8: * Refactored to operate in SDP server setup dir for more deterministic/reliable operation. * Fixed trust mechanism to trust both 'p4d' and 'p4broker'. |
||
#9 | 21839 | C. Thomas Tyler |
reset_sdp.sh v2.8.23: * Added exlusions to 'preserve dirs' feature to avoid copying DVCS repo files. * Reduced excessive noise losing the '-v' flag for tarfile extraction. * Enhanced to use 'curl' rather than 'wget', as it is functionally equivalent for our purposes, and is more ubiquituous. configure_sample_depot_for_sdp.sh v1.4.7: * Enhanced environment isolation. |
||
#8 | 21830 | C. Thomas Tyler |
Adjusted production-grade values for Helix Server 'filesys.*.min' configurables to values more appropriate for a minimally-provisioned demo-grade installation, reducing safety check from 5G to 10M. #review-21831 @russell_jackson |
||
#7 | 21500 | C. Thomas Tyler |
Updated to latest SDP. Fixed bug where spec depot files from Sample Depot were lost. Removed SSL_config.txt, as it is now redundant with SDP udpates. |
||
#6 | 21349 | C. Thomas Tyler |
Updated SDP version. Updated default release for P4/P4D to r16.2. Updated default Swarm to 2016.3. Adapated to new volume layout structure. Added support for generating SSL autogen cert file. |
||
#5 | 20357 | C. Thomas Tyler |
configure_sample_depot_for_sdp.sh v1.4.3: * Adapted to SDP change in password file location. * Adapted to SDP change setting P4ENVIRO. |
||
#4 | 19363 | C. Thomas Tyler |
Comment cleanup. No functional changes. |
||
#3 | 19316 | C. Thomas Tyler | Updated to account for removal of P4DPORT in SDP. | ||
#2 | 19261 | C. Thomas Tyler | Accounted for move to //guest/perforce_software from //guest/tom_tyler. | ||
#1 | 19230 | C. Thomas Tyler |
Populate -o //guest/tom_tyler/helix-installer/... //guest/perforce_software/helix-installer/.... |
||
//guest/tom_tyler/helix-installer/main/src/configure_sample_depot_for_sdp.sh | |||||
#7 | 16606 | C. Thomas Tyler |
Improved handling of simulated mail from SDP scripts, so that live_checkpoint succeeds on all platforms; it had failed previosuly on Mac/Darwin due to failure sending email. Improved backup handling. |
||
#6 | 16603 | C. Thomas Tyler | Fixed issue working with SSL-enabled servers. | ||
#5 | 16403 | C. Thomas Tyler |
Added comments detailing various SDP 'p4 login' calls; tweaked calls. |
||
#4 | 16398 | C. Thomas Tyler | Added 'p4d -xu' to upgrade databases. | ||
#3 | 16394 | C. Thomas Tyler | Added logins with various ways P4PORT may be specified. | ||
#2 | 16360 | C. Thomas Tyler | Minor code and copyright cleanup. | ||
#1 | 15829 | C. Thomas Tyler |
Moved Helix Installer to new Workshop Standard for personal projects. |
||
//guest/tom_tyler/sw/main/hi/src/configure_sample_depot_for_sdp.sh | |||||
#16 | 15060 | C. Thomas Tyler | Rollback of Copy Up done with a new utility that apparently needs some work. | ||
#15 | 15051 | C. Thomas Tyler | Copy Up using 'p4 copy -r -b tom_tyler-hi-dev' | ||
#14 | 14987 | C. Thomas Tyler |
Reduced excessive noise from from rsync and tar commands. Added '--delete' to rsync commands handling the Sample Depot to force a clean reset. Added clear completion messages. |
||
#13 | 14985 | C. Thomas Tyler | Added exit 0 to keep Vagrant happy. | ||
#12 | 14982 | C. Thomas Tyler | Fixed embarrasing typo. | ||
#11 | 14974 | C. Thomas Tyler | Fixed issues handling non-SSL servers. | ||
#10 | 14972 | C. Thomas Tyler | Doc tweaks. | ||
#9 | 14961 | C. Thomas Tyler | Fixed optimization to avoid bogus rsync commands. | ||
#8 | 14958 | C. Thomas Tyler | Modified to do 'p4 trust' and 'p4 info' via-broker in addition to direct. | ||
#7 | 14957 | C. Thomas Tyler | Fixed 'tar' command, now cross-platform smart and PATH-dependent. | ||
#6 | 14955 | C. Thomas Tyler | Made 'sed' PATH-dependent, removing hard coding. | ||
#5 | 14120 | C. Thomas Tyler |
Moved sample depot checkpoint load to configurator script. Added '-fast' flag to speed up testing. Various minor tweaks. |
||
#4 | 14109 | C. Thomas Tyler |
Tweaked copy of spec depot files to get a clean verify. Added detection of rsync failure. |
||
#3 | 14106 | C. Thomas Tyler |
Implemented Sample Depot SDP configurator script. Tightened logic around process kill statement, and made it work for Linux/Mac. |
||
#2 | 14062 | C. Thomas Tyler | Added super user and set password. | ||
#1 | 14033 | C. Thomas Tyler |
Added script to transform stock Sample Depot data set to be SDP compliant (mostly a stub at this point). Various improvements to reset_sdp.sh and the wrapper r.sh. |