#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ set -u # Change the first three variables below to match the volume names on the machine # you are configuring the SDP on. # Set the remaining variables appropriately. The meaning of each should be obvious. # Before running this script, create a configuration file. You can use a # single configuration file named mkdirs.cfg in the same directory as this # mkdirs.sh script. Alternately, you can create a file named mkdirs.N.cfg, # where 'N' is the name of the instance you intend to configure. # # Run this script as root, and pass in the instance number you are configuring. # # If you do not have root access, then some commands will be skipped, and you # will need to have the defined $P4DIR directory already existing (created by # the root user and the ownership changed to the OS user that you are planning # to run Perforce under). The /hxmetadata*, /hxdepots and /hxlogs volumes will # also need to be owned by the OS user as well in order for the script to work. # You can then run this script as the OS user for Perforce and everything # should work fine. # # This script creates an init script in the $P4DIR/$SDP_INSTANCE/bin directory. # You can use it from there if you are configuring a cluster, or you can link # it in /etc/init.d if you are setting up a stand alone machine. Sample # systemd service files are included in the systemd folder. # # After running this script, set up the crontab based on templates # generated in $P4DIR/common/etc/cron.d. For convenience, a sample generated # is to $P4DIR/p4.crontab. # # Next, put the license file in place in the P4ROOT dir, and launch the server with the init # script. # # Then run $P4DIR/common/bin/p4master_run <instance> $P4DIR/common/bin/live_checkpoint.sh # and then run both the daily_checkpoint.sh and recreate_db_checkpoint.sh to # make sure everything is working before setting up the crontab. # # Also run $P4DIR/common/bin/p4master_run <instance> $P4DIR/common/bin/p4review.py <instance> # to make sure the review script is working properly. If you intend to use # Swarm, you can skip configuration of the review daemon, and instead configure # Swarm to handle review-style email notifications. # # UPGRADING SDP # Specify the -test parameter to the script. # In this case the script will NOT make the various directories under $P4DIR, # but will instead create /tmp/p4 directory structure with the various files # processed via templates etc. You can then manually compare this directory # structure with your existing $P4DIR structure and manually copy the various # files into it. function msg () { echo -e "$*"; } function errmsg () { msg "\\nError: (line: ${BASH_LINENO[0]}) $*" >&2; } function warnmsg () { msg "\\nWarning: (line: ${BASH_LINENO[0]}) $*"; } function bail() { errmsg "(line: ${BASH_LINENO[0]}) ${1:-Unknown Error}\n" >&2; exit "${2:-1}"; } function usage () { msg "Usage:\\n\\t${0##*/} <instance> [-test]\\n" exit 1 } declare -i DoChownCommands=1 if [ $# -lt 1 ] || [ $# -gt 2 ]; then usage fi # Verify instance value. SDP_INSTANCE=$1 if [[ "$SDP_INSTANCE" == "-test" ]]; then bail "An instance argument is required." fi # Note that if the following is set to 1 then a trial run is done, into /tmp/p4 TEST=0 if [[ $# -gt 1 && "$2" == "-test" ]]; then TEST=1 fi # Check for config file - same dir as this script if [[ -r "${0%/*}/mkdirs.${SDP_INSTANCE}.cfg" ]]; then mkdirs_config="${0%/*}/mkdirs.${SDP_INSTANCE}.cfg" elif [[ -r "${0%/*}/mkdirs.cfg" ]]; then mkdirs_config="${0%/*}/mkdirs.cfg" else bail "Missing mkdirs config file $mkdirs_config." fi # shellcheck disable=SC1090 source "$mkdirs_config" || bail "Failed to source $mkdirs_config." if [[ -r $SDP/Version ]]; then # shellcheck disable=SC2086 SDP_VERSION="$(cat $SDP/Version)" else SDP_VERSION="Rev. SDP/Unix/UNKNOWN" fi P4DIR=/p4 P4SERVER="p4_$SDP_INSTANCE" export AWK=awk export ID=id export MAIL=mail OS=$(uname) if [[ "${OS}" = "SunOS" ]] ; then export AWK=/usr/xpg4/bin/awk export ID=/usr/xpg4/bin/id export MAIL=mailx elif [[ "${OS}" = "AIX" ]] ; then export AWK=awk export ID=id export MAIL=mail fi if [[ $($ID -u) -eq 0 ]]; then msg "Verified: Running as root." elif [[ $(whoami) == "$OSUSER" ]]; then warnmsg "Not running as root; chown commands will be skipped and basic directories must exist.\n" DoChownCommands=0 else bail "$0 must be run as as root or $OSUSER.\n" fi if [[ $TEST -eq 1 ]]; then # CN has prefix CN=/tmp DB1=tmp/hxmetadata1 DB2=tmp/hxmetadata2 DD=tmp/hxdepots LG=tmp/hxlogs mkdir -p /$DB1 /$DB2 /$DD /$LG P4DIR=/tmp/p4 msg "\n********* -test specified - will install to $P4DIR *********\n" fi SDP_COMMON=$SDP/Server/Unix/p4/common declare -i PreflightOK=1 [[ -f $SDP_COMMON/bin/p4 ]] || { errmsg "No p4 in $SDP_COMMON/bin"; PreflightOK=0; } [[ -f $SDP_COMMON/bin/p4d ]] || { errmsg "No p4d in $SDP_COMMON/bin"; PreflightOK=0; } [[ -f $SDP_COMMON/bin/p4p ]] || warnmsg "No p4p in $SDP_COMMON/bin" [[ -f $SDP_COMMON/bin/p4broker ]] || warnmsg "No p4broker in $SDP_COMMON/bin" if [[ $TEST -eq 1 && ! -d "$P4DIR" ]]; then mkdir "$P4DIR" || bail "Failed to create $P4DIR." fi # If we aren't running as root, extend the preflight checks to verify basic dirs # exist. if [[ $($ID -u) -ne 0 && $TEST -eq 0 ]]; then for d in "$P4DIR" "/$DD" "/$DB1" "/$DB2" "/$LG"; do if [[ -d "$d" ]]; then # shellcheck disable=SC2012 disable=SC2016 dirOwner=$(ls -ld "$d" | $AWK '{print $3}') # shellcheck disable=SC2012 disable=SC2016 dirGroup=$(ls -ld "$d" | $AWK '{print $4}') if [[ "$dirOwner" != "$OSUSER" ]]; then errmsg "Dir [$d] exists but with wrong owner, $dirOwner instead of $OSUSER." PreflightOK=0 fi if [[ "$dirGroup" != "$OSGROUP" ]]; then warnmsg "Dir [$d] exists but with wrong group, $dirGroup instead of $OSGROUP." fi else errmsg "Dir must exist if not running as root: $d" PreflightOK=0 fi done fi if [[ $PreflightOK -eq 1 ]]; then msg "Verified: Preflight checks passed." else bail "Aborting due to failed preflight checks." fi chmod 755 "$SDP_COMMON/bin/p4" chmod 700 "$SDP_COMMON/bin/p4d" [[ -f "$SDP_COMMON/bin/p4broker" ]] && chmod 700 "$SDP_COMMON/bin/p4broker" [[ -f "$SDP_COMMON/bin/p4web" ]] && chmod 700 "$SDP_COMMON/bin/p4web" [[ -f "$SDP_COMMON/bin/p4p" ]] && chmod 700 "$SDP_COMMON/bin/p4p" # Make sure you update the p4 and p4d to the latest versions in the # sdp/Server/Unix/p4/common/bin directory when making new instances at a later date. # shellcheck disable=SC2016 P4RELNUM=$("$SDP_COMMON/bin/p4" -V | grep -i Rev. | $AWK -F / '{print $3}') # shellcheck disable=SC2016 P4BLDNUM=$("$SDP_COMMON/bin/p4" -V | grep -i Rev. | $AWK -F / '{print $4}' | $AWK '{print $1}') # shellcheck disable=SC2016 P4DRELNUM=$("$SDP_COMMON/bin/p4d" -V | grep -i Rev. | $AWK -F / '{print $3}') # shellcheck disable=SC2016 P4DBLDNUM=$("$SDP_COMMON/bin/p4d" -V | grep -i Rev. | $AWK -F / '{print $4}' | $AWK '{print $1}') [[ -d "$P4DIR" ]] || mkdir "$P4DIR" [[ -d "/$DD/p4" ]] || mkdir "/$DD/p4" [[ -d "/$DB1/p4" ]] || mkdir "/$DB1/p4" [[ -d "/$DB2/p4" ]] || mkdir "/$DB2/p4" [[ -d "/$LG/p4" ]] || mkdir "/$LG/p4" mkdir -p "/$LG/p4/$SDP_INSTANCE/tmp" mkdir -p "/$DD/p4/$SDP_INSTANCE/depots" mkdir -p "/$DD/p4/$SDP_INSTANCE/checkpoints" case "$SERVER_TYPE" in (p4d_standby) mkdir -p "/$LG/p4/$SDP_INSTANCE/journals.rep" if [[ $DoChownCommands -eq 1 ]]; then chown "$OSUSER:$OSGROUP" "/$LG/p4/$SDP_INSTANCE/journals.rep" fi ;; (p4d_replica|p4d_edge|p4d_ha) mkdir -p "/$DD/p4/$SDP_INSTANCE/checkpoints.${REPLICA_ID#p4d_}" if [[ $DoChownCommands -eq 1 ]]; then chown "$OSUSER:$OSGROUP" "/$DD/p4/$SDP_INSTANCE/checkpoints.${REPLICA_ID#p4d_}" fi ;; esac [[ -d "$P4DIR/ssl" ]] || mkdir -p "$P4DIR/ssl" [[ -f "$SDP/Server/Unix/p4/ssl/config.txt" && ! -f "$P4DIR/ssl/config.txt" ]] &&\ cp "$SDP/Server/Unix/p4/ssl/config.txt" "$P4DIR/ssl/" if [[ "$CN" == "" ]]; then # When blank we put on "local" drive [[ -d "$P4DIR/common/bin" ]] || mkdir -p "$P4DIR/common/bin" [[ -d "$P4DIR/common/config" ]] || mkdir -p "$P4DIR/common/config" else # (default) put it on hxdepots or where specified [[ -d "$CN/p4/common" ]] || mkdir -p "$CN/p4/common" [[ -L "$P4DIR/common" ]] || ln -s "$CN/p4/common" "$P4DIR/common" [[ -d "$CN/p4/common/bin" ]] || mkdir -p "$CN/p4/common/bin" [[ -d "$CN/p4/common/config" ]] || mkdir -p "$CN/p4/common/config" fi mkdir -p "/$DB1/p4/$SDP_INSTANCE/db1/save" mkdir -p "/$DB2/p4/$SDP_INSTANCE/db2/save" mkdir -p "/$LG/p4/$SDP_INSTANCE/logs" cd "/$DD/p4/$SDP_INSTANCE" || bail "Could not cd to /$DD/p4/$SDP_INSTANCE" # Create /p4/1 on local disk as a directory # Then the links within it cd "$P4DIR" || bail "Could not cd to $P4DIR" [[ -d "$SDP_INSTANCE" ]] || mkdir "$SDP_INSTANCE" cd "$SDP_INSTANCE" || bail "Could not cd to $SDP_INSTANCE" [[ -d bin ]] || mkdir bin [[ -L root ]] || ln -s "/$DB1/p4/$SDP_INSTANCE/db1" root [[ -L offline_db ]] || ln -s "/$DB2/p4/$SDP_INSTANCE/db2" offline_db if [[ ! -d logs ]]; then # shellcheck disable=SC2226 [[ -L logs ]] || ln -s "/$LG/p4/$SDP_INSTANCE/logs" fi # shellcheck disable=SC2226 [[ -L depots ]] || ln -s "/$DD/p4/$SDP_INSTANCE/depots" # shellcheck disable=SC2226 [[ -L checkpoints ]] || ln -s "/$DD/p4/$SDP_INSTANCE/checkpoints" # shellcheck disable=SC2226 [[ -L tmp ]] || ln -s "/$LG/p4/$SDP_INSTANCE/tmp" case "$SERVER_TYPE" in (p4d_standby) # shellcheck disable=SC2226 [[ -L journals.rep ]] || ln -s "/$LG/p4/$SDP_INSTANCE/journals.rep" ;; (p4d_replica|p4d_edge|p4d_ha) # shellcheck disable=SC2226 [[ -L "checkpoints.${REPLICA_ID#p4d_}" ]] || ln -s "/$DD/p4/$SDP_INSTANCE/checkpoints.${REPLICA_ID#p4d_}" esac [[ -L sdp ]] || ln -s "$SDP" "$P4DIR/sdp" case "$SERVER_TYPE" in (p4d_master|p4d_edge) SERVERID="$MASTER_ID" echo "$SERVERID" > "$P4DIR/$SDP_INSTANCE/root/server.id" ;; (p4d_replica|p4d_standby) SERVERID="$REPLICA_ID" echo "$SERVERID" > "$P4DIR/$SDP_INSTANCE/root/server.id" ;; (*) SERVERID="$SERVER_TYPE" ;; esac [[ -f "$P4DIR/common/bin/p4_$P4RELNUM.$P4BLDNUM" ]] || cp "$SDP_COMMON/bin/p4" "$P4DIR/common/bin/p4_$P4RELNUM.$P4BLDNUM" [[ -f "$P4DIR/common/bin/p4d_$P4DRELNUM.$P4DBLDNUM" ]] || cp "$SDP_COMMON/bin/p4d" "$P4DIR/common/bin/p4d_$P4DRELNUM.$P4DBLDNUM" if [[ ! -f "$P4DIR/common/bin/p4_vars" ]]; then cp -R "$SDP_COMMON"/bin/* "$P4DIR/common/bin" # Copy certain subdirs of /p4/common if don't already exist. for dir in etc hms lib; do if [[ ! -d "$P4DIR/common/$dir" ]]; then cp -pr "$SDP_COMMON/$dir" "$P4DIR/common/." fi done cd "$P4DIR/common/bin" || bail "Could not cd to $P4DIR/common/bin" ln -s "p4_$P4RELNUM.$P4BLDNUM" "p4_${P4RELNUM}_bin" ln -s "p4d_$P4DRELNUM.$P4DBLDNUM" "p4d_${P4DRELNUM}_bin" ln -s "p4_${P4RELNUM}_bin" "p4_bin" sed -e "s:REPL_SDPVERSION:${SDP_VERSION}:g" \ -e "s/REPL_OSUSER/${OSUSER}/g" \ "$SDP_COMMON/config/p4_vars.template" > p4_vars fi cd "$P4DIR/common/bin" || bail "Could not cd to $P4DIR/common/bin" ln -s "p4d_${P4DRELNUM}_bin" "p4d_${SDP_INSTANCE}_bin" if [[ ! -e "$P4DIR/common/config/.p4passwd.${P4SERVER}.admin" ]]; then echo "$P4ADMINPASS" > "$P4DIR/common/config/.p4passwd.${P4SERVER}.admin" else warnmsg "Skipping update of existing admin password file $P4DIR/common/config/.p4passwd.${P4SERVER}.admin" fi if [[ ! -e "$P4DIR/common/config/.p4passwd.${P4SERVER}.service" ]]; then echo "$P4SERVICEPASS" > "$P4DIR/common/config/.p4passwd.${P4SERVER}.service" else warnmsg "Skipping update of existing service password file $P4DIR/common/config/.p4passwd.${P4SERVER}.service" fi # Create broker links if broker exists if [[ -f "$SDP_COMMON/bin/p4broker" ]]; then # shellcheck disable=SC2016 P4BRELNUM=$("$SDP_COMMON/bin/p4broker" -V | grep -i Rev. | $AWK -F / '{print $3}') # shellcheck disable=SC2016 P4BBLDNUM=$("$SDP_COMMON/bin/p4broker" -V | grep -i Rev. | $AWK -F / '{print $4}' | $AWK '{print $1}') [[ -f "$P4DIR/common/bin/p4broker_$P4BRELNUM.$P4BBLDNUM" ]] || cp "$SDP_COMMON/bin/p4broker" "$P4DIR/common/bin/p4broker_$P4BRELNUM.$P4BBLDNUM" [[ -L "p4broker_${P4BRELNUM}_bin" ]] && unlink "p4broker_${P4BRELNUM}_bin" ln -s "p4broker_$P4BRELNUM.$P4BBLDNUM" "p4broker_${P4BRELNUM}_bin" [[ -L "p4broker_${SDP_INSTANCE}_bin" ]] && unlink "p4broker_${SDP_INSTANCE}_bin" ln -s "p4broker_${P4BRELNUM}_bin" "p4broker_${SDP_INSTANCE}_bin" cd "$P4DIR/$SDP_INSTANCE/bin" || \ bail "Could not cd to $P4DIR/$SDP_INSTANCE/bin" [[ -L "p4broker_${SDP_INSTANCE}" ]] || ln -s "$P4DIR/common/bin/p4broker_${SDP_INSTANCE}_bin" "p4broker_${SDP_INSTANCE}" sed "s/REPL_SDP_INSTANCE/${SDP_INSTANCE}/g" "$SDP_COMMON/etc/init.d/p4broker_instance_init.template" > "p4broker_${SDP_INSTANCE}_init" chmod +x "p4broker_${SDP_INSTANCE}_init" fi # Create p4p links if p4p exists cd "$P4DIR/common/bin" || bail "Could not cd to $P4DIR/common/bin" if [[ -x "$SDP_COMMON/bin/p4p" ]]; then # shellcheck disable=SC2016 P4PRELNUM=$("$SDP_COMMON/bin/p4p" -V | grep -i Rev. | $AWK -F / '{print $3}') # shellcheck disable=SC2016 P4PBLDNUM=$("$SDP_COMMON/bin/p4p" -V | grep -i Rev. | $AWK -F / '{print $4}' | $AWK '{print $1}') [[ -f "$P4DIR/common/bin/p4p_$P4PRELNUM.$P4PBLDNUM" ]] || cp "$SDP_COMMON/bin/p4p" "$P4DIR/common/bin/p4p_$P4PRELNUM.$P4PBLDNUM" [[ -L "p4p_${P4PRELNUM}_bin" ]] && unlink "p4p_${P4PRELNUM}_bin" ln -s "p4p_$P4PRELNUM.$P4PBLDNUM" "p4p_${P4PRELNUM}_bin" [[ -L "p4p_${SDP_INSTANCE}_bin" ]] && unlink "p4p_${SDP_INSTANCE}_bin" ln -s "p4p_${P4PRELNUM}_bin" "p4p_${SDP_INSTANCE}_bin" cd "$P4DIR/$SDP_INSTANCE/bin" || \ bail "Could not cd to $P4DIR/$SDP_INSTANCE/bin" [[ -L "p4p_${SDP_INSTANCE}" ]] || ln -s "$P4DIR/common/bin/p4p_${SDP_INSTANCE}_bin" "p4p_${SDP_INSTANCE}" sed -e "s/REPL_SDP_INSTANCE/${SDP_INSTANCE}/g" \ -e "s/REPL_DNSNAME/${P4DNSNAME}/g" \ "$SDP_COMMON/etc/init.d/p4p_instance_init.template" > "p4p_${SDP_INSTANCE}_init" chmod +x "p4p_${SDP_INSTANCE}_init" mkdir -p "/$DD/p4/$SDP_INSTANCE/cache" fi cd "$P4DIR/$SDP_INSTANCE/bin" || bail "Could not cd to $P4DIR/$SDP_INSTANCE/bin" ln -s "$P4DIR/common/bin/p4_bin" "p4_$SDP_INSTANCE" sed "s/REPL_SDP_INSTANCE/${SDP_INSTANCE}/g" \ "$SDP_COMMON/etc/init.d/p4d_instance_init.template" > "p4d_${SDP_INSTANCE}_init" chmod +x "p4d_${SDP_INSTANCE}_init" # Moved the less commonly used, but always created init scripts to an init directory. mkdir init cd init || bail "Could not cd to $PWD/init" sed "s/REPL_SDP_INSTANCE/${SDP_INSTANCE}/g" \ "$SDP_COMMON/etc/init.d/p4dtg_instance_init.template" > "p4dtg_${SDP_INSTANCE}_init" chmod +x "p4dtg_${SDP_INSTANCE}_init" cd .. || bail "Could not cd to .. from $PWD" if [[ $CASE_SENSITIVE -eq 1 ]]; then ln -s "$P4DIR/common/bin/p4d_${SDP_INSTANCE}_bin" "p4d_$SDP_INSTANCE" else echo '#!/bin/bash' > "p4d_$SDP_INSTANCE" echo "P4D=/p4/common/bin/p4d_${SDP_INSTANCE}_bin" >> "p4d_$SDP_INSTANCE" # shellcheck disable=SC2016 echo 'exec $P4D -C1 "$@"' >> "p4d_$SDP_INSTANCE" chmod +x "p4d_$SDP_INSTANCE" fi cd "$P4DIR/common/config" || bail "Could not cd to $P4DIR/common/config" sed -e "s/REPL_MAILTO/${MAILTO}/g" \ -e "s/REPL_MAILFROM/${MAILFROM}/g" \ -e "s/REPL_ADMINUSER/${ADMINUSER}/g" \ -e "s/REPL_MASTER_ID/${MASTER_ID}/g" \ -e "s/REPL_SSLPREFIX/${SSL_PREFIX}/g" \ -e "s/REPL_P4PORT/${P4_PORT}/g" \ -e "s/REPL_P4BROKERPORT/${P4BROKER_PORT}/g" \ -e "s/REPL_P4WEBPORT/${P4WEB_PORT}/g" \ -e "s/REPL_P4FTPPORT/${P4FTP_PORT}/g" \ -e "s/REPL_DNSNAME/${P4DNSNAME}/g" \ -e "s/REPL_P4P_TARGET_PORT/${P4P_TARGET_PORT}/g" \ "$SDP_COMMON/config/instance_vars.template" > "p4_${SDP_INSTANCE}.vars" sed -e "s/REPL_ADMINISTRATOR/${MAILTO}/g" \ -e "s/REPL_COMPLAINFROM/${COMPLAINFROM}/g" \ -e "s/REPL_MAILHOST/${MAILHOST}/g" \ -e "s/REPL_DNSNAME/${P4DNSNAME}/g" \ "$SDP_COMMON/config/p4review.cfg.template" > "p4_${SDP_INSTANCE}.p4review.cfg" cd "$P4DIR" || bail "Could not cd to $P4DIR" if [[ ! -f "${P4DIR}/p4.crontab" ]]; then CRONTAB_NAME=p4.crontab else CRONTAB_NAME=p4.crontab.new rm -f "${P4DIR}/$CRONTAB_NAME" > /dev/null 2>&1 msg "Existing crontab found, writing new crontab to ${P4DIR}/${CRONTAB_NAME}" fi sed -e "s/REPL_MAILTO/${MAILTO}/g" \ -e "s/REPL_MAILFROM/${MAILFROM}/g" \ -e "s/REPL_INSTANCE/${SDP_INSTANCE}/g" \ "$SDP_COMMON/etc/cron.d/template.crontab.combined" > "$CRONTAB_NAME" cd "$P4DIR/${SDP_INSTANCE}/bin" || \ bail "Could not cd to $P4DIR/${SDP_INSTANCE}/bin" if [[ "$SHAREDDATA" == "TRUE" ]]; then if [[ "$SERVER_TYPE" == p4d_replica || "$SERVER_TYPE" == p4d_standby || "$SERVER_TYPE" == p4d_ha ]]; then msg "Configuring Replica sharing depot data with master, skipping chown/chmod of depot files" DoChownCommands=0 fi fi if [[ $DoChownCommands -eq 1 ]]; then if [[ $TEST -eq 0 ]]; then chown "$OSUSER:$OSGROUP" "/$DD" chown "$OSUSER:$OSGROUP" "/$LG" chown "$OSUSER:$OSGROUP" "/$DB1" chown "$OSUSER:$OSGROUP" "/$DB2" fi chown "$OSUSER:$OSGROUP" "/$DD/p4" chown "$OSUSER:$OSGROUP" "/$LG/p4" chown "$OSUSER:$OSGROUP" "/$DB1/p4" chown "$OSUSER:$OSGROUP" "/$DB2/p4" chown -h "$OSUSER:$OSGROUP" "$P4DIR" chown -h "$OSUSER:$OSGROUP" "$P4DIR/$SDP_INSTANCE" chown -h "$OSUSER:$OSGROUP" "$P4DIR/common" [[ $TEST -eq 0 ]] && chown -h "$OSUSER:$OSGROUP" "$P4DIR/sdp" chown "$OSUSER:$OSGROUP" "$P4DIR/*" chown -Rh "$OSUSER:$OSGROUP" "$P4DIR/common/" [[ $TEST -eq 0 ]] && chown -Rh "$OSUSER:$OSGROUP" "$P4DIR/sdp" chown -Rh "$OSUSER:$OSGROUP" "$P4DIR/common" chown -Rh "$OSUSER:$OSGROUP" "/$DB1/p4/$SDP_INSTANCE" chown -Rh "$OSUSER:$OSGROUP" "/$DB2/p4/$SDP_INSTANCE" chown -Rh "$OSUSER:$OSGROUP" "/$LG/p4/$SDP_INSTANCE" if [[ "$SHAREDDATA" == "FALSE" ]]; then msg "Setting ownership on depot files - this may take some time ..." chown -Rh "$OSUSER:$OSGROUP" "/$DD/p4/$SDP_INSTANCE" fi chown -Rh "$OSUSER:$OSGROUP" "$P4DIR/$SDP_INSTANCE/bin" else msg "Skipped chown/chmod commands for large directory trees." fi chmod 700 "/$DB1/p4" chmod 700 "/$DB2/p4" chmod 700 "/$DD/p4" chmod 700 "/$LG/p4" chmod -R 700 "/$DB1/p4/$SDP_INSTANCE" chmod -R 700 "/$DB2/p4/$SDP_INSTANCE" chmod -R 700 "$P4DIR/common" chmod -R 700 "/$LG/p4/$SDP_INSTANCE" if [[ "$SHAREDDATA" == "FALSE" ]]; then msg "Setting permissions on depot files - this may take some time ..." chmod -R 700 "/$DD/p4/$SDP_INSTANCE" fi if [[ "$SDP_INSTANCE" != "$MASTERINSTANCE" ]]; then if [[ -f "$P4DIR/$MASTERINSTANCE/root/license" ]]; then ln -s "$P4DIR/$MASTERINSTANCE/root/license" "$P4DIR/$SDP_INSTANCE/root/license" chown -h "$OSUSER:$OSGROUP" "$P4DIR/$SDP_INSTANCE/root/license" fi fi chmod 755 "$P4DIR/${SDP_INSTANCE}"/bin/*_init chmod 755 "$P4DIR/${SDP_INSTANCE}"/bin/init/*_init chmod 600 "$P4DIR/common/config/.p4passwd.${P4SERVER}.admin" chmod 600 "$P4DIR/common/config/.p4passwd.${P4SERVER}.service" find $P4DIR/common/bin/ -name "*.cfg" -exec chmod 600 {} \; find $P4DIR/common/bin/ -name "*.html" -exec chmod 600 {} \; chmod 700 $P4DIR/ssl find $P4DIR/ssl/ -type f -exec chmod 600 {} \; if [[ $TEST -eq 1 ]]; then msg " This was done in TEST mode - please run the following command to see any changes should be applied to your live environment (manually): diff -r /p4/$SDP_INSTANCE/bin $P4DIR/$SDP_INSTANCE/bin diff -r /p4/common $P4DIR/common If upgrading an older SDP version then be careful to ensure files in /p4/common/config are correct and update that /p4/common/bin/p4_vars is appropriate.\n" fi exit 0
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#135 | 30991 | C. Thomas Tyler | Fixed sestatus call (typo). | ||
#134 | 30936 | C. Thomas Tyler |
Enhanced SELinux detection logic to so semanage and restorecon are called only if it is beneficial to do so. |
||
#133 | 30892 | C. Thomas Tyler | Added usage error for unkown option/flag. | ||
#132 | 30883 | C. Thomas Tyler | Corrected text in message confusing ufw with firewalld firewalls. | ||
#131 | 30839 | C. Thomas Tyler |
Adjusted mkdirs.sh to be backward compatible with old structure installs (not using package dirs). |
||
#130 | 30782 | C. Thomas Tyler |
Added new install_sdp.sh script and supporting documentation. The new install_sdp.sh makes SDP independent of the separate Helix Installer software (the reset_sdp.sh script). The new script greatly improves the installation experience for new server machines. It is ground up rewrite of the reset_sdp.sh script. The new script preserves the desired behaviors of the original Helix Installer script, but is focused on the use case of a fresh install on a new server machine. With this focus, the scripts does not have any "reset" logic, making it completely safe. Added various files and functionalityfrom Helix Installer into SDP. * Added firewalld templates to SDP, and added ufw support. * Improved sudoers generation. * Added bash shell templates. This script also installs in the coming SDP Package structure. New installs use a modified SDP structure that makes it so the /p4/sdp and /p4/common now point to folders on the local OS volume rather than the /hxepots volume. The /hxdepots volume, which is often NFS mounted, is still used for depots and checkpoints, and for backups. The new structure uses a new /opt/perforce/helix-sdp structure under which /p4/sdp and /p4/common point. This structure also contains the expaneded SDP tarball, downloads, helix_binaries, etc. This change represents the first of 3-phase rollout of the new package structure. In this first phase, the "silent beta" phase, the new structure is used for new installations only. This phase requires no changes to released SDP scripts except for mkdirs.sh, and even that script remains backward-compatible with the old structure if used independently of install_sdp.sh. If used with install_sdp.sh, the new structure is used. In the second phase (targeted for SPD 2024.2 release), the sdp_upgrade.sh script will convert existing installations to the new structure. In the third phase (targeted for SDP 2025.x), this script will be incorporated into OS pacakge installations for the helix-sdp package. Perforce internal wikis have more detail on this change. #review-30783 |
||
#129 | 30681 | C. Thomas Tyler |
Added gen_sudoers.sh script to generate a sudoers file for perforce OSUSER. This generates a more secure limited sudoers file. Previously, adding a sudoers entry for the OSUSER (usually 'perforce') was done only by the Helix Installer. In the Helix Installer variant, a single "one-size-filts-all" sudoers file was used, with the following characteristics: * The instances for Helix Core services were referenced with a '*' wildcard to match all SDP instances, which has since been determined to introduce a vulnerability. In this new script, the wildcard is replaced with separate entries for each SDP instance. * There were entries for all known paths of utilities like lslocks, setcap, and getcap. This new script generates the correct path valid for the current machine. With this change, the functionality will be available in the SDP directly. This new gen_sudoers.sh script can be called by mkdirs.sh directly to update the sudoers file each time a new SDP instance is added, if the new '-fs' (full sudo) or '-ls' (limited sudoers) entries are used. There is no change to the default behavior of mkdirs.sh; only a change if new options are utilized. This script comes with docs and examples for the new script as well as doc changes for mkdirs.sh. (Also added missing documentation for the '-no_enable' option). Further changes needed: * Add doc reference in SDP_Guide.Unix.adoc |
||
#128 | 30527 | C. Thomas Tyler |
Corrected issue where systemd service is configured with P4USER rather than OSUSER. #review-30528 |
||
#127 | 30503 | C. Thomas Tyler | Completed earlier work to use TarRootUnsupported variable to reduce hard-coding. | ||
#126 | 30502 | C. Thomas Tyler | Added SDP Version file to set of files copied to local, non-package area. | ||
#125 | 30474 | C. Thomas Tyler | Fixed typo/bug. | ||
#124 | 30473 | C. Thomas Tyler | Made it easier to call mkdirs.sh without explicit './' from setup dir. | ||
#123 | 30471 | C. Thomas Tyler | Added '-no_enable' option to avoid enabling systemd services. | ||
#122 | 30469 | C. Thomas Tyler |
Refined '-f' (fast) mode in mkdirs.sh. As before, we skip the big 'chown -R' commands in fast mode. But with this change, we do targeted 'chown' commands to ensure key directories like /hxdepots/p4/N and /hxdepots/p4/N/depots and /hxcheckpionts/p4/N/checkpoints have correct ownership. |
||
#121 | 30454 | C. Thomas Tyler |
mkdirs.sh v6.2.0: Added in-code docs for TarRoot, and corrected TarRoot to work seamlessly with both package and non-package installations. Fixed issue with SDPVersion file in non-package installs. Added logic to avoid generating calling 'semanage' unnecessarily to do SELInux configuration if it was allready done. This removes a source of most harmless non-zero exit codes. |
||
#120 | 30452 | C. Thomas Tyler | Refined symlink chown logic for /p4/common. | ||
#119 | 30449 | C. Thomas Tyler | Fixed bug with ownership chown targeting wrong path. | ||
#118 | 30448 | C. Thomas Tyler | Fixed doc typo. | ||
#117 | 30447 | C. Thomas Tyler | Added '-c <cfg_file>'. | ||
#116 | 30446 | C. Thomas Tyler |
Change to be backward-compatible with mkdirs.cfg files that do not define a value for CD. Default for CD is same was DD. |
||
#115 | 30445 | C. Thomas Tyler |
Added undocumented support for /opt/perforce/helix-sdp package directory. #review With this change, mkdirs.sh behavior is changed depending on the pre-existence of the directory /opt/perforce/helix-sdp, the future package install location. The /opt/perforce/helix-sdp folder is not created by design, effectively making support for the package structure a Tech Preview (i.e. Beta) feature. This change provides behaviors needed to support package installation that also dovetail with goals of decoupling the /p4/common and /p4/sdp folders from NFS in NFS environments. If the /opt/perforce/helix-sdp is created with proper permissions and user/group ownersihp before mkdirs.sh is run, new behaviors apply. The proper ownership and permissions are: /opt/perforce/helix-sdp - root:perforce, 775 perms. Allows perforce user to manage subdirs, e.g. common, downloads, helix_binaries, p4/sdp. /opt/perforce/helix-sdp/sdp - root:root, 755 perms. This is the immutable area, an extract of the SDP tarball and no additional files. This root-managed area is intended to be updated only by OS package upgrades, and cannot be affected by the perforce user. These folders in the structure are modifiable by the perforce user (owned by perforce:perforce), and will contain generated files, symlinks, and possibly even local custom files. * /opt/perforce/helix-sdp/downloads, for unpacking SDP tarballs for upgrades. * /opt/perforce/helix-sdp/helix_binaries, contains get_helix_binaries.sh, p4*. * /opt/perforce/helix-sdp/p4/sdp - A copy of /opt/perforce/helix-sdp/sdp, owned by perforce:perforce rather than root:root. In this version, the following behaviour changes occur if /opt/perforce/helix-sdp exists when this script is run: * The /p4/sdp symlink points to /opt/perforce/helix-sdp/p4/sdp rather than /hxdepots/sdp. * The /p4/common symlink points to /opt/perforce/helix-sdp/p4/sdp/Server/Unix/p4/common rather than /hxdepots/p4/common. In this beta version, if run in an environment where those /p4/common and /p4/sdp symlinks already exist, the symlinks are NOT changed even if /opt/perforce/helix-sdp exists. This is Phase 1 if a multi-phased release of this feature. Phase 1 will support only greenfield installations where /opt/perforce/helix-sdp is created externally to this script. (Targeted for SDP 2024.1 Patch 1, July 2024) In Phase 2, the /opt/perforce/helix-sdp will be created by this script in greenfield installations with a documented '-pkg' option (if run as root), and sdp_upgrade.sh will migrate to this structure if and only if /opt/perforce/helix-sdp exists. Targeted for SDP 2024.1 Patch 2 (Summer 2024). In Phase 3, this will be integrated into the OS package installation mechanism for Rocky9+ and Ubuntu 22.04+, possibly others. (Targeted for SDP 2025.1+.) SDP-1069: Decouple /p4/common and /p4/sdp from NFS. The above captures the initial plan. See this wiki page with more updated information as it evolves: https://perforce.atlassian.net/wiki/spaces/PROS/pages/936542265/SDP-1069+Decouple+SDP+from+NFS |
||
#114 | 30433 | C. Thomas Tyler | Corrected chown command for a symlink. | ||
#113 | 30431 | C. Thomas Tyler |
Fixed issue with some dbg() calls not displaying output. Fixed issue with command line did not display correctly. |
||
#112 | 30430 | C. Thomas Tyler |
Enhanced logic determining owner/group to use 'stat' rather than 'ls' with sed/awk. This is more portable, clearer, and likely faster. |
||
#111 | 30362 | C. Thomas Tyler | Removed a line of dead code. | ||
#110 | 30333 | C. Thomas Tyler |
Added HxCheckpoints, optional setting to store checkpoints on a separate volume from depots. Default is same value as HxDepots. This changes makes the SPD more adaptable to customers, especially using NFS, that prefer to have separate mounts for checkpoints vs. those for archive files. The original SDP design was to have everything that needed to be backed up (i.e. metadata checkpoints and the versioned file tree) on a single volume. This is still the preferred mode, but allowing checkpoints and archives to be separate is useful for some. #review-30334 |
||
#109 | 30036 | C. Thomas Tyler |
mkdirs.sh tweaks: * The configurables.cfg file is deployed if it doesn't already exist. * The SiteTags.cfg.sample file is deployed if neither SiteTags.cfg.sample nor SiteTags.cfg exists. |
||
#108 | 30020 | C. Thomas Tyler | Changed mkdirs.sh to not require p4d on a non-p4d server. | ||
#107 | 29802 | C. Thomas Tyler |
Fixed typo/bug. #review-29803 @robert_cowham |
||
#106 | 29691 | C. Thomas Tyler |
Added logic to avoid applying the 'setcap' (the OOM Killer defense) in Docker environments (e.g. for testing). Similar to @296666, but that changeg was for upgrade.sh, and this is for mkdirs.sh. Info: Using the 'setcap' command on a binary in a Docker container renders the binary inoperable. Attempts to execute the binary result in an 'Operation not permitted' error. The workaround: Don't use 'setcap' on binaries in a Docker container. #review-29692 |
||
#105 | 29620 | C. Thomas Tyler | Added setcap capability to mkdirs.sh. | ||
#104 | 29388 | C. Thomas Tyler | Added crontab for broker-only installs. | ||
#103 | 29305 | C. Thomas Tyler |
Fixed typo. #review-29300 |
||
#102 | 29299 | C. Thomas Tyler |
Fixed bug that caused crontab not to be generated for proxy-only installs. Also: Cosmetic change: Normalized NO_OP message prefix. #review-29300 |
||
#101 | 29156 | C. Thomas Tyler |
Fixed minor bugs resulting in harmless errors when mkdirs.sh is used for proxy-only or broker-only hosts. Enhanced clarity of error messages related to 'cd' failures. #review-29157 |
||
#100 | 28397 | C. Thomas Tyler |
Enhanced support for proxy-only and broker-only hosts. #review-28398 |
||
#99 | 28192 | C. Thomas Tyler |
Added new optional PERMS setting in mkdirs.cfg, to allow opening of SDP perms when mkdirs.sh is used. This is useful for customer environments (mainly test environments). Documentation for the feature is included in the mkdirs.cfg file. Default behavior is unchanged. |
||
#98 | 27992 | C. Thomas Tyler |
Added DEFAULT_DOMAIN to list of required values that must be defined in mkdirs.cfg in the checks executed in mkdirs.sh. |
||
#97 | 27886 | Domenic | Add closing brace for TargetServerID variable in the error message. | ||
#96 | 27802 | C. Thomas Tyler |
Doc correction to mkdirs.sh. No functional change. |
||
#95 | 27782 | Domenic |
Add a check for REPLICA_ID matching MASTER_ID to ensure that replica/edge servers don't use the same setting as the master server. Anchor the checks for SERVER_TYPE so that leading/trailing characters aren't included. This also meant adding an explicit entry for p4d_edge_replica as it was previously implicit from p4d_edge. Update version to v4.6.0. |
||
#94 | 27729 | C. Thomas Tyler | Fixed issue causing some chown commands not to run reliably. | ||
#93 | 27722 | C. Thomas Tyler |
Refinements to @27712: * Resolved one out-of-date file (verify_sdp.sh). * Added missing adoc file for which HTML file had a change (WorkflowEnforcementTriggers.adoc). * Updated revdate/revnumber in *.adoc files. * Additional content updates in Server/Unix/p4/common/etc/cron.d/ReadMe.md. * Bumped version numbers on scripts with Version= def'n. * Generated HTML, PDF, and doc/gen files: - Most HTML and all PDF are generated using Makefiles that call an AsciiDoc utility. - HTML for Perl scripts is generated with pod2html. - doc/gen/*.man.txt files are generated with .../tools/gen_script_man_pages.sh. #review-27712 |
||
#92 | 27385 | ashaikh |
Fix mkdirs.sh install errors on replica servers When you run the mkdirs.sh script on an edge server, you get a symlink error for the checkpoints directory: Running: ln -s /tmp/hxmounts/hxdepots/p4/1/checkpoints.edge.1 /tmp/p4/1/checkpoints.edge.1 ln: failed to create symbolic link ‘/tmp/p4/1/checkpoints.edge.1’: No such file or directory Error: (line: 735) Failed to create symlink. You also get an error for the chown commands on the checkpoints directory because there is an extra / slash in the path. SDP Job - https://swarm.workshop.perforce.com/jobs/SDP-596 |
||
#91 | 27369 | C. Thomas Tyler |
mkdirs.sh v4.5.0: * p4p no longer installed by default unless ServerType is p4proxy. * p4dtg_init no longer installed by default. * If p4dtg installed, warning message indicates more setup needed. * Added '-I <svc>' to install p4dtg and/or p4p. * p4d no longer installed if ServerType is p4proxy or p4broker. * Fixed issue where '-n' (NoOp mode) missed displaying some commands that would execute during proxy initialization. * Fixed harmless error if run from /p4/sdp/Server/Unix/setup rather than /hxdepots/p4/sdp/Server/Unix/setup after the initial install. * Allowed for several aliases to the server type flag, e.g. 'p4p', 'p4d', 'p4b', 'edge', etc, with corresponding doc chnages. * Fixed an issue where a 'chmod' command would attempt to execute in NoOp mode. * Enhanced some 2nd-pass workflows, removing harmless but unnecessary errors. * Enhanced docs to indicate where init script and systemd templates can be found. #review-27370 |
||
#90 | 27307 | C. Thomas Tyler |
mkdirs.sh v4.4.1: * Fixed issue with generated root-owned files after initial install. * Fixed dependency on USER environment variable, which is not guaranteed to be defined on all platforms. Now 'whoami' is used. |
||
#89 | 27250 | C. Thomas Tyler |
Adjusted JournalPrefix standard to account for shared /hxdepots. The JournalPrefix standard now allows for unfiltered replicas (such as HA/DR replicas) to use same journalPrefix value as filtered replicas and edge servers, using per-ServerID checkpoints folder, if they share the same /hxdepots (e.g. NFS-mounted) with the master (e.g. when lbr.replication=shared). Related code change made to mkdirs.sh and mkrep.sh to support the tweaks to the standard. #review-27251 |
||
#88 | 27099 | C. Thomas Tyler |
Eliminated extra /p4/Version file. The standard for determining the SDP version for 2020.1 and forward will be the file /p4/sdp/Version. This is more clear. Job SDP-564 evolved slightly (per DevNotes in the job). The goal remains the same, to standardize the method of determining the SDP version for SDP 2020.1+. It was deemed that having an extra copy in /p4/Version will not help with that, and instead would introduce more failure modes and possibilities for out-of-sync files. This does mean the 'tarball extraction' sdp folder that is symlinked to from /p4/sdp is now a critical part of the SDP installation. This is normal and as documented, though there have been cases where SDP is copied from one machine to another in some incomplete way, e.g. rysnc of /p4/common but not /hxdepots/sdp and the symlink to it from /p4/sdp. However, the verify_sdp.sh will catch that form of misconfiguration. #review-27100 |
||
#87 | 27061 | Robert Cowham | mkdirs.sh now respects the MASTER_ID value from mkdirs.cfg (or mkdirs.N.cfg). | ||
#86 | 27022 | C. Thomas Tyler | Changed 'exes' to 'helix_binaries' in folder and script name, per review. | ||
#85 | 26986 | C. Thomas Tyler |
Refinements to mkdirs.sh and mkdirs.cfg, per code review: https://swarm.workshop.perforce.com/reviews/26962 In mkdirs.cfg: * Added comments about proper use of DB1/DB2. * Added comment about coming removal of P4FTP/P4WEB settings. In mkdirs.sh v4.1.1: * Replaced -M with set of -M* flags to specify mount points. * Completed implementation of -M* flags. * Enhanced examples in usage function. * Added clarification of difference between '-n' and '-p'. * Fixed some typos. #review @rcowham |
||
#84 | 26982 | C. Thomas Tyler |
mkdirs.sh v4.1.0: * Accounted for directory structure change of Maintenance to Unsupported. * Added standard command line processing with '-h' and '-man' doc flags, and other flags (all documented). * Added in-code docs and updated AsciiDoc. * Enhanced '-test' mode to simulate /hx* mounts. * Enhanced preflight testing, and fixed '-test' mode installs. * Added support for installing to an alternate root directory. * Added '-s <ServerID>' option to override REPLICA_ID. * Added '-S <TargetServerID>' used for replicas of edge servers. * Added '-t <server_type>' option to override SERVER_TYPE. * Added '-M' option to override mount points. * Added '-f' fast option to skip big chown/chmod commands, and moved those commands near the end as well. verify_sdp.sh v5.9.0: * Added check for /p4/Version file, and checked that other legacy SDP methods of checking version * Added sanity check for crontab. * Added 'test skip' mechanism to skip certain tests: - crontab: Skip crontab check. Use this if you do not expect crontab to be configured, perhaps if a different scheduler is used. - license: Skip license related checks. - version: Skip version checks. - excess: Skip checks for excess copies of p4d/p4p/p4broker in PATH. * Added VERIFY_SDP_SKIP_TEST_LIST setting ton instance_vars.template, to define a standard way to have verify_sdp.sh always skip certain tests for a site. * Extended '-online' checks to check for bogus P4MASTERPORT, a common config error. Update test_SDP.py: * Adjusted test suite to account for various changes in mkdirs.sh. * Added 'dir' parameter to run_cmd() and sudo_cmd(), to run a command from a specified directory (as required to test new mkdirs.sh) * Added check_links() similar to existing check_dirs() function. === Upgrade Process Changes === Made /p4/common/bin/p4d/p4/p4broker/p4p shell script rather than binary. This changes the way SDP new binaries are staged for upgrade. For safety, exes are now staged to a director outside the PATH, the /p4/sdp/exes folder. A new 'get_latest_exes.sh' script simplifies the task of pulling executables from the Perforce FTP server. This can be used 'as is' for environments with outbound internet access, and is useful in any case to describe now to acquire binaries. This addresses an issue where a p4d binary staged for a future upgrade might be called before the actual upgrade is performed. upgrade.sh v4.0.0: * All preflight checks are now done first. Added '-p' to abort after preflight. * Added '-n' to show what would be done before anything is executed. * Minimalist logic to start/stop only servers that are upgrade, and apply upgrades only as needed. * Staging of exes for upgrade is now separate from /p4/common/bin * Improved in-code docs, added '-h' and '-man' options. * Retained pre/post P4D 2019.1 upgrade logic. |
||
#83 | 26718 | Robert Cowham |
Rename P4MASTER to P4MASTERHOST for clarity with comments in: - mkdirs.cfg/mkdirs.sh - p4_<instance>.vars - other files which reference Remove unnecessary sed for p4p.template |
||
#82 | 26185 | C. Thomas Tyler |
Standby replicas no longer need to use an idiosyncratic journals.rep folder, and now use the same checkpoints dir naming convention as edge servers and other replica types. Back story: Prior to P4D 2018.1, standby replicas follow a different convention for placement of journal files. With the introduction of the rpl.journalcopy.location configurable in P4D 2018.1, standby replicas now set that configurable and use the same convention as other replica types. |
||
#81 | 26080 | Robert Cowham | Fix typo and failing test | ||
#80 | 26079 | C. Thomas Tyler |
Fixed typo. Thanks, Robert! |
||
#79 | 26060 | C. Thomas Tyler |
This supercedes review-24715. Bringing over recent bug fixes and code clean up from Rusty Jackson's local branch. Adds an edge maintenance script to unload clients on edge servers. Automated setting up maintenance.cfg from mkdirs.cfg settings. Cleaned up sdputils.py - It had a logic bug, and fixed all scripts based on changes. == Merge Details == Merge strategy: All the fixes to Maintenance scripts were brought in, but not the changes to active them in crontab that would enable them. Rusty's original change had crontab changes to call maintenance scripts, and also calls to 'p4 unlock -x' as a workround to p4d bugs related to orphanged locks on edge servers. Those bugs are already fixed. Also, adding anything new to cron requires a test suite addition and doc updates. So changes to the template.crontab.combined were ignored. However, a new file was added to the Maintenace folder, sample_cron_entries.txt, to make it easy for folks to see those changes. Rusty's original change had mkdirs.sh changes to generate maintenance.cfg. This change was enhanced during the merge. Rather than overwriting maintenance.cfg (potentially blowing away configuration for other instances), a new section for the new instance is appended to maintenance.cfg when 'mkdirs.sh <instance>' is run. Future considerations: Before adding anything from Maintenance to cron, we need to do the following: * Have mkdirs.sh install the Maintenance folder to the as-deployed structure under /p4/common, i.e. to /p4/common/maintenance, for HMS management compliance. * Add tests and docs for each script called (directly or indirectly). |
||
#78 | 25922 | C. Thomas Tyler |
Fixed root-owned symlinks in /p4/N after running mkdirs.sh. This harmless-but-ugly issue caused no functional issues, but is not as intended. |
||
#77 | 25575 | Robert Cowham |
Updates to SDP and tests for 19.1 upgrades #review @tom_tyler |
||
#76 | 25568 | Robert Cowham | Wasn't processing server type p4d_ham | ||
#75 | 25551 | Robert Cowham | Fix test broken by 25550 when hms removed. | ||
#74 | 25546 | Robert Cowham | Fix failing test - check properly for link existence before creating. | ||
#73 | 25450 | Robert Cowham | Fix failing test - over enthusiastic string quoting (with wildcard)... | ||
#72 | 25427 | C. Thomas Tyler | Fixed root ownership of generated crontab file. | ||
#71 | 25386 | C. Thomas Tyler |
Fixed mkdirs.sh issue with SSL-enabled installs due to missing a 'chown'. Bypassing pre-commit to patch. #review-25387 |
||
#70 | 25363 | C. Thomas Tyler |
Fixed bug where /p4/N/bin ownership is not set correctly (owned by root). Fixed issue with missing symlink creation for replicas. Made compliant with shellcheck.sh v0.6.0, driving many changes and fixing potential glob-expansion issues. Fixed an issue with '-test' not setting DB1/DB2. Removed unused/obsolete MD def'n. Added 'p4d_ha' server type to distinguish from existing 'p4d_standby' type, and added comments to mkdirs.cfg to note that p4d_standby is for pre-2018.1 servers, and p4d_ha is for 2018.1+. The distinction is that P4D 2018.1 introduced rpl.journalcopy.location, which changes P4D behaviors and corresponding best practice for placement of the actively pulled journal file for 'journalcopy' replicas. For pre-2018.1, we use /hxlogs/p4/N/journals.rep, and for 2018.1+ with rpl.journalcopy.location=1 set, we us /hxdepots/p4/N/checkpoints.<tag>. Normalized inconsistent indentation. |
||
#69 | 25316 | Robert Cowham | Fix breaking test - logic around $CN was misunderstood. | ||
#68 | 25306 | ashaikh |
Fix syntax error in mkdirs.sh script Currently the mkdirs.sh script will not create the /p4/common/bin or the /p4/common/config folders due to a syntax error and failure in the conditional check. This leads to a cascading failure in other parts of the script. From my limited testing, the CN value is never an empty string because if mkdirs.sh is not run in test mode, the value is imported from the mkdirs.cfg file. |
||
#67 | 25231 | Robert Cowham |
Output line numbers on mkdirs errors Tweak tests to cope when parsing output |
||
#66 | 25190 | Robert Cowham |
Change to make instance dir a directory under /p4 (so on root volume). This avoids potential problems where /hxdepots is shared on multiple instances. It also improves performance often. Left the option to put /p4/common on a shared volume if required. |
||
#65 | 25147 | Robert Cowham |
Made -test parameter work again and included tests to detect future breakage! |
||
#64 | 25005 | C. Thomas Tyler | Removed hms reference. | ||
#63 | 23580 | C. Thomas Tyler |
Fixed a typo that would cause a chown to fail. Thanks to @rmarin for spotting that the first attempt to fix this was wrong! #review @rmarin |
||
#62 | 23456 | C. Thomas Tyler |
Fixed mkdirs.sh bug where 'chown' was trying to change ownership of the wrong dir. Bypassing pre-commit review as this is an obvious typo fix. #review-23457 |
||
#61 | 23354 | C. Thomas Tyler |
Fixed bug where checkpoints.* dir for replicas was created on /hxlogs rather than /hxdepots volume. Removed unnecessary logic to create a symlink for checkpoints.* dir for replicas. Fixed ownership of new checkpoints.* dir on creation. PATCH: Bypassing pre-commit review. #review-23355 |
||
#60 | 23266 | C. Thomas Tyler |
Fixes and Enhancements: * Enabled daily_checkpoint.sh operate on edge servers, to keep /p4/N/offline_db current on those hosts for site-local recovery w/o requiring a site-local replica (though having a site-local replica can still be useful). * Disabled live_checkpoint.sh for edge servers. * More fully support topologies using edge severs, in both geographically distributed and horizaontal scaling "wokspace server" solutions. * Fix broken EDGESERVER value definition. * Modified name of SDP counter that gets set when a checkpoint is taken to incorporate ServerID, so now the counter name will look like lastSDPCheckpoint.master.1, or lastSDPCheckpoint.p4d_edge_sfo, rather than just lastSDPCheckpoint. There will be multiple such counters in a topology that uses edge servers, and/or which takes checkpoints on replicas. * Added comments for all functions. For the master server, journalPrefix remains: /p4/N/checkpoints/p4_N The /p4/N/checkpoints is reserved for writing by the master/commit server only. For non-standby (possibly filtered) replicas and edge serves, journalPrefix is: /p4/N/checkpoints.<ShortServerID>/p4_N.<ShortServerID> Here, ShortServerID is just the ServerID with the 'p4d_' prefix trimmed, since it is redundant in this context. See mkrep.sh, which enshines a ServerID (server spec) naming standard, with values like 'p4d_fr_bos' (forwarding replica in Boston) and p4d_edge_blr (Edge server in Bangalore). So the journalPrefix for the p4d_edge_bos replica would be: /p4/N/checkpoints.edge_bos/p4_N.edge_bos For "standby" (aka journalcopy) replicas, journalPrefix is set to /p4/N/journals.rep. which is written to the $LOGS volume, due to the nature of standby replicas using journalPrefix to write active server logs to pre-rotated journals. Some take-away to be updated in docs: * The /p4/N/checkpoints folder must be reserved for checkpoints that originate on the master. It should be safe to rsync this folder (with --delete if desired) to any replica or edge server. This is consistent with the current SDP. * I want to change 'journals.rep' to 'checkpoints.<ShortServerID>' for non-standby replicas, to ensure that checkpoints and journals taken on those hosts are written to a volume where they are backed up. * In sites with multiple edge serves, some sharing achive files ('workspace servers'), multiple edge servers will share the same SAN. So we one checkpoints dir per ServerID, and we want that dir to be on the /hxdepots volume. Note that the journalPrefix for replicas was a fixed /p4/N/journals.rep. This was on the /hxlogs volume - a presumably fast-for-writes volume, but typically NOT backed up and not very large. This change puts it under /p4/N/checkpoints.* for edge servers and non-standby replicas, but ensures other replica types and edge servers can generate checkpoints to a location that is backed up and has plenty of storage capacity. For standby replicas only (which cannot be filtered), the journalPrefix remains /p4/N/journals.rep on the /hxlogs volume. |
||
#59 | 23202 | C. Thomas Tyler | Fixed an obvious typo in mkdirs.sh. | ||
#58 | 22957 | C. Thomas Tyler |
Merged two changes initiated on main down to: Change @22826 by robert_cowham: Ignore some files on Mac. Change @22950 by awkan/ttyler: Provide an override for difference between proxy listening port and target port |
||
#57 | 22788 | C. Thomas Tyler | Backed out the backout; moving forward with this change. | ||
#56 | 22694 | Russell C. Jackson (Rusty) | Backout accidental submit of shelved change. | ||
#55 | 22693 | Russell C. Jackson (Rusty) |
Branched a Unix only version of the SDP. Removed extra items to create a cleaner tree. Moved a few items around to make more sense without Windows in the mix. |
||
#54 | 22625 | Russell C. Jackson (Rusty) | Approved in Review 22364 | ||
#53 | 22623 | Russell C. Jackson (Rusty) | Update to account for Tom's name change. | ||
#52 | 22201 | C. Thomas Tyler | Fixed typo. | ||
#51 | 22200 | C. Thomas Tyler |
More elegant and portable implementation of logic to find config file in same dir as mkdirs.sh script. Works on Mac, too. This works for various means of calling mkdirs.sh, e.g. ./mkdirs.sh /full/path/to/mkdirs.sh /path/with/symlink/to/mkdirs.sh mkdirs.sh (found in PATH) Approving my own review here for this minor change so I can deploy, as others are on holiday or traveling. |
||
#50 | 22145 | Robert Cowham | Update mkdirs to help testing. | ||
#49 | 22070 | Russell C. Jackson (Rusty) |
Pull the configuration items out into mkdirs.cfg and source that file in mkdirs.sh. Makes it a little cleaner, and less likely for someone to mess up the mkdirs.sh script. Also preps for potential future configure_sdp.sh script to ask questions and create the mkdirs.cfg file. |
||
#48 | 21418 | C. Thomas Tyler |
Changed CASEINSENSITIVE to CASE_SENSITIVE, to make it harder to pick the wrong value by miscalculating the double-negative. Changed the default to be case-sensitive, for several reasons: * Case-sensitive is better for git interoperability. Most git repos in the wild, especially those used in corporate environments, are case sensitive. * Case-sensitive is better for importing from most legacy SCM systems, as it can handle a wider array of data sets. * Case-sensitive is required for some data sets that have case-only variants. * The SDP now includes a case check trigger to help avoid the worst perils of a case-sensitive server. * While confusion is always possible if the client OS doesn't match the case sensitivty setting of the server, the least harmful permuatations of problems occur with the widest array of client platforms when you have a case-senstive server. * Case-insensitive is really only best suited to pure-Windows sites. Also added a comment indicating you don't need the Review daemon if you intend to handle review-style email notifications with Swarm. |
||
#47 | 21327 | Russell C. Jackson (Rusty) | Update to have mkdirs put the correct instance name into the crontab files. | ||
#46 | 21267 | Robert Cowham |
Fix failing tests. Changed default filesystem names to: DB1=hxmetadata1 DB2=hxmetadata2 DD=hxdepots LG=hxlogs hx=Helix. Updated docs to reflect this |
||
#45 | 21192 | Russell C. Jackson (Rusty) | Fixed issue with root and offline_db links. | ||
#44 | 21178 | Russell C. Jackson (Rusty) |
Change the SDP so that root and offline_db can be on different volumes and still accomplish a fast database recovery using recreate_db_checkpoint.sh and recreate_db_sync_replica.sh. This is done by switching the links now rather than moving the db files. |
||
#43 | 21123 | Russell C. Jackson (Rusty) | Corrected cp path for ssl directory. | ||
#42 | 21049 | Russell C. Jackson (Rusty) |
Added config.txt to /p4/ssl so we and customers don't have to go find it in the admin guide. Added comments and cp command for config.txt and setting up ssl certs. Uncommented the journalcopy echo commands at the bottom of mkdirs.sh per discussion with Tom. |
||
#41 | 20982 | Russell C. Jackson (Rusty) | Added the new CN setting for /p4/common to the test folder path resets. | ||
#40 | 20905 | Russell C. Jackson (Rusty) |
Defaulted the SDP to be on the same volume as $CN since that is where it is usually located. Changed MAILTO to default to $MAILFROM since that is what it is most of the time and I am lazy and don't like typing the email address twice. |
||
#39 | 20904 | Russell C. Jackson (Rusty) |
Added CN=$DD and changed all $DD/p4/common entries to $CN/p4/common in order to allow you to install an instance on a different depotdata volume than the 1st instance. Before this change, doing that would incorrectly create a common folder on the 2nd depotdata volume. Also removed the lines that copy the binaries from an existing /p4/common/bin folder back to the SDP. It is perfectly acceptable to install another instance that is using a newer version than the existing instance. |
||
#38 | 20790 | C. Thomas Tyler | Added '/p4/common/hms' to list of dirs to install.' | ||
#37 | 20708 | C. Thomas Tyler |
Per discussion: s/checkpoints.rep/journals.rep/g This directory name changed, used in the journalPrefix configurable, is intended to clarify that the should be targeted to for a FAST volume for use with journalcopy, rather than the LARGE volume as would be implied when using a directory with "checkpoints" in the name. |
||
#36 | 20432 | C. Thomas Tyler |
Improved edge & daisy chained replica support in instance_vars.template. Changed so P4MASTER is set dynamically, based on how/whether the P4TARGET of the current ServerID is set. This also eliminates a possible discrepancey beteween P4MASTER as defined in the p4_N.vars/mkdirs.sh and the master hostname as defined in P4TARGET configurables. The value defined with P4TARGET must also work with SSH keys. (As a best practice P4TARGET should be a host alias so that it doesn't need to be changed in case of failvoer of your P4TARGET server). Changed so SHAREDDATA is set dynamically, based on how/whether the lbr.replication of the current ServerID is set. If it is unset, set to none, ondemand, or cache, then SHAREDDATA is set to TRUE, otherwise FALSE. Dynamic queries use 'p4d -cset' so they work regardless of whether the p4d process is up or not. Some internal refactoring was necessary to ensure all variables are set before they are used. This involved a minor tweak to mkdirs.sh to remove the now-unnecessary 'sed' for SHAREDDATA when generating p4_N.vars from the template. SHAREDDATA must still be configured in mkdirs.sh because it can be run before a replica is fully configured. Goals: * Simplfy SDP configuration for complex topologies by eliminating configuration external to p4d where possible/practical. * Reduce chances for discrepancies and errors as topologies evolve over time. * Allow the p4_1.vars file to identical on all hosts in the topology, an HMS requirement. |
||
#35 | 20395 | C. Thomas Tyler | Fixed typo. | ||
#34 | 20376 | C. Thomas Tyler |
Incorporated HMS service user naming standard into the SDP, i.e. "svc_<serverid>." Removed SVCUSER setting from mkdirs.sh accordingly. Fixed mkdirs.sh so /p4/common/etc is created if it doesn't already exist, just as the 'lib' dir is handled. Also a minor structural enhacement in instance_vars.template. Added SDP_ALWAYS_LOGIN setting to instance_vars.template, setting the default to 0 to prevent unnecessary logins. |
||
#33 | 20363 | C. Thomas Tyler |
Removed references to legacy names for checkpoint scripts. No functional changes. Bypassing pre-commit code review. #review-20364 |
||
#32 | 20217 | Russell C. Jackson (Rusty) | Needed move password creation outside of common area since it is instance specific now. | ||
#31 | 20170 | Russell C. Jackson (Rusty) |
Moved password and users into the config directory to allow for instance specific users and passwords. Ran into a case where two different teams were sharing the same server hardware and needed this type of differentiation. Surprised that we haven't hit this sooner. Also defaulted mkdirs to use the numeric ports since this is the most common installation. |
||
#30 | 19983 | Russell C. Jackson (Rusty) | Commented out the echo around using journalcopy until we are ready to support that. | ||
#29 | 19410 | Russell C. Jackson (Rusty) |
Made out on setting permissions consistent in mkdirs. Added the new output as valid in test_SDP. |
||
#28 | 19314 | Russell C. Jackson (Rusty) |
Change p4verify.sh to use -S to verify shelves on a replica instead of printing the files on the shelf. Removed the HOST_IP settings from mkdirs and instance_vars since it causes problems in a shared depotdata environment, and it was a temporary fix to work around a bug with auth.id that is being fixed. |
||
#27 | 19311 | adrian_waters | If creating a replica that shares depot files with master, don't do chown/chmod on the depot files as this can take a significant time when migrating existing Helix servers into the SDP; In addition, issue 'warning' message that chown/chmod could take some time to complete so user is aware of processing being carried out | ||
#26 | 18952 | C. Thomas Tyler |
Tweaks to mkdirs.sh: * Replaced ADMINEMAIL with MAILTO and MAILFROM settings. This allows MAILTO to be a distribution list or a comma-delimited list of email addresses, while MAILFROM must always be exactly one email address. * Added quotes to allow handling of adresses like '#P4AdminTeam," where the '#' character designates a distribution list. |
||
#25 | 18930 | C. Thomas Tyler |
Fixed typo; caught by test suite. Bypassing pre-commit review 'cuz this is an obivious fix. #review-18931 |
||
#24 | 18925 | C. Thomas Tyler |
Enhanced p4_vars.template to support operating on an SDP-managed host where no p4d process runs, such as a p4p (proxy) host, where no /p4/n/root/server.id file exists. The $SERVERID value will be empty (but defined) in this case. Enhanced to better support operating with replicas that share /depotdata with their master servers, by making P4TRUST and P4TICKETS values contain $SERVERID. Moved SHAREDDATA from p4_vars to instance_vars, since it is not inherently a global setting. In sophisticated enterprise environments, it can vary on a per-replica basis. Adjusted mkdirs.sh accordingly. |
||
#23 | 18618 | Russell C. Jackson (Rusty) | Added a crontab for the edge servers. | ||
#22 | 18203 | richard_baum |
Removed hard-coded /p4 path for server.id file location. This fixes "-test" mode so server.id file is written to the correct location. Without this it is written to /p4 where there could be an installation with the same instance ID. In that case the server.id file could get overwritten with different data. If there is not an existing instance an error message would be displayed. Now neither of these should occur :-) Also fixed some typos. |
||
#21 | 16563 | C. Thomas Tyler |
Routine Merge Down to dev from main using: p4 merge -b perforce_software-sdp-dev p4 resolve -as |
||
#20 | 16373 | C. Thomas Tyler |
Routine Merge Down to dev from main using: p4 merge -b perforce_software-sdp-dev |
||
#19 | 16335 | C. Thomas Tyler |
Routine Merge Down to dev from main using: p4 merge -b perforce_software-sdp-dev |
||
#18 | 16029 | C. Thomas Tyler |
Routine merge to dev from main using: p4 merge -b perforce_software-sdp-dev |
||
#17 | 15701 | C. Thomas Tyler | Routine merge down using 'p4 merge -b perforce_software-sdp-dev'. | ||
#16 | 14136 | C. Thomas Tyler |
Routine merge down to dev from main for SDP using perforce_software-sdp-dev. |
||
#15 | 14038 | C. Thomas Tyler | Routine merge-down to SDP dev from main. | ||
#14 | 13906 | C. Thomas Tyler |
Normalized P4INSTANCE to SDP_INSTANCE to get Unix/Windows implementations in sync. Reasons: 1. Things that interact with SDP in both Unix and Windows environments shoudn't have to account for this obscure SDP difference between Unix and Windows. (I came across this doing CBD work). 2. The Windows and Unix scripts have different variable names for defining the same concept, the SDP instance. Unix uses P4INSTANCE, while Windows uses SDP_INSTANCE. 3. This instance tag, a data set identifier, is an SDP concept. I prefer the SDP_INSTANCE name over P4INSTANCE, so I prpose to normalize to SDP_INSTANCE. 4. The P4INSTANCE name makes it look like a setting that might be recognized by the p4d itself, which it is not. (There are other such things such as P4SERVER that could perhaps be renamed as a separate task; but I'm not sure we want to totally disallow the P4 prefix for variable names. It looks too right to be wrong in same cases, like P4BIN and P4DBIN. That's a discussion for another day, outside the scope of this task). Meanwhile: * Fixed a bug in the Windows 2013.3 upgrade script that was referencing undefined P4INSTANCE, as the Windows environment defined only SDP_INSTANCE. * Had P4INSTANCE been removed completely, this change would likely cause trouble for users doing updates for existing SDP installations. So, though it involves slight technical debt, I opted to keep a redundant definition of P4INSTANCE in p4_vars.template, with comments indicating SDP_INSTANCE should be used in favor of P4INSTANCE, with a warning that P4INSTANCE may go away in a future release. This should avoid unnecessary upgrade pain. * In mkdirs.sh, the varialbe name was INSTANCE rather than SDP_INSTANCE. I changed that as well. That required manual change rather than sub/replace to avoid corrupting other similar varialbe names (e.g. MASTERINSTANCE). This is a trivial change technically (a substitute/replace, plus tweaks in p4_vars.template), but impacts many files. |
||
#13 | 13586 | C. Thomas Tyler |
Routine merge down from main -> dev. Trivial merges, all resolved with 'p4 resolve -as.' |
||
#12 | 12923 | C. Thomas Tyler |
Routine merge down from main to dev. Resolved with 'p4 resolve -as', no merges or conflicts. |
||
#11 | 12170 | Russell C. Jackson (Rusty) | Merged in changes in Main | ||
#10 | 12116 | Russell C. Jackson (Rusty) | Update dev from main. | ||
#9 | 12107 | C. Thomas Tyler |
Routine merge down from 'main' to 'dev', resolved with 'p4 resolve -as'. |
||
#8 | 12028 | C. Thomas Tyler | Refreshed SDP dev branch, merging down from main. | ||
#7 | 11523 | Russell C. Jackson (Rusty) |
Modified P4BROKERPORTNUM to just be the port number and added P4BROKERPORT to instance_vars to be consistent with P4PORT. Also makes it easier to modify p4review.py to use P4BROKERPORT rather than P4PORT for the subject line when needed. |
||
#6 | 11490 | Russell C. Jackson (Rusty) |
Added SSL_PREFIX back and P4MASTERPORTNUM in order to support the sync_replica.sh and weekly_sync_replica.sh scripts. |
||
#5 | 11477 | Russell C. Jackson (Rusty) |
Updated to use /usr/bin/env python Added workshop header. Changed cfg to config. |
||
#4 | 11474 | Russell C. Jackson (Rusty) |
Had to move the cfg directory to the metadata volume and link it under the instance directory to provide the proper separation in a shared volume environment. The instance specific vars cannot be in a shared directory since they need to be different on each node using the shared volume. Since the files moved back to the instance directory, I changed the names back to: instance_vars p4review.cfg to keep things simple. Also moved p4_vars.template to SDP/Server/Unix/p4/common/cfg so that it doesn't get copied to the /p4/common/bin folder. Plus, it makes more sense for it to be in that directory in the SDP structure. |
||
#3 | 11468 | Russell C. Jackson (Rusty) | Added comments regarding configuring Edge and Edge replicas. | ||
#2 | 11466 | Russell C. Jackson (Rusty) |
Initial work to simplify p4_vars and remove cluster stuff. Testing of named instances surfaced some bugs that are in prod sdp, now fixed in dev. Added three triggers from RCJ SDP Moved p4review.cfg into the new /p4/common/cfg to go along with the instance_vars files. mkdirs.sh now generates an instance_p4review.cfg as well. Removed incremental p4verify to clean up a bit. It didn't support replicas and was really never used. All port settings now live in <instance>_vars file. You set what you want the ports to be in mkdirs.sh. There is no more fancy logic to try to guess what the port should be. You set it, and that is what it is. Remaining to do is to updated scripts to not need p4master_run. Saved that work for later since this is tested and works. |
||
#1 | 10638 | C. Thomas Tyler | Populate perforce_software-sdp-dev. | ||
//guest/perforce_software/sdp/main/Server/Unix/setup/mkdirs.sh | |||||
#1 | 10148 | C. Thomas Tyler | Promoted the Perforce Server Deployment Package to The Workshop. |