#!/bin/bash #------------------------------------------------------------------------------ set -u #============================================================================== # Declarations and Environment declare ThisScript="${0##*/}" declare Version=1.5.1 declare SDPRoot= declare DocDir= declare -i DoP4Reconcile=0 declare -i DoP4Submit=0 declare -i CancelP4Submit=0 declare -i Debug="${DEBUG:-0}" declare -i i declare -i ErrorCount=0 declare -i WarningCount=0 declare -i ScriptCount=0 declare -a ScriptList declare -a ScriptInterpreter declare -a ScriptUsageFlag declare Script= declare UserSpecifiedScripts= declare Cmd= declare DocGenDir= declare ManPageFile= declare Log=Unset # Store location of all known scripts, with paths relative to # the SDP branch root directory. ScriptList[$ScriptCount]="doc/gen/gen_script_man_pages.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="helix_binaries/get_helix_binaries.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/bin/ccheck.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/bin/load_checkpoint.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/bin/mkrep.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/bin/p4login" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/bin/p4pcm.pl" ScriptInterpreter[$ScriptCount]="perl" ScriptUsageFlag[$ScriptCount]="h" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/bin/p4verify.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/bin/sdp_health_check.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/bin/upgrade.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/bin/verify_sdp.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/sdp_upgrade/sdp_upgrade.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/setup/mkdirs.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 ScriptList[$ScriptCount]="Server/Unix/p4/common/sdp_upgrade/clear_depot_Map_fields.sh" ScriptInterpreter[$ScriptCount]="bash" ScriptUsageFlag[$ScriptCount]="man" ScriptCount+=1 #============================================================================== # Local Functions function msg () { echo -e "$*"; } function dbg () { [[ "$Debug" -eq 1 ]] || return; msg "$*"; } function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; } function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; } function bail () { errmsg "${1:-Unknown Error}\\n"; exit "${2:-1}"; } #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). # The default is -h. # # $2 - error message (optional). Specify this if usage() is called due to # user error, in which case the given message displayed first, followed by the # standard usage message (short or long depending on $1). If displaying an # errror, usually $1 should be -h so that the longer usage message doesn't # obsure the error message. # # Sample Usage: # usage # usage -h # usage -man # usage -h "Incorrect command line usage." #------------------------------------------------------------------------------ function usage { declare style="${1:--h}" declare errorMessage="${2:-Unset}" if [[ "$errorMessage" != Unset ]]; then msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" fi echo "USAGE for $ThisScript v$Version: $ThisScript [<Script> [<Script2> ...]] [-rec | -submit] [-L <log>] [-d|-D] or $ThisScript [-h|-man|-V] " if [[ $style == -man ]]; then msg " DESCRIPTION: This script generates man pages for SDP shell scripts. This is done by executing scripts with the '-man' argument for each script (or '-h' depending on the script). This helps ensure the latest in-code documentation from the scripts is included in the SDP Guide using an include directive in the AsciiDoc. This script generates docs in SDP doc/gen folder, where they are referenced in AsciiDoc (*.adoc) files. It can update files in version control (using 'p4 rec') if the '-rec' option is used. Alternately, if '-submit' is specified (which implies '-rec'), any files opened by reconcile are automatically submitted with a description of 'Updated generated script man pages.' After this script is executed, the AsciiDoc can be updated by executing the following in the SDP 'doc' folder, the local folder corresponding to //guest/perforce_software/sdp/dev/doc: make clean make OPTIONS: <Script> [<Script2> ...] Specify a space-delimited list of scripts to generate docs for. If no script is specified, man pages are generated for all configured scripts. Only the script basename, e.g. mkrep.sh or load_checkpoint.sh, should be given. This script knows where all documented scripts exist in the SDP structure, so the user need only specify the basename. -rec Specify '-rec' to reconcile of generated files. -s[ubmit] Specify '-s' (or '-submit') to submit any generated doc files with a changelist description of: 'Updated generate script man pages.' The '-submit' is ignored if associated script are checked out. The '-submit' option implies '-rec'. Files may submitted to: //guest/perforce_software/sdp/dev/doc/gen/... -L <log> Specify the path to a log file, or the special value 'off' to disable logging. By default, all output (stdout and stderr) goes to /tmp/gen_script_man_page.<datestamp>.log NOTE: This script is self-logging. That is, output displayed on the screen is simultaneously captured in the log file. Do not run this script with redirection operators like '> log' or '2>&1', and do not use 'tee.' -d Debug mode, generates more verbose output. -D Set extreme debugging verbosity using bash 'set -x' mode. Implies '-d'. HELP OPTIONS: -h Display short help message -man Display man-style help message -V Dispay version info for this script and its libraries. " fi exit 1 } #============================================================================== # Command Line Processing declare -i shiftArgs=0 set +u while [[ $# -gt 0 ]]; do case $1 in (-L) Log="$2"; shiftArgs=1;; (-h) usage -h;; (-rec) DoP4Reconcile=1;; (-s|-submit) DoP4Submit=1;; (-man) usage -man;; (-V) msg "\\n$ThisScript v$Version\\n"; exit 1;; (-d) Debug=1;; # Debug, call dbg() functions with more verbose debug output. (-D) Debug=1; set -x;; # Extreme debug; use bash 'set -x' mode. (-*) usage -h "Unknown option '$1'.";; (*) if [[ -z "$UserSpecifiedScripts" ]]; then UserSpecifiedScripts="$1" else UserSpecifiedScripts="$UserSpecifiedScripts 1" fi ;; esac # Shift (modify $#) the appropriate number of times. shift; while [[ $shiftArgs -gt 0 ]]; do [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments." shiftArgs=$shiftArgs-1 shift done done set -u #============================================================================== # Command Line Verification [[ "$PWD" == *"/doc/gen" ]] || \ usage -h "\\n\\tRun this from the 'doc/gen' dir under the SDP root dir, not \\n\\t$PWD" [[ "${Log:-Unset}" == Unset ]] && \ Log="/tmp/${ThisScript%.sh}.$(date +'%Y%m%d-%H%M%S').log" [[ -n "$UserSpecifiedScripts" ]] || UserSpecifiedScripts=all # The '-submit' option implies '-rec'. [[ "$DoP4Submit" -eq 1 ]] && DoP4Reconcile=1 #============================================================================== # Main Program if [[ "$Log" != "off" ]]; then touch "$Log" || bail "Couldn't touch log file [$Log]." # Redirect stdout and stderr to a log file. exec > >(tee "$Log") exec 2>&1 msg "\\nLog is: $Log\\n" fi msg "Running $ThisScript v$Version as $USER@${HOSTNAME%%.*} at $(date)." # This runs from the SDP 'doc/gen' folder for whatever branch we are in. # dir. if [[ "$PWD" == *"/doc/gen" ]]; then # Calculate the 'SDPRoot', up one dir level if we started in 'doc'. DocGenDir="$PWD" DocDir="${PWD%/gen}" SDPRoot="${DocDir%/*}" else bail "Bad start dir [$PWD], should be 'doc/gen' subdirector under SDP branch root." fi # Setup just enough environment to be able to run the scripts with the '-man' option # to generate a man page from in-script docs. These values must be exported. export P4U_HOME="${SDPRoot}/Server/Unix/p4/common/bin" export P4U_LIB="${SDPRoot}/Server/Unix/p4/common/lib" dbg "P4U_HOME=$P4U_HOME" dbg "P4U_LIB=$P4U_LIB" msg "Generating docs in dir: $DocGenDir" if [[ "$UserSpecifiedScripts" == "all" ]]; then i=0; while [[ "$i" -lt "${#ScriptList[@]}" ]]; do ManPageFile="${ScriptList[$i]}.man.txt" ManPageFile="${DocGenDir}/${ManPageFile##*/}" msg "Generating doc for: ${ScriptList[$i]}" Cmd="${ScriptInterpreter[$i]} $SDPRoot/${ScriptList[$i]} -${ScriptUsageFlag[$i]}" # Abort submit of the generated script man page if the script itself # is opened for edit. if [[ -n "$(p4 -ztag -F %rev% opened "$SDPRoot/${ScriptList[$i]}")" && "$DoP4Submit" -eq 1 ]]; then warnmsg "The '-submit' will be ignored because this file is checked out: $SDPRoot/${ScriptList[$i]}" CancelP4Submit=1 fi dbg "Running: $Cmd" # Note: Several scripts generate a non-zero exit code when '-man' or '-h' # are used, and so checking the exit code isn't helpful in determining if # we successfully generated docs. Instead we check if the generated doc # file is empty or not, with stderr discarded. $Cmd > "$ManPageFile" 2>/dev/null if [[ -s "$ManPageFile" ]]; then dbg "Doc content generated." else errmsg "No doc content generated for: ${ScriptList[$i]}" fi i+=1 done else for Script in $UserSpecifiedScripts; do dbg "Searching for docs for script: $Script" Cmd= i=0; while [[ "$i" -lt "${#ScriptList[@]}" ]]; do dbg "COMPARING [${ScriptList[i]}] vs. [*$Script]" if [[ "${ScriptList[i]}" == *"$Script" ]]; then Cmd="SDPRoot/${ScriptInterpreter[$i]} ${ScriptList[$i]} -${ScriptUsageFlag[$i]}" # Abort submit of the generated script man page if the script itself # is opened for edit. if [[ -n "$(p4 -ztag -F %rev% opened "$SDPRoot/${ScriptList[$i]}")" && "$DoP4Submit" -eq 1 ]]; then warnmsg "The '-submit' will be ignored because this file is checkec out: $SDPRoot/${ScriptList[$i]}" CancelP4Submit=1 fi break fi i+=1 done if [[ -n "$Cmd" ]]; then msg "Generating doc for $Script" dbg "Running: $Cmd" ManPageFile="${DocGenDir}/${Script}.man.txt" $Cmd > "$ManPageFile" 2>&1 if [[ -s "$ManPageFile" ]]; then dbg "Doc content generated." else errmsg "No doc content generated for: ${ScriptList[$i]}" fi else errmsg "No script docs available for script: $Script" fi done fi if [[ "$ErrorCount" -eq 0 ]]; then msg "\\nInterim script docs generated successfully." else bail "Encountered $ErrorCount errors generating intermim script docs." fi if [[ "$DoP4Reconcile" -eq 1 ]]; then Cmd="p4 -d $DocGenDir rec" msg "Reconciling in $DocGenDir with: $Cmd" if ! $Cmd; then bail "Reconcile in $DocGenDir failed!" fi if [[ -n "$(p4 -ztag -F %rev% opened "$DocGenDir/..." | head -1)" ]]; then Cmd="p4 -s reopen -t +w $DocGenDir/..." msg "Ensuring generated files have +w file type modifier with:\\n$Cmd" if ! $Cmd; then errmsg "Error running 'p4 reopen' command." fi Cmd="p4 -s -d $DocGenDir submit -d \"Updated generated script man pages.\" ..." if [[ "$DoP4Submit" -eq 1 ]]; then if [[ "$CancelP4Submit" -eq 1 ]]; then msg "Opened files detected in $DocGenDir, but '-submit' ignored due to checkout out script files. When ready and after submitting script files, submit with:\\n\\t$Cmd" else # shellcheck disable=SC2090 if p4 -s -d "$DocGenDir" submit -d "Updated generated script man pages." ...; then msg "\\nSubmit complete." else bail "Submit of genreated man pages in $DocGenDir failed." fi fi else msg "Opened files detected in $DocGenDir. Submit with:\\n\\t$Cmd" fi else msg "No versioned files need to be updated in: $DocGenDir" fi else msg "\\nNo reconcile or submit attempted. Consider running with '-rec' and/or '-submit' options." fi exit "$ErrorCount"
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#5 | 30782 | C. Thomas Tyler |
Added new install_sdp.sh script and supporting documentation. The new install_sdp.sh makes SDP independent of the separate Helix Installer software (the reset_sdp.sh script). The new script greatly improves the installation experience for new server machines. It is ground up rewrite of the reset_sdp.sh script. The new script preserves the desired behaviors of the original Helix Installer script, but is focused on the use case of a fresh install on a new server machine. With this focus, the scripts does not have any "reset" logic, making it completely safe. Added various files and functionalityfrom Helix Installer into SDP. * Added firewalld templates to SDP, and added ufw support. * Improved sudoers generation. * Added bash shell templates. This script also installs in the coming SDP Package structure. New installs use a modified SDP structure that makes it so the /p4/sdp and /p4/common now point to folders on the local OS volume rather than the /hxepots volume. The /hxdepots volume, which is often NFS mounted, is still used for depots and checkpoints, and for backups. The new structure uses a new /opt/perforce/helix-sdp structure under which /p4/sdp and /p4/common point. This structure also contains the expaneded SDP tarball, downloads, helix_binaries, etc. This change represents the first of 3-phase rollout of the new package structure. In this first phase, the "silent beta" phase, the new structure is used for new installations only. This phase requires no changes to released SDP scripts except for mkdirs.sh, and even that script remains backward-compatible with the old structure if used independently of install_sdp.sh. If used with install_sdp.sh, the new structure is used. In the second phase (targeted for SPD 2024.2 release), the sdp_upgrade.sh script will convert existing installations to the new structure. In the third phase (targeted for SDP 2025.x), this script will be incorporated into OS pacakge installations for the helix-sdp package. Perforce internal wikis have more detail on this change. #review-30783 |
||
#4 | 30681 | C. Thomas Tyler |
Added gen_sudoers.sh script to generate a sudoers file for perforce OSUSER. This generates a more secure limited sudoers file. Previously, adding a sudoers entry for the OSUSER (usually 'perforce') was done only by the Helix Installer. In the Helix Installer variant, a single "one-size-filts-all" sudoers file was used, with the following characteristics: * The instances for Helix Core services were referenced with a '*' wildcard to match all SDP instances, which has since been determined to introduce a vulnerability. In this new script, the wildcard is replaced with separate entries for each SDP instance. * There were entries for all known paths of utilities like lslocks, setcap, and getcap. This new script generates the correct path valid for the current machine. With this change, the functionality will be available in the SDP directly. This new gen_sudoers.sh script can be called by mkdirs.sh directly to update the sudoers file each time a new SDP instance is added, if the new '-fs' (full sudo) or '-ls' (limited sudoers) entries are used. There is no change to the default behavior of mkdirs.sh; only a change if new options are utilized. This script comes with docs and examples for the new script as well as doc changes for mkdirs.sh. (Also added missing documentation for the '-no_enable' option). Further changes needed: * Add doc reference in SDP_Guide.Unix.adoc |
||
#3 | 30114 | C. Thomas Tyler | Refined, and updated script to generate docs for itself. | ||
#2 | 30112 | C. Thomas Tyler | Adapted gen_script_man_pages.sh to run from SDP doc/gen dir. | ||
#1 | 30110 | C. Thomas Tyler | Added gen_script_man_pages.sh to SDP package. |