#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ set -u #============================================================================== # Declarations and Environment declare ThisScript="${0##*/}" declare Version="1.3.3" declare Log=off declare -i SilentMode=0 declare -i ErrorCount=0 declare -i WarningCount=0 declare -i Debug=${DEBUG:-0} declare -a ImageFiles declare -i i=0 #============================================================================== # Local Functions #------------------------------------------------------------------------------ function msg () { echo -e "$*"; } function dbg () { [[ "$Debug" -eq 0 ]] || msg "$*" >&2; } function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; } function warnmsg () { msg "\\nWarning: ${1:-Unknown Warning}\\n"; WarningCount+=1; } function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; } #------------------------------------------------------------------------------ # Function dsi_create ($tag, $version) # # Return success (0) upon successful DSI creation, non-zero on error. #------------------------------------------------------------------------------ function dsi_create () { dbg "CALL: dsi_create ($*)" local tag=${1:-} local imageVersion=${2:-1.0.1} local headRev= local newRev= local imageFile= local dirsFile= local filesFile= local linksFile= local element= local output= local owner= local group= local perms= local timestamp= local checksum= [[ -n "$tag" ]] || bail "Internal error; tag unset." msg "Creating image for DSI tag $tag in $ImageRoot" load_images "$tag" headRev=$(get_head_rev "$tag") newRev=$((headRev+1)) imageFile="$StorageDir/$tag.$newRev.dsi" [[ -e "$imageFile" ]] && bail "Image file already exists: $imageFile" touch "$imageFile" || bail "Could not initialize image file: $imageFile" if ! echo "i $tag $imageVersion" > "$imageFile"; then bail "Failed to write first line to: $imageFile" fi cd "$ImageRoot" || bail "Could not cd to image root dir: $ImageRoot" dirsFile=$(mktemp) filesFile=$(mktemp) linksFile=$(mktemp) # $ sec=$(TZ="Australia/Sydney" date +'%s' -d "2015-05-20 18:05:02") # $ TZ="America/Los_Angeles" date -d "@$sec" # d <RelPath>|<Owner>:<Group>|<Perms>|<Timestamp> dbg "Generating directory entries." if find ./ -type d -print > "$dirsFile"; then while read -r d; do # Normalize timezone to UTC and store as seconds for image storage. dbg "TZ=UTC ls -ld --time-style +'%s' \"$d\"" output=$(TZ=UTC ls -ld --time-style +'%s' "$d") element="${d#./}" perms="${output%% *}" owner=$(echo "$output"|awk '{print $3}') group=$(echo "$output"|awk '{print $4}') timestamp=$(echo "$output"|awk '{print $6}') echo "d ${element}|${owner}:${group}|${perms}|${timestamp}" >> "$imageFile" ||\ bail "Could not update image file: $imageFile" done < "$dirsFile" else errmsg "Errors encountered attempting 'find' command for directories in $PWD. Check for ownership issues prevetning user $USER from accessing folders." rm -f "$dirsFile" return 1 fi # f <RelPath>|<Owner>:<Group>|<Perms>|<Timestamp>|<Checksum> dbg "Generating file entries." if find ./ -type f -print > "$filesFile"; then while read -r f; do # Normalize timezone to UTC and store as seconds for image storage. dbg "TZ=UTC ls -ld --time-sytle +'%s' \"$f\"" output=$(TZ=UTC ls -ld --time-style +'%s' "$f") element="${f#./}" perms="${output%% *}" owner=$(echo "$output"|awk '{print $3}') group=$(echo "$output"|awk '{print $4}') timestamp=$(echo "$output"|awk '{print $6}') checksum=$(sha256sum "$f"|awk '{print $1}') echo "f ${element}|${owner}:${group}|${perms}|${timestamp}|${checksum}" >> "$imageFile" ||\ bail "Could not update image file: $imageFile" done < "$filesFile" else errmsg "Errors encountered attempting 'find' command for files in $PWD. Check for ownership issues prevetning user $USER from accessing folders." rm -f "$filesFile" return 1 fi # s <RelPath>|<Owner>:<Group>|<Perms>|<LinkTarget> dbg "Generating symlink entries." if find ./ -type l -print > "$linksFile"; then while read -r s; do dbg "TZ=UTC ls -ld --time-sytle +'%s' \"$s\"" output=$(TZ=UTC ls -ld --time-style +'%s' "$s") element="${s#./}" perms="${output%% *}" owner=$(echo "$output"|awk '{print $3}') group=$(echo "$output"|awk '{print $4}') echo "s ${element}|${owner}:${group}|${perms}|${timestamp}|${checksum}" >> "$imageFile" ||\ bail "Could not update image file: $imageFile" done < "$linksFile" else errmsg "Errors encountered attempting 'find' command for symlinks in $PWD. Check for ownership issues prevetning user $USER from accessing folders." rm -f "$filesFile" return 1 fi msg "Image $tag@$newRev created successfully." rm -f "$dirsFile" "$filesFile" "$linksFile" return 0 } #------------------------------------------------------------------------------ function dsi_diff () { dbg "CALL: dsi_diff ($*)" msg "Diffing image against ${Tag}@${Rev}" } #------------------------------------------------------------------------------ function dsi_list () { dbg "CALL: dsi_list ($*)" local tag=${1:-} msg "Listing images for $tag" load_images "$tag" if [[ ${#ImageFiles[@]} -gt 0 ]]; then for imageFile in "${ImageFiles[@]}"; do msg "$imageFile" done else msg "No images found for $tag." fi } #------------------------------------------------------------------------------ function dsi_remove () { dbg "CALL: dsi_remove ($*)" msg "Removing image ${Tag}@${Rev}" } #------------------------------------------------------------------------------ function dsi_remove_all () { dbg "CALL: dsi_remove_all ($*)" msg "Removing all images for ${Tag}" } #------------------------------------------------------------------------------ function dsi_tags () { dbg "CALL: dsi_tags ()" msg "Listing tags:" } #------------------------------------------------------------------------------ function load_images () { dbg "CALL: load_images ($*)" local tag=${1:-} local fileList= local -i i=0 ImageFiles=() fileList=$(mktemp) cd "$StorageDir" || bail "Could not cd to storage dir: $StorageDir" ls -t "$tag".*.dsi > "$fileList" 2>/dev/null if [[ -s "$fileList" ]]; then while read -r dsiFile; do ImageFiles[$i]="$dsiFile" i+=1 done < "$fileList" fi cd - > /dev/null || bail "Could not cd back to $OLDPWD." rm -f "$fileList" } #------------------------------------------------------------------------------ function get_head_rev () { dbg "CALL: get_head_rev ($*)" local tag=${1:-} local headRev=1 local rev= local imageFile= [[ -n "$tag" ]] || bail "Internal error; tag unset." if [[ ${#ImageFiles[@]} -eq 0 ]]; then echo 0 return else # Get the higest rev, using the file naming convention for image # files, "<tag>.<rev>.dsi" for imageFile in "${ImageFiles[@]}"; do rev=${imageFile%.dsi} rev=${rev##*.} [[ "$rev" -gt "$headRev" ]] && headRev="$rev" done fi echo "$headRev" } #------------------------------------------------------------------------------ # Function: verify_rev_format #------------------------------------------------------------------------------ function verify_rev_format () { declare rev=${1:-Unset} [[ "$rev" == "Unset" ]] && return 2 # Value can be special value LATEST, or a positive integer. [[ "$rev" == "Unset" ]] && return 0 [[ "$rev" == "LATEST" || "$rev" == "NEXT" ]] && return 0 [[ "$rev" =~ ^[1-9]{1}[0-9]*$ ]] && return 0 return 1 } #------------------------------------------------------------------------------ # Function: terminate function terminate { # Disable signal trapping. trap - EXIT SIGINT SIGTERM dbg "$ThisScript: EXITCODE: $ErrorCount" # Stop logging. [[ "${Log}" == "off" ]] || stoplog # With the trap removed, exit. exit "$ErrorCount" } #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). # The default is -h. # # $2 - error message (optional). Specify this if usage() is called due to # user error, in which case the given message displayed first, followed by the # standard usage message (short or long depending on $1). If displaying an # errror, usually $1 should be -h so that the longer usage message doesn't # obsure the error message. # # Sample Usage: # usage # usage -h # usage -man # usage -h "Incorrect command line usage." #------------------------------------------------------------------------------ function usage { declare style="${1:--h}" declare errorMessage="${2:-Unset}" if [[ "$errorMessage" != "Unset" ]]; then msg "\\n\\nUsage Error:\\n\\n$errorMessage\\n\\n" fi msg "USAGE for $ThisScript v$Version: Usage varies based on mode, which can be one of: create, diff, info, list, rm, rmall, tags $ThisScript create [<tag>] $ThisScript diff [<tag>[@<rev>]] $ThisScript info $ThisScript list [<tag>] $ThisScript rm [<tag>@<rev>] $ThisScript rmall [<tag>] $ThisScript tags Key environment variables: * DSI_TAG defines a default for <tag>. * DSI_STORAGE, or specify as '-s <storage_dir>'. * DSI_IMAGE_ROOT, defaults to \$PWD, or specify as '-d <image_root>'. or $ThisScript [-h|-man|-V] " if [[ $style == -man ]]; then msg " DESCRIPTION: Directory Structure Image (DSI) is a comprehensive comparison utility, used primarily to detect changes to a directory tree since the initial installation. A DSI can be generated as part of the release process for a software product, and then used subsequently to detect changes. Detection of changes includes: * Last-modified Timestamp * Newly added files/directories * Deleted files/directories * Changes to contents of files, including showing diffs * Changes in file/directory owner * Changes in file/directory group * Changes in file/directory permissions (e.g. +x bit, etc.) Command line flags can suppress certain types of diffs that may not be important in certain scenarios, such as group/owner changes, as well as special verifications such as 'owner can be anything except root.' This software is powered by the Perforce Helix version management software using Helix native DVCS features to show file and content differences. TAGS AND REVISIONS: A DSI is assigned a tag on creation, which is generally the name of the product and optionally product version ID, e.g. 'fgs' or 'fgs-1.0'. DSI tag names can include alphanumeric characters, dashes, underscores, and dots. DSI revisions are positive integers associated with a tag and assigned upon creation. MODES: Create, specified with 'create [<tag>]' (or 'c' for short) Create and store a DSI image with a given tag name. Diff, specified with 'diff [<tag>]' (or 'd' for short) Diff a DSI image with the given tag, using the latest version or a specified revision tag. Optionally, specify the @<rev> to compare against a revision other than the head revision. If no flags are specified to limit the output, a comprehensive diff will report: * Timestamp differences * Checksum differences * Size Differences The Diff mode supports flags to exlcude certain things from comparison: Fix, specified with 'fix [<tag>]' (or 'f' for short) Fix a directory structure as it appears on disk, focing it to match the image. This will overwrite files in the target structure. Info, specified with 'info' (or 'i' for short) Show DSI environment variables, default settings, and version info. List, specified as 'list [<tag>]' (or 'l' for short) List stored DSI images for the given tag. Remove, specified as 'rm [<tag>]@<rev>' Remove the specified revision for the given tag. For this usage the revision must be specified. RemoveAll, specified as 'rmall <tag>' Remove all images for the given tag. Tags, specify as 'tags' (or 't' for short) List available image tags. <tag>[@<rev>] Specify the DSI tag and revision. A default tag value can be defined by setting the DSI_TAG environment variable. For Diff mode, the following can be specified: <tag> Explicit value for <tag>, overriding DSI_TAG. Revision defaults to @LATEST <tag>@<rev> Explicit definition of tag and revision @<rev> Uses DSI_TAG for the tag, with explicit rev. For Create or List modes, the following can be specified: <tag> Explicit value for <tag>, overriding DSI_TAG. Revision is not set with Create and List modes. For Remove mode the tag may be implied (with DSI_TAG) or specified explicitly. The revision is required. For RemoveAll mode the tag must be specified explicitly. A revision is not allowed. OPTIONS: -r <image_root> Specify the root directory in which to create a DSI, or in which to do a comparsion against stored DSIs. The default is the current working directory as indicated by \$PWD. -s <storage_dir> Specify the DSI storage dir. A default can be defined by setting the DSI_STORAGE environment variable GENERAL OPTIONS: -L <log> Specify the path to a log file to enable logging. Or, specify '-L on' to enable logging using a default log file name: /tmp/$ThisScript.v$Version.<datestamp>.log By default, logging is disabled. NOTE: This script is self-logging if logging is enabled. That is, output displayed on the screen is simultaneously captured in the log file. Do not run this script with redirection operators like '> log' or '2>&1', and do not use 'tee.' -si Operate silently. All output (stdout and stderr) is redirected to the log only; no output appears on the terminal. This cannot be used with '-L off'. -D Set extreme debugging verbosity. HELP OPTIONS: -h Display short help message -man Display man-style help message -V Dispay version info for this script and its libraries. DSI FILE FORMAT: The directory structure image file format is defined in the documentation. FILES: DSI package definition files are stored in the DSI_STORAGE directory, and have a .dsi file name suffix. EXAMPLES: Example 1: Create a DSI of /p4/common dsi c -r /p4/common " | less fi exit 1 } #============================================================================== # Command Line Processing declare Mode="Help" declare Tag="${DSI_TAG:-Unset}" declare Rev="Unset" declare TagAndRev="Unset" declare StorageDir="${DSI_STORAGE:-Unset}" declare ImageRoot="${DSI_IMAGE_ROOT:-$PWD}" declare ImageVersion=1.0.1 declare -i InitStorageDir=0 declare -i shiftArgs=0 set +u while [[ $# -gt 0 ]]; do case $1 in (c|create|Create) Mode="Create";; (d|diff|Diff) Mode="Diff";; (f|fix|Fix) Mode="Fix";; (i|info|Info) Mode="Info";; (l|list|List) Mode="List";; (rm) Mode="Remove";; (rmall) Mode="RemoveAll";; (t|tags|Tags) Mode="Tags";; (-i) InitStorageDir=1;; (-s) StorageDir="$2"; shiftArgs=1;; (-r) ImageRoot="$2"; shiftArgs=1;; (-h) usage -h;; (-man) usage -man;; (-V) msg "$ThisScript v$Version"; exit 1;; (-v) ImageVersion="$2"; shiftArgs=1;; (-L) Log="$2"; shiftArgs=1;; (-si) SilentMode=1;; (-d) Debug=1;; (-D) Debug=1; set -x;; # Use bash 'set -x' exteme debug mode. (-*) usage -h "Unknown arg ($1).";; (*) TagAndRev="$1";; esac # Shift (modify $#) the appropriate number of times. shift; while [[ $shiftArgs -gt 0 ]]; do [[ $# -eq 0 ]] && usage -h "Incorrect number of arguments." shiftArgs=$shiftArgs-1 shift done done set -u #============================================================================== # Command Line Verification [[ "$Mode" == "Help" ]] && usage [[ "$Log" == "on" ]] && Log="${LOGS:-/tmp}/$ThisScript.v$Version.$(date +'%Y%m%d-%H%M%S').log" if [[ "$Mode" == "Create" ]]; then [[ "$TagAndRev" != "Unset" ]] && Tag="$TagAndRev" ### EDITME - Verify ImageVersion is alphanumber plus dash/underbar. ###[[ "$ImageVersion" =~ [[: [[ "$Tag" == "Unset" ]] && \ bail "For Create mode, specify '<tag>' or set DSI_TAG." fi if [[ "$Mode" == "List" ]]; then [[ "$TagAndRev" != "Unset" ]] && Tag="$TagAndRev" [[ "$Tag" == "Unset" ]] && \ bail "For List mode, specify '<tag>' or set DSI_TAG." fi if [[ "$Mode" == "Diff" ]]; then Rev="LATEST" if [[ "$TagAndRev" =~ @ ]]; then Tag="${TagAndRev%%@*}" Rev="${TagAndRev##*@}" elif [[ "$TagAndRev" != "Unset" ]]; then Tag="$TagAndRev" fi [[ "$Tag" == "Unset" ]] && \ bail "For Diff mode, specify '<tag>[@<rev>]' or set DSI_TAG." fi if [[ "$Mode" == "Remove" ]]; then Tag="$TagAndRev" if [[ "$TagAndRev" =~ @ ]]; then Tag="${TagAndRev%%@*}" Rev="${TagAndRev##*@}" else bail "For Remove mode specify <tag>@<rev> or @<rev> (if DSI_TAG is set)." fi [[ "$Tag" == "Unset" ]] && \ bail "The Tag is unset. For Remove mode, specify <tag>@<rev> or @<rev> if DSI_TAG is set." fi if [[ "$Mode" == "RemoveAll" ]]; then Tag="$TagAndRev" if [[ "$Tag" =~ @ ]]; then bail "For Remove All mode specify only <tag>, not <tag>@<rev>." fi [[ "$Tag" == "Unset" ]] && \ bail "The Tag is unset. For RemoveAll mode specify <tag> or set DSI_TAG." fi if [[ "$Mode" == "Tags" ]]; then [[ "$TagAndRev" != "Unset" ]] && \ bail "A tag value was specified [$TagAndRev] with Tags mode. Don't specify an individual tag name in Tags mode." fi [[ "$Rev" == "Unset" ]] && \ Rev="NEXT" verify_rev_format "$Rev" ||\ bail "Value for revision [$Rev] must be LATEST or a positive integer." [[ "$SilentMode" -eq 1 && "$Log" == off ]] && \ usage -h "Cannot use '-si' with '-L off'." [[ "$StorageDir" == "Unset" && "$Mode" != "Info" ]] && \ bail "No storage dir defined. Set DSI_STORAGE or specify '-s <storage_dir>'." if [[ ! -d "$StorageDir" && "$Mode" != "Info" ]]; then if [[ "$InitStorageDir" -eq 1 ]]; then msg "Initializing DSI Storage Dir: $StorageDir" if mkdir -p "$StorageDir"; then msg "Initialized storage dir: $StorageDir" else bail "Failed to create DSI storage dir [$StorageDir]. Aborting." fi else bail "DSI Storage Dir [$StorageDir] does not exist. Add '-i' flag to create it." fi fi #============================================================================== # Main Program trap terminate EXIT SIGINT SIGTERM if [[ "${Log}" != off ]]; then touch "${Log}" || bail "Couldn't touch log file [${Log}]." # Redirect stdout and stderr to a log file. if [[ $SilentMode -eq 0 ]]; then exec > >(tee "${Log}") exec 2>&1 else exec >"${Log}" exec 2>&1 fi msg "Logging to: $Log" fi case "$Mode" in (Create) dsi_create "$Tag" "$StorageDir" "$ImageVersion" ||\ bail "Failed to create DSI. Aborting." ;; (Diff) dsi_diff "${Tag}@${Rev}" ;; (Info) msg "DSI system info: $ThisScript v$Version Tag ${Tag:-Unset} DSI Storage Dir: ${StorageDir:-Unset} Package Root: ${ImageRoot:-Unset} " ;; (List) dsi_list "$Tag" ;; (Remove) dsi_remove "${Tag}@${Rev}" ;; (RemoveAll) dsi_remove_all "${Tag}" ;; (Tags) dsi_tags ;; (*) bail "Invalid operational mode [$Mode] specified. Aborting.";; esac # See the terminate() function, which is really where this script exits. exit 0
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#12 | 28904 | C. Thomas Tyler |
Fixed issue with uninitialized ImageFiles array. Adjusted format of 'ls' command getting timestamp values to a hopefully more portable style. Changed shasum to sha256sum. |
||
#11 | 28669 | C. Thomas Tyler | Initial images created. | ||
#10 | 28668 | C. Thomas Tyler | Started DSI generation work. | ||
#9 | 28630 | C. Thomas Tyler |
Refined and simplified image file format and related terminology. Implemented rudimentary file creation/listing logic. |
||
#8 | 28093 | C. Thomas Tyler | Added stub functions. | ||
#7 | 27620 | C. Thomas Tyler |
Experimental: Change shebang from #!/bin/bash to #!/usr/bin/env bash. Not sure this is a good idea, since it relies on the /usr/bin/env being the absolute path to 'env', and also that the UNIX variants and Linux distros agree on hte location of 'env'. It's 100% reliably either /bin/env or /usr/bin/env. Alas, the shebang line can't exercise logic to check; the shebang line reference a single hard-coded path. Some considerations: * As of this writing, Mac OSX "Big Sur" 11.2 stubbornly still ships with the outdated-by-a-decade bash 3.x, without support for key features like associative arrays and thus useless (and insecure, too). * OSX Support would be a (very) nice-to-have, but not a hard requirement. If we stick with a shebang of #!/bin/bash, it will work OK on all required platforms (where bash is 4.x or 5.x), and only require manual operational idiosyncracies on Mac (.e.g. running "bash dsi" to get the one in /usr/local/bin/bash installed with brew, which is a modern and useful 5.x. And one day, it will magically start working on OSX assuming Apple eventually ships with a modern bash. (If they did, we wouldn't need to think about this). * The choice of #!/bin/bash vs. #!/usr/bin/env bash is a matter of priority. Do we want portability to more systems (preferring #!/usr/bin/env) or more predictable behavior on a given system or system of a known platform (preferring #!/bin/bash). * The checks for $BASH_VERSION in the code already prevent issues unexpectedly inconsistent results; that will work with either #!/bin/bash or #!/usr/bin/env bash. * That a /bin/bash of some version will exist is a safe assumption; any system for which that isn't true isn't something we need to support. |
||
#6 | 27130 | C. Thomas Tyler | Added AsciiDoc. | ||
#5 | 26937 | C. Thomas Tyler | Added packge format file, other work in progess. | ||
#4 | 26936 | C. Thomas Tyler |
Added 'Info' and 'Tags' modes, and tests for each. Enahnced fast usage message showing details of each mode. Enhanced command line processing and commmad line verification. |
||
#3 | 26816 | C. Thomas Tyler | Fixed info message erroneously classified as an error message. | ||
#2 | 26752 | C. Thomas Tyler |
Adjustements to pass shellcheck. Test suite output is now more verbose. |
||
#1 | 26750 | C. Thomas Tyler |
Populate -f -o ... //dsi/dev/.... |
||
//guest/tom_tyler/sw/main/dsi/src/dsi | |||||
#6 | 24918 | C. Thomas Tyler |
Converted mode and tag/rev arguments to not require flags, to simplfy usage. Adjusted tests accordingly. |
||
#5 | 24917 | C. Thomas Tyler | Removed dependency on SDP /p4/common/lib files. | ||
#4 | 24894 | Perforce maintenance |
Added DSI_TAG and DSI_DIR vers. Added verify_rev_format() and a test for it. Added more tests, including test for new verify_rev_format() function. |
||
#3 | 24891 | Perforce maintenance |
Enhanced docs for '-L' logging. Added implied 'less' when using '-man' option. Usage with no args generates usage error. Code style enhancement to help with shellcheck compliance. |
||
#2 | 24888 | Perforce maintenance | Fixed typo in usage error message. | ||
#1 | 24886 | Perforce maintenance | Added baseline dsi utility, doc-only. |