#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ #============================================================================== # Declarations and Environment export P4U_HOME=${P4U_HOME:-/p4/common/bin} export P4U_LIB=${P4U_LIB:-/p4/common/lib} export P4U_ENV=$P4U_LIB/p4u_env.sh export P4U_LOG=Unset export VERBOSITY=${VERBOSITY:-3} # Environment isolation. For stability and security reasons, prepend # PATH to include dirs where known-good scripts exist. # known/tested PATH and, by implication, executables on the PATH. export PATH=$P4U_HOME:$PATH:~/bin:. [[ -r "$P4U_ENV" ]] || { echo -e "\nError: Cannot load environment from: $P4U_ENV\n\n" exit 1 } declare BASH_LIBS=$P4U_ENV BASH_LIBS+=" $P4U_LIB/libcore.sh" BASH_LIBS+=" $P4U_LIB/libp4u.sh" for bash_lib in $BASH_LIBS; do source $bash_lib ||\ { echo -e "\nFATAL: Failed to load bash lib [$bash_lib]. Aborting.\n"; exit 1; } done declare Version=2.1.3 declare -i SilentMode=0 declare ReportOnly=1 declare WorkingDir="/offline" declare WorkspaceStorage= export VERBOSITY=3 #============================================================================== # Local Functions #------------------------------------------------------------------------------ # Function: find_evil_twins_between_streams # Find evil twins between a stream and its parent. function find_evil_twins_between_streams () { declare stream=${1:-Unset} declare parent= declare integCmd= declare changelist= declare clientName= declare clientRoot= declare -i count parent=$(p4 -ztag -F %Parent% stream -o $stream) [[ $stream == Unset ]] && bail "find_evil_twins_between_streams(): BAD USAGE." msg "${H2}\nSearching for evil twins between [$stream] and its parent [$parent]:" clientName="tmp.auto.etd.$P4USER.${stream//\//_}.TO.${parent//\//_}" clientRoot="$WorkspaceStorage/${clientName#tmp.auto.}" export P4CLIENT="$clientName" echo -e "Client: $P4CLIENT\n\nOwner: $P4USER\n\nDescription:\n\tUsed by $THISSCRIPT\n\nRoot: $clientRoot\n\nStream: $parent\n\n" > $TmpFile msg "Using generated temporary workspace $P4CLIENT." p4 -s client -i < $TmpFile || \ bail "Failed to create/update temp client using this spec:\n$(cat $TmpFile)\n" if [[ ! -d "$clientRoot" ]]; then run "mkdir -p $clientRoot" ||\ bail "Failed to create workspace root dir." fi cd "$clientRoot" ||\ bail "Failed to cd to $clientRoot." vmsg "Operating in: $PWD" echo -e ".p4ignore*\n.p4config*\nEvilTwin.notes" >> .p4ignore echo -e "P4CLIENT=$clientName\nP4USER=$P4USER\nP4IGNORE=.p4ignore\n" > .p4config export P4CONFIG=.p4config run "p4 opened -m 1" \ "Checking for opened files in workspace $P4CLIENT." 0 0 \ "not opened on this client" ||\ bail "Failed to verify that no files are opened in workspace $P4CLIENT." vmsg "Verified: No files opened in workspace $P4CLIENT." run "p4 -s client -f -s -S $parent" \ "Switching workspace to target/parent stream [$parent]." run "p4 -s sync -q" \ "Syncing workspace." integCmd="p4 -s integ -Ro -n -S $stream" msg "Checking for evil twins with: $integCmd | grep 'without -i flag'" echo -e "$integCmd | grep \'without -i flag\'" >> EvilTwin.notes $integCmd | grep 'without -i flag' > $TmpFile 2>&1 count=0 if [[ -s "$TmpFile" ]]; then count=$(wc -l $TmpFile | cut -d ' ' -f 1) msg "Found $count evil twins." if [[ $VERBOSITY -gt 4 ]]; then msg "Contents of $TmpFile:" cat $TmpFile fi EvilTwinCount=$((EvilTwinCount+count)) fi rm -f $TmpFile2 while read integData; do targetFile=${integData%% - *} targetFile=${targetFile#info: } sourceFile=${integData##*t integrate from } sourceFile=${sourceFile% without -i flag} echo "${sourceFile}__TO__${targetFile}" >> $TmpFile2 done < $TmpFile if [[ $VERBOSITY -gt 4 ]]; then msg "Contents of $TmpFile2:" cat $TmpFile2 fi if [[ ! -s "$TmpFile2" ]]; then msg "No evil twins detected between $stream and $parent." return 0 fi integCmd="p4 -s integ -i" # If we're fixing rather than merely reporting, generate a pending changelist number. if [[ $ReportOnly -eq 0 ]]; then changelist=$(echo -e "Change: new\n\nClient: $P4CLIENT\n\nUser: $P4USER\n\nStatus: new\n\nDescription:\n\tFixing evil twins between $stream and $parent.\n\n" | p4 change -i | awk '{print $2}') if [[ $changelist =~ ^[0-9]+$ ]]; then integCmd+=" -c $changelist" else bail "Failed to generate a pending changelist." fi fi [[ $ReportOnly -eq 1 ]] && integCmd+=" -n" msg "Fixing evil twins between $stream and $parent." while read integPaths; do sourceFile=${integPaths%%__TO__*} targetFile=${integPaths##*__TO__} echo $integCmd "$sourceFile" "$targetFile" echo "$integCmd \"$sourceFile\" \"$targetFile\"" >> EvilTwin.notes $integCmd "$sourceFile" "$targetFile" ||\ bail "This integ command failed: $integCmd $integPaths $sourceFile $targetFile" done < $TmpFile2 if [[ $ReportOnly -eq 0 ]]; then run "p4 -s resolve -ay" "Resolving accepting target." ||\ "Resolve failed!" run "p4 -s submit -c $changelist" "Submitting change $changelist." ||\ bail "Failed to submit change $changelist." fi return 0 } #------------------------------------------------------------------------------ # Function: terminate function terminate { # Disable signal trapping. trap - EXIT SIGINT SIGTERM # Don't litter. cleanTrash vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus" # Stop logging. [[ "${P4U_LOG}" == off ]] || stoplog # With the trap removed, exit. exit $OverallReturnStatus } #------------------------------------------------------------------------------ # Function: usage (required function) # # Input: # $1 - style, either -h (for short form) or -man (for man-page like format). #------------------------------------------------------------------------------ function usage { declare style=${1:--h} echo "USAGE for $THISSCRIPT v$Version: $THISSCRIPT [-i <instance>] { -d <stream_depot> | -s //source/stream } [-w <work_dir>] [-f] [-L <log>] [-si] [-v<n>] [-D] or $THISSCRIPT [-h|-man|-V] " if [[ $style == -man ]]; then echo -e " DESCRIPTION: Detect and optionally fix \"evil twins\" in a given stream depot. The term \"evil twin\" has origins in the ClearCase version control system, the first to have truly sophistcated branching and merging capability. With this sophistication came compelixty, and an evil twin is one such complexity. An evil twin can occur in any version control system with sophisticated branching and merging (Perforce, Git, ClearCase, AccuRev, etc.). In Perforce, an evil twin occurs when two files with the same name relative to the root of a stream are created in both streams with 'p4 add', rather than the preferred workflow of creating src/foo.c in one stream and branching it into another. When a file is created with 'p4 add' twice rather than the preferred 'add then branch' flow, there is no integration history connecting the files. This creates a problem when merging across streams, as their is no 'base' or common ancestor that is needed to calculate merge results. In ClearCase, this was a bad sitution indeed - one such file needed to be designated as the \"evil\" twin, and that file and its history would need to be obliterated ('rmelem' in ClearCase parlance). In Perforce, the situation is not nearly so bad. Rather than designating one of the twins as evil, we simply do a 'baseless integration' and establish a family history, after which point the two twins can have a happy family reunion. Note that many ClearClase sites deployed an Evil Twin prevention trigger that could prevent evil twins from occuring in the first place, and deploying such a trigger is a best practice in ClearCase environments. There is a similar trigger for Perforce, but it is not commonly deployed, since evil twins aren't nearly as problematic in Perforce. They don't occur often. At a high level: * Evil twin detection means doing an integrate preview, and checking for the \"can't integrate without -i flag\" warning. The -i flag is for baseless integrations, essentially evil twins. * Evil twin correction means doing an integrate with the -i flag on the individual files, and then doing a 'p4 resolve -ay' (accept \"yours,\" i.e. accept target), and submitting. That creates an integration record, visible as a merge arrow in P4V. This solution using 'p4 resolve -ay' is the best choice for mass detection and resolution of evil twins. The assumption here is that the contents are already correct, and you only want to draw merge arrows and avoid changing content in the processing. For evil twins that occur organically in natural development, e.g. due to developer error or miscoordination among developers working in different streams, you might choose to resolve with -ay or -at (accept \"theirs,\" i.e. accept source). Natural evil twins are uncommon and occur typically only for individual files. (And if they are common, that could be a sign of a larger process/communication issue among teams). === WARNING WARNING WARING === This script may require a large amount of temporary storage, as it creates a large number of stream workspaces and does a sync in them. See the '-w' flag. OPTIONS: -i <instance> Specify the SDP instance name. This is required unless the SDP environment has previously been loaded in the current shell, i.e. by sourcing the $P4CBIN/p4_vars file and specifying the instance, e.g. : source $P4CBIN 1 -d <stream_sdepot> Specify a stream depot to process, e.g. -d fgs to process the //fgs stream depot. This argument is required unless '-s //source/stream' is specified, in which case the stream depot is inferred from the stream path. -s //source/stream Specify a particular source stream to search. The specified stream and its parent will be checked for evil twins. -w <work_dir> Specify a working directory, used for temporary workspace storage, etc. The default is \"/scratch\". === WARNING WARNING WARING === This script may require a large amount of temporary storage, as it creates a large number of stream workspaces and does a sync in them. This flag controls the scratch storage area used by this script. The workspace storage directory will be in an \"etd\" directory below the specified <work_dir> So by default, workspace root directories will appear one directory level below \"/scratch/etd\". This script will attempt to create the \"etd\" directory under specified <work_dir> if it does not already exist, and abort if it cannot be created. -f Fix evil twins with forced integrates doing baseless merges and resolving with '-ay', effectively drawing merge arrows to establish a branching relationshiop, but not affecting target file content. By default, a report shows the commands that would be run, but they are not executed. -v<n> Set verbosity 1-5 (-v1 = quiet, -v5 = highest). -L <log> Specify the path to a log file, or the special value 'off' to disable logging. All output (stdout and stderr) are captured in the log. NOTE: This script is self-logging. That is, output displayed on the screen is simultaneously captured in the log file. Do not run this script with redirection operators like '> log' or '2>&1', and do not use 'tee.' -si Operate silently. All output (stdout and stderr) is redirected to the log only; no output appears on the terminal. This cannot be used with '-L off'. -D Set extreme debugging verbosity. HELP OPTIONS: -h Display short help message -man Display man-style help message -V Dispay version info for this script and its libraries. EXAMPLES: First, setup environment: cd $P4CBIN source ./p4_vars 1 Then run as per the following examples, suitable for a long-running script. Example 1: Search for all Evil Twins in the fgs depot: nohup ./$THISSCRIPT -i 1 -w /big/scratch -d fgs < /dev/null > /tmp/etd.log 2>&1 & tailf \$(ls -t \$LOGS/${THISSCRIPT%.sh}.*.log|head -1) Example 2: Find and fix all Evil Twins in the fgs depot: nohup ./$THISSCRIPT -i 1 -w /big/scratch -d fgs -f < /dev/null > /tmp/etd.log 2>&1 & tailf \$(ls -t \$LOGS/${THISSCRIPT%.sh}.*.log|head -1) Example 3: Find and fix all Evil twins between //fgs/dev and its parent. nohup ./$THISSCRIPT -i 1 -w /big/scratch -s //fgs/dev < /dev/null > /tmp/etd.log 2>&1 & tailf \$(ls -t \$LOGS/${THISSCRIPT%.sh}.*.log|head -1) " fi exit 1 } #============================================================================== # Command Line Processing declare StreamListFile=/tmp/tmp.StreamList.$$.$RANDOM declare TmpFile=/tmp/tmp.EvilTwins.$$.$RANDOM declare TmpFile2=/tmp/tmp.EvilTwins2.$$.$RANDOM declare StreamDepot=Unset declare StreamData=Unset declare SourceStream=Unset declare DepotTypeCheck=Unset declare -i PathwayCount=0 declare -i EvilTwinCount=0 GARBAGE+=" $StreamListFile $TmpFile $TmpFile2" declare -i shiftArgs=0 set +u while [[ $# -gt 0 ]]; do case $1 in (-d) StreamDepot=$2; shiftArgs=1;; (-w) WorkingDir=$2; shiftArgs=1;; (-s) SourceStream=$2; shiftArgs=1;; (-f) ReportOnly=0;; (-h) usage -h;; (-man) usage -man;; (-V) show_versions; exit 1;; (-v1) export VERBOSITY=1;; (-v2) export VERBOSITY=2;; (-v3) export VERBOSITY=3;; (-v4) export VERBOSITY=4;; (-v5) export VERBOSITY=5;; (-i) export SDP_INSTANCE=$2; shiftArgs=1;; (-L) export P4U_LOG=$2; shiftArgs=1;; (-si) SilentMode=1;; (-n) export NO_OP=1;; (-D) set -x;; # Debug; use 'set -x' mode. (*) usageError "Unknown arg ($1).";; esac # Shift (modify $#) the appropriate number of times. shift; while [[ $shiftArgs -gt 0 ]]; do [[ $# -eq 0 ]] && usageError "Bad usage." shiftArgs=$shiftArgs-1 shift done done set -u #============================================================================== # Command Line Verification [[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \ usageError "Cannot use '-si' with '-L off'." if [[ "$SourceStream" != Unset ]]; then [[ "$SourceStream" != "//"* ]] && \ usageError "The source stream specified with '-s' must be of the form //depot/stream." # Determine stream depot from source stream if it wasn't provided with '-d'. if [[ $StreamDepot == Unset ]]; then StreamDepot=${SourceStream#//} StreamDepot=${StreamDepot%%/*} fi fi [[ $StreamDepot == Unset ]] && \ usageError "The '-d <stream_depot>' parameter is required." export SDP_INSTANCE=${SDP_INSTANCE:-Unset} [[ $SDP_INSTANCE == Unset ]] && \ usageError "The SDP environment must be loaded, or the '-i <instance>' parameter provided." # Load and then tweak SDP environment. source p4_vars "$SDP_INSTANCE" export P4ENVIRO=/dev/null/.p4enviro unset P4CONFIG [[ $P4U_LOG == Unset ]] && \ export P4U_LOG="${LOGS}/${THISSCRIPT%.sh}.$(date +'%Y%m%d-%H%M').log" #============================================================================== # Main Program trap terminate EXIT SIGINT SIGTERM declare -i OverallReturnStatus=0 if [[ "${P4U_LOG}" != off ]]; then touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]." # Redirect stdout and stderr to a log file. if [[ $SilentMode -eq 0 ]]; then exec > >(tee ${P4U_LOG}) exec 2>&1 else exec >${P4U_LOG} exec 2>&1 fi initlog fi msg "$THISSCRIPT v$Version started at $(date)." # If depot was specified as '//x', normalize to 'x'. [[ "$StreamDepot" == "//"* ]] && StreamDepot=${StreamDepot#//} DepotTypeCheck=$(p4 -ztag -F %Type% depot -o $StreamDepot 2>/dev/null) if [[ -n "$DepotTypeCheck" ]]; then [[ "$DepotTypeCheck" != "stream" ]] && \ bail "The depot specified by '-d' must be of type 'stream', not $DepotTypeCheck." else bail "Could not determine depot type for depot $StreamDepot. Aborting." fi WorkspaceStorage="$WorkingDir/etd" # Avoid issues with dirs with spaces. if [[ "$WorkspaceStorage" != *" "* ]]; then vmsg "Verified: The working dir [$WorkingDir] does not contain spaces." else bail "The '-w <work_dir>' specified cannot contain spaces." fi if [[ ! -d "$WorkspaceStorage" ]]; then run "/bin/mkdir -p $WorkspaceStorage" ||\ bail "Could not create dir: $WorkspaceStorage" fi if [[ $SourceStream == Unset ]]; then msg "Getting list of streams in //$StreamDepot." p4 -ztag -F "%Type%:%Stream%:%Parent%" streams "//$StreamDepot/*" > $StreamListFile # Process all non-virtual, non-mainline, non-task streams. while read StreamData; do [[ $StreamData == "mainline:"* || $StreamData == "virtual:"* || $StreamData == "task:"* ]] && continue SourceStream=${StreamData#*:} SourceStream=${SourceStream%:*} find_evil_twins_between_streams "$SourceStream" ||\ OverallReturnStatus=1 PathwayCount=$((PathwayCount+1)) done < $StreamListFile else find_evil_twins_between_streams "$SourceStream" ||\ OverallReturnStatus=1 PathwayCount=1 fi if [[ $OverallReturnStatus -eq 0 ]]; then msg "${H}\nAll processing completed successfully.\n" else msg "${H}\nProcessing completed, but with errors. Scan above output carefully.\n" fi # Illustrate using $SECONDS to display runtime of a script. msg "${H2}Summary:\n\tFound $EvilTwinCount evil twins searching $PathwayCount pathway(s). Note that this may contain duplicates across multiple streams.\n" msg "That took $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n" # See the terminate() function, which is really where this script exits. exit $OverallReturnStatus
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#4 | 26652 | Robert Cowham |
This is Tom's change: Introduced new 'Unsupported' directory to clarify that some files in the SDP are not officially supported. These files are samples for illustration, to provide examples, or are deprecated but not yet ready for removal from the package. The Maintenance and many SDP triggers have been moved under here, along with other SDP scripts and triggers. Added comments to p4_vars indicating that it should not be edited directly. Added reference to an optional site_global_vars file that, if it exists, will be sourced to provide global user settings without needing to edit p4_vars. As an exception to the refactoring, the totalusers.py Maintenance script will be moved to indicate that it is supported. Removed settings to support long-sunset P4Web from supported structure. Structure under new .../Unsupported folder is: Samples/bin Sample scripts. Samples/triggers Sample trigger scripts. Samples/triggers/tests Sample trigger script tests. Samples/broker Sample broker filter scripts. Deprecated/triggers Deprecated triggers. To Do in a subsequent change: Make corresponding doc changes. |
||
#3 | 25193 | C. Thomas Tyler |
EvilTwinDetector.sh v2.1.3: * Narrowed focus to stream depots only, as only in stream depots is there sufficient metadata detect evil twins. * Updated evil twin detection logic. * Updated docs and command line usage. * Better reporting. * Added capability to fix evil twins with baseless integrations. |
||
#2 | 24781 | C. Thomas Tyler |
EvilTwinDetector.sh v1.1.0: * Implemented trigger mode to prevent evil triggers. * Refined output. * Enhanced docs. * Fixed typos/grammatical errors in a few error messages. Bypassing code review for now as this change is about to be merged with and succeeded by a different, bigger change made elsewhere. |
||
#1 | 20062 | C. Thomas Tyler | Added Evil Twin Detector maintenance script. |