EvilTwinDetector.sh #2

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

export P4U_HOME=${P4U_HOME:-/p4/common/bin}
export P4U_LIB=${P4U_LIB:-/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=Unset
export VERBOSITY=${VERBOSITY:-3}

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=$P4U_HOME:$PATH:~/bin:.

[[ -r "$P4U_ENV" ]] || {
   echo -e "\nError: Cannot load environment from: $P4U_ENV\n\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   source $bash_lib ||\
      { echo -e "\nFATAL: Failed to load bash lib [$bash_lib]. Aborting.\n"; exit 1; }
done

declare Version=1.1.0
declare -i SilentMode=0
export VERBOSITY=3

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: find_evil_twins_between_paths
#
# Input:
# $1 - Path 1, should look like //stream_depot/stream/...
# $2 - Path 2, should look like //stream_depot/stream/...
#
# Behaviours:
# Sets Problems=1 if check couldn't be done right for some reason.
# Increments $EvilTwinCount if evil twins were detected.
#
# Returns: Count of evil twins found, zero if none are found.
#
#------------------------------------------------------------------------------
function find_evil_twins_between_paths ()
{
   vvmsg "CALL: find_evil_twins_between_paths ($*)"
   declare p1=${1:-Unset}
   declare p2=${2:-Unset}
   declare s2=
   declare -i count
   [[ $p1 == Unset || $p2 == Unset ]] && bail "find_evil_twins_between_paths(): BAD USAGE."
   s2=${p2%/...}

   vmsg "Switching workspace to stream $s2"
   p4 client -f -s -S "$s2"

   msg "Searching for evil twins between [$p1] and [$p2] ..."
   vmsg "Executing: p4 integ -Ro -n $p1 $p2"

   p4 -s integ -Ro -n "$p1" "$p2" | grep "without -i flag" > $TmpFile 2>&1
   [[ $? -ne 0 ]] && Problems=1

   count=0

   if [[ -s "$TmpFile" ]]; then

      msg "The following files are evil twins:"

      while read line; do
         [[ "$line" == "info:"* ]] || continue
         line=${line#info: }
         line=${line%% - can\'t integrate from *}
         echo $line
         count=$((count+1))
      done < $TmpFile

      msg "$count evil twins were found here."

      EvilTwinCount=$((EvilTwinCount+count))
   fi

   return 0
}

#------------------------------------------------------------------------------
# Function: find_evil_twins_in_changelist
#
# Input:
# $1 - Changelist
# $2 - User
# $3 - Workspace
#
# Behaviours:
# Sets Problems=1 if check couldn't be done right for some reason.
# Increments $EvilTwinCount if evil twins were detected.
#
# Returns: Count of evil twins found, zero if none are found.
#
#------------------------------------------------------------------------------
function find_evil_twins_in_changelist ()
{
   vvmsg "CALL: find_evil_twins_in_changelist ($*)"
   declare c=${1:-Unset}
   declare u=${2:-Unset}
   declare w=${3:-Unset}
   declare stream= currentStream= currentStreamType=
   declare path=

   # Return immediately if the user worksapce is not a stream worksapce.
   currentStream=$(p4 -ztag -F %Stream% client -o $w)
   [[ -z "$currentStream" ]] && return 0

   currentStreamType=$(p4 -ztag -F %Type% stream -o $currentStream)

   # If our current stream is a virtual stream, go thru the stream
   # parent heirarchy until we hit the first non-virtual stream.
   while [[ "$currentStreamType" == virtual ]]; do
      currentStream=$(p4 -ztag -F %Parent% stream -o $currentStream)
      currentStreamType=$(p4 -ztag -F %Type% stream -o $currentStream)
   done

   p4 -ztag -F "%Type%:%Stream%" streams "//$StreamDepot/*" > $StreamListFile

   export P4CLIENT=tmp.auto.$u.${THISSCRIPT%.sh}

   echo -e "Client: $P4CLIENT\n\nOwner: $P4USER\n\nDescription:\n\tUsed by $THISSCRIPT\n\nRoot: $P4TMP/${THISSCRIPT%.sh}\n\nStream: $currentStream\n" > $TmpFile

   p4 client -f -i < $TmpFile > /dev/null 2>&1 ||\
      bail "$THISSCRIPT (Internal Trigger Error): Failed to create temp client using this spec:\n$(cat $TmpFile)\n"

   while read stream; do
      [[ $stream == "virtual:"* ]] && continue
      stream=${stream##*:}
      path="$stream/..."
   done < $StreamListFile

}

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $Problems"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit $Problems
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

Stand-Alone (Detection) Mode:
$THISSCRIPT -d <stream_depot> [-s //source/stream] [-t //target/stream] [-i <instance>] [-L <log>] [-si] [-v<n>] [-n] [-D]

Trigger (Prevention) Mode:
$THISSCRIPT -c <changelist> -u <user> -w <workspace> [-L <log>] [-si] [-v<n>] [-n] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	Detect \"evil twins\" in a given stream depot.  This can be a stand-alone reporting script,
	or can operate as a trigger to prevent evil twins from being created in stream depots.


	An evil twin is a file with the path relative to the root of the stream as some previously
	existing file, but which was created by a separate 'p4 add' command rather than being
	created in the normal way, by merging from stream to stream.  When viewed in the P4V
	Revision Graph, normally related files would appear on the same Revision graph, while
	evil twins would each have their own Revision Graph, with disconnected histories.

	Evil twins cause a bit of complexity when merging, since the merge must be done as a
	baseless merge, requiring the '-i' flag to the 'p4 merge' command.  After doing a
	baseless merge and then resolving and submitting, the evil twin, actually a long lost
	relative, is reunited with its siblings.

	This script finds evil twins that already exist, as a reporting script.  As a trigger, it
	prevents newly added files from becoming evil in the first place.

ARGUMENTS (Prevention/Trigger Mode Only):
 -c <changelist>
	Specify the changelist in trigger mode.

	This argument is required in trigger mode; usage of '-c change' implies trigger mode.

 -u <user>
	Specify the user that owns the changelist.

	This argument is required in trigger mode.  (It could be looked up from the changelist,
	but passing it in is faster, and trigger mode optimizes for speed).

 -w <workspace>
	Specify the workspace associated with the changelist.

	This argument is required in trigger mode.  (It could be looked up from the changelist,
	but passing it in is faster, and trigger mode optimizes for speed).

ARGUMENTS (Detection/Stand Alone Mode Only):
 -d <stream_sdepot>
	Specify a stream depot to process, e.g. -d fgs to process the //fgs stream depot.

	This argument is required.

 -s //source/stream
	Specify a particular source stream to search.

	By default, all non-virtual streams are checked.

	Be aware that if neither -s nor -t are given, a very large numbers of streams may be checked,
	as all possible combinations of source and target stream are checked.

 -t //target/stream
	Specify a particular target stream to search.

	By default, all non-virtual streams are checked.

	Be aware that if neither -s nor -t are given, a very large numbers of streams may be checked,
	as all possible combinations of source and target stream are checked.

 -i <instance>
	Specify the SDP instance name.

	This is required unless the SDP environment has previously been loaded in the current
	shell, i.e. by sourcing the SDP p4_vars file and specifying the instance.

GENERAL ARGUMENTS:
 -v<n>	Set verbosity 1-5 (-v1 = quiet, -v5 = highest).

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  All output (stdout and stderr) are captured in the log.

	NOTE: This script is self-logging.  That is, output displayed on the screen
	is simultaneously captured in the log file.  Do not run this script with
	redirection operators like '> log' or '2>&1', and do not use 'tee.'

-si	Operate silently.  All output (stdout and stderr) is redirected to the log
	only; no output appears on the terminal.  This cannot be used with
	'-L off'.
      
 -n	No-Op.  Prints commands instead of running them.

 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

EXAMPLES:
	Stand-Alone (Detection) Mode:
	Search for evil twins between streams //fgs/DevA and //fgs/DevB.
	$THISSCRIPT -s //fgs/DevA -t //fgs/DevB

	Search for all possible evil twins in the //fgs stream depot:
	$THISSCRIPT -d fgs

	Trigger (Prevention) Mode:
	Enable this with an entry in the triggers table like this example:
	NoEvilTwins change-submit //... \"/p4/common/bin/triggers/EvilTwinDetector.sh -c %changelist% -u %user% -w %client%\"

	To test, apply the trigger to a more narrow path than //...

"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare Change=Unset
declare User=Unset
declare Workspace=Unset
declare StreamListFile=/tmp/tmp.StreamList.$$.$RANDOM
declare SearchedPathsFile=/tmp/tmp.SearchedPaths.$$.$RANDOM
declare TmpFile=/tmp/tmp.EvilTwins.$$.$RANDOM
declare TmpFile2=/tmp/tmp.EvilTwins.2.$$.$RANDOM
declare StreamDepot=Unset
declare SourceStream=Unset
declare SourcePath=
declare TargetStream=Unset
declare TargetPath=
declare DepotTypeCheck=Unset
declare -i PathwayCount=0
declare -i EvilTwinCount=0
GARBAGE+=" $StreamListFile $SearchedPathsFile $TmpFile $TmpFile2"

declare -i shiftArgs=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-c) Change=$2; export P4U_LOG=off; shiftArgs=1;;
      (-u) User=$2; shiftArgs=1;;
      (-w) Workspace=$2; shiftArgs=1;;
      (-s) SourceStream=$2; shiftArgs=1;;
      (-t) TargetStream=$2; shiftArgs=1;;
      (-d) StreamDepot=$2; shiftArgs=1;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-v1) export VERBOSITY=1;;
      (-v2) export VERBOSITY=2;;
      (-v3) export VERBOSITY=3;;
      (-v4) export VERBOSITY=4;;
      (-v5) export VERBOSITY=5;;
      (-i) export SDP_INSTANCE=$2; shiftArgs=1;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-n) export NO_OP=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

if [[ $Change == Unset ]]; then
   if [[ "$SourceStream" != Unset ]]; then
      [[ "$SourceStream" != "//"* ]] && \
         usageError "The source stream specified with '-s' must be of the form //depot/stream."

      # Determine stream depot from source stream if it wasn't provided with '-d'.
      if [[ $StreamDepot == Unset ]]; then
         StreamDepot=${SourceStream#//}
         StreamDepot=${StreamDepot%%/*}
      fi
   fi

   if [[ "$TargetStream" != Unset ]]; then
      [[ "$TargetStream" != "//"* ]] && \
         usageError "The target stream specified with '-t' must be of the form //depot/stream."

      # Determine stream depot from target stream if it wasn't provided with '-d'.
      if [[ $StreamDepot == Unset ]]; then
         StreamDepot=${TargetStream#//}
         StreamDepot=${StreamDepot%%/*}
      fi
   fi

   [[ $StreamDepot == Unset ]] && \
      usageError "The '-d <stream_depot>' parameter is required."
else
   [[ $Change =~ ^[1-9]{1}[0-9]*$ ]] ||\
      usageError "The value specified with '-c' must be a purely numeric integer value; [$Change] is not valid."
   [[ $User == Unset ]] &&\
      usageError "The '-u <user>' parameter is required when '-c <change>' is specified, in trigger mode."
   [[ $Workspace == Unset ]] &&\
      usageError "The '-w <workspace>' parameter is required when '-c <change>' is specified, in trigger mode."
fi

export SDP_INSTANCE=${SDP_INSTANCE:-Unset}
[[ $SDP_INSTANCE == Unset ]] && \
   usageError "The SDP environment must be loaded, or the '-i <instance>' parameter provided."

# Load and then tweak SDP environment.
source p4_vars "$SDP_INSTANCE"
export P4ENVIRO=/dev/null/.p4enviro
unset P4CONFIG

[[ $P4U_LOG == Unset ]] && \
   export P4U_LOG="${LOGS}/${THISSCRIPT%.sh}.$(date +'%Y%m%d-%H%M').log"

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i Problems=0

if [[ "${P4U_LOG}" != off ]]; then
   touch ${P4U_LOG} || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee ${P4U_LOG})
      exec 2>&1
   else
      exec >${P4U_LOG}
      exec 2>&1
   fi

   initlog
fi

if [[ $Change != Unset ]]; then
   find_evil_twins_in_changelist "$Change" "$User" "$Workspace"

else
   msg "$THISSCRIPT v$Version started at $(date)."

   # If depot was specified as '//x', normalize to 'x'.
   [[ "$StreamDepot" == "//"* ]] && StreamDepot=${StreamDepot#//}

   DepotTypeCheck=$(p4 -ztag -F %Type% depot -o $StreamDepot 2>/dev/null)

   if [[ -n "$DepotTypeCheck" ]]; then
      [[ "$DepotTypeCheck" != "stream" ]] && \
         bail "The depot specified by '-d' must be of type 'stream', not $DepotTypeCheck."
   else
      bail "Could not determine depot type for depot $StreamDepot. Aborting."
   fi

   msg "Getting list of streams in //$StreamDepot."
   p4 -ztag -F "%Type%:%Stream%" streams "//$StreamDepot/*" > $StreamListFile

   export P4CLIENT=tmp.auto.$P4USER.${THISSCRIPT%.sh}

   # Generate a temp workspace.  Set the view to the //spec depot just because
   # we know it exists.  The view is reset later when the workspace is
   # associated with a stream.
   echo -e "Client: $P4CLIENT\n\nOwner: $P4USER\n\nDescription:\n\tUsed by $THISSCRIPT\n\nRoot: $P4TMP/${THISSCRIPT%.sh}\n\nView:\n\t//spec/... //$P4CLIENT/spec/..." > $TmpFile

   msg "Using generated temporary workspace $P4CLIENT."
   p4 -s client -f -i < $TmpFile ||\
      bail "Failed to create temp client using this spec:\n$(cat $TmpFile)\n"

   if [[ $SourceStream == Unset && $TargetStream == Unset ]]; then
      touch $SearchedPathsFile || bail "Could not touch temp file $SearchedPathsFile."
      while read SourceStream; do
         [[ $SourceStream == "virtual:"* ]] && continue
         SourceStream=${SourceStream##*:}
         SourcePath="$SourceStream/..."

         while read TargetStream; do
            [[ "$TargetStream" == "virtual:"* ]] && continue
            TargetStream=${TargetStream##*:}
            [[ "$SourceStream" == "$TargetStream" ]] && continue
            TargetPath="$TargetStream/..."

            # Direction doesn't matter for evil twin detection, so we only need to search
            # in a single direction.  Avoid redundant checking by checknig the
            # "searched paths" temp file, which contains a list of already searched paths.
            grep "${SourceStream#//}:${TargetStream#//}\$" $SearchedPathsFile > /dev/null 2>&1
            [[ $? -eq 0 ]] && continue
            grep "${TargetStream#//}:${SourceStream#//}\$" $SearchedPathsFile > /dev/null 2>&1
            [[ $? -eq 0 ]] && continue

            find_evil_twins_between_paths "$SourcePath" "$TargetPath" ||\
               Problems=1

            PathwayCount=$((PathwayCount+1))
            echo "${SourceStream#//}:${TargetStream#//}" >> $SearchedPathsFile
         done < $StreamListFile
      done < $StreamListFile
   elif [[ $SourceStream != Unset && $TargetStream != Unset ]]; then
      SourcePath="$SourceStream/..."
      TargetPath="$TargetStream/..."

      find_evil_twins_between_paths "$SourcePath" "$TargetPath" ||\
         Problems=1

      PathwayCount=1
   else
      bail "Specifying only '-s' or '-t', but not both, is not (yet?) supported."
   fi

   if [[ $Problems -eq 0 ]]; then
      msg "${H}\nAll processing completed successfully.\n"
   else
      msg "${H}\nProcessing completed, but with errors.  Scan above output carefully.\n" 
   fi

   # Illustrate using $SECONDS to display runtime of a script.
   msg "Found $EvilTwinCount evil twins searching $PathwayCount pathway(s) in $(($SECONDS/3600)) hours $(($SECONDS%3600/60)) minutes $(($SECONDS%60)) seconds.\n"

fi

# See the terminate() function, which is really where this script exits.
exit $EvilTwinCount