DANGER_CLEAN.sh #1

#!/bin/bash

#==============================================================================
# Declarations and Environment
set -u

declare ThisScript="${0##*/}"
declare Version=2.5.1
declare ThisHost=${HOSTNAME%%.*}
declare ThisUser=
declare CmdLine="$ThisScript $*"
declare RunUser=
declare RunUserList="perforce p4admin p4dude dude"
declare RunGroup=
declare RunGroupList="perforce p4admin"
declare OtherGroup=
declare OtherGroupList="p4staff friends"
declare SudoersFile=
declare SystemFilesToClean="/etc/cron.allow /etc/needrestart/conf.d/99-no-auto-restart.conf"
declare P4PrometheusRepoFile=
declare P4PrometheusBinaries="/usr/local/bin/p4prometheus /usr/local/bin/p4metrics /usr/local/bin/node_exporter"
declare P4PrometheusConfigDirs="/etc/p4prometheus"
declare RunUserHomeDir=
# Include standard mount points and alternate mount points for testing non-default install.
declare SDPMountPoints="/hxdepots /mnt/p4depots /hxcheckpoints /mnt/p4checkpoints /hxmetadata /mnt/p4db /hxmetadata1 /mnt/p4db-1 /hxmetadata2 /mnt/p4db-2 /hxmetadata3 /mnt/p4db-3 /hxlogs /mnt/p4logs /BigDisk /DBs /Journals"
declare SDPBackupDirs="/backup /hxdepots/backup /mnt/p4depots/backup /BigDisk/backup /root/opt_perforce_sdp_backup"
declare OtherDirsToClean="/var/log/opt_perforce_sdp_backup"
declare SDPStructureDirs="/p4 /opt/perforce/helix-sdp /opt/perforce/p4-sdp"
declare SDPInstallDir="/root/install_sdp"
# BSWLabSpecificDir contains the set of dirs to cleanup between labs in BSW mode,
# i.e. those extra directories created in certain labs.
declare BSWLabSpecificDirs="/old_checkpoints /tape_backup /mnt/p4depots/p4/1/depot_files /tmp/depot/Jam/MAIN"
declare Log=
declare LogLink=
declare -i UseExistingMountPoints=0
declare -i ErrorCount=0
declare -i Debug=${DEBUG:-0}
declare -i NoOp=1
declare -i ItemsCleanedCount=0
declare -i RetryCount=0
declare -i RetryMax=10
declare -i UserRemoved=0
declare -i BSWMode=0
declare -i DisableFirewall=1
declare -i DisablePerforcePackageRepo=1
declare -i RemoveSystemdServices=1
declare SDPInstance=
declare SDPInstanceList="1 fgs"
declare P4ROOT=
declare P4DPidFile=
declare P4DPid=
declare ServiceFile=
declare ServiceName=
declare FirewallDir=
declare TmpFile=
declare PackageRepoFile=
declare GPGKey=
declare PackageManager=
declare -i ServicesRemoved=0
declare -i i=0
declare H1="=============================================================================="

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Functions msg(), dbg(), and bail().
# Sample Usage:
#    bail "Missing something important. Aborting."
#    bail "Aborting with exit code 3." 3
function msg () { echo -e "$*"; }
function errmsg () { msg "\\nError: ${1:-Unknown Error}\\n"; ErrorCount+=1; }
function dbg () { [[ "$Debug" -eq 0 ]] || msg "DEBUG: $*"; }
function bail () { errmsg "${1:-Unknown Error}"; exit "${2:-1}"; }

#------------------------------------------------------------------------------
# Functions run($cmd, $desc, $ignoreNoOp, $incrementItemCount)
#
# This function is similar to functions defined in SDP core libraries, but we
# need to duplicate them here since this script runs before the SDP is
# available on the machine (and thus this script must limited dependencies).
#
# This runs commands or merely displays them if NoOp is 1.
#
# Arguments:
# $1 - command to execute.
# $2 - description of command to run
# $3 - ignoreNoOp. If 1, always execute command, even if NoOP is 1.
# $4 - incrementItemCount. If 1, increment global ItemsCleanedCount
# to count actions involved in cleanup.
function run {
   local cmd="${1:-echo Testing run}"
   local desc="${2:-}"
   local -i ignoreNoOp=${3:-0}
   local -i incrementItemCount=${4:-1}

   [[ -n "$desc" ]] && msg "$desc"
   if [[ "$NoOp" -eq 0 || "$ignoreNoOp" -eq 1 ]]; then
      msg "Running: $cmd"
      $cmd
      CMDEXITCODE=$?
   else
      msg "NO_OP: Would have run: $cmd"
      CMDEXITCODE=0
   fi

   [[ "$incrementItemCount" -eq 1 ]] && ItemsCleanedCount+=1

   return $CMDEXITCODE
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style="${1:--h}"
   declare errorMessage="${2:-}"

   if [[ -n "$errorMessage" ]]; then
      msg "\\nUSAGE ERROR: $errorMessage\\n"
   fi

   msg "USAGE for $ThisScript v$Version:

$ThisScript [-y] [-ued] [-bsw] [-d|-D]

or

$ThisScript [-bsw] [-h|-man]
"
   if [[ $style == -man ]]; then
      msg "
DESCRIPTION:
	This script undoes what install_sdp.sh does to enable a series of clean
	start-from-scratch tests.

OPTIONS:
 -y
 	Specify live operation mode. By default, this script operates in Preview
 	mode where it takes no actions that affect data, but instead only displays
	commands that would be executed with '-y'.

 -ued
 	Specify '-ued' to leave existing SDP mount points from a prior install in
	place. The intent is to enable testing the '-ued' option to install_sdp.sh.
	All other folders including '/p4' are still removed.

	This option also prevents cleanup of:
	* The /root/install_sdp directory.
	* The /hxdepots/backup directory.

 -bsw
 	This changes behaviors in ways optimized for usage in Battle School Workshop
 	(BSW).  In mode, the goal is to save time by doing only cleanup needed between
	executions of the 'lab 0' (or 'lab N') commands in the BSW Lab Environment. In
    BSW, we can assume that a baseline install was done during the AMI creation
    phase of lab environment setup. With '-bsw', this script cleans up bits and pieces
    of the lab environment between user labs, and then the Lab Engine recreates the
    environment with a Battle School optimized call to 'install_sdp.sh' to reset the
    lab environment.

 	The default cleaning method of this script cleans more things than are needed
	for BSW. In BSW, the following are not cleaned up:
	* The Perforce Package Repository is left alone.
	* The OSUSER perforce is left intact.
    * No attempt is made to remove firewall entries.
    * No attempt is made to remove systemd services.
	* The directory structure cleanup is more narrow in the /mnt/p4depots area.
    * The directory structure cleanup is extended to include BSW Lab-specific dirs:
   $BSWLabSpecificDirs

DEBUGGING OPTIONS:
 -d	Enable debug message.
 -D	Enable extreme debugging with bash 'set -x'. Implies '-d'.

HELP OPTIONS:
 -h	Display short help message.
 -man	Display this full manual page.
 -V	Display script and version.
"
   fi

   exit 2
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0
set +u

while [[ $# -gt 0 ]]; do
   case $1 in
      (-y) NoOp=0;;
      (-ued)
         # For '-ued', DO NOT clean up /root/install_sdp directory.
         UseExistingMountPoints=1
         SDPInstallDir=
      ;;
      (-bsw)
         BSWMode=1
         DisableFirewall=0
         DisablePerforcePackageRepo=0
         RemoveSystemdServices=0
         SDPMountPoints="/mnt/p4depots /mnt/p4db /mnt/p4logs"
         # Avoid removal of OS users and groups.
         RunUserList=
         RunGroupList=
         OtherGroupList=
         # For BSW mode, DO NOT clean up /root/install_sdp directory.
         SDPInstallDir=
      ;;
      (-h) usage -h;;
      (-man|--help) usage -man;;
      (-V|--version) msg "$ThisScript v$Version"; exit 2;;
      (-d) Debug=1;;
      (-D) Debug=1; set -x;; # Debug; use 'set -x' mode.
      (-*) usage -h "Unknown flag/option ($1).";;
      (*) usage -h "Unknown parameter ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && bail "Usage Error: Wrong number of parameters to flags/options."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Main Program

Log="${ThisScript%.sh}.$(date +'%Y-%m-%d-%H%M%S').log"
LogLink="${ThisScript%.sh}.log"

touch "$Log" || bail "Could not initialize log with: touch \"$Log\""

exec > >(tee "$Log")
exec 2>&1

if [[ -e "$LogLink" ]]; then
   if [[ -L "$LogLink" ]]; then
      rm -f "$LogLink"
   else
      # If the name that should be a symlink is not a symlink, move it aside before
      # creating the symlink.
      mv -f "$LogLink" "${LogLink%.log}.old.log"
   fi
fi

# Point LogLink symlink to current log.
ln -f -s "$Log" "$LogLink"

msg "${H1}\\nLog is: $Log\\n"

ThisUser=$(whoami)
msg "Started $ThisScript v$Version on host $ThisHost as user $ThisUser at $(date), called as:\\n\\t$CmdLine\\n"

if [[ "$NoOp" -eq 0 ]]; then
   msg "Live Operation Mode."
else
   msg "Dry Run / Preview Mode."
fi

command -v getent > /dev/null || bail "Can't find utility getent in PATH."

#------------------------------------------------------------------------------
# Determine OS Package Manager, apt-get, yum, or zypper.
# TO DO: Ponder if/when to prefer dnf over yum.
if command -v yum > /dev/null; then
   PackageManager="yum"
elif command -v apt-get > /dev/null; then
   PackageManager="apt-get"
elif command -v zypper > /dev/null; then
   PackageManager="zypper"
else
   errmsg "Unknown Package Manager [$PackageManager]. Not attempting Perforce Package Repo removal."
fi

#------------------------------------------------------------------------------
# Remove connection to Perforce Package Repository (if necessary).

if [[ "$DisablePerforcePackageRepo" -eq 1 ]]; then
   if [[ -n "$PackageManager" ]]; then
      case "$PackageManager" in
         (yum)
            PackageRepoFile="/etc/yum.repos.d/perforce.repo"
            if [[ -e "$PackageRepoFile" ]]; then
               if run "rm -f $PackageRepoFile" "Removing package repo file: $PackageRepoFile"; then
                  msg "Perforce Packages Repo removed successfully."
                  ItemsCleanedCount+=1
               else
                  errmsg "Failed to remove Perforce Package Repo."
               fi
            else
               msg "Verified: No Perforce Package Repo exists: $PackageRepoFile"
            fi

            GPGKey=$(rpm -q gpg-pubkey --qf "%{NAME}-%{VERSION}-%{RELEASE} %{SUMMARY}\n" 2>/dev/null|grep -E "Perforce Software"|head -1|awk '{print $1}')

            if [[ -n "$GPGKey" ]]; then
               if run "rpm -e $GPGKey" "Removing Perforce Packages Public Key."; then
                  msg "Perforce Packages Public Key removed successfully."
                  ItemsCleanedCount+=1
               else
                  errmsg "Failed to remove Perforce Packages Public Key."
               fi
            else
               msg "Verified: No GPG Key exists for Perforce Software."
            fi
         ;;
         (apt-get)
            PackageRepoFile="/etc/apt/sources.list.d/perforce.list"
            GPGKey="/usr/share/keyrings/perforce.gpg"

            if [[ -e "$PackageRepoFile" ]]; then
               if run "rm -f $PackageRepoFile" "Removing package repo file: $PackageRepoFile"; then
                  msg "Perforce Packages Repo removed successfully."
                  ItemsCleanedCount+=1

                  if run "apt-get update" "Doing apt update after removing ${PackageRepoFile%%*/} file."; then
                     ItemsCleanedCount+=1
                     msg "The apt-get package manager was updated successfully."
                  else
                     errmsg "An error occurred updating the apt-get package manager."
                  fi
               else
                  errmsg "Failed to remove Perforce Package Repo."
               fi
            else
               msg "Verified: No Perforce Package Repo exists: $PackageRepoFile"
            fi

            if [[ -e "$GPGKey" ]]; then
               if run "rm -f $GPGKey" "Removing GPG Key for Perforce Software: $GPGKey"; then
                  msg "Successfully removed GPG Key."
                  ItemsCleanedCount+=1
               else
                  errmsg "Failed to remove Perforce Packages Public Key."
               fi
            else
               msg "Verified: No GPG Key exists for Perforce Software."
            fi
         ;;
         (zypper)
            msg "Zypper package manager detected. Skipping Perforce Package Repo removal."
         ;;
         (*)
            errmsg "Unknown Package Manager [$PackageManager]. Not attempting Perforce Package Repo removal."
         ;;
      esac
   else
      msg "No OS Package Manager (yum/apt-get/zypper) detected. Skipping Perforce Package Repo removal."
   fi
else
   msg "Skipping Perforce Package Repo removal."
fi

#------------------------------------------------------------------------------
# Remove p4prometheus-node package and related components

msg "Checking for p4prometheus-node package."
if dpkg -l | grep -q p4prometheus-node 2>/dev/null || \
   rpm -q p4prometheus-node > /dev/null 2>&1; then
   
   # Stop and disable p4prometheus-specific services first
   for ServiceName in p4prometheus p4metrics node_exporter monitor_locks.timer monitor_locks.service; do
      if systemctl is-active --quiet "$ServiceName" 2>/dev/null; then
         run "systemctl stop $ServiceName" "Stopping p4prometheus service: $ServiceName" ||\
            errmsg "Failed to stop service $ServiceName"
      fi
      if systemctl is-enabled --quiet "$ServiceName" 2>/dev/null; then
         run "systemctl disable $ServiceName" "Disabling p4prometheus service: $ServiceName" ||\
            errmsg "Failed to disable service $ServiceName"
      fi
   done
   
   # Remove the package
   case "$PackageManager" in
      (apt-get)
         run "apt-get remove --purge -y p4prometheus-node" "Removing p4prometheus-node package" ||\
            errmsg "Failed to remove p4prometheus-node package"
      ;;
      (yum|dnf)
         run "yum remove -y p4prometheus-node" "Removing p4prometheus-node package" ||\
            errmsg "Failed to remove p4prometheus-node package"
      ;;
      (zypper)
         run "zypper remove -y p4prometheus-node" "Removing p4prometheus-node package" ||\
            errmsg "Failed to remove p4prometheus-node package"
      ;;
   esac
   
   # Remove binaries that were installed by install_p4prom.sh
   for binary in $P4PrometheusBinaries; do
      if [[ -f "$binary" ]]; then
         run "rm -f $binary" "Removing p4prometheus binary: $binary" ||\
            errmsg "Failed to remove binary: $binary"
      fi
   done
   
   # Remove systemd service files
   for ServiceName in p4prometheus p4metrics node_exporter monitor_locks; do
      for ext in service timer; do
         ServiceFile="/etc/systemd/system/${ServiceName}.${ext}"
         if [[ -f "$ServiceFile" ]]; then
            run "rm -f $ServiceFile" "Removing systemd file: $ServiceFile" ||\
               errmsg "Failed to remove $ServiceFile"
         fi
      done
   done
   
   # Reload systemd
   run "systemctl daemon-reload" "Reloading systemd after p4prometheus cleanup" 0 0 ||\
      errmsg "Failed to reload systemd"
   
   # Remove configuration directories
   for dir in $P4PrometheusConfigDirs; do
      if [[ -d "$dir" ]]; then
         run "rm -rf $dir" "Removing p4prometheus config dir: $dir" ||\
            errmsg "Failed to remove config dir: $dir"
      fi
   done
   
   # Remove node_exporter user
   if id -u "node_exporter" > /dev/null 2>&1; then
      run "userdel node_exporter" "Removing node_exporter user" ||\
         errmsg "Failed to remove node_exporter user"
   fi
   
   # Remove package repository config
   case "$PackageManager" in
      (apt-get)
         P4PrometheusRepoFile="/etc/apt/sources.list.d/p4prometheus.list"
         if [[ -f "$P4PrometheusRepoFile" ]]; then
            run "rm -f $P4PrometheusRepoFile" "Removing p4prometheus repo file" ||\
               errmsg "Failed to remove repo file: $P4PrometheusRepoFile"
            run "apt-get update" "Updating apt after removing p4prometheus repo" 0 0 ||\
               errmsg "Failed to update apt"
         fi
      ;;
      (yum|dnf)
         P4PrometheusRepoFile="/etc/yum.repos.d/p4prometheus.repo"
         if [[ -f "$P4PrometheusRepoFile" ]]; then
            run "rm -f $P4PrometheusRepoFile" "Removing p4prometheus repo file" ||\
               errmsg "Failed to remove repo file: $P4PrometheusRepoFile"
         fi
      ;;
      (zypper)
         if zypper lr | grep -q p4prometheus 2>/dev/null; then
            run "zypper removerepo p4prometheus" "Removing p4prometheus zypper repo" ||\
               errmsg "Failed to remove zypper repo"
         fi
      ;;
   esac
   
else
   msg "Verified: p4prometheus-node package is not installed."
fi

#------------------------------------------------------------------------------
# Stop, disable, and remove all p4* systemd services.
if [[ "$RemoveSystemdServices" -eq 1 ]]; then
   msg "Stopping, disabling, and removing any p4* systemd services."
   ServicesRemoved=0
   for ServiceFile in /etc/systemd/system/p4*.service; do
      [[ "$ServiceFile" == *'*'* ]] && continue
      ServiceName="${ServiceFile##*/}"

      run "systemctl stop $ServiceName" "Issuing stop to service $ServiceName." ||\
         errmsg "Error calling: systemctl stop $ServiceName"
      run "systemctl disable $ServiceName" "Issuing disable to service $ServiceName." ||\
         errmsg "Error calling: systemctl stop $ServiceName"
      run "rm -f $ServiceFile" "Removing $ServiceFile" ||\
         errmsg "Could not do: rm -f $ServiceFile"
      ServicesRemoved+=1
   done

   if [[ "$ServicesRemoved" -eq 1 ]]; then
      run "systemctl daemon-reload" "Reloading systemd after removing $ServicesRemoved p4* systemd services." 0 0 ||\
         errmsg "Error encountered doing: systemctl daemon-reload"
   fi
else
   msg "Skipping removal of systemd services."
fi

#------------------------------------------------------------------------------
# Remove local OSUSER (default 'perforce') account.

for SDPInstance in $SDPInstanceList; do
   P4ROOT="/p4/$SDPInstance/root"
   P4DPidFile="$P4ROOT/server.pid"
   if [[ -r "$P4DPidFile" ]]; then
      P4DPid=$(cat "$P4DPidFile")
         if [[ -n $(ps -o pid= "$P4DPid") ]]; then
         run "kill $(cat "$P4DPidFile")" "Gently killing p4d to avoid errant systemd issues." ||\
            errmsg "Error killing pid in $P4DPidFile. If the process is still, a SIGKILL will be sent."
         sleep 1
      else
         msg "The server.pid file [$P4DPidFile] contains pid $P4DPid, but that process does not exist. Removing PID file."
         rm -f "$P4DPidFile"
      fi
   fi
done

for RunUser in $RunUserList; do
   if id -u "$RunUser" > /dev/null 2>&1; then
      if crontab -l -u "$RunUser" > /dev/null 2>&1; then
         run "crontab -r -u $RunUser" "Removing crontab for user $RunUser" ||\
            errmsg "Error removing crontab for user $RunUser."
      fi

      # If anything remains running, kill away.
      if run  "pgrep -u $RunUser" "Checking for processes by user $RunUser." 1 0 > /dev/null; then
         run "pkill -u $RunUser -KILL" \
            "Killing all processes by OSUSER $RunUser" ||\
            errmsg "Failed to kill all processes by user $RunUser."
      else
         msg "No OS processes detected for user $RunUser."
      fi

      UserRemoved=0
      # Remove user only if local; avoid attempting 'userdel' for an LDAP user.
      if grep -Eq "^$RunUser:" /etc/passwd; then
         for ((RetryCount=0; i<RetryMax; i++)); do
            if run "userdel $RunUser" "Removing user $RunUser."; then
               msg "Verified: User $RunUser does not exist."
               UserRemoved=1
               break
            else
               RetryCount+=1
               msg "Could not remove user $RunUser on try $RetryCount. Waiting a bit and trying (up to $RetryMax times)."
               sleep 2
            fi
         done

         [[ "$UserRemoved" -eq 1 ]] || \
            errmsg "Failed to remove user $RunUser after $RetryMax tries."
      else
         msg "Not removing non-local OS user $RunUser."
      fi

   else
      msg "Verified: User $RunUser does not exist."
   fi

   SudoersFile="/etc/sudoers.d/$RunUser"
   if [[ -r "$SudoersFile" ]]; then
      run "rm -f $SudoersFile" ||\
         errmsg "Failed to do: rm -f $SudoersFile"
   else
      msg "Verified: No sudoers file for user $RunUser exists."
   fi

   RunUserHomeDir="/home/$RunUser"
   if [[ -d "$RunUserHomeDir" ]]; then
      run "rm -rf $RunUserHomeDir" "Removing user home dir: $RunUserHomeDir" ||\
         errmsg "Failed to remove user home dir [$RunUserHomeDir]."
   else
      msg "Verified: Home dir for user $RunUser [$RunUserHomeDir] does not exist."
   fi
done

for RunGroup in $RunGroupList; do
   if getent group "$RunGroup" > /dev/null 2>&1; then
      run "groupdel $RunGroup" "Removing run group $RunGroup" ||\
         errmsg "Failed to remove run group $RunGroup."
   else
      msg "Verified: Run Group $RunGroup does not exist."
   fi
done

for OtherGroup in $OtherGroupList; do
   if getent group "$OtherGroup" > /dev/null 2>&1; then
      run "groupdel $OtherGroup" "Removing other group $OtherGroup" ||\
         errmsg "Failed to remove other group $OtherGroup."
   else
      msg "Verified: Other Group $OtherGroup does not exist."
   fi
done

if [[ "$UseExistingMountPoints" -eq 0 ]]; then
   msg "Removing SDP mount points or files under mount points."
   TmpFile=$(mktemp)
   # Get a list of mount points.
   mount | awk '{print $3}' > "$TmpFile"

   # For directories that might be mount points, check to see if they are indeed
   # mount points.  If they not, simply run 'rm -rf' on the directory. If they are
   # mount points, do 'rm -rf' on anything reported by 'ls -A' in the mount point
   # directory.
   for d in $SDPMountPoints; do
      if [[ -d "$d" ]]; then
         if [[ "$BSWMode" -eq 1 && "$d" =~ p4depots ]]; then
            # For BSW, cleanup only the p4 directory under the depots mount.
            run "rm -rf $d/p4" "BSW Mode: Removing p4 directory under SDP mount point $d/p4" ||\
               errmsg "Failed to remove p4 directory under SDP mount point: $d/p4"
         else
            if grep -q "^$d$" "$TmpFile"; then
               if [[ -n "$(cd "$d"; ls -A)" ]]; then
                  run "rm -rf $d/* $d/.*" "Removing directories under SDP mount point: $d" ||\
                     errmsg "Failed to remove directories under SDP mount point: $d/*"
               else
                  msg "Verified: Nothing exists under mount point: $d"
               fi
            else
               run "rm -rf $d" "Removing SDP simulated mount point dir: $d" ||\
                  errmsg "Failed to remove SDP simulated mount point dir: $d"
            fi
         fi
      else
         msg "Verified: SDP dir [$d] does not exist."
      fi
   done

   msg "Cleaniup SDP Backup Dirs."
   for d in $SDPBackupDirs; do
      if [[ -d "$d" ]]; then
         run "rm -rf $d" "Removing SDP backup dir: $d" ||\
            errmsg "Failed to remove SDP backup dir [$d]."
      else
         msg "Verified: SDP backup dir [$d] does not exist."
      fi
   done

   rm -f "$TmpFile"
else
   msg "Skipping cleanup of mount points and backup dirs due to '-ued'."
fi

for d in $SDPStructureDirs; do
   if [[ -d "$d" ]]; then
      run "rm -rf $d" "Removing SDP dir: $d" ||\
         errmsg "Failed to remove SDP dir [$d]."
   else
      msg "Verified: SDP dir [$d] does not exist."
   fi
done

for d in $SDPInstallDir $OtherDirsToClean; do
   if [[ -d "$d" ]]; then
      run "rm -rf $d" "Removing test setup dir: $d" ||\
         errmsg "Failed to remove test setup dir [$d]."
   else
      msg "Verified: Test setup dir [$d] does not exist."
   fi
done

FirewallDir=/etc/firewalld/services
ServicesRemoved=0

if [[ "$DisableFirewall" -eq 1 ]]; then
   if [[ -d "$FirewallDir" ]]; then
      cd "$FirewallDir" || bail "Could not do: cd $FirewallDir"
      msg "Checking for firewalld service files in: $PWD"

      if [[ -n "$(command -v firewall-cmd)" ]]; then
         for ServiceFile in p4*.xml; do
            [[ -r "$ServiceFile" ]] || continue
            ServiceName="${ServiceFile%.xml}"
            run "firewall-cmd --permanent --delete-service=$ServiceName" \
               "Deleting firewall entry for $ServiceName" ||\
               errmsg "Deleting firewall entry for $ServiceName failed."

            ServicesRemoved=1

            # Firewalld renames the *.xml files to *.xml.old upon deletion of the
            # rule.
            if [[ -r "$PWD/${ServiceFile}.old" ]]; then
               run "rm -f $PWD/${ServiceFile}.old" "Removing $PWD/${ServiceFile}.old" ||\
                  errmsg "Deleting file $PWD/${ServiceFile}.old failed."
            fi
         done

         if [[ "$ServicesRemoved" -eq 1 ]]; then
            run "firewall-cmd --reload" "Firewall reload after firewalld service cleanup." 0 0 ||\
               errmsg "Firewall reload failed after firewalld service cleanup."
         else
            dbg "No firewalld services for P4 found to remove."
         fi
      else
         msg "No firewall-cmd firewall utility found. Skipping firewall cleanup."
      fi
   fi

   FirewallDir=/etc/ufw/applications.d
   ServicesRemoved=0
   if [[ -d "$FirewallDir" ]]; then
      cd "$FirewallDir" || bail "Could not do: cd $FirewallDir"
      msg "Checking for ufw firewall application files in: $PWD"

      if [[ -n "$(command -v ufw)" ]]; then
         if ls -d p4* > /dev/null 2>&1; then
            for ServiceFile in p4*; do
               run "rm -f $PWD/${ServiceFile}" "Removing $PWD/${ServiceFile}" ||\
                  errmsg "Failed to remove file: $PWD/${ServiceFile}"
               ServicesRemoved=1
            done

            if [[ "$ServicesRemoved" -eq 1 ]]; then
               run "ufw reload" "Firewall reload after ufw application cleanup." 0 0 ||\
                  errmsg "Firewall reload failed after ufw application cleanup."
            else
               dbg "No ufw applications for P4 found to remove."
            fi
         else
            msg "Verified: No p4* ufw firewall services defined."
         fi
      else
         msg "No ufw firewall utility found. Skipping firewall cleanup."
      fi
   fi
else
   msg "Skipping disable of firewall."
fi

msg "Removing system files."
for f in $SystemFilesToClean; do
   if [[ -e "$f" ]]; then
      run "rm -f $f" "Removing system file: $f" ||\
         errmsg "Failed to remove system file: $f"
   else
      msg "Verified: System file $f does not exist."
   fi
done

if [[ "$ErrorCount" -eq 0 ]]; then
   msg "\\nAll processing completed OK. $ItemsCleanedCount items needed cleaning."
else
   msg "\\nProcessing completed, but with $ErrorCount errors. $ItemsCleanedCount items needed cleaning."
fi

msg "\\nLog is: $Log\\n${H1}"

exit "$ErrorCount"