edge_maintenance #11

#!/bin/bash
#
# weekly_maintenance.sh – perform routine Perforce cleanup and notifications.
#
# This script performs a series of maintenance tasks against a Helix Core
# installation using the SDP environment.  It logs all output to a file
# and emails the results to the configured MAILTO address.  If a large
# number of clients are queued for unloading, it alerts the administrator
# instead of attempting the unload.

# Fail fast on errors, unset variables and pipeline failures.
set -uo pipefail

# Determine the SDP instance either from the first positional parameter or
# the environment.  Require it to be set to avoid accidental misuse.
P4INSTANCE="${1:-${P4INSTANCE:-}}"
if [[ -z "$P4INSTANCE" ]]; then
  echo "Instance parameter not supplied."
  echo "You must supply the Perforce instance as a parameter to this script or set \$P4INSTANCE in the environment."
  exit 1
fi

# Source SDP environment variables and ensure we can talk to Perforce.
if ! source "/p4/common/bin/p4_vars" "$P4INSTANCE"; then
  echo "Unable to source p4_vars for instance $P4INSTANCE" >&2
  exit 1
fi

# Authenticate using the SDP helper.  Suppress output but do not fail the script if login fails.
/p4/common/bin/p4login >/dev/null 2>&1 || true

# Prepare our log file and client names.  Use the hostname and server name for uniqueness.
LOG="/p4/sdp/Maintenance/${HOSTNAME}_${P4SERVER}_log.txt"
P4CLIENT="${HOSTNAME}_unload"

# Ensure the maintenance working directory exists and change into it for relative paths.
MAINT_DIR="/p4/sdp/Maintenance"
if [[ ! -d "$MAINT_DIR" ]]; then
  echo "Maintenance directory $MAINT_DIR does not exist." >&2
  exit 1
fi
cd "$MAINT_DIR"

# Helper function to timestamp and append a message to the log.
log_msg() {
  local msg="${1:-}"
  printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$msg" >>"$LOG"
}

# Start logging.
log_msg "--- Weekly maintenance started for instance $P4INSTANCE ---"

# Remove any leftover server lock files.  This can prevent DB replays from
# succeeding if the previous run terminated uncleanly.
log_msg "Removing server lock files from /p4/${P4INSTANCE}/root/server.locks"
rm -rf "/p4/${P4INSTANCE}/root/server.locks" >>"$LOG" 2>&1 || true

# Generate lists of inactive clients.  accessdates.py produces clients.txt
# based on the configured number of weeks.
log_msg "Generating client inactivity report via accessdates.py"
accessdates.py "$P4INSTANCE" >>"$LOG" 2>&1 || true

# Count the number of clients to be unloaded and decide what to do.  Use
# wc -l with a redirection to avoid counting the filename itself.  Fall back
# to zero if the file is missing.
client_count=0
if [[ -f clients.txt ]]; then
  client_count=$(wc -l < clients.txt || echo 0)
fi
log_msg "Inactive client count: $client_count"

# If there are fewer than 200 clients queued, unload them.  Otherwise alert
# the administrator by emailing the list of clients rather than unloading
# them, as processing a very large set may impact server performance.
if (( client_count > 0 )); then
  if (( client_count < 200 )); then
    log_msg "Unloading $client_count clients via unload_clients.py"
    unload_clients.py >>"$LOG" 2>&1 || true
  else
    log_msg "Too many clients ($client_count) to unload automatically; sending alert email instead."
    if [[ -n "${MAILTO:-}" ]]; then
      mail -s "${HOSTNAME} ${P4SERVER} client unload threshold exceeded" "$MAILTO" < clients.txt || true
    fi
  fi
fi

# Notify users about pending client deletions.  This script emails owners of
# workspaces that will be removed soon.  Only run once per maintenance window.
log_msg "Running email_pending_client_deletes.py"
email_pending_client_deletes.py "$P4INSTANCE" >>"$LOG" 2>&1 || true

# Remove empty pending changelists.  These accumulate over time and can slow
# down server operations.
log_msg "Removing empty pending changelists via remove_empty_pending_changes.py"
remove_empty_pending_changes.py "$P4INSTANCE" >>"$LOG" 2>&1 || true

# Finish logging and send the report.
log_msg "--- Weekly maintenance completed ---"
if [[ -n "${MAILTO:-}" ]]; then
  mail -s "${HOSTNAME} ${P4SERVER} Weekly maintenance log" "$MAILTO" <"$LOG" || true
fi