#!/bin/bash #============================================================================== # Copyright and license info is available in the LICENSE file included with # the Server Deployment Package (SDP), and also available online: # https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE #------------------------------------------------------------------------------ # This script will allow you to archive files and optionally purge files based # on a configurable number of days and minimum revisions that you want to keep. # This is useful if you want to keep a certain number of days worth of files # instead of a specific number of revisions. This script currently only accepts # paths to specific files, it does not support globbing or wildcards. # # Note: If you run this script with purge mode disabled, and then enable it after # the fact, all previously archived files specified in the configuration file will # be purged if the configured criteria is met. # # Prior to running this script, you may want to disable server locks for archive # to reduce impact to end users. # https://www.perforce.com/perforce/doc.current/manuals/cmdref/Content/CmdRef/configurables.configurables.html#server.locks.archive # # SDP_INSTANCE - The instance of Perforce that is being backed up. If not # set in environment, pass in as argument to script. # # P4_ARCHIVE_CONFIG - The location of the config file used to determine retention. # If not set in environment, pass in as argument to script. This can be stored on # a physical disk or somewhere in perforce. # # P4_ARCHIVE_DEPOT - Depot to archive the files in (string) # # P4_ARCHIVE_REPORT_MODE - Do not archive revisions; report on which revisions would # have been archived (bool - default: true) # # P4_ARCHIVE_TEXT - Archive text files (or other revisions stored in delta format, # such as files of type binary+D) (bool - default: false) # # P4_PURGE_MODE - Enables purging of files after they are archived (bool - default: false) # # CONFIG FILE FORMAT: # The config file should contain a list of file paths, number of days and minimum # of revisions to keep in a tab delimited format. # # <PATH> <DAYS> <MINIMUM REVISIONS> # # Example: # //test/1.txt 10 1 # //test/2.txt 1 3 # //test/3.txt 10 10 # //test/4.txt 30 3 # //test/5.txt 30 8 # # USAGE: ./purge_revisions.sh <SDP_INSTANCE> <P4_ARCHIVE_CONFIG> <P4_ARCHIVE_DEPOT> <P4_ARCHIVE_REPORT_MODE (Optional)> <P4_ARCHIVE_TEXT (Optional)> <P4_PURGE_MODE (Optional)> # # EXAMPLES: # Run from CLI that will archive files as defined in the config file # ./purge_revisions.sh 1 /p4/common/config/p4_1.p4purge.cfg archive FALSE # # Cron job that will will archive files as defined in the config file, including text files # 30 0 * * * [ -e /p4/common/bin ] && /p4/common/bin/run_if_master.sh ${INSTANCE} /p4/common/bin/purge_revisions.sh ${INSTANCE} /p4/common/config/p4_1.p4purge.cfg archive FALSE FALSE # # Regex pattern for patterns you don't want to allow in paths separated by `|` EXCLUDE_PATH_CHARS='\*|\.\.\.' P4_ARCHIVE_ARGS='-h' export SDP_INSTANCE=${SDP_INSTANCE:-Undefined} export SDP_INSTANCE=${1:-$SDP_INSTANCE} if [[ $SDP_INSTANCE == Undefined ]]; then echo "Instance parameter not supplied." echo "You must supply the Perforce instance as a parameter to this script." exit 1 fi export P4_ARCHIVE_CONFIG=${P4_ARCHIVE_CONFIG:-Undefined} export P4_ARCHIVE_CONFIG=${2:-$P4_ARCHIVE_CONFIG} if [[ $P4_ARCHIVE_CONFIG == Undefined ]]; then echo "Location of the config file not supplied." echo "You must supply the location of the config file to this script." exit 1 fi export P4_ARCHIVE_DEPOT=${P4_ARCHIVE_DEPOT:-Undefined} export P4_ARCHIVE_DEPOT=${3:-$P4_ARCHIVE_DEPOT} if [[ $P4_ARCHIVE_DEPOT == Undefined ]]; then echo "Archive depot not supplied." echo "You must supply a name of the archive depot to archive files to." exit 1 fi export P4_ARCHIVE_REPORT_MODE=${P4_ARCHIVE_REPORT_MODE:-true} export P4_ARCHIVE_REPORT_MODE=${4:-$P4_ARCHIVE_REPORT_MODE} export P4_ARCHIVE_TEXT=${P4_ARCHIVE_TEXT:-false} export P4_ARCHIVE_TEXT=${5:-$P4_ARCHIVE_TEXT} export P4_PURGE_MODE=${P4_PURGE_MODE:-false} export P4_PURGE_MODE=${6:-$P4_PURGE_MODE} . /p4/common/bin/p4_vars $SDP_INSTANCE . /p4/common/bin/backup_functions.sh LOGFILE=$LOGS/purge_revisions.log ######### Validate Config File ########## # Check if config file exists and is readable if [[ "${P4_ARCHIVE_CONFIG:0:2}" == '//' ]]; then P4_ARCHIVE_CONFIG_CONTENTS=($($P4BIN print -q "${P4_ARCHIVE_CONFIG}" 2>&1)) # If p4 command returns the archive file path, then the file was not found if [[ "${P4_ARCHIVE_CONFIG_CONTENTS}" == "${P4_ARCHIVE_CONFIG}" ]]; then echo "Config file ${P4_ARCHIVE_CONFIG} does not exist or is not readable!" fi # Read contents of config file stored in perforce and get duplicates P4_ARCHIVE_CONFIG_DUPES=($($P4BIN print -q "${P4_ARCHIVE_CONFIG}" | $AWK '{ print $1 }' | uniq -D)) else if ! [[ -r "$P4_ARCHIVE_CONFIG" ]]; then die "Config file ${P4_ARCHIVE_CONFIG} does not exist or is not readable!" fi # Read contents of config file on disk and get duplicates P4_ARCHIVE_CONFIG_DUPES=($(cat ${P4_ARCHIVE_CONFIG} | $AWK '{ print $1 }' | uniq -D)) fi # Ensure no duplicate paths exist if [[ "${#P4_ARCHIVE_CONFIG_DUPES[@]}" -gt 0 ]]; then log "Duplicate path(s) found, please check config: ${P4_ARCHIVE_CONFIG}" log ${P4_ARCHIVE_CONFIG_DUPES[@]} die "Aborting, please remove duplicate path(s) from config file!" fi declare -a P4_ARCHIVE # Load config file into array if [[ "${P4_ARCHIVE_CONFIG:0:2}" == '//' ]]; then readarray P4_ARCHIVE <<< "$($P4BIN print -q "${P4_ARCHIVE_CONFIG}")" else readarray P4_ARCHIVE < $P4_ARCHIVE_CONFIG fi ######### Start of Script ########## check_vars set_vars rotate_last_run_logs log "Start $P4SERVER Purge Revisions" check_uid check_dirs $P4CBIN/p4login # Build P4 archive args based on flags if [[ "${P4_ARCHIVE_REPORT_MODE,,}" == "true" ]]; then log "########## RUNNING IN REPORT MODE ##########" P4_ARCHIVE_ARGS="${P4_ARCHIVE_ARGS} -n" fi if [[ "${P4_ARCHIVE_TEXT,,}" == "true" ]]; then log "########## TEXT ARCHIVE MODE ENABLED ##########" P4_ARCHIVE_ARGS="${P4_ARCHIVE_ARGS} -t" fi if [[ "${P4_PURGE_MODE,,}" == "true" ]]; then log "########## PURGE MODE ENABLED ##########" fi # Parse archive revision config let i=0 while (( ${#P4_ARCHIVE[@]} > i )); do # Split each line into separate vars IFS=$'\t' read -r ARCHIVE_PATH ARCHIVE_DAYS ARCHIVE_MIN_REV <<< "${P4_ARCHIVE[i]}" unset IFS # Ensure there are no empty vars if [[ -z "$ARCHIVE_PATH" || -z "$ARCHIVE_DAYS" || -z "$ARCHIVE_MIN_REV" ]]; then log "One or more vars found to be empty, make sure config file is tab delimited, skipping ${P4_ARCHIVE[i]}" # Look for excluded characters in path elif [[ "$ARCHIVE_PATH" =~ ($EXCLUDE_PATH_CHARS) ]]; then log "Invalid characters found in path ${ARCHIVE_PATH}, skipping." elif ! [[ "$ARCHIVE_DAYS" =~ ^[0-9]+$ ]]; then log "Invalid characters found in days ${ARCHIVE_DAYS} for ${ARCHIVE_PATH}, skipping." elif ! [[ "$ARCHIVE_MIN_REV" =~ ^[0-9]+$ ]]; then log "Invalid characters found in min rev ${ARCHIVE_MIN_REV} for ${ARCHIVE_PATH}, skipping." else # Calculate archive date based on number of days specified in config file ARCHIVE_DATE=$(date -d "-${ARCHIVE_DAYS} days" +%Y/%m/%d) # Get head revision info for file ARCHIVE_HEAD_INFO=$($P4BIN -F "%headRev% %headTime%" fstat "${ARCHIVE_PATH}") # Split head revision info into separate vars IFS=$' ' read -r ARCHIVE_HEAD_REV ARCHIVE_HEAD_TIME <<< "${ARCHIVE_HEAD_INFO}" unset IFS # Convert epoch time to standard time ARCHIVE_HEAD_TIME=$(date -d @${ARCHIVE_HEAD_TIME} +%Y/%m/%d) # If the head revision is less than min revision, there's nothing to do if [[ "$ARCHIVE_HEAD_REV" -le "$ARCHIVE_MIN_REV" ]]; then log "Head revision of ${ARCHIVE_HEAD_REV} for ${ARCHIVE_PATH} is less than or equal to ${ARCHIVE_MIN_REV}, skipping." (( i++ )) continue fi # Get total number of revisions that are not archived or purged ARCHIVE_TOTAL_REVS=($($P4BIN -ztag -F "%rev%,%time%" files -a -e "${ARCHIVE_PATH}")) # If the total revisions is less than min revision, there's nothing to do, even though files are older # than the number of days specified if [[ ${#ARCHIVE_TOTAL_REVS[@]} -le "$ARCHIVE_MIN_REV" ]]; then log "Total revisions of ${#ARCHIVE_TOTAL_REVS[@]} for ${ARCHIVE_PATH} is less than or equal to ${ARCHIVE_MIN_REV}, skipping." # If purge mode is enabled, and we previously archived files, then we'll need to see if there is anything to purge if [[ "${P4_PURGE_MODE,,}" == "true" ]]; then # Generate path of file in archive depot ARCHIVE_DEPOT_PATH="//${P4_ARCHIVE_DEPOT}/${ARCHIVE_PATH:2}" # Get max rev info for file in archive ARCHIVE_MAX_REV=$($P4BIN -ztag -F "%rev%" files -A "${ARCHIVE_DEPOT_PATH}") # Get the most recent revision number of file that is not archived IFS=$',' read -ra ARCHIVE_MIN_REV_TO_KEEP <<< ${ARCHIVE_TOTAL_REVS[-1]} unset IFS # If the most recent archive rev of the file is less than the lowest minimum revision to keep, then file can be purged if [[ "$ARCHIVE_MAX_REV" != '' && "$ARCHIVE_MAX_REV" -lt "${ARCHIVE_MIN_REV_TO_KEEP[0]}" ]]; then log "########## Purging previously archived files for ${ARCHIVE_PATH} before ${ARCHIVE_DATE} ##########" { time $P4BIN archive -D $P4_ARCHIVE_DEPOT $P4_ARCHIVE_ARGS -p "${ARCHIVE_PATH}"\#${ARCHIVE_MAX_REV}; } >> "$LOGFILE" 2>&1 fi fi (( i++ )) continue fi # Get number of revisions before the number of days specified ARCHIVE_REVS=($($P4BIN -ztag -F "%rev%,%time%" files -a -e "${ARCHIVE_PATH}@${ARCHIVE_DATE}")) if [[ $(( ${#ARCHIVE_TOTAL_REVS[@]} - ${#ARCHIVE_REVS[@]} )) -ge "$ARCHIVE_MIN_REV" ]]; then # Archiving files since there are more revisions available than the minimum revisions specified log "########## Archiving files for ${ARCHIVE_PATH} before ${ARCHIVE_DATE} ##########" { time $P4BIN archive -D $P4_ARCHIVE_DEPOT $P4_ARCHIVE_ARGS "${ARCHIVE_PATH}"@${ARCHIVE_DATE}; } >> "$LOGFILE" 2>&1 if [[ "${P4_PURGE_MODE,,}" == "true" ]]; then log "########## Purging files for ${ARCHIVE_PATH} before ${ARCHIVE_DATE} ##########" { time $P4BIN archive -D $P4_ARCHIVE_DEPOT $P4_ARCHIVE_ARGS -p "${ARCHIVE_PATH}"@${ARCHIVE_DATE}; } >> "$LOGFILE" 2>&1 fi else # Since the difference of total revisions available and number of revisions before the archive date is less # than minimum revisions, archive up to the minimum number of revisions log "########## Archiving files for ${ARCHIVE_PATH} before ${ARCHIVE_DATE} ##########" log "Minimum revisions specified in config: ${ARCHIVE_MIN_REV}" # Get the most recent revision number to keep by using the minimum number of revisions as the index IFS=$',' read -ra ARCHIVE_MIN_REV_TO_KEEP <<< ${ARCHIVE_TOTAL_REVS[${ARCHIVE_MIN_REV}]} unset IFS # Grab the revision number which is the first value in the revision;epoch time array log "Calculated number of revisions to archive before: ${ARCHIVE_MIN_REV_TO_KEEP[0]}" { time $P4BIN archive -D $P4_ARCHIVE_DEPOT $P4_ARCHIVE_ARGS "${ARCHIVE_PATH}"\#${ARCHIVE_MIN_REV_TO_KEEP[0]}; } >> "$LOGFILE" 2>&1 if [[ "${P4_PURGE_MODE,,}" == "true" ]]; then log "########## Purging files for ${ARCHIVE_PATH} before ${ARCHIVE_DATE} ##########" { time $P4BIN archive -D $P4_ARCHIVE_DEPOT $P4_ARCHIVE_ARGS -p "${ARCHIVE_PATH}"\#${ARCHIVE_MIN_REV_TO_KEEP[0]}; } >> "$LOGFILE" 2>&1 fi fi fi (( i++ )) done check_disk_space remove_old_logs log "End $P4SERVER Purge Revisions"
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#4 | 27610 | Robert Cowham |
Rename/move file(s) to unsupported. User contributed and doesn't have full test harness/docs. |
||
#3 | 25592 | C. Thomas Tyler |
chmod +x for shell scripts. To Do: These scripts need tests and docs added. #review @ashaikh |
||
#2 | 25579 | ashaikh |
Add support for wildcards/globbing to purge_revisions.sh script This change allows you to specify wildcard paths in the purge revisions config file in case you want to apply the same revisions policy to multiple files. |
||
#1 | 24330 | ashaikh |
Add a script to archive/purge revisions based on number of days. This script will allow you to archive files and optionally purge files based on a configurable number of days and minimum revisions that you want to keep. This is useful if you want to keep a certain number of days worth of files instead of a specific number of revisions. This script currently only accepts paths to specific files, it does not support globbing or wildcards. |