package_downloads.sh #14

#!/bin/bash
#==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
#------------------------------------------------------------------------------

#==============================================================================
# Declarations and Environment

# Allow override of P4U_HOME, which is set only when testing P4U scripts.
export P4U_HOME=${P4U_HOME:-../main/Server/Unix/p4/common/bin}
export P4U_LIB=${P4U_LIB:-../main/Server/Unix/p4/common/lib}
export P4U_ENV=$P4U_LIB/p4u_env.sh
export P4U_LOG=
P4U_LOG="/tmp/package_downloads.sdp.$(date +'%Y%m%d-%H%M%S').log"

# Environment isolation.  For stability and security reasons, prepend
# PATH to include dirs where known-good scripts exist.
# known/tested PATH and, by implication, executables on the PATH.
export PATH=~/bin:$P4U_HOME:$PATH:.
export P4CONFIG=${P4CONFIG:-.p4config}

[[ -r "$P4U_ENV" ]] || {
   echo -e "\\nError: Cannot load environment from: $P4U_ENV\\n\\n"
   exit 1
}

declare BASH_LIBS=$P4U_ENV
BASH_LIBS+=" $P4U_LIB/libcore.sh"
BASH_LIBS+=" $P4U_LIB/libp4u.sh"

for bash_lib in $BASH_LIBS; do
   # shellcheck disable=SC1090
   source "$bash_lib"
done

declare Version=1.4.4
declare -i SilentMode=0
declare AppHome=${PWD%/*}
declare ExcludesCfg=$AppHome/tools/excludes.downloads.cfg
declare CommonFilesList=$AppHome/tools/common_files.txt
declare SDPBranch=main
declare TmpDir=
declare SDPVersion=
declare SDPUnixVersion=
declare SDPWindowsVersion=
declare DownloadsDir=
declare TarFile=
declare StaticTarFile=
declare ZipFile=
declare StaticZipFile=
declare UnixWorkingDir=
declare WindowsWorkingDir=
declare ChangeFile=
declare OutFile=
declare Changelist=

export VERBOSITY=3

#==============================================================================
# Local Functions

#------------------------------------------------------------------------------
# Function: terminate
function terminate
{
   # Disable signal trapping.
   trap - EXIT SIGINT SIGTERM

   # Don't litter.
   cleanTrash

   vvmsg "$THISSCRIPT: EXITCODE: $OverallReturnStatus"

   # Stop logging.
   [[ "${P4U_LOG}" == off ]] || stoplog

   # With the trap removed, exit.
   exit "$OverallReturnStatus"
}

#------------------------------------------------------------------------------
# Function: usage (required function)
#
# Input:
# $1 - style, either -h (for short form) or -man (for man-page like format).
#------------------------------------------------------------------------------
function usage
{
   declare style=${1:--h}

   echo "USAGE for $THISSCRIPT v$Version:

$THISSCRIPT [-f] [-s] [-o] [-P] [-L <log>] [-si] [-D]

or

$THISSCRIPT [-h|-man|-V]
"
   if [[ $style == -man ]]; then
      echo -e "
DESCRIPTION:
	Create an SDP distribution package files, *.tgz files for
	UNIX/Linux and *.zip for Windows, from the latest files
	in the SDP main branch (or dev branch for a pre-release).

	Prior to packaging, a variety of checks are done to ensure
	the latest files are synced, the user is logged in, no
	files are checked out in the main branch, etc.

	An exclusion config file defines files and directories
	to be excluded from packaging.  This exclusion config
	file has file name per line, which may include the
	'*' wild card. This helps ensure only files that belong
	in the package are shipped.

	The exclusion config file can also specify directories to be
	excluded, which start with a 'D:' prefix.

	The exclusion config file is:
	$ExcludesCfg

	The generated tar files for UNIX/Linux are:
	* sdp.Unix.<YYYY>.<N>.<BUILD>.tgz, the version-identified file.
	* sdp.Unix.tgz, the same contents in a consistently named file.

	The generated zip files Windows are:
	* sdp.Windows.<YYYY>.<N>.<BUILD>.zip, the version-identified file.
	* sdp.Windows.zip, the same contents in a consistently name file.

OPTIONS:
 -P	Do a pre-release version.  With this flag, the 'dev' branch
 	is packaged instead of 'main', and the preflight checks for
	checked out files are also done in 'dev'. The generated tarfile
	name format also changes, from sdp.<Platform>.<YYYY>.<N>.<tgz_or_zip>
	to sdp.<Platform>.<YYYY>.<N>-DEV-PRE-RELEASE.<BUILD>.<tgz_or_zip>.

	With '-P' the sdp.Unix.tgz and sdp.Windows.zip tar files, which
	always represent the latest GA release, are not updated.

 -f	Force creation of target tar/zip file, replacing an existing
	one if necessary.  The default behavior is to abort if
	the target package files already exists.

 -o	Proceed even if files are opened in Perforce and/or not synced
 	to the head revision.  This may be useful for testing purposes.

	By default, we abort if any files are checked out, or if files
	are not syncd to the head revision.  This option is not suitable
	for creating a distribution file that will be submitted.

 -L <log>
	Specify the path to a log file, or the special value 'off' to disable
	logging.  By default, all output (stdout and stderr) goes to:
	$(dirname "${P4U_LOG}").

	NOTE: This script is self-logging.  That is, output displayed on the
	screen is simultaneously captured in the log file.  Do not run this
	script with redirection operators like '> log' or '2>&1', and do not
	use 'tee.'

-si	Operate silently.  All output (stdout and stderr) is redirected to
	the log only; no output appears on the terminal.  This cannot be
	used with '-L off'.
      
 -D     Set extreme debugging verbosity.

HELP OPTIONS:
 -h	Display short help message
 -man	Display man-style help message
 -V	Dispay version info for this script and its libraries.

FILES:
	Common Files List file, listing files common to both UNIX/Linux and
	Windows packages:
	$CommonFilesList

	Exclusions Config File, listing files to exclude from all packages:
	$ExcludesCfg

EXAMPLES:
	Typical Usage is with no arguments:
	cd $AppHome/tools
	./package_downloads.sh

	Thus must be run from a workspace that maps at least (and ideally
	only) the folder //guest/perforce_software/sdp/...

SEE ALSO:
"
   fi

   exit 1
}

#==============================================================================
# Command Line Processing

declare -i shiftArgs=0
declare -i ForceTarCreation=0
declare -i AllowOpenedFiles=0
declare -i PreRelease=0

set +u
while [[ $# -gt 0 ]]; do
   case $1 in
      (-f) ForceTarCreation=1;;
      (-o) AllowOpenedFiles=1;;
      (-P) PreRelease=1; SDPBranch=dev;;
      (-h) usage -h;;
      (-man) usage -man;;
      (-V) show_versions; exit 1;;
      (-L) export P4U_LOG=$2; shiftArgs=1;;
      (-si) SilentMode=1;;
      (-D) set -x;; # Debug; use 'set -x' mode.
      (*) usageError "Unknown arg ($1).";;
   esac

   # Shift (modify $#) the appropriate number of times.
   shift; while [[ $shiftArgs -gt 0 ]]; do
      [[ $# -eq 0 ]] && usageError "Bad usage."
      shiftArgs=$shiftArgs-1
      shift
   done
done
set -u

#==============================================================================
# Command Line Verification

[[ $SilentMode -eq 1 && $P4U_LOG == off ]] && \
   usageError "Cannot use '-si' with '-L off'."

#==============================================================================
# Main Program

trap terminate EXIT SIGINT SIGTERM

declare -i OverallReturnStatus=0

if [[ "${P4U_LOG}" != off ]]; then
   touch "${P4U_LOG}" || bail "Couldn't touch log file [${P4U_LOG}]."

   # Redirect stdout and stderr to a log file.
   if [[ $SilentMode -eq 0 ]]; then
      exec > >(tee "${P4U_LOG}")
      exec 2>&1
   else
      exec >"${P4U_LOG}"
      exec 2>&1
   fi

   initlog
fi

TmpDir=$(mktemp -d)

# A General Availability (GA) release tarball looks something like:
# sdp.Unix.2019.3.26571.gz.
#
# A Pre-Release tarball looks like:
# sdp.Unix.2020.1-DEV-PRE-RELEASE.27141.gz.
#
if [[ "$PreRelease" -eq 0 ]]; then
   SDPVersion=$(cut -d '/' -f 3 < ../main/Version).$(cut -d '/' -f 4 < ../main/Version | cut -d ' ' -f 1)
else
   SDPVersion=$(cut -d '/' -f 3 < ../dev/Version)-DEV-PRE-RELEASE.$(cut -d '/' -f 4 < ../dev/Version | cut -d ' ' -f 1)
fi

# shellcheck disable=SC2116 disable=SC2086
SDPVersion=$(echo $SDPVersion)
SDPUnixVersion=sdp.Unix.$SDPVersion
SDPWindowsVersion=sdp.Windows.$SDPVersion
DownloadsDir=$AppHome/downloads
TarFile=$DownloadsDir/$SDPUnixVersion.tgz
StaticTarFile=$DownloadsDir/sdp.Unix.tgz
ZipFile=$DownloadsDir/$SDPWindowsVersion.zip
StaticZipFile=$DownloadsDir/sdp.Windows.zip
UnixWorkingDir=/tmp/$SDPUnixVersion
WindowsWorkingDir=/tmp/$SDPWindowsVersion
ChangeFile=$TmpDir/change.p4s
OutFile=$TmpDir/output.txt
Changelist=

runCmd "p4 -ztag -F %TicketExpiration% login -s" \
   "Checking login status." 0

if [[ -n "$CMDOUTPUT" ]]; then
   msg "Verified: Logged into Perforce."
else
   bail "Not logged into Perforce."
fi

runCmd "p4 -F %depotFile% -ztag opened ../$SDPBranch/..." \
   "Checking for opened files in SDP $SDPBranch branch."

if [[ -n "$CMDOUTPUT" ]]; then
   if [[ $AllowOpenedFiles -eq 1 ]]; then
      msg "Proceeding despite opened files being detected, due to -o flag."
   else
      if [[ $CMDOUTPUT == *"//"* ]]; then
         bail "These files are checked out in the $SDPBranch branch:\\n$CMDOUTPUT\\n\\nAborting."
      else
         bail "Could not determine if files are checked out in the $SDPBranch branch. Output was:\\n$CMDOUTPUT\\n\\nAborting."
      fi
   fi
else
   msg "Verified: No files checked out in SDP $SDPBranch branch."
fi

runCmd "p4 -F %depotFile% -ztag sync -n ../..." \
   "Checking for unsynced SDP files."

if [[ -n "$CMDOUTPUT" ]]; then
   if [[ $AllowOpenedFiles -eq 1 ]]; then
      msg "Proceeding despite unsynced files, due to -o flag."
   else
      if [[ $CMDOUTPUT == *"//"* ]]; then
         bail "These files need to be synced:\\n$CMDOUTPUT\\n\\nAborting."
      else
         bail "Could not determine if files need to be synced. Output was:\\n$CMDOUTPUT\\n\\nAborting."
      fi
   fi
else
   msg "Verified: No unsynced SDP files."
fi

msg "AppHome=[$AppHome] TF=[$TarFile] ZF=[$ZipFile] UWD=[$UnixWorkingDir] WWD=[$WindowsWorkingDir]."

# Cleanup from prior half-start runs if needed.
for d in "$UnixWorkingDir" "$WindowsWorkingDir"; do
   runCmd "rm -rf $d" "Removing dir from earlier run: $d"
done

if [[ -r $TarFile ]]; then
   if [[ $ForceTarCreation -eq 1 ]]; then
      runCmd "rm -f $TarFile" \
         "Removing existing tar file [$TarFile] due to '-f'." ||\
            bail "Failed to remove existing tar file [$TarFile].  Aborting."
   else
      bail "Tar file already exists: [$TarFile].  Use '-f' to forcibly replace.  Aborting."
   fi
fi

if [[ -r $ZipFile ]]; then
   if [[ $ForceTarCreation -eq 1 ]]; then
      runCmd "rm -f $ZipFile" \
         "Removing existing zip file [$ZipFile] due to '-f'." ||\
            bail "Failed to remove existing zip file [$ZipFile].  Aborting."
   else
      bail "Zip file already exists: [$ZipFile].  Use '-f' to forcibly replace.  Aborting."
   fi
fi

msg "Creating empty working directories."
for workingDir in $UnixWorkingDir $WindowsWorkingDir; do
   if [[ -d "$workingDir" ]]; then
      runCmd "rm -rf $workingDir" \
         "Removing old working dir [$workingDir]." ||\
         bail "Failed to remove old working dir [$workingDir]. Aborting."
   fi

   runCmd "mkdir -p $workingDir" "Initializing empty working dir [$workingDir]." 0 ||\
      bail "Failed to create empty working dir [$workingDir]. Aborting."
done

if [[ ! -d "$DownloadsDir" ]]; then
   runCmd "mkdir -p $DownloadsDir" "Initializing empty downloads dir [$DownloadsDir]." 0 ||\
      bail "Failed to create downloads dir [$DownloadsDir]. Aborting."
fi

#------------------------------------------------------------------------------
# Unix *.tgz file.
SDPRoot="$UnixWorkingDir/sdp"

runCmd "cp -pr ${AppHome}/$SDPBranch $SDPRoot" \
   "Copying SDP package from $SDPBranch branch to Unix working dir." 0 ||\
   bail "The copy to the working dir failed.  Aborting."

cd "$SDPRoot" || bail "Failed to cd to SDP root in Working Dir [$SDPRoot]."

if [[ -r "$ExcludesCfg" ]]; then
   while read -r excludeFile; do
      [[ "$excludeFile" == "" || "$excludeFile" == "#"* ]] && continue
      if [[ "$excludeFile" == "D:"* ]]; then
         excludeDir=${excludeFile#D:}
         msg "find $SDPRoot/ -type d -name \"$excludeDir\" -print -exec rm -r -f {} \\;"
         find "$SDPRoot/" -type d -name "$excludeDir" -print -exec rm -r -f {} \;
      else
         msg "find $SDPRoot/ -type f -name \"$excludeFile\" -print -exec rm -f {} \\;"
         find "$SDPRoot/" -type f -name "$excludeFile" -print -exec rm -f {} \;
      fi
   done < "$ExcludesCfg"
fi

cd "$UnixWorkingDir" || bail "Failed to cd to Working Dir [$UnixWorkingDir]."

# shellcheck disable=SC2071
if [[ "$(uname -s)" == Darwin && "$(uname -r)" > "18" ]]; then
   runCmd "tar -cz --no-xattrs -f $TarFile sdp" ||\
      bail "Failed to create tar file [$TarFile]. Aborting."
else
   runCmd "tar -cz -f $TarFile sdp" ||\
      bail "Failed to create tar file [$TarFile]. Aborting."
fi

#------------------------------------------------------------------------------
# Windows *.zip file.
SDPRoot="$WindowsWorkingDir/sdp"

runCmd "cp -pr ${AppHome}/$SDPBranch $SDPRoot" \
   "Copying SDP package from $SDPBranch branch to Windows working dir." 0 ||\
   bail "The copy to the working dir failed.  Aborting."

cd "$SDPRoot" || bail "Failed to cd to SDP root in Working Dir [$SDPRoot]."

msg "\\nProcessing excludes from $ExcludesCfg:\\n$(cat "$ExcludesCfg")\\n"

if [[ -r "$ExcludesCfg" ]]; then
   while read -r excludeFile; do
      [[ "$excludeFile" == "" || "$excludeFile" == "#"* ]] && continue
      # For Windows, exclude Unix stuff in places where Windows would otherwise be excluded.
      excludeFile=${excludeFile//Windows/Unix}
      if [[ "$excludeFile" == "D:"* ]]; then
         excludeDir=${excludeFile#D:}
         msg "find $SDPRoot/ -type d -name \"$excludeDir\" -print -exec rm -r -f {} \\;"
         find "$SDPRoot/" -type d -name "$excludeDir" -print -exec rm -r -f {} \;
      else
         msg "find $SDPRoot/ -type f -name \"$excludeFile\" -print -exec rm -f {} \\;"
         find "$SDPRoot/" -type f -name "$excludeFile" -print -exec rm -f {} \;
      fi
   done < "$ExcludesCfg"
fi

cd "$WindowsWorkingDir" || bail "Failed to cd to Working Dir [$WindowsWorkingDir]."

# Copy common files.
if [[ -r "$CommonFilesList" ]]; then
   while read -r srcFileSpec; do
      [[ "$srcFileSpec" == "" || "$srcFileSpec" == "#"* ]] && continue
      # Prefix Unix working dir to relative path specified in the Common Files List file.
      srcFileSpec="$UnixWorkingDir/sdp/$srcFileSpec"
      tgtDir=${srcFileSpec//Unix/Windows}
      msg "SFS=[$srcFileSpec] TD=[$tgtDir]"
      tgtDir=${tgtDir%\*}
      msg "TD2=[$tgtDir]"
      tgtDir=${tgtDir%\/*}
      msg "TD3=[$tgtDir]"
      for srcFile in $srcFileSpec; do
         msg "Copying [$srcFile] to [$tgtDir]."
         if [[ -d "$srcFile" ]]; then
            cp -pr "$srcFile" "$tgtDir" ||\
               bail "Failure performing copy of common subdirs. Aborting."
         else
            cp -p "$srcFile" "$tgtDir" ||\
               bail "Failure performing copy of common files. Aborting."
         fi
      done
   done < "$CommonFilesList"
else
   warnmsg "Skipping copy of common files from Unix to Windows, due to missing common files list [$CommonFilesList]."
   OverallReturnStatus=1
fi

# Convert ASCII text files to Windows line endings before packaging.
for textFile in $(find . -type f -exec file {} \; | grep -E -i '(ascii|text)'| cut -d: -f1); do
   convertedTextFile=.tmp.converted
   awk 'sub("$", "\r")' "$textFile" > "$convertedTextFile"
   mv -f "$convertedTextFile" "$textFile"
done

runCmd "zip -r $ZipFile sdp" ||\
   bail "Failed to create tar file [$ZipFile]. Aborting."

runCmd "tar -tzf $TarFile" \
   "Tar file [$TarFile] contains these elements:"

if [[ "$PreRelease" -eq 0 ]]; then
   runCmd "cp -f -p $TarFile $StaticTarFile" \
      "Updated static tar file [$StaticTarFile]." ||\
      bail "Failed to update static tar file [$StaticTarFile]."

   runCmd "cp -f -p $ZipFile $StaticZipFile" \
      "Updated static zip file [$StaticZipFile]." ||\
      bail "Failed to update static zip file [$StaticZipFile]."
fi

cd "$DownloadsDir" || bail "Failed to cd to downloads dir [$DownloadsDir]."

echo -e "Change: new\\n
Description:\\n\\tPackaged SDP $SDPVersion." > "$ChangeFile"

p4 change -i < "$ChangeFile" > "$OutFile" 2>&1

if ! grep -E 'Change \d+ created.' "$OutFile"; then
   cat "$OutFile"
   rm -f "$OutFile"
   bail "Failed to create pending changelist. Aborting."
fi

# Parse the generated changelist number from the output file.
Changelist=$(cat "$OutFile")
Changelist=${Changelist#Change }
Changelist=${Changelist%% *}
# shellcheck disable=SC2086 disable=SC2116
Changelist=$(echo $Changelist)

runCmd "p4 -s rec -c $Changelist" \
   "Reconciling in downloads dir [$DownloadsDir]." ||\
   bail" Failed to reconcile changes."

if [[ "$PreRelease" -eq 0 ]]; then
   runCmd "p4 -s delete -c $Changelist ...tgz ...zip" \
      "Cleaning up old zip/tar files in downloads dir." ||\
      bail" Failed to cleanup old zip/tar files."

   runCmd "p4 describe -s $Changelist" \
      "Showing generated change $Changelist." 1 1 0 ||\
      bail" Failed to describe change $Changelist."
fi

msg "To complete packaging, run:\\n\\tp4 -s submit -c $Changelist\\n"

[[ -d "$TmpDir" ]] && rm -rf "$TmpDir"

if [[ $OverallReturnStatus -eq 0 ]]; then
   msg "${H}\\nAll processing completed successfully.\\n"
else
   msg "${H}\\nProcessing completed, but with errors.  Scan above output carefully.\\n" 
fi

msg "That took about $((SECONDS/3600)) hours $((SECONDS%3600/60)) minutes $((SECONDS%60)) seconds.\\n"

# See the terminate() function, which is really where this script exits.
exit $OverallReturnStatus