run_docker_tests.sh #31

  • //
  • guest/
  • perforce_software/
  • sdp/
  • dev/
  • test/
  • run_docker_tests.sh
  • View
  • Commits
  • Open Download .zip Download (5 KB)
#!/bin/bash
#------------------------------------------------------------------------------
set -u

#------------------------------------------------------------------------------
# Run the Docker tests for the SDP - updated to use podman

# Usage Examples from 'test' dir under workspace branch root:
#    sdp/test/run_docker_tests.sh
#    sdp/test/run_docker_tests.sh ubuntu20
#    sdp/test/run_docker_tests.sh rocky9
#    sdp/test/run_docker_tests.sh centos7 ubuntu22
#    sdp/test/run_docker_tests.sh all
#    sdp/test/run_docker_tests.sh ALL
#
# Goes together with run_docker_tests.sh
#
# Note that this file is expecting to be mapped into the root of the workspace
# and with the sdp directory in the same root.
#
# So workspace view should look something like:
#    View:
#        //guest/perforce_software/sdp/main/... //myws.sdp/sdp/...

declare oses=

function msg () { echo -e "$*"; }
function bail () { msg "\\nError: ${1:-Unknown Error}\\n"; exit "${2:-1}"; }

# This file should be in <workspace-root>/sdp/test/
# We calculate dir relative to directory of script
script_dir="${0%/*}"
root_dir="$(cd "$script_dir/../.."; pwd -P)"

if [[ "${1:-Unset}" == "Unset" ]]; then
   # Set Default.
   oses="rocky9"
else
   for os in $(echo "$@" | tr ',' ' '); do
      case "$os" in
         (all) oses="rocky8 rocky9 ubuntu20 ubuntu22";;
         (ALL) oses="rocky8 rocky9 ubuntu20 ubuntu22";;
         (ubuntu20) oses+="ubuntu20";;
         (ubuntu22) oses+="ubuntu22";;
         (rocky8) oses+="rocky8";;
         (rocky9) oses+="rocky9";;
         
         (*)
            msg "Warning: Unknown OS [$os]."
            oses+="$os"
         ;;
      esac
   done
fi

if command -v getenforce > /dev/null; then
   selinux=$(getenforce)
   msg "Detected SELinux mode: $selinux"
   # If enforcing mode then you have to allow container to write to the output dir below - for now just bail.
   if [[ "$selinux" == "Enforcing" ]]; then
      msg "Warning: SELinux is enabled in Enforcing mode on this machine $(hostname), IP $(hostname -I). Disabling it."
      msg "Changing /etc/selinux/config:"
      cp -p /etc/selinux/config /etc/selinux/config.bak
      sed -i -e s@SELINUX=enforcing@SELINUX=disabled@g /etc/selinux/config
      diff /etc/selinux/config.bak /etc/selinux/config
      msg "Running: setenforce 0"
      setenforce 0
      msg "Running: getenforce"
      getenforce
   else
      msg "Verified: SELinux is not in enforcing mode on this machine $(hostname), IP $(hostname -I)."
   fi
fi

# Directory where test output is put by the container
# Easier to make it under sdp which is a mounted volume
test_output_dir="$script_dir/output"
[[ -d "$test_output_dir" ]] || mkdir "$test_output_dir"
all_test_output="$test_output_dir/alltests.out"
if [[ -f "$all_test_output" ]]; then
   rm "$all_test_output"
fi

msg "Running SDP tests"
tests_failed=0
for os in $oses
do
   test_output="$test_output_dir/test-${os}.out"
   if [[ -f $test_output ]]; then
       rm "$test_output"
   fi
   docker_dir="$root_dir/sdp/test/docker"

   # Either a single $dockerfile will exist, or both $dockerfile_base and dockerfile_sdp
   # will exist.
   dockerfile_base="${docker_dir}/Dockerfile.${os}.base"
   dockerfile_sdp="${docker_dir}/Dockerfile.${os}.sdp"
   dockerfile="${docker_dir}/Dockerfile.${os}"

   # If there is a Dockerfile.${os} file, do a single podman build with that file.
   # If instead there are Dockerfile.${os}.base and Dockerfile.${os}.sdp files,
   # build the base Docker for the OS first, and then the SDP variant on top.
   if [[ -r "$dockerfile" ]]; then
      podman build --rm=true -t="perforce/${os}-sdp" -f "${dockerfile}" "${docker_dir}"
   elif [[ -r "$dockerfile_base" && -r "$dockerfile_sdp" ]]; then
      podman build --rm=true -t="perforce/${os}-base" -f "${dockerfile_base}" "${docker_dir}"
      podman build --rm=true -t="perforce/${os}-sdp" -f "${dockerfile_sdp}" "${docker_dir}"
   else
      bail "Can't find needed Docker files. Must have either $dockerfile OR ($dockerfile_base AND dockerfile_sdp)."
   fi

   # Run the Docker image, mounting the /sdp directory within it. The SDP image
   # has a default RUN command which is configured within it.
   # We don't directly stop on error but process a little later below so that nice
   # messages are written to Jenkins console output.
   # Note to use setcap with p4d >= 2023.1 we need --cap-add=SYS_RESOURCE for podman!
   set +e -x
   podman run --replace --detach --rm --cap-add=SYS_RESOURCE --cap-add=CAP_AUDIT_WRITE -v "${root_dir}/sdp:/sdp" -e "TESTOS=${os}" --name "${os}-sdp" "localhost/perforce/${os}"
   podman exec "${os}-sdp" /p4/docker_entry.sh
   tests_failed=$?
   set +x
   set -e
   echo "$os" >> "$all_test_output"
   # Avoid Jenkins immediately failing job without letting us cat the output
   set +e
   cat "$test_output" >> "$all_test_output"
   set -e
   if [[ "$tests_failed" -ne 0 ]]; then
      break
   fi
   set -x
   podman kill "${os}-sdp"
   # podman rm "${os}-sdp"
   set +x
done
cat "$all_test_output"

exit "$tests_failed"
# Change User Description Committed
#33 30904 Robert Cowham Fix container name for run command
#32 30903 C. Thomas Tyler Added '-f' to 'rm' for all test output file cleanup.
#31 30902 C. Thomas Tyler Added '--replace' option to 'podman run' command.
#30 30879 Robert Cowham Update tests to use systemctl (remove hack) and fix some env problems.
#29 30868 C. Thomas Tyler Fixed typo in container name.
#28 30867 C. Thomas Tyler Removed '--replace' option, changed back to single docker file.
#27 30866 C. Thomas Tyler Adjusted '/Users/ttyler/pub/sdp/dev/test' to '' as an experiment.
#26 30865 C. Thomas Tyler Added '--replace' option to 'podman run' command.
#25 30864 C. Thomas Tyler Reverting to older mechanisn by re-adding Dockerfile.rocky9.base and
Dockerfile.rocky9.sdp files, but keeping the Dockerfile.rocky9 file
in place as the path forward. The intent is to see if the old
mechanism still works.

Modified run_docker_tests.sh so it'll use the old 2-image mechanism
if the *{os}.base and *.{os}.sdp files are found. Otherwise, if
the {os} file is found, it'll use that.  So both old and new styles
are supported until fully deprecate one for the other.
#24 30863 C. Thomas Tyler Adapted to work with combos of existing Docker files.
#23 30862 C. Thomas Tyler Displays detected value of SELinux.
#22 30861 C. Thomas Tyler Adjusted so SELinux, if found in Enforcing mode, is disabled rather than
just bailing.
#21 30845 Robert Cowham Base the podman/docker image on rocky9-ubi-init to get systemd
This supports systemd within the container if run under podman instead of docker
Remove centos7 docker files
#20 30356 C. Thomas Tyler Added Rocky 9 and Ubuntu 22 to test suite.

Dropped CentOS 6.

Changed default OS for tests from CentOS 7 to Rocky 9.

Adapted to using podman for the test suite.

Reduced minimum disk space requirement for testing
from 1G to 10M.

To be revisited:
* Disabled OOM killer defenese feature and systemd due to
compatibility issues with the test environment.
* Disabled systemd by moving systemctl aside

Enhanced build_docker_image.sh; added '-clean' option and added
support for more platforms.

Updated OS-specific packages as needed for all platforms, e.g.
adding 'file' and 'rsync' utils as needed in Docker definitions.

Added minimum performance recommendations for test containers in
config.

Updated env.sh utilities.

This change test on:
[X] centos7
[X] rocky8
[X] rocky9
[X] ubuntu20
[X] ubuntu22

#review-30357 @robert_cowham

TO DO:
* This builds using the official SDP test suite run manually on
Tom's laptop. Next step: Get it working on our shiny new Jenkins
Rocky 9 server machine.
#19 30295 Robert Cowham Fix tests to run under podman SDP-1106
#18 28850 C. Thomas Tyler Test suite tweaks:
* Added command line usage notes in scripts.
* Enhanced comand line processing.
* Added aliases for building on Rocky Linux 8.

#review-28851
#17 27722 C. Thomas Tyler Refinements to @27712:
* Resolved one out-of-date file (verify_sdp.sh).
* Added missing adoc file for which HTML file had a change (WorkflowEnforcementTriggers.adoc).
* Updated revdate/revnumber in *.adoc files.
* Additional content updates in Server/Unix/p4/common/etc/cron.d/ReadMe.md.
* Bumped version numbers on scripts with Version= def'n.
* Generated HTML, PDF, and doc/gen files:
  - Most HTML and all PDF are generated using Makefiles that call an AsciiDoc utility.
  - HTML for Perl scripts is generated with pod2html.
  - doc/gen/*.man.txt files are generated with .../tools/gen_script_man_pages.sh.

#review-27712
#16 27002 C. Thomas Tyler Updated ubuntu tests to add needed utilities, and change from
testing Ubuntu 16.04 to 20.04.  Changed 'centos' to imply cento7
rather than centos6 (which we shoud drop soon).
#15 26987 Robert Cowham Change order of tests so centos7 is first
#14 26982 C. Thomas Tyler mkdirs.sh v4.1.0:
* Accounted for directory structure change of Maintenance to Unsupported.
* Added standard command line processing with '-h' and '-man' doc flags,
and other flags (all documented).
* Added in-code docs and updated AsciiDoc.
* Enhanced '-test' mode to simulate /hx* mounts.
* Enhanced preflight testing, and fixed '-test' mode installs.
* Added support for installing to an alternate root directory.
* Added '-s <ServerID>' option to override REPLICA_ID.
* Added '-S <TargetServerID>' used for replicas of edge servers.
* Added '-t <server_type>' option to override SERVER_TYPE.
* Added '-M' option to override mount points.
* Added '-f' fast option to skip big chown/chmod commands, and
moved those commands near the end as well.

verify_sdp.sh v5.9.0:
* Added check for /p4/Version file, and checked that other legacy
SDP methods of checking version
* Added sanity check for crontab.
* Added 'test skip' mechanism to skip certain tests:
 - crontab: Skip crontab check. Use this if you do not expect crontab to be configured, perhaps if a different scheduler is used.
 - license: Skip license related checks.
 - version: Skip version checks.
 - excess: Skip checks for excess copies of p4d/p4p/p4broker in PATH.
* Added VERIFY_SDP_SKIP_TEST_LIST setting ton instance_vars.template,
to define a standard way to have verify_sdp.sh always skip certain
tests for a site.
* Extended '-online' checks to check for bogus P4MASTERPORT, a common
config error.

Update test_SDP.py:
* Adjusted test suite to account for various changes in mkdirs.sh.
* Added 'dir' parameter to run_cmd() and sudo_cmd(), to run a
command from a specified directory (as required to test new
mkdirs.sh)
* Added check_links() similar to existing check_dirs() function.

=== Upgrade Process Changes ===

Made /p4/common/bin/p4d/p4/p4broker/p4p shell script rather than binary.

This changes the way SDP new binaries are staged for upgrade.  For
safety, exes are now staged to a director outside the PATH, the
/p4/sdp/exes folder. A new 'get_latest_exes.sh' script simplifies
the task of pulling executables from the Perforce FTP server. This
can be used 'as is' for environments with outbound internet access,
and is useful in any case to describe now to acquire binaries.

This addresses an issue where a p4d binary staged for a future
upgrade might be called before the actual upgrade is performed.

upgrade.sh v4.0.0:
* All preflight checks are now done first. Added '-p' to abort after preflight.
* Added '-n' to show what would be done before anything is executed.
* Minimalist logic to start/stop only servers that are upgrade, and apply
upgrades only as needed.
* Staging of exes for upgrade is now separate from /p4/common/bin
* Improved in-code docs, added '-h' and '-man' options.
* Retained pre/post P4D 2019.1 upgrade logic.
#13 25554 Robert Cowham Automated tests for calling upgrade.sh
#12 25153 Robert Cowham Change scripts to be able to be run from workspace root
sdp/test/run_docker_tests.sh

So no longer requires funny mapping in workspace view (see README.md for view)
#11 25151 Robert Cowham Update p4d version to 18.2
#10 25149 Robert Cowham Changes to docker files as part of prep for multi-container work
Centos6 & 7 now install python 3.6 from ISU Community as pre-built
Refer to p4python==17.2 to avoid need to compile
Changes to test infrastructure only
#9 24581 Robert Cowham Fix test version for Perforce
#8 22199 C. Thomas Tyler Updating to 2017.1 as it is now GA.
#7 20880 Robert Cowham Improve error handling - tail the log file so errors more visible in Jenkins failures
#6 19407 Russell C. Jackson (Rusty) Updated p4 version to 16.1
#5 19367 Robert Cowham Improve error handling for Jenkinsß
#4 19366 Robert Cowham Improve error messages hopefully when run by Jenkins
#3 19099 Robert Cowham Minor refactor to make things a little clearer as to test file location.
#2 19045 Robert Cowham Add docker file for Centos7
Refactor to move common stuff to a shell script.
#1 19019 Robert Cowham Run ubuntu