[tests] and different operation modes to integration testing script

Previously, `./run_integration_tests.sh` would build a release tarball
of all the provisioning binaries, along with container images containing
each provisioning binary, and then unpack the binaries into the deployment
staging area, even if only the containers were run. This was slow when
debugging because compressing and uncompressing binaries (that are not
even used) is slow.

This adds a `-c` mode to the `./run_integration_tests.sh` to only build
an deploy the container images to save time when debugging.

Additionally, a `-d` (debug) mode is added that prevents the containers
from being torn down on script exit to enable debugging test failures.

Signed-off-by: Tim Trippel <ttrippel@google.com>

config/dev/deploy.sh

@@ -4,55 +4,80 @@
 # SPDX-License-Identifier: Apache-2.0
 set -e
 
+################################################################################
+# Check usage.
+################################################################################
 usage() {
     echo >&2 "ERROR: $1"
     echo >&2 ""
     echo >&2 "Usage: $0 <release-dir>"
     exit 1
 }
 
+################################################################################
+# Parse args.
+################################################################################
 if [ $# != 1 ]; then
     usage "Unexpected number of arguments"
 fi
-
 RELEASE_DIR=$1
 if [ ! -d "${RELEASE_DIR}" ]; then
     usage "RELEASE_DIR: ${RELEASE_DIR} does not exist"
 fi
-
 CONFIG_DIR="$(dirname "$0")"
 
+################################################################################
+# Source envars.
+################################################################################
 source "${CONFIG_DIR}/env/spm.env"
 
+################################################################################
+# Create deployment dir structure.
+################################################################################
+echo "Staging deployment directory structure ..."
 if [ ! -d "${OPENTITAN_VAR_DIR}" ]; then
     echo "Creating config directory: ${OPENTITAN_VAR_DIR}. This requires sudo."
     sudo mkdir -p "${OPENTITAN_VAR_DIR}"
     sudo chown "${USER}" "${OPENTITAN_VAR_DIR}"
 fi
-
-echo "Staging envars and configuration files"
 cp -r "${CONFIG_DIR}/env" "${OPENTITAN_VAR_DIR}"
-
 mkdir -p "${OPENTITAN_VAR_DIR}/spm/config"
 cp -Rf ${CONFIG_DIR}/spm/* "${OPENTITAN_VAR_DIR}/spm/config"
+echo "Done."
 
-echo "Installing and configuring SoftHSM"
-
+################################################################################
+# Install SoftHSM2 to deployment dir and initialize it.
+################################################################################
+echo "Installing and configuring SoftHSM2 ..."
 if [ ! -d "${OPENTITAN_VAR_DIR}/softhsm2" ]; then
     mkdir -p "${OPENTITAN_VAR_DIR}/softhsm2"
     tar -xvf "${RELEASE_DIR}/softhsm_dev.tar.xz" \
         --directory "${OPENTITAN_VAR_DIR}/softhsm2"
 fi
-
 ${CONFIG_DIR}/softhsm/init.sh "${CONFIG_DIR}" \
     "${OPENTITAN_VAR_DIR}/softhsm2/softhsm2" \
     "${OPENTITAN_VAR_DIR}"
+echo "Done."
 
-echo "Unpacking release binaries"
+################################################################################
+# Unpack the infrastructure release binaries (PA, SPM, etc.).
+################################################################################
+echo "Unpacking release binaries and container images ..."
 mkdir -p "${OPENTITAN_VAR_DIR}/release"
-tar -xvf "${RELEASE_DIR}/provisioning_appliance_binaries.tar.xz" \
-    --directory "${OPENTITAN_VAR_DIR}/release"
+if [ -z "${CONTAINERS_ONLY}" ]; then
+    tar -xvf "${RELEASE_DIR}/provisioning_appliance_binaries.tar.xz" \
+        --directory "${OPENTITAN_VAR_DIR}/release"
+else
+    sudo cp "${RELEASE_DIR}/provisioning_appliance_containers.tar" \
+        "${OPENTITAN_VAR_DIR}/release/"
+    echo "Skipping unpacking raw binaries; deploying containers only ..."
+fi
+echo "Done."
 
+################################################################################
+# Load and configure infrastructure containers.
+################################################################################
+echo "Loading containers to podman local registry ..."
 # Configure podman to use the local k8s pause container.
 mkdir -p ~/.config/containers
 cat << EOF > ~/.config/containers/containers.conf
@@ -62,11 +87,15 @@ cat << EOF > ~/.config/containers/containers.conf
 infra_image = "podman_pause:latest"
 
 EOF
-
-echo "Loading containers to podman local registry"
 podman load \
     -i "${OPENTITAN_VAR_DIR}/release/provisioning_appliance_containers.tar"
+echo "Done."
 
-echo "Launching containers"
+################################################################################
+# Launch containers with podman.
+################################################################################
+echo "Launching containers ..."
 podman play kube "${CONFIG_DIR}/containers/provapp.yml" \
     --configmap "${CONFIG_DIR}/env/spm.yml"
+echo "Done."
+

run_integration_tests.sh

@@ -7,11 +7,37 @@ set -e
 
 readonly REPO_TOP=$(git rev-parse --show-toplevel)
 
+# Parse command line options.
+for i in "$@"; do
+  case $i in
+    # -c option: Only build/deploy container images, not raw binaries.
+    # Saves time when running this script if not permanently deploying infra.
+    -c|--containers-only)
+      export CONTAINERS_ONLY="yes"
+      shift
+      ;;
+    # -d option: Activate debug mode, which will not tear down containers if
+    # there is a failure so the failure can be inspected.
+    -d|--debug)
+      export DEBUG="yes"
+      shift
+      ;;
+    *)
+      echo "Unknown option $i"
+      exit 1
+      ;;
+  esac
+done
+
 # Build release binaries.
 # TODO: Build inside util/containers/build container to be able to consistently
 # reproduce the runtime environment for targets that leak outside the Bazel
 # sandbox (e.g. "@softhsm2//:softhsm2").
-bazelisk build --stamp //release:provisioning_appliance_binaries
+if [ -z "${CONTAINERS_ONLY}" ]; then
+  bazelisk build --stamp //release:provisioning_appliance_binaries
+else
+  bazelisk build --stamp //release:provisioning_appliance_containers_tar
+fi
 bazelisk build --stamp //release:softhsm_dev
 
 # Deploy the provisioning appliance services
@@ -23,13 +49,15 @@ shutdown_containers() {
   podman pod stop provapp
   podman pod rm provapp
 }
-trap shutdown_containers EXIT
+if [ -z "${DEBUG}" ]; then
+  trap shutdown_containers EXIT
+fi
 
 ${REPO_TOP}/config/dev/deploy.sh ${REPO_TOP}/bazel-bin/release
 
 bazelisk run //src/spm:spmutil -- \
-    --hsm_pw=${SPM_HSM_PIN_USER} \
-    --hsm_so=${OPENTITAN_VAR_DIR}/softhsm2/libsofthsm2.so \
+    --hsm_pw="${SPM_HSM_PIN_USER}" \
+    --hsm_so="${OPENTITAN_VAR_DIR}/softhsm2/libsofthsm2.so" \
     --hsm_type=0 \
     --hsm_slot=0 \
     --force_keygen \
@@ -39,7 +67,7 @@ bazelisk run //src/spm:spmutil -- \
     --low_sec_ks="0x23df79a8052010ef6e3d49255b606f871cff06170247c1145ebb71ad23834061" \
     --load_high_sec_ks \
     --high_sec_ks="0xaba9d5616e5a7c18b9a41d8a22f42d4dc3bafa9ca1fad01e404e708b1eab21fd" \
-    --ca_outfile=${OPENTITAN_VAR_DIR}/spm/config/certs/NuvotonTPMRootCA0200.cer
+    --ca_outfile="${OPENTITAN_VAR_DIR}/spm/config/certs/NuvotonTPMRootCA0200.cer"
 
 bazelisk run //src/pa:loadtest -- \
     --pa_address="localhost:5001" \