Skip to content

Commit

Permalink
ci: Add test cases for CoCo image pulling without forked containerd
Browse files Browse the repository at this point in the history
Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo.

Fixes kata-containers#5763

Depends:
kata-containers/kata-containers#7688
kata-containers/kata-containers#7676

Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>
  • Loading branch information
ChengyuZhu6 committed Sep 8, 2023
1 parent 021bf58 commit 6fb1af5
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 2 deletions.
69 changes: 68 additions & 1 deletion integration/confidential/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"

CONTAINERD_CONFIG="/etc/containerd/config.toml"
NYDUS_SNAPSHOTTER_BINARY="/home/zcy/workspace/image_sharing/bin/containerd-nydus-grpc"
NYDUS_SNAPSHOTTER_CONFIG="/etc/nydus/config-tarfs.toml"
NYDUS_SNAPSHOTTER_TARFS_CONFIG="/etc/nydus/config-tarfs.toml"
NYDUS_SNAPSHOTTER_GUEST_CONFIG="/etc/nydus/config-guest.toml"
# Toggle between true and false the service_offload configuration of
# the Kata agent.
#
Expand Down Expand Up @@ -440,3 +444,66 @@ EOF
EOF
fi
}

###############################################################################

# remote-snapshotter

configure_remote_snapshotter() {
case "${SNAPSHOTTER:-}" in
"nydus")
configure_nydus_snapshotter
;;
*) ;;

esac
}
check_containerd_version() {
containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
if echo $containerd_version | grep -q "^v1.7"; then
return 1
else
return 0
fi
}
configure_containerd_for_nydus_snapshotter() {
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG"
if check_containerd_version; then
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n' "$CONTAINERD_CONFIG"
else
sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
fi
}
remove_nydus_snapshotter_from_containerd() {
sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG"
if check_containerd_version; then
sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG"
else
sudo sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
fi
}
remove_test_image() {
local test_image = "$1"
sudo crictl rmi "$1"
pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
sudo crictl rmi "$pause_name"
}

restart_nydus_snapshotter() {
echo "Kill nydus snapshotter"
bin="containerd-nydus-grpc"
sudo kill -9 $(pidof $bin) || true
echo "Restart nydus snapshotter"
sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" &
}

configure_nydus_snapshotter() {
echo "Configure nydus snapshotter"
if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
else
NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
fi
sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
restart_nydus_snapshotter
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
metadata:
name: nydus-container$INDEX
image:
image: $IMAGE
log_path: container.1.log
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
apiVersion: v1
kind: Pod
metadata:
name: busybox-cc
name: busybox-cc$INDEX
spec:
runtimeClassName: $RUNTIMECLASS
containers:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
#!/usr/bin/env bats
# Copyright (c) 2022 IBM Corporation
#
# SPDX-License-Identifier: Apache-2.0
#

load "${BATS_TEST_DIRNAME}/lib.sh"
load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"

tag_suffix=""
if [ "$(uname -m)" != "x86_64" ]; then
tag_suffix="-$(uname -m)"
fi

# Images used on the tests.

image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"

original_kernel_params=$(get_kernel_params)
# Allow to configure the runtimeClassName on pod configuration.
RUNTIMECLASS="${RUNTIMECLASS:-kata}"
test_tag="[cc][agent][kubernetes][containerd]"

# Create the test pod.
#
# Note: the global $sandbox_name, $pod_config should be set
# already. It also relies on $CI and $DEBUG exported by CI scripts or
# the developer, to decide how to set debug flags.
#
create_test_pod() {
local pod_config="$1"

echo "Create the test sandbox"
echo "Pod config is: $pod_config"
crictl crictl run --with-pull -r kata-qemu $pod_config nydus-sandbox.yaml
}

# Create a pod configuration out of a template file.
#
# Parameters:
# $1 - the container image.
# Return:
# the path to the configuration file. The caller should not care about
# its removal afterwards as it is created under the bats temporary
# directory.
#
# Environment variables:
# RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS.
#
new_pod_config() {
local base_config="${FIXTURES_DIR}/cri-pod-config.yaml.in"
local image="$1"
local index="$2"

local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$2" envsubst <"$base_config" >"$new_config"
echo "$new_config"
}

setup() {
start_date=$(date +"%Y-%m-%d %H:%M:%S")
setup_proxy
switch_measured_rootfs_verity_scheme none
}

@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
if [ "$SNAPSHOTTER" = "nydus" ]; then
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
pod_config="$(new_pod_config "$image_unsigned_unprotected")"
echo $pod_config
create_test_pod "$pod_config"
remove_test_image "$image_unsigned_unprotected"
fi
}

@test "$test_tag Test can create two pods with pulling the image only once" {
if [ "$SNAPSHOTTER" = "nydus" ]; then
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter

pod_config_1="$(new_pod_config "$image_unsigned_unprotected" "1")"
echo $pod_config_1
create_test_pod $pod_config_1
pod_config_2="$(new_pod_config "$image_unsigned_unprotected" "2")"
echo $pod_config_2
create_test_pod $pod_config_2

pull_times=$(journalctl -g "PullImage \"$image_unsigned_unprotected\" with snapshotter nydus" | wc -l)
[ ${#pull_times[@]} -eq 1 ]
remove_test_image "$image_unsigned_unprotected"
fi
}

@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" {
skip
switch_image_service_offload on
if [ "$SNAPSHOTTER" = "nydus" ]; then
EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
create_test_pod
remove_test_image "$image_unsigned_unprotected"
fi
}

teardown() {
remove_nydus_snapshotter_from_containerd
}

0 comments on commit 6fb1af5

Please sign in to comment.