diff --git a/Makefile b/Makefile index b75e427a6..5a193c0b5 100644 --- a/Makefile +++ b/Makefile @@ -145,6 +145,7 @@ kind-e2e-image: docker-build .PHONY: e2e-test e2e-test: export KUBECONFIG = $(KIND_KUBECONFIG) +e2e-test: export BATS_FILES := $(BATS_FILES) e2e-test: e2e-setup docker-build install ## Run the e2e tests @$(e2e_make) test diff --git a/Makefile.vars.mk b/Makefile.vars.mk index 282990f58..7645b3605 100644 --- a/Makefile.vars.mk +++ b/Makefile.vars.mk @@ -1,7 +1,8 @@ IMG_TAG ?= latest -BIN_FILENAME ?= $(PROJECT_ROOT_DIR)/k8up -TESTBIN_DIR ?= $(PROJECT_ROOT_DIR)/testbin/bin +CURDIR ?= $(shell pwd) +BIN_FILENAME ?= $(CURDIR)/$(PROJECT_ROOT_DIR)/k8up +TESTBIN_DIR ?= $(CURDIR)/$(PROJECT_ROOT_DIR)/testbin/bin CRD_FILE ?= k8up-crd.yaml CRD_FILE_LEGACY ?= k8up-crd-legacy.yaml @@ -24,6 +25,7 @@ SHASUM ?= $(shell command -v sha1sum > /dev/null && echo "sha1sum" || echo "shas E2E_TAG ?= e2e_$(shell $(SHASUM) $(BIN_FILENAME) | cut -b-8) E2E_REPO ?= local.dev/k8up/e2e E2E_IMG = $(E2E_REPO):$(E2E_TAG) +BATS_FILES ?= . INTEGRATION_TEST_DEBUG_OUTPUT ?= false diff --git a/README.md b/README.md index 6b3254014..5183efee9 100644 --- a/README.md +++ b/README.md @@ -36,11 +36,13 @@ You'll need: To run the end-to-end test (e.g. `make e2e-test`), you additionally need: -- node / npm -- bash (installed, doesn't have to be your default shell) -- shasum / sha1sum -- helm 3 -- jq +- `helm` (version 3) +- `jq` +- `node` and `npm` +- `bash` (installed, doesn't have to be your default shell) +- `shasum` or `sha1sum` +- `base64` +- `find` These are the most common make targets: `build`, `test`, `docker-build`, `run`, `kind-run`. Run `make help` to get an overview over the relevant targets and their intentions. @@ -78,7 +80,7 @@ K8up supports both OpenShift 3.11 clusters and newer Kubernetes clusters 1.16+. However, to support OpenShift 3.11 a legacy CRD definition with `apiextensions.k8s.io/v1beta1` is needed, while K8s 1.22+ only supports `apiextensions.k8s.io/v1`. You need `node` and `npm` to run the tests, as it runs with [DETIK][detik]. -To run e2e tests run: +To run e2e tests, execute: ```bash make e2e-test @@ -90,7 +92,19 @@ To test compatibility of k8up with OpenShift 3.11 (or any other specific K8s ver make e2e-test -e CRD_SPEC_VERSION=v1beta1 -e KIND_NODE_VERSION=v1.13.12 -e KIND_KUBECTL_ARGS=--validate=false -e BACKUP_ENABLE_LEADER_ELECTION=false ``` -To remove the local KIND cluster and other resources, run +To test just a specific e2e test, run: + +```bash +make e2e-test -e BATS_FILES=test-00-deployment.bats +``` + +To remove the local KIND cluster and other e2e resources, run: + +```bash +make e2e-clean +``` + +To cleanup all created artifacts, there's always: ```bash make clean diff --git a/docs/modules/ROOT/pages/references/object-specifications.adoc b/docs/modules/ROOT/pages/references/object-specifications.adoc index 306d755e4..806362868 100644 --- a/docs/modules/ROOT/pages/references/object-specifications.adoc +++ b/docs/modules/ROOT/pages/references/object-specifications.adoc @@ -108,7 +108,7 @@ This will restore the latest snapshot from `\http://10.144.1.224:9000` to the PV * `backend`: see <> for further explanation * `restoreMethod`: is either `s3` or `folder`. For s3 please see `backend` for `folder` you just need to provide a valid claim name as shown in the example above -* `restoreFiler`: a filter passed to the underlying Restic, which will be used. Please consult the https://restic.readthedocs.io/en/latest/050_restore.html[Restic docs] for valid path filters. +* `restoreFilter`: a filter passed to the underlying Restic, which will be used. Please consult the https://restic.readthedocs.io/en/latest/050_restore.html[Restic docs] for valid path filters. * `snapshot`: valid snapshot ID that should get restored. If not provided, the most recent one will be restored. * `keepJobs`: amount of jobs that should be left after cleanup, for example how many job/pod objects should be left after they finished. Defaults to 6. Only applicable when used within a <>. * `tags`: list of tags to be considered for the restore. They're ignored if a snapshot ID is provided. diff --git a/e2e/Makefile b/e2e/Makefile index a867cb2c0..bc3bd0ebd 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -28,7 +28,7 @@ test: export KUBECONFIG = $(KIND_KUBECONFIG) test: export E2E_IMAGE = $(E2E_IMG) test: export WRESTIC_IMAGE = $(WRESTIC_IMG) test: setup kind-e2e-image ## Run the E2E tests - @$(bats) . $(bats_args) + @$(bats) $(bats_args) $(BATS_FILES) kind-e2e-image: kind-setup ## Load the e2e container image onto e2e cluster @$(KIND) load docker-image --name $(KIND_CLUSTER) $(E2E_IMG) diff --git a/e2e/lib/k8up.bash b/e2e/lib/k8up.bash index 1e7af6c3e..6181a793b 100644 --- a/e2e/lib/k8up.bash +++ b/e2e/lib/k8up.bash @@ -1,5 +1,7 @@ #!/bin/bash +export MINIO_NAMESPACE=${MINIO_NAMESPACE-minio} + errcho() { >&2 echo "${@}" } @@ -29,10 +31,11 @@ kustomize() { } restic() { - kubectl run wrestic \ + kubectl run "wrestic" \ --rm \ --attach \ --restart Never \ + --wait \ --namespace "${DETIK_CLIENT_NAMESPACE-"k8up-system"}" \ --image "${WRESTIC_IMAGE}" \ --env "AWS_ACCESS_KEY_ID=myaccesskey" \ @@ -48,6 +51,26 @@ restic() { --json } +mc() { + minio_access_key=$(kubectl -n "${MINIO_NAMESPACE}" get secret minio -o jsonpath="{.data.accesskey}" | base64 --decode) + minio_secret_key=$(kubectl -n "${MINIO_NAMESPACE}" get secret minio -o jsonpath="{.data.secretkey}" | base64 --decode) + minio_url=http://${minio_access_key}:${minio_secret_key}@minio.minio.svc.cluster.local:9000 + kubectl run "minio" \ + --rm \ + --attach \ + --stdin \ + --restart Never \ + --wait \ + --namespace "${DETIK_CLIENT_NAMESPACE-"k8up-system"}" \ + --image "${MINIO_IMAGE-minio/mc:latest}" \ + --env "MC_HOST_s3=${minio_url}" \ + --pod-running-timeout 10s \ + --quiet=true \ + --command -- \ + mc \ + "${@}" +} + replace_in_file() { VAR_NAME=${1} VAR_VALUE=${2} @@ -74,11 +97,16 @@ apply() { kubectl apply -f "debug/${1}/main.yml" } -given_a_subject() { - kubectl delete namespace k8up-e2e-subject --ignore-not-found +given_a_clean_ns() { + kubectl delete namespace "${DETIK_CLIENT_NAMESPACE}" --ignore-not-found kubectl delete pv subject-pv --ignore-not-found - kubectl create namespace k8up-e2e-subject || true + kubectl create namespace "${DETIK_CLIENT_NAMESPACE}" + echo "✅ The namespace '${DETIK_CLIENT_NAMESPACE}' is ready." +} + +given_a_subject() { apply definitions/subject + echo "✅ The subject is ready" } given_s3_storage() { @@ -87,16 +115,24 @@ given_s3_storage() { helm upgrade --install minio \ --values definitions/minio/helm.yaml \ --create-namespace \ - --namespace minio \ + --namespace "${MINIO_NAMESPACE}" \ minio/minio + + echo "✅ S3 Storage is ready" } -given_running_operator() { +given_a_running_operator() { apply definitions/k8up + + NAMESPACE=k8up-system \ + wait_until deployment/k8up-operator available + echo "✅ A running operator is ready" } wait_until() { object=${1} condition=${2} - kubectl -n "${DETIK_CLIENT_NAMESPACE}" wait --timeout 1m --for "condition=${condition}" "${object}" + ns=${NAMESPACE=${DETIK_CLIENT_NAMESPACE}} + echo "Waiting for '${object}' in namespace '${ns}' to become '${condition}' ..." + kubectl -n "${ns}" wait --timeout 1m --for "condition=${condition}" "${object}" } diff --git a/e2e/test-01-backup.bats b/e2e/test-01-backup.bats index e879ae0ff..8660434b6 100644 --- a/e2e/test-01-backup.bats +++ b/e2e/test-01-backup.bats @@ -12,7 +12,8 @@ DETIK_CLIENT_NAMESPACE="k8up-e2e-subject" DEBUG_DETIK="true" @test "verify a backup" { - given_running_operator + given_a_running_operator + given_a_clean_ns given_s3_storage given_a_subject @@ -25,6 +26,8 @@ DEBUG_DETIK="true" run restic snapshots 2>/dev/null + echo "Restic output: '${output}'" + echo -n "Number of Snapshots >= 1? " jq -e 'length >= 1' <<< "${output}" # Ensure that there was actually a backup created }