Add a smoke-test for controller swap

Signed-off-by: Kimmo Lehto <klehto@mirantis.com>

.github/workflows/smoke.yml

@@ -364,6 +364,34 @@ jobs:
         env:
           LINUX_IMAGE: ${{ matrix.image }}
         run: make smoke-backup-restore
+
+  smoke-controller-swap:
+    strategy:
+      matrix:
+        image:
+          - quay.io/k0sproject/bootloose-alpine3.18
+
+    name: Controller swap
+    needs: build
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          check-latest: true
+
+      - {"name":"Compiled binary cache","uses":"actions/download-artifact@v4","with":{"name":"k0sctl","path":"."}}
+      - {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
+      - {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
+      - {"name":"Make binaries executable","run":"chmod +x k0sctl || true\nchmod +x smoke-test/kubectl || true"}
+
+      - name: Run smoke tests
+        env:
+          LINUX_IMAGE: ${{ matrix.image }}
+        run: make smoke-controller-swap
 
   smoke-init:
     name: Init sub-command smoke test

Makefile

@@ -51,7 +51,7 @@ build-all: $(addprefix bin/,$(bins)) bin/checksums.md
 clean:
 	rm -rf bin/ k0sctl
 
-smoketests := smoke-basic smoke-basic-rootless smoke-files smoke-upgrade smoke-reset smoke-os-override smoke-init smoke-backup-restore smoke-dynamic smoke-basic-openssh smoke-dryrun smoke-downloadurl
+smoketests := smoke-basic smoke-basic-rootless smoke-files smoke-upgrade smoke-reset smoke-os-override smoke-init smoke-backup-restore smoke-dynamic smoke-basic-openssh smoke-dryrun smoke-downloadurl smoke-controller-swap
 .PHONY: $(smoketests)
 $(smoketests): k0sctl
 	$(MAKE) -C smoke-test $@

smoke-test/Makefile

@@ -55,5 +55,8 @@ smoke-downloadurl: $(bootloose) id_rsa_k0s k0sctl
 smoke-backup-restore: $(bootloose) id_rsa_k0s k0sctl
 	./smoke-backup-restore.sh
 
+smoke-controller-swap: $(bootloose) id_rsa_k0s k0sctl
+	BOOTLOOSE_TEMPLATE=bootloose-controller-swap.yaml.tpl K0SCTL_CONFIG=k0sctl-controller-swap.yaml ./smoke-controller-swap.sh
+
 %.iid: Dockerfile.%
 	docker build --iidfile '$@' - < '$<'

smoke-test/bootloose-controller-swap.yaml.tpl

@@ -0,0 +1,23 @@
+cluster:
+  name: k0s
+  privateKey: ./id_rsa_k0s
+machines:
+- count: 3
+  backend: docker
+  spec:
+    image: $LINUX_IMAGE
+    name: manager%d
+    privileged: true
+    volumes:
+    - type: bind
+      source: /lib/modules
+      destination: /lib/modules
+    - type: volume
+      destination: /var/lib/k0s
+    portMappings:
+    - containerPort: 22
+      hostPort: 9022
+    - containerPort: 443
+      hostPort: 443
+    - containerPort: 6443
+      hostPort: 6443

smoke-test/k0sctl-controller-swap.yaml

@@ -0,0 +1,29 @@
+apiVersion: k0sctl.k0sproject.io/v1beta1
+kind: cluster
+spec:
+  hosts:
+    - role: controller
+      uploadBinary: true
+      ssh:
+        address: "127.0.0.1"
+        port: 9022
+        keyPath: ./id_rsa_k0s
+    - role: controller
+      uploadBinary: true
+      ssh:
+        address: "127.0.0.1"
+        port: 9023
+        keyPath: ./id_rsa_k0s
+    - role: controller
+      uploadBinary: true
+      ssh:
+        address: "127.0.0.1"
+        port: 9024
+        keyPath: ./id_rsa_k0s
+  k0s:
+    version: "${K0S_VERSION}"
+    config:
+      spec:
+        telemetry:
+          enabled: false
+

smoke-test/smoke-controller-swap.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env sh
+
+K0SCTL_CONFIG=${K0SCTL_CONFIG:-"k0sctl-controller-swap.yaml"}
+
+set -ex
+
+
+. ./smoke.common.sh
+trap cleanup EXIT
+
+deleteCluster
+createCluster
+
+echo "* Starting apply"
+../k0sctl apply --config "${K0SCTL_CONFIG}" --debug
+echo "* Apply OK"
+
+echo "* Get the ip of the last controller"
+controllerip=$(bootloose show manager2 -o json | grep '"ip"' | head -1 | cut -d'"' -f4)
+
+echo "* Wipe controller 3"
+docker rm -fv "$(bootloose show manager2 -o json | grep '"container"' | head -1 | cut -d'"' -f4)"
+
+echo "* Verify its gone"
+bootloose show manager2 | grep "Not created"
+
+echo "* Recreate controller2"
+createCluster
+
+echo "* Verify its back and IP is the same"
+bootloose show manager2 | grep "Running"
+newip=$(bootloose show manager2 -o json | grep '"ip"' | head -1 | cut -d'"' -f4)
+if [ "$controllerip" != "$newip" ]; then
+  echo "IP mismatch: $controllerip != $newip - ip should get reused"
+  exit 1
+fi
+
+echo "* Re-apply should fail because of known hosts"
+if ../k0sctl apply --config "${K0SCTL_CONFIG}" --debug; then
+  echo "Re-apply should have failed because of known hosts"
+  exit 1
+fi
+
+echo "* Clear known hosts"
+truncate -s 0 ~/.ssh/known_hosts
+
+echo "* Re-apply should fail because of replaced controller"
+if ../k0sctl apply --config "${K0SCTL_CONFIG}" --debug; then
+  echo "Re-apply should have failed because of replaced controller"
+  exit 1
+fi
+
+echo "* Perform etcd member removal"
+bootloose ssh root@manager0 -- k0s etcd leave --peer-address "$controllerip"
+
+echo "* Re-apply should succeed"
+../k0sctl apply --config "${K0SCTL_CONFIG}" --debug
+
+echo "* Done"