Skip to content

Commit

Permalink
Add a dry-run smoke-test
Browse files Browse the repository at this point in the history
Signed-off-by: Kimmo Lehto <klehto@mirantis.com>
  • Loading branch information
kke committed Nov 6, 2023
1 parent b1c5de7 commit 4699fc3
Show file tree
Hide file tree
Showing 7 changed files with 177 additions and 14 deletions.
1 change: 1 addition & 0 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ on:
- '**.go'
- go.mod
- go.sum
- Makefile

jobs:
gotest:
Expand Down
22 changes: 11 additions & 11 deletions .github/workflows/smoke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run smoke tests
Expand All @@ -82,7 +82,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Build image
Expand Down Expand Up @@ -112,7 +112,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run smoke tests
Expand Down Expand Up @@ -140,7 +140,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run smoke tests
Expand All @@ -165,7 +165,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run smoke tests
Expand All @@ -186,7 +186,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run OS override smoke test
Expand Down Expand Up @@ -216,7 +216,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run smoke tests
Expand Down Expand Up @@ -247,7 +247,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run smoke tests
Expand Down Expand Up @@ -277,7 +277,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run smoke tests
Expand Down Expand Up @@ -307,7 +307,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run smoke tests
Expand All @@ -330,7 +330,7 @@ jobs:

- {"name":"Compiled binary cache","uses":"actions/download-artifact@v3","with":{"name":"k0sctl","path":"."}}
- {"name":"K0sctl cache","uses":"actions/cache@v3","with":{"path":"/var/cache/k0sctl/k0s\n~/.cache/k0sctl/k0s\n","key":"k0sctl-cache"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh) }}","restore-keys":"kubectl-"}}
- {"name":"Kubectl cache","uses":"actions/cache@v3","with":{"path":"smoke-test/kubectl\n","key":"kubectl-${{ hashFiles('smoke-test/smoke.common.sh') }}","restore-keys":"kubectl-"}}
- {"name":"Make binaries executable","run":"chmod +x k0sctl\nchmod +x smoke-test/kubectl"}

- name: Run init smoke test
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ build-all: $(addprefix bin/,$(bins)) bin/checksums.md
clean:
rm -rf bin/ k0sctl

smoketests := smoke-basic smoke-files smoke-upgrade smoke-reset smoke-os-override smoke-init smoke-backup-restore smoke-dynamic smoke-basic-openssh
smoketests := smoke-basic smoke-files smoke-upgrade smoke-reset smoke-os-override smoke-init smoke-backup-restore smoke-dynamic smoke-basic-openssh smoke-dryrun
.PHONY: $(smoketests)
$(smoketests): k0sctl
$(MAKE) -C smoke-test $@
Expand Down
4 changes: 2 additions & 2 deletions phase/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,9 @@ func (m *Manager) Run() error {

fmt.Println(Colorize.BrightRed("dry-run: cluster state altering actions would be performed:"))
for host, msgs := range m.dryMessages {
fmt.Println(Colorize.Bold(fmt.Sprintf("* %s :", host)))
fmt.Println(Colorize.BrightRed("dry-run:"), Colorize.Bold(fmt.Sprintf("* %s :", host)))
for _, msg := range msgs {
fmt.Println(Colorize.Red(" -"), msg)
fmt.Println(Colorize.BrightRed("dry-run:"), Colorize.Red(" -"), msg)
}
}
return
Expand Down
3 changes: 3 additions & 0 deletions smoke-test/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ smoke-init: $(bootloose) id_rsa_k0s k0sctl
smoke-upgrade: $(bootloose) id_rsa_k0s k0sctl
./smoke-upgrade.sh

smoke-dryrun: $(bootloose) id_rsa_k0s k0sctl
./smoke-dryrun.sh

smoke-reset: $(bootloose) id_rsa_k0s k0sctl
./smoke-reset.sh

Expand Down
24 changes: 24 additions & 0 deletions smoke-test/k0sctl-dryrun.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
apiVersion: k0sctl.k0sproject.io/v1beta1
kind: cluster
spec:
hosts:
- role: controller
uploadBinary: true
os: "$OS_OVERRIDE"
ssh:
address: "127.0.0.1"
port: 9022
keyPath: ./id_rsa_k0s
- role: worker
uploadBinary: true
os: "$OS_OVERRIDE"
ssh:
address: "127.0.0.1"
port: 9023
keyPath: ./id_rsa_k0s
k0s:
version: "${K0S_VERSION}"
config:
spec:
telemetry:
enabled: false
135 changes: 135 additions & 0 deletions smoke-test/smoke-dryrun.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
#!/usr/bin/env bash

# Default values for environment variables
K0SCTL_CONFIG=${K0SCTL_CONFIG:-"k0sctl-dryrun.yaml"}
K0S_FROM=${K0S_FROM:-"v1.21.6+k0s.0"}
K0S_TO=${K0S_TO:-"$(curl -s "https://docs.k0sproject.io/stable.txt")"}

log="smoke-dryrun.log"

# Source common functions
. ./smoke.common.sh
trap cleanup EXIT

# Define functions
remoteCommand() {
local userhost="$1"
shift
bootloose ssh "${userhost}" -- "$*"
}

colorEcho() {
local color=$1
shift
echo -e "\033[1;3${color}m************************************************************\033[0m"
echo -e "\033[1;3${color}m$*\033[0m"
echo -e "\033[1;3${color}m************************************************************\033[0m"
}

checkDryRunLines() {
local mode=$1
local expected=$2
local count
count=$(grep -c "dry-run" "${log}")
case "${mode}" in
min)
if [ "${count}" -lt "${expected}" ]; then
colorEcho 1 "Expected at least ${expected} dry-run lines, got ${count}"
exit 1
fi
;;
none)
if [ "${count}" -ne 0 ]; then
colorEcho 1 "Expected zero dry-run lines, got ${count}"
exit 1
fi
;;
*)
echo "Unknown mode for checkDryRunLines"
exit 1
;;
esac
}

dryRunNoChanges() {
if ! grep -q "no cluster state altering actions" "${log}"; then
colorEcho 1 "Expected dry-run to have no changes"
exit 1
fi
}

dumpDryRunLines() {
colorEcho 2 "Dry-run filtered log:"
grep "dry-run" "${log}"
}

expectK0sVersion() {
local expected=$1
local remote
remote=$(remoteCommand "root@manager0" "k0s version")
if [ "${remote}" != "${expected}" ]; then
colorEcho 1 "Expected k0s version ${expected}, got ${remote}"
exit 1
fi
}

expectNoK0s() {
echo "Expecting no k0s on controller"
if remoteCommand "root@manager0" "test -d /etc/k0s"; then
colorEcho 1 "Expected no /etc/k0s on controller"
exit 1
fi
if remoteCommand "root@manager0" "test -f /etc/k0s/k0s.yaml"; then
colorEcho 1 "Expected no /etc/k0s/k0s.yaml on controller"
exit 1
fi
if remoteCommand "root@manager0" "ps -ef" | grep -q "k0s controller"; then
colorEcho 1 "Expected no k0s controller process on controller"
exit 1
fi
}

applyConfig() {
local extra_flag=$1
../k0sctl apply --config "${K0SCTL_CONFIG}" --debug "${extra_flag}" | tee "${log}"
}

deleteCluster
createCluster

K0S_VERSION="${K0S_FROM}"

colorEcho 3 "Installing ${K0S_VERSION} with --dry-run"
applyConfig "--dry-run"
expectNoK0s
checkDryRunLines min 3
dumpDryRunLines

colorEcho 3 "Installing ${K0S_VERSION}"
applyConfig
expectK0sVersion "${K0S_FROM}"
checkDryRunLines none

colorEcho 3 "Installing ${K0S_VERSION} with --dry-run again"
applyConfig "--dry-run"
expectK0sVersion "${K0S_FROM}"
dryRunNoChanges

colorEcho 4 "Succesfully installed ${K0S_FROM}, moving on to upgrade to ${K0S_TO}"
K0S_VERSION="${K0S_TO}"

colorEcho 3 "Upgrading to ${K0S_VERSION} with --dry-run"
applyConfig "--dry-run"
expectK0sVersion "${K0S_FROM}"
checkDryRunLines min 3
dumpDryRunLines

colorEcho 3 "Upgrading to ${K0S_VERSION}"
applyConfig
expectK0sVersion "${K0S_TO}"
checkDryRunLines none

colorEcho 3 "Upgrading to ${K0S_VERSION} with --dry-run again"
applyConfig "--dry-run"
expectK0sVersion "${K0S_TO}"
dryRunNoChanges

0 comments on commit 4699fc3

Please sign in to comment.