ci: build and use temporary bastion for each vsphere CI test run (#850)

* ci: github action workflow to create vsphere bastion host

* deprecate scripts that creates permanent bastion

* ci: allow overriding bastion ssh port

* ci: refactor scripts to create bastion host

* test using airgapped bastion

* ci: use temp bastion host for vsphere e2e tests

* use datastore input from github action secrets

* correct public key reference

* update vsphre release flow to use temporary bastion

* fix linter errors

.github/workflows/release-vsphere-template.yaml

@@ -59,17 +59,36 @@ jobs:
         with:
           ssh-private-key: ${{ secrets.SSH_BASTION_KEY_CONTENTS }}
 
+      # configure git to access private repo hosting vsphere module mesosphere/vcenter-tools
+      - name: Configure git to clone private registry from mesosphere org
+        run: |
+          git config --global url."https://git:${{ secrets.MESOSPHERECI_USER_TOKEN }}@github.com/mesosphere".insteadOf "https://github.com/mesosphere"
+          git config --global url."https://${{ secrets.MESOSPHERECI_USER_TOKEN }}:x-oauth-basic@github.com/mesosphere".insteadOf ssh://git@github.com/mesosphere
+
       - name: Build vSphere template for ${{ matrix.os }} with ${{ matrix.buildConfig }} configuration
         uses: magefile/mage-action@v2
         with:
           version: latest
           args: runE2e "${{ matrix.os }}" "${{ matrix.buildConfig }}" ova false
         env:
           SSH_BASTION_KEY_CONTENTS: ${{ secrets.SSH_BASTION_KEY_CONTENTS }}
-          SSH_BASTION_HOST: ${{ secrets.SSH_BASTION_HOST }}
-          SSH_BASTION_USERNAME: ${{ secrets.SSH_BASTION_USERNAME }}
+          SSH_BASTION_PUBLIC_KEY_CONTENTS: ${{ secrets.SSH_BASTION_PUBLIC_KEY_CONTENTS }}
+          VSPHERE_USERNAME: ${{ secrets.VSPHERE_USERNAME }}
+          VSPHERE_USER: ${{ secrets.VSPHERE_USERNAME }} # required for terraform
+          VSPHERE_PASSWORD: ${{ secrets.VSPHERE_PASSWORD }}
+          GITHUB_TOKEN: ${{ secrets.MESOSPHERECI_USER_TOKEN }}
+          VSPHERE_SERVER: ${{ secrets.VSPHERE_SERVER }}
+          VSPHERE_DATASTORE: ${{ secrets.VSPHERE_DATASTORE }}
+
+      - name: Run make destroy to clean up failed tests
+        if: ${{ always() }}
+        run: make infra.vsphere.destroy || true
+        env:
+          SSH_BASTION_PUBLIC_KEY_CONTENTS: ${{ secrets.SSH_BASTION_PUBLIC_KEY_CONTENTS }}
           VSPHERE_USERNAME: ${{ secrets.VSPHERE_USERNAME }}
+          VSPHERE_USER: ${{ secrets.VSPHERE_USERNAME }} # required for terraform
           VSPHERE_PASSWORD: ${{ secrets.VSPHERE_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.MESOSPHERECI_USER_TOKEN }}
           VSPHERE_SERVER: ${{ secrets.VSPHERE_SERVER }}
           VSPHERE_DATASTORE: ${{ secrets.VSPHERE_DATASTORE }}
+          VSPHERE_DATACENTER: ${{ secrets.VSPHERE_DATACENTER }}

.github/workflows/vsphere-e2e.yaml

@@ -66,6 +66,12 @@ jobs:
         uses: webfactory/ssh-agent@v0.8.0
         with:
           ssh-private-key: ${{ secrets.SSH_BASTION_KEY_CONTENTS }}
+
+      # configure git to access private repo hosting vsphere module mesosphere/vcenter-tools
+      - name: Configure git to clone private registry from mesosphere org
+        run: |
+          git config --global url."https://git:${{ secrets.MESOSPHERECI_USER_TOKEN }}@github.com/mesosphere".insteadOf "https://github.com/mesosphere"
+          git config --global url."https://${{ secrets.MESOSPHERECI_USER_TOKEN }}:x-oauth-basic@github.com/mesosphere".insteadOf ssh://git@github.com/mesosphere
 
       - name: Run E2E test for ${{ matrix.os }} with ${{ matrix.buildConfig }} configuration
         uses: magefile/mage-action@v2
@@ -74,11 +80,24 @@ jobs:
           args: runE2e "${{ matrix.os }}" "${{ matrix.buildConfig }}" ova true
         env:
           SSH_BASTION_KEY_CONTENTS: ${{ secrets.SSH_BASTION_KEY_CONTENTS }}
-          SSH_BASTION_HOST: ${{ secrets.SSH_BASTION_HOST }}
-          SSH_BASTION_USERNAME: ${{ secrets.SSH_BASTION_USERNAME }}
+          SSH_BASTION_PUBLIC_KEY_CONTENTS: ${{ secrets.SSH_BASTION_PUBLIC_KEY_CONTENTS }}
           VSPHERE_USERNAME: ${{ secrets.VSPHERE_USERNAME }}
+          VSPHERE_USER: ${{ secrets.VSPHERE_USERNAME }} # required for terraform
           VSPHERE_PASSWORD: ${{ secrets.VSPHERE_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.MESOSPHERECI_USER_TOKEN }}
           VSPHERE_SERVER: ${{ secrets.VSPHERE_SERVER }}
           VSPHERE_DATASTORE: ${{ secrets.VSPHERE_DATASTORE }}
           VSPHERE_DATACENTER: ${{ secrets.VSPHERE_DATACENTER }}
+
+      - name: Run make destroy to clean up failed tests
+        if: ${{ always() }}
+        run: make infra.vsphere.destroy || true
+        env:
+          SSH_BASTION_PUBLIC_KEY_CONTENTS: ${{ secrets.SSH_BASTION_PUBLIC_KEY_CONTENTS }}
+          VSPHERE_USERNAME: ${{ secrets.VSPHERE_USERNAME }}
+          VSPHERE_USER: ${{ secrets.VSPHERE_USERNAME }} # required for terraform
+          VSPHERE_PASSWORD: ${{ secrets.VSPHERE_PASSWORD }}
+          GITHUB_TOKEN: ${{ secrets.MESOSPHERECI_USER_TOKEN }}
+          VSPHERE_SERVER: ${{ secrets.VSPHERE_SERVER }}
+          VSPHERE_DATASTORE: ${{ secrets.VSPHERE_DATASTORE }}
+          VSPHERE_DATACENTER: ${{ secrets.VSPHERE_DATACENTER }}

Makefile

@@ -156,8 +156,7 @@ $(ENVSUBST_ASSETS)/envsubst:
 
 
 include hack/pip-packages/Makefile
-include test/infra/aws/Makefile
-include test/infra/vsphere/Makefile
+include test/infra/Makefile
 
 github-token.txt:
 	echo $(GITHUB_TOKEN) >> github-token.txt

pkg/packer/manifests/vsphere/packer.pkr.hcl

@@ -194,6 +194,12 @@ variable "ssh_bastion_host" {
   type = string
   default = ""
 }
+
+variable "ssh_bastion_port" {
+  type = string
+  default = "22"
+}
+
 variable "ssh_bastion_password" {
   type = string
   default = ""
@@ -362,6 +368,7 @@ locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
 locals {
   build_timestamp              = local.timestamp
   ssh_bastion_host             = var.ssh_bastion_host
+  ssh_bastion_port             = var.ssh_bastion_port
   ssh_bastion_password         = var.ssh_bastion_password
   ssh_bastion_private_key_file = var.ssh_bastion_private_key_file
   ssh_bastion_username         = var.ssh_bastion_username
@@ -416,6 +423,7 @@ source "vsphere-clone" "kib_image" {
   password                     = var.vsphere_password
   ssh_agent_auth               = var.ssh_agent_auth
   ssh_bastion_host             = local.ssh_bastion_host
+  ssh_bastion_port             = local.ssh_bastion_port
   ssh_bastion_password         = local.ssh_bastion_password
   ssh_bastion_private_key_file = local.ssh_bastion_private_key_file
   ssh_bastion_username         = local.ssh_bastion_username

test/infra/aws/Makefile → test/infra/Makefile

@@ -11,6 +11,15 @@ export TERRAFORM_BIN = $(TERRAFORM_ASSETS)/terraform
 TEST_MODULES_DIR = $(CURDIR)/test/infra
 TEST_INFRA_STATE_DIR ?= $(CURDIR)/.local/infra
 
+# export for local testing
+SSH_BASTION_PUBLIC_KEY_FILE ?= ""
+VSPHERE_DATASTORE ?= "ci-kib"
+
+ifeq ($(CI),true)
+  $(shell echo "$(SSH_BASTION_PUBLIC_KEY_CONTENTS)" > $(CURDIR)/vsphere-bastion.pub)
+  SSH_BASTION_PUBLIC_KEY_FILE = $(CURDIR)/vsphere-bastion.pub
+endif
+
 .PHONY: install-terraform
 install-terraform: ## Download and unpack terraform binary
 install-terraform: $(TERRAFORM_BIN)
@@ -30,19 +39,48 @@ infra.aws.init: install-terraform
 	$(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/aws init -from-module=$(TEST_MODULES_DIR)/aws -force-copy || true
 	cd $(TEST_INFRA_STATE_DIR)/aws && $(TEST_INFRA_STATE_DIR)/aws/init.sh
 
+.PHONY: infra.vsphere.init
+infra.vsphere.init: ## Initialize infrastructure metadata
+infra.vsphere.init: install-terraform
+	$(call print-target)
+	mkdir -p $(TEST_INFRA_STATE_DIR)/vsphere
+	$(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/vsphere init -from-module=$(TEST_MODULES_DIR)/vsphere -force-copy || true
+
 .PHONY: infra.aws.create
 infra.aws.create: ## Create infrastructure
-infra.aws.create: install-terraform infra.aws.init
+infra.aws.create: infra.aws.init
 	$(call print-target)
 	TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/aws plan
 	TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/aws apply -auto-approve
 
+PHONY: infra.vsphere.create
+infra.vsphere.create: ## Create infrastructure
+infra.vsphere.create: TF_VAR_ssh_public_key=$(SSH_BASTION_PUBLIC_KEY_FILE)
+infra.vsphere.create: TF_VAR_datastore_name=$(VSPHERE_DATASTORE)
+infra.vsphere.create: infra.vsphere.init
+	$(call print-target)
+	TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/vsphere plan
+	TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/vsphere apply -auto-approve
+
 packer-aws-offline-override.yaml: infra.aws.create install-envsubst
 	@$(TEST_INFRA_STATE_DIR)/aws/export.sh $@
 
+packer-ova-offline-override.yaml: infra.vsphere.create install-envsubst
+	@$(TEST_INFRA_STATE_DIR)/vsphere/export.sh $@
+
 .PHONY: infra.aws.destroy
 infra.aws.destroy: ## Destroy infrastructure
 infra.aws.destroy: install-terraform infra.aws.init
 	$(call print-target)
 	TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/aws destroy -auto-approve
 	rm -r $(TEST_INFRA_STATE_DIR)/aws
+
+
+.PHONY: infra.vsphere.destroy
+infra.vsphere.destroy: ## Destroy infrastructure
+infra.vsphere.destroy: TF_VAR_ssh_public_key=$(SSH_BASTION_PUBLIC_KEY_FILE)
+infra.vsphere.destroy: TF_VAR_datastore_name=$(VSPHERE_DATASTORE)
+infra.vsphere.destroy: infra.vsphere.init
+	$(call print-target)
+	TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/vsphere destroy -auto-approve
+	rm -r $(TEST_INFRA_STATE_DIR)/vsphere

test/infra/vsphere/export.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
+set -eu
 # shellcheck disable=SC2001
 echo "${SSH_BASTION_KEY_CONTENTS}" | sed 's/\\n/\n/g' >> vsphere-tests.pem
 chmod 600 vsphere-tests.pem
+
+SSH_BASTION_HOST="$("${TERRAFORM_BIN}" -chdir=.local/infra/vsphere output -raw bastion_node_ssh_nat_address)"
+export SSH_BASTION_HOST
+
+SSH_BASTION_PORT="$("${TERRAFORM_BIN}" -chdir=.local/infra/vsphere output -raw bastion_node_ssh_nat_port)"
+export SSH_BASTION_PORT
+
+SSH_BASTION_USERNAME="$("${TERRAFORM_BIN}" -chdir=.local/infra/vsphere output -raw bastion_node_ssh_user)"
+export SSH_BASTION_USERNAME
+
 "${ENVSUBST_ASSETS}"/envsubst < test/infra/vsphere/packer-vsphere-airgap.yaml.tmpl >> "$1"

test/infra/vsphere/main.tf

@@ -0,0 +1,32 @@
+resource "random_id" "build_id" {
+  byte_length = 8
+}
+module "bastion_node" {
+  source = "github.com/mesosphere/vcenter-tools/modules/vmclone"
+
+  node_name      = "konvoy-image-builder-bastion-${random_id.build_id.hex}"
+  ssh_public_key = file(var.ssh_public_key)
+
+  datastore_name              = var.datastore_name
+  datastore_is_cluster        = false
+  vm_template_name            = var.bastion_base_template
+  resource_pool_name          = var.resource_pool_name
+  root_volume_size            = var.root_volume_size #80
+  vsphere_folder              = var.vsphere_folder
+  ssh_user                    = var.ssh_user
+  custom_attribute_owner      = var.bastion_owner
+  custom_attribute_expiration = "4h"
+  vsphere_network             = var.vsphere_network
+}
+
+output "bastion_node_ssh_user" {
+  value = var.ssh_user
+}
+
+output "bastion_node_ssh_nat_address" {
+  value = module.bastion_node.nat_address
+}
+
+output "bastion_node_ssh_nat_port" {
+  value = module.bastion_node.nat_ssh_port
+}

test/infra/vsphere/packer-vsphere-airgap.yaml.tmpl

@@ -7,6 +7,7 @@
 packer:
   ssh_bastion_username: "${SSH_BASTION_USERNAME}"
   ssh_bastion_host: "${SSH_BASTION_HOST}"
+  ssh_bastion_port: "${SSH_BASTION_PORT}"
   ssh_bastion_private_key_file: vsphere-tests.pem
   cluster: "zone1"
   datacenter: "dc1"