From ec5e9932ff2b52c5f75a0119c06c2300ac38d6b7 Mon Sep 17 00:00:00 2001 From: Shalin Patel Date: Thu, 6 Jul 2023 14:40:57 -0700 Subject: [PATCH] ci: build and use temporary bastion for each vsphere CI test run (#850) * ci: github action workflow to create vsphere bastion host * deprecate scripts that creates permanent bastion * ci: allow overriding bastion ssh port * ci: refactor scripts to create bastion host * test using airgapped bastion * ci: use temp bastion host for vsphere e2e tests * use datastore input from github action secrets * correct public key reference * update vsphre release flow to use temporary bastion * fix linter errors --- .../workflows/release-vsphere-template.yaml | 23 +++- .github/workflows/vsphere-e2e.yaml | 23 +++- Makefile | 3 +- hack/configure-vsphere-bastion.yaml | 36 ------ hack/vsphere/main.tf | 110 ------------------ pkg/packer/manifests/vsphere/packer.pkr.hcl | 8 ++ test/infra/{aws => }/Makefile | 40 ++++++- test/infra/vsphere/Makefile | 8 -- test/infra/vsphere/export.sh | 11 ++ test/infra/vsphere/main.tf | 32 +++++ .../vsphere/packer-vsphere-airgap.yaml.tmpl | 1 + test/infra/vsphere/variables.tf | 46 ++++++++ 12 files changed, 180 insertions(+), 161 deletions(-) delete mode 100644 hack/configure-vsphere-bastion.yaml delete mode 100644 hack/vsphere/main.tf rename test/infra/{aws => }/Makefile (56%) delete mode 100644 test/infra/vsphere/Makefile create mode 100644 test/infra/vsphere/main.tf create mode 100644 test/infra/vsphere/variables.tf diff --git a/.github/workflows/release-vsphere-template.yaml b/.github/workflows/release-vsphere-template.yaml index 52fbd8b57..4e3380ff8 100644 --- a/.github/workflows/release-vsphere-template.yaml +++ b/.github/workflows/release-vsphere-template.yaml @@ -59,6 +59,12 @@ jobs: with: ssh-private-key: ${{ secrets.SSH_BASTION_KEY_CONTENTS }} + # configure git to access private repo hosting vsphere module mesosphere/vcenter-tools + - name: Configure git to clone private registry from mesosphere org + run: | + git config --global url."https://git:${{ secrets.MESOSPHERECI_USER_TOKEN }}@github.com/mesosphere".insteadOf "https://github.com/mesosphere" + git config --global url."https://${{ secrets.MESOSPHERECI_USER_TOKEN }}:x-oauth-basic@github.com/mesosphere".insteadOf ssh://git@github.com/mesosphere + - name: Build vSphere template for ${{ matrix.os }} with ${{ matrix.buildConfig }} configuration uses: magefile/mage-action@v2 with: @@ -66,10 +72,23 @@ jobs: args: runE2e "${{ matrix.os }}" "${{ matrix.buildConfig }}" ova false env: SSH_BASTION_KEY_CONTENTS: ${{ secrets.SSH_BASTION_KEY_CONTENTS }} - SSH_BASTION_HOST: ${{ secrets.SSH_BASTION_HOST }} - SSH_BASTION_USERNAME: ${{ secrets.SSH_BASTION_USERNAME }} + SSH_BASTION_PUBLIC_KEY_CONTENTS: ${{ secrets.SSH_BASTION_PUBLIC_KEY_CONTENTS }} + VSPHERE_USERNAME: ${{ secrets.VSPHERE_USERNAME }} + VSPHERE_USER: ${{ secrets.VSPHERE_USERNAME }} # required for terraform + VSPHERE_PASSWORD: ${{ secrets.VSPHERE_PASSWORD }} + GITHUB_TOKEN: ${{ secrets.MESOSPHERECI_USER_TOKEN }} + VSPHERE_SERVER: ${{ secrets.VSPHERE_SERVER }} + VSPHERE_DATASTORE: ${{ secrets.VSPHERE_DATASTORE }} + + - name: Run make destroy to clean up failed tests + if: ${{ always() }} + run: make infra.vsphere.destroy || true + env: + SSH_BASTION_PUBLIC_KEY_CONTENTS: ${{ secrets.SSH_BASTION_PUBLIC_KEY_CONTENTS }} VSPHERE_USERNAME: ${{ secrets.VSPHERE_USERNAME }} + VSPHERE_USER: ${{ secrets.VSPHERE_USERNAME }} # required for terraform VSPHERE_PASSWORD: ${{ secrets.VSPHERE_PASSWORD }} GITHUB_TOKEN: ${{ secrets.MESOSPHERECI_USER_TOKEN }} VSPHERE_SERVER: ${{ secrets.VSPHERE_SERVER }} VSPHERE_DATASTORE: ${{ secrets.VSPHERE_DATASTORE }} + VSPHERE_DATACENTER: ${{ secrets.VSPHERE_DATACENTER }} diff --git a/.github/workflows/vsphere-e2e.yaml b/.github/workflows/vsphere-e2e.yaml index 383e76915..391e2c8e5 100644 --- a/.github/workflows/vsphere-e2e.yaml +++ b/.github/workflows/vsphere-e2e.yaml @@ -66,6 +66,12 @@ jobs: uses: webfactory/ssh-agent@v0.8.0 with: ssh-private-key: ${{ secrets.SSH_BASTION_KEY_CONTENTS }} + + # configure git to access private repo hosting vsphere module mesosphere/vcenter-tools + - name: Configure git to clone private registry from mesosphere org + run: | + git config --global url."https://git:${{ secrets.MESOSPHERECI_USER_TOKEN }}@github.com/mesosphere".insteadOf "https://github.com/mesosphere" + git config --global url."https://${{ secrets.MESOSPHERECI_USER_TOKEN }}:x-oauth-basic@github.com/mesosphere".insteadOf ssh://git@github.com/mesosphere - name: Run E2E test for ${{ matrix.os }} with ${{ matrix.buildConfig }} configuration uses: magefile/mage-action@v2 @@ -74,11 +80,24 @@ jobs: args: runE2e "${{ matrix.os }}" "${{ matrix.buildConfig }}" ova true env: SSH_BASTION_KEY_CONTENTS: ${{ secrets.SSH_BASTION_KEY_CONTENTS }} - SSH_BASTION_HOST: ${{ secrets.SSH_BASTION_HOST }} - SSH_BASTION_USERNAME: ${{ secrets.SSH_BASTION_USERNAME }} + SSH_BASTION_PUBLIC_KEY_CONTENTS: ${{ secrets.SSH_BASTION_PUBLIC_KEY_CONTENTS }} VSPHERE_USERNAME: ${{ secrets.VSPHERE_USERNAME }} + VSPHERE_USER: ${{ secrets.VSPHERE_USERNAME }} # required for terraform VSPHERE_PASSWORD: ${{ secrets.VSPHERE_PASSWORD }} GITHUB_TOKEN: ${{ secrets.MESOSPHERECI_USER_TOKEN }} VSPHERE_SERVER: ${{ secrets.VSPHERE_SERVER }} VSPHERE_DATASTORE: ${{ secrets.VSPHERE_DATASTORE }} VSPHERE_DATACENTER: ${{ secrets.VSPHERE_DATACENTER }} + + - name: Run make destroy to clean up failed tests + if: ${{ always() }} + run: make infra.vsphere.destroy || true + env: + SSH_BASTION_PUBLIC_KEY_CONTENTS: ${{ secrets.SSH_BASTION_PUBLIC_KEY_CONTENTS }} + VSPHERE_USERNAME: ${{ secrets.VSPHERE_USERNAME }} + VSPHERE_USER: ${{ secrets.VSPHERE_USERNAME }} # required for terraform + VSPHERE_PASSWORD: ${{ secrets.VSPHERE_PASSWORD }} + GITHUB_TOKEN: ${{ secrets.MESOSPHERECI_USER_TOKEN }} + VSPHERE_SERVER: ${{ secrets.VSPHERE_SERVER }} + VSPHERE_DATASTORE: ${{ secrets.VSPHERE_DATASTORE }} + VSPHERE_DATACENTER: ${{ secrets.VSPHERE_DATACENTER }} \ No newline at end of file diff --git a/Makefile b/Makefile index 01aa1d8ac..80511b0b9 100644 --- a/Makefile +++ b/Makefile @@ -156,8 +156,7 @@ $(ENVSUBST_ASSETS)/envsubst: include hack/pip-packages/Makefile -include test/infra/aws/Makefile -include test/infra/vsphere/Makefile +include test/infra/Makefile github-token.txt: echo $(GITHUB_TOKEN) >> github-token.txt diff --git a/hack/configure-vsphere-bastion.yaml b/hack/configure-vsphere-bastion.yaml deleted file mode 100644 index a95f757f1..000000000 --- a/hack/configure-vsphere-bastion.yaml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- hosts: bastion - become: yes - become_user: root - become_method: sudo - - tasks: - - name: Set authorized key for builder user - authorized_key: - user: "{{ item.name }}" - key: "{{ lookup('file', '{{ item.key }}') }}" - with_items: "{{ ssh_users }}" - - - name: Disable Password Authentication - lineinfile: - dest: /etc/ssh/sshd_config - regexp: '^PasswordAuthentication' - line: "PasswordAuthentication no" - state: present - backup: yes - - - name: Disable Root Login - lineinfile: - dest: /etc/ssh/sshd_config - regexp: '^PermitRootLogin' - line: "PermitRootLogin no" - state: present - backup: yes - notify: - - restart ssh - - handlers: - - name: restart ssh - service: - name: sshd - state: restarted diff --git a/hack/vsphere/main.tf b/hack/vsphere/main.tf deleted file mode 100644 index 82f6c30b2..000000000 --- a/hack/vsphere/main.tf +++ /dev/null @@ -1,110 +0,0 @@ -terraform { - required_providers { - vsphere = { - version = "1.15.0" - } - } -} -provider "vsphere" { - allow_unverified_ssl = false - -} -variable "datacenter_name" { - description = "The datacenter name" - default = "dc1" -} - -variable "datastore_name" { - description = "The datastore name" - default = "tmp-iscsi-ovh" -} - -variable "resource_pool_name" { - description = "The resource pool name" - default = "cluster-api" -} - -variable "network_name_airgapped" { - description = "The network name" - default = "Airgapped" -} - -variable "network_name_public" { - description = "The network name" - default = "Public" -} - -variable "bastion_vm_template" { - description = "The VM template name for the bastion machine" - default = "base-centos-7" -} - - -variable "root_user" { - description = "The root user" - default = "centos" -} - -data "vsphere_datacenter" "dc" { - name = var.datacenter_name -} - -data "vsphere_datastore" "datastore" { - name = var.datastore_name - datacenter_id = data.vsphere_datacenter.dc.id -} - -data "vsphere_resource_pool" "pool" { - name = var.resource_pool_name - datacenter_id = data.vsphere_datacenter.dc.id -} - -data "vsphere_network" "network_public" { - name = var.network_name_public - datacenter_id = data.vsphere_datacenter.dc.id -} - -data "vsphere_network" "network_airgapped" { - name = var.network_name_airgapped - datacenter_id = data.vsphere_datacenter.dc.id -} - -data "vsphere_virtual_machine" "bastion_template" { - name = var.bastion_vm_template - datacenter_id = data.vsphere_datacenter.dc.id -} - -resource "vsphere_virtual_machine" "konvoy-e2e-bastion" { - name = "bastion-host" - resource_pool_id = data.vsphere_resource_pool.pool.id - datastore_id = data.vsphere_datastore.datastore.id - - num_cpus = 4 - memory = 6144 - guest_id = "centos64Guest" - - - network_interface { - network_id = data.vsphere_network.network_public.id - } - - network_interface { - network_id = data.vsphere_network.network_airgapped.id - } - - - clone { - template_uuid = data.vsphere_virtual_machine.bastion_template.id - linked_clone = false - } - - disk { - label = "disk0" - datastore_id = data.vsphere_datastore.datastore.id - size = 80 - } -} - -output "bastion_ip" { - value = vsphere_virtual_machine.konvoy-e2e-bastion.guest_ip_addresses -} diff --git a/pkg/packer/manifests/vsphere/packer.pkr.hcl b/pkg/packer/manifests/vsphere/packer.pkr.hcl index 5680fddf7..5bd5d5a66 100644 --- a/pkg/packer/manifests/vsphere/packer.pkr.hcl +++ b/pkg/packer/manifests/vsphere/packer.pkr.hcl @@ -194,6 +194,12 @@ variable "ssh_bastion_host" { type = string default = "" } + +variable "ssh_bastion_port" { + type = string + default = "22" +} + variable "ssh_bastion_password" { type = string default = "" @@ -362,6 +368,7 @@ locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } locals { build_timestamp = local.timestamp ssh_bastion_host = var.ssh_bastion_host + ssh_bastion_port = var.ssh_bastion_port ssh_bastion_password = var.ssh_bastion_password ssh_bastion_private_key_file = var.ssh_bastion_private_key_file ssh_bastion_username = var.ssh_bastion_username @@ -416,6 +423,7 @@ source "vsphere-clone" "kib_image" { password = var.vsphere_password ssh_agent_auth = var.ssh_agent_auth ssh_bastion_host = local.ssh_bastion_host + ssh_bastion_port = local.ssh_bastion_port ssh_bastion_password = local.ssh_bastion_password ssh_bastion_private_key_file = local.ssh_bastion_private_key_file ssh_bastion_username = local.ssh_bastion_username diff --git a/test/infra/aws/Makefile b/test/infra/Makefile similarity index 56% rename from test/infra/aws/Makefile rename to test/infra/Makefile index 4b5594f54..25fe65667 100644 --- a/test/infra/aws/Makefile +++ b/test/infra/Makefile @@ -11,6 +11,15 @@ export TERRAFORM_BIN = $(TERRAFORM_ASSETS)/terraform TEST_MODULES_DIR = $(CURDIR)/test/infra TEST_INFRA_STATE_DIR ?= $(CURDIR)/.local/infra +# export for local testing +SSH_BASTION_PUBLIC_KEY_FILE ?= "" +VSPHERE_DATASTORE ?= "ci-kib" + +ifeq ($(CI),true) + $(shell echo "$(SSH_BASTION_PUBLIC_KEY_CONTENTS)" > $(CURDIR)/vsphere-bastion.pub) + SSH_BASTION_PUBLIC_KEY_FILE = $(CURDIR)/vsphere-bastion.pub +endif + .PHONY: install-terraform install-terraform: ## Download and unpack terraform binary install-terraform: $(TERRAFORM_BIN) @@ -30,19 +39,48 @@ infra.aws.init: install-terraform $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/aws init -from-module=$(TEST_MODULES_DIR)/aws -force-copy || true cd $(TEST_INFRA_STATE_DIR)/aws && $(TEST_INFRA_STATE_DIR)/aws/init.sh +.PHONY: infra.vsphere.init +infra.vsphere.init: ## Initialize infrastructure metadata +infra.vsphere.init: install-terraform + $(call print-target) + mkdir -p $(TEST_INFRA_STATE_DIR)/vsphere + $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/vsphere init -from-module=$(TEST_MODULES_DIR)/vsphere -force-copy || true + .PHONY: infra.aws.create infra.aws.create: ## Create infrastructure -infra.aws.create: install-terraform infra.aws.init +infra.aws.create: infra.aws.init $(call print-target) TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/aws plan TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/aws apply -auto-approve +PHONY: infra.vsphere.create +infra.vsphere.create: ## Create infrastructure +infra.vsphere.create: TF_VAR_ssh_public_key=$(SSH_BASTION_PUBLIC_KEY_FILE) +infra.vsphere.create: TF_VAR_datastore_name=$(VSPHERE_DATASTORE) +infra.vsphere.create: infra.vsphere.init + $(call print-target) + TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/vsphere plan + TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/vsphere apply -auto-approve + packer-aws-offline-override.yaml: infra.aws.create install-envsubst @$(TEST_INFRA_STATE_DIR)/aws/export.sh $@ +packer-ova-offline-override.yaml: infra.vsphere.create install-envsubst + @$(TEST_INFRA_STATE_DIR)/vsphere/export.sh $@ + .PHONY: infra.aws.destroy infra.aws.destroy: ## Destroy infrastructure infra.aws.destroy: install-terraform infra.aws.init $(call print-target) TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/aws destroy -auto-approve rm -r $(TEST_INFRA_STATE_DIR)/aws + + +.PHONY: infra.vsphere.destroy +infra.vsphere.destroy: ## Destroy infrastructure +infra.vsphere.destroy: TF_VAR_ssh_public_key=$(SSH_BASTION_PUBLIC_KEY_FILE) +infra.vsphere.destroy: TF_VAR_datastore_name=$(VSPHERE_DATASTORE) +infra.vsphere.destroy: infra.vsphere.init + $(call print-target) + TF_LOG=INFO $(TERRAFORM_BIN) -chdir=$(TEST_INFRA_STATE_DIR)/vsphere destroy -auto-approve + rm -r $(TEST_INFRA_STATE_DIR)/vsphere diff --git a/test/infra/vsphere/Makefile b/test/infra/vsphere/Makefile deleted file mode 100644 index 453031ecf..000000000 --- a/test/infra/vsphere/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -# Infra related goals -# --------------------------------------------------------------------- -export SCRIPT_DIR ?= $(CURDIR)/hack - -TEST_SCRIPTS_DIR ?= $(CURDIR)/test/infra/vsphere - -packer-ova-offline-override.yaml: install-envsubst - @$(TEST_SCRIPTS_DIR)/export.sh $@ diff --git a/test/infra/vsphere/export.sh b/test/infra/vsphere/export.sh index fa8f4dd9f..717ed8770 100755 --- a/test/infra/vsphere/export.sh +++ b/test/infra/vsphere/export.sh @@ -1,5 +1,16 @@ #!/bin/bash +set -eu # shellcheck disable=SC2001 echo "${SSH_BASTION_KEY_CONTENTS}" | sed 's/\\n/\n/g' >> vsphere-tests.pem chmod 600 vsphere-tests.pem + +SSH_BASTION_HOST="$("${TERRAFORM_BIN}" -chdir=.local/infra/vsphere output -raw bastion_node_ssh_nat_address)" +export SSH_BASTION_HOST + +SSH_BASTION_PORT="$("${TERRAFORM_BIN}" -chdir=.local/infra/vsphere output -raw bastion_node_ssh_nat_port)" +export SSH_BASTION_PORT + +SSH_BASTION_USERNAME="$("${TERRAFORM_BIN}" -chdir=.local/infra/vsphere output -raw bastion_node_ssh_user)" +export SSH_BASTION_USERNAME + "${ENVSUBST_ASSETS}"/envsubst < test/infra/vsphere/packer-vsphere-airgap.yaml.tmpl >> "$1" diff --git a/test/infra/vsphere/main.tf b/test/infra/vsphere/main.tf new file mode 100644 index 000000000..8e36ce77c --- /dev/null +++ b/test/infra/vsphere/main.tf @@ -0,0 +1,32 @@ +resource "random_id" "build_id" { + byte_length = 8 +} +module "bastion_node" { + source = "github.com/mesosphere/vcenter-tools/modules/vmclone" + + node_name = "konvoy-image-builder-bastion-${random_id.build_id.hex}" + ssh_public_key = file(var.ssh_public_key) + + datastore_name = var.datastore_name + datastore_is_cluster = false + vm_template_name = var.bastion_base_template + resource_pool_name = var.resource_pool_name + root_volume_size = var.root_volume_size #80 + vsphere_folder = var.vsphere_folder + ssh_user = var.ssh_user + custom_attribute_owner = var.bastion_owner + custom_attribute_expiration = "4h" + vsphere_network = var.vsphere_network +} + +output "bastion_node_ssh_user" { + value = var.ssh_user +} + +output "bastion_node_ssh_nat_address" { + value = module.bastion_node.nat_address +} + +output "bastion_node_ssh_nat_port" { + value = module.bastion_node.nat_ssh_port +} diff --git a/test/infra/vsphere/packer-vsphere-airgap.yaml.tmpl b/test/infra/vsphere/packer-vsphere-airgap.yaml.tmpl index 72114379e..8ce5aba44 100644 --- a/test/infra/vsphere/packer-vsphere-airgap.yaml.tmpl +++ b/test/infra/vsphere/packer-vsphere-airgap.yaml.tmpl @@ -7,6 +7,7 @@ packer: ssh_bastion_username: "${SSH_BASTION_USERNAME}" ssh_bastion_host: "${SSH_BASTION_HOST}" + ssh_bastion_port: "${SSH_BASTION_PORT}" ssh_bastion_private_key_file: vsphere-tests.pem cluster: "zone1" datacenter: "dc1" diff --git a/test/infra/vsphere/variables.tf b/test/infra/vsphere/variables.tf new file mode 100644 index 000000000..28f084cea --- /dev/null +++ b/test/infra/vsphere/variables.tf @@ -0,0 +1,46 @@ +variable "datastore_name" { + description = "The datastore name" + default = "ci-kib" +} + +variable "bastion_base_template" { + description = "base template name" + default = "os-qualification-templates/d2iq-base-RockyLinux-9.1" +} + +variable "resource_pool_name" { + description = "The resource pool name" + default = "cluster-api" +} + +variable "root_volume_size" { + description = "Disk size for root volume" + default = 80 +} + +variable "vsphere_folder" { + description = "folder name to store the VM" + default = "cluster-api" +} + +variable "ssh_user" { + description = "The root user" + default = "centos" +} + + +variable "bastion_owner" { + description = "The root user" + default = "Konvoy image builder" +} + +variable "ssh_public_key" { + description = "Path to the SSH Public key. for example: ~/.ssh/id_rsa.pub" + type = string +} + +variable "vsphere_network" { + description = "vsphere network" + type = string + default = "Airgapped" +}