[PART-2]vsan stretch testcase automation for TKG, WCP and VMService VMs

go.sum

@@ -963,4 +963,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+s
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
-sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

tests/e2e/crypto.go

@@ -123,7 +123,7 @@ var _ = ginkgo.Describe("[csi-supervisor] [encryption] Block volume encryption",
 		vmImageName := GetAndExpectStringEnvVar(envVmsvcVmImageName)
 		framework.Logf("Waiting for virtual machine image list to be available in namespace '%s' for image '%s'",
 			namespace, vmImageName)
-		vmi = waitNGetVmiForImageName(ctx, vmopClient, namespace, vmImageName)
+		vmi = waitNGetVmiForImageName(ctx, vmopClient, vmImageName)
 		gomega.Expect(vmi).NotTo(gomega.BeEmpty())
 	})
 

tests/e2e/csi_static_provisioning_basic.go

@@ -2430,5 +2430,4 @@ var _ = ginkgo.Describe("Basic Static Provisioning", func() {
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 	})
-
 })

tests/e2e/e2e_common.go

@@ -279,38 +279,42 @@ customPort -> label include the testcases running on vCenter custom port <VC:444
 deprecated ->label include the testcases which are no longer in execution
 */
 const (
-	flaky               = "flaky"
-	disruptive          = "disruptive"
-	wcp                 = "wcp"
-	tkg                 = "tkg"
-	vanilla             = "vanilla"
-	preferential        = "preferential"
-	vsphereConfigSecret = "vsphereConfigSecret"
-	snapshot            = "snapshot"
-	stable              = "stable"
-	newTest             = "newTest"
-	multiVc             = "multiVc"
-	block               = "block"
-	file                = "file"
-	core                = "core"
-	hci                 = "hci"
-	p0                  = "p0"
-	p1                  = "p1"
-	p2                  = "p2"
-	vsanStretch         = "vsanStretch"
-	longRunning         = "longRunning"
-	deprecated          = "deprecated"
-	vmc                 = "vmc"
-	tkgsHA              = "tkgsHA"
-	thickThin           = "thickThin"
-	customPort          = "customPort"
-	windows             = "windows"
-	semiAutomated       = "semiAutomated"
-	level2              = "level2"
-	level5              = "level5"
-	negative            = "negative"
-	listVolume          = "listVolume"
-	multiSvc            = "multiSvc"
+	flaky                 = "flaky"
+	disruptive            = "disruptive"
+	wcp                   = "wcp"
+	tkg                   = "tkg"
+	vanilla               = "vanilla"
+	preferential          = "preferential"
+	vsphereConfigSecret   = "vsphereConfigSecret"
+	snapshot              = "snapshot"
+	stable                = "stable"
+	newTest               = "newTest"
+	multiVc               = "multiVc"
+	block                 = "block"
+	file                  = "file"
+	core                  = "core"
+	hci                   = "hci"
+	p0                    = "p0"
+	p1                    = "p1"
+	p2                    = "p2"
+	vsanStretch           = "vsanStretch"
+	longRunning           = "longRunning"
+	deprecated            = "deprecated"
+	vmc                   = "vmc"
+	tkgsHA                = "tkgsHA"
+	thickThin             = "thickThin"
+	customPort            = "customPort"
+	windows               = "windows"
+	semiAutomated         = "semiAutomated"
+	level2                = "level2"
+	level5                = "level5"
+	negative              = "negative"
+	listVolume            = "listVolume"
+	multiSvc              = "multiSvc"
+	primaryCentric        = "primaryCentric"
+	controlPlaneOnPrimary = "controlPlaneOnPrimary"
+	distributed           = "distributed"
+	vmsvc                 = "vmsvc"
 )
 
 // The following variables are required to know cluster type to run common e2e

tests/e2e/hci.go

@@ -462,7 +462,7 @@ var _ bool = ginkgo.Describe("hci", func() {
 		}()
 
 		ginkgo.By("Wait for 5-10 mins, verify that the k8s-worker is restarted and brought up on another host")
-		wait4AllK8sNodesToBeUp(ctx, client, nodes)
+		wait4AllK8sNodesToBeUp(nodes)
 		gomega.Expect(waitForAllNodes2BeReady(ctx, client)).To(gomega.Succeed())
 
 		ginkgo.By("Scale up statefulset2 and scale down statefulset1 to 2 replicas")

tests/e2e/hci_mesh_rwx_disruptive.go

@@ -677,7 +677,7 @@ var _ = ginkgo.Describe("[rwx-hci-singlevc-disruptive] RWX-Topology-HciMesh-Sing
 
 		ginkgo.By("PSOD all host in remote cluster4 and when psod is triggered, create new set of rwx pvc")
 		for i := 0; i < len(hostListCluster4); i++ {
-			err = psodHost(hostListCluster4[i])
+			err = psodHost(hostListCluster4[i], "")
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 			if i == 0 {
@@ -729,7 +729,7 @@ var _ = ginkgo.Describe("[rwx-hci-singlevc-disruptive] RWX-Topology-HciMesh-Sing
 		ginkgo.By("PSOD again all host in remote cluster4 and perform scaleup " +
 			"operation on deployment and statefulset")
 		for i := 0; i < len(hostListCluster4); i++ {
-			err = psodHost(hostListCluster4[i])
+			err = psodHost(hostListCluster4[i], "")
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 			if i == 0 {
@@ -1873,7 +1873,7 @@ var _ = ginkgo.Describe("[rwx-hci-singlevc-disruptive] RWX-Topology-HciMesh-Sing
 
 		ginkgo.By("PSOD all host in local cluster2 and when psod is triggered, create new set of rwx pvc")
 		for i := 0; i < len(hostListCluster2); i++ {
-			err = psodHost(hostListCluster2[i])
+			err = psodHost(hostListCluster2[i], "")
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 			if i == 0 {
@@ -1929,7 +1929,7 @@ var _ = ginkgo.Describe("[rwx-hci-singlevc-disruptive] RWX-Topology-HciMesh-Sing
 		ginkgo.By("PSOD all host in local cluster3 and perform scaleup " +
 			"operation on deployment and statefulset")
 		for i := 0; i < len(hostListCluster3); i++ {
-			err = psodHost(hostListCluster3[i])
+			err = psodHost(hostListCluster3[i], "")
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 			if i == 0 {

tests/e2e/multi_vc_operation_storm.go

@@ -287,7 +287,7 @@ var _ = ginkgo.Describe("[multivc-operationstorm] MultiVc-OperationStorm", func(
 			}
 
 			ginkgo.By("Wait for k8s cluster to be healthy")
-			wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+			wait4AllK8sNodesToBeUp(nodeList)
 			err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 		}()
@@ -349,7 +349,7 @@ var _ = ginkgo.Describe("[multivc-operationstorm] MultiVc-OperationStorm", func(
 
 		// Wait for k8s cluster to be healthy
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 

tests/e2e/multi_vc_site_down.go

@@ -234,14 +234,14 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
 			}
 
 			ginkgo.By("Wait for k8s cluster to be healthy")
-			wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+			wait4AllK8sNodesToBeUp(nodeList)
 			err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 		}()
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -383,14 +383,14 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
 			}
 
 			ginkgo.By("Wait for k8s cluster to be healthy")
-			wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+			wait4AllK8sNodesToBeUp(nodeList)
 			err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 		}()
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -493,7 +493,7 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
 			}
 
 			ginkgo.By("Wait for k8s cluster to be healthy")
-			wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+			wait4AllK8sNodesToBeUp(nodeList)
 			err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -505,7 +505,7 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
 		}
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -632,7 +632,7 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
 			}
 
 			ginkgo.By("Wait for k8s cluster to be healthy")
-			wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+			wait4AllK8sNodesToBeUp(nodeList)
 			err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -644,7 +644,7 @@ var _ = ginkgo.Describe("[multivc-sitedown] MultiVc-SiteDown", func() {
 		}
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 

tests/e2e/no_hci_mesh_rwx_multivc_disruptive.go

@@ -292,7 +292,7 @@ var _ = ginkgo.Describe("[rwx-multivc-operationstorm] RWX-MultiVc-OperationStorm
 			}
 
 			ginkgo.By("Wait for k8s cluster to be healthy")
-			wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+			wait4AllK8sNodesToBeUp(nodeList)
 			err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -517,7 +517,7 @@ var _ = ginkgo.Describe("[rwx-multivc-operationstorm] RWX-MultiVc-OperationStorm
 			}
 
 			ginkgo.By("Wait for k8s cluster to be healthy")
-			wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+			wait4AllK8sNodesToBeUp(nodeList)
 			err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -787,7 +787,7 @@ var _ = ginkgo.Describe("[rwx-multivc-operationstorm] RWX-MultiVc-OperationStorm
 			}
 
 			ginkgo.By("Wait for k8s cluster to be healthy")
-			wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+			wait4AllK8sNodesToBeUp(nodeList)
 			err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 		}()
@@ -824,7 +824,7 @@ var _ = ginkgo.Describe("[rwx-multivc-operationstorm] RWX-MultiVc-OperationStorm
 			}
 
 			ginkgo.By("Wait for k8s cluster to be healthy")
-			wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+			wait4AllK8sNodesToBeUp(nodeList)
 			err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 

tests/e2e/no_hci_mesh_rwx_singlevc_topology_disruptive.go

@@ -752,7 +752,7 @@ var _ = ginkgo.Describe("[rwx-nohci-singlevc-disruptive] RWX-Topology-NoHciMesh-
 
 		ginkgo.By("PSOD all host")
 		for i := 0; i < len(hostList); i++ {
-			err = psodHost(hostList[i])
+			err = psodHost(hostList[i], "")
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 			if i == 2 {

tests/e2e/preferential_topology_disruptive.go

@@ -365,7 +365,7 @@ var _ = ginkgo.Describe("[preferential-disruptive] Preferential-Topology-Disrupt
 		}()
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -429,7 +429,7 @@ var _ = ginkgo.Describe("[preferential-disruptive] Preferential-Topology-Disrupt
 		}
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 

tests/e2e/rwx_topology_utils.go

@@ -886,7 +886,7 @@ func verifyK8sNodeStatusAfterSiteRecovery(client clientset.Interface, ctx contex
 	time.Sleep(pollTimeoutShort)
 
 	ginkgo.By("Wait for k8s cluster to be healthy")
-	wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+	wait4AllK8sNodesToBeUp(nodeList)
 	err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 	if err != nil {
 		return fmt.Errorf("error waiting for all nodes to be ready: %w", err)
@@ -895,18 +895,24 @@ func verifyK8sNodeStatusAfterSiteRecovery(client clientset.Interface, ctx contex
 }
 
 /* This util will perform psod operation on a host */
-func psodHost(hostIP string) error {
+func psodHost(hostIP string, psodTimeOut string) error {
 	ginkgo.By("PSOD")
-	sshCmd := fmt.Sprintf("vsish -e set /config/Misc/intOpts/BlueScreenTimeout %s", psodTime)
-	op, err := runCommandOnESX("root", hostIP, sshCmd)
+	var timeout string
+	if psodTimeOut != "" {
+		timeout = psodTimeOut
+	} else {
+		timeout = psodTime
+	}
+	sshCmd := fmt.Sprintf("vsish -e set /config/Misc/intOpts/BlueScreenTimeout %s", timeout)
+	op, err := runCommandOnESX(rootUser, hostIP, sshCmd)
 	framework.Logf(op)
 	if err != nil {
 		return fmt.Errorf("failed to set BlueScreenTimeout: %w", err)
 	}
 
 	ginkgo.By("Injecting PSOD")
-	psodCmd := "vsish -e set /reliability/crashMe/Panic 1"
-	op, err = runCommandOnESX("root", hostIP, psodCmd)
+	psodCmd := "vsish -e set /reliability/crashMe/Panic 1; exit"
+	op, err = runCommandOnESX(rootUser, hostIP, psodCmd)
 	framework.Logf(op)
 	if err != nil {
 		return fmt.Errorf("failed to inject PSOD: %w", err)

tests/e2e/snapshot_vmservice_vm.go

@@ -145,7 +145,7 @@ var _ bool = ginkgo.Describe("[snapshot-vmsvc] Snapshot VM Service VM", func() {
 		vmImageName := GetAndExpectStringEnvVar(envVmsvcVmImageName)
 		framework.Logf("Waiting for virtual machine image list to be available in namespace '%s' for image '%s'",
 			namespace, vmImageName)
-		vmi = waitNGetVmiForImageName(ctx, vmopC, namespace, vmImageName)
+		vmi = waitNGetVmiForImageName(ctx, vmopC, vmImageName)
 		gomega.Expect(vmi).NotTo(gomega.BeEmpty())
 
 		// Get snapshot client using the rest config

tests/e2e/topology_operation_strom_cases.go

@@ -223,7 +223,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
 		}()
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -261,7 +261,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
 		}
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -287,7 +287,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
 		}()
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -326,7 +326,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
 		}
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
@@ -557,7 +557,7 @@ var _ = ginkgo.Describe("[topology-operationstorm] Topology-OperationStorm", fun
 		}
 
 		ginkgo.By("Wait for k8s cluster to be healthy")
-		wait4AllK8sNodesToBeUp(ctx, client, nodeList)
+		wait4AllK8sNodesToBeUp(nodeList)
 		err = waitForAllNodes2BeReady(ctx, client, pollTimeout*4)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())