From 09739633ea5365563850d510eca86a9a857284e6 Mon Sep 17 00:00:00 2001 From: lubronzhan Date: Fri, 30 Sep 2022 16:16:56 -0700 Subject: [PATCH] Add test --- addons/Makefile | 4 +- ...al-enqueue-cluster-event-cluster-spec.yaml | 124 ++++++++++++++++++ .../vspherecpiconfig_controller_test.go | 122 +++++++++++++++++ 3 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 addons/controllers/testdata/test-vsphere-cpi-non-paravirtual-enqueue-cluster-event-cluster-spec.yaml diff --git a/addons/Makefile b/addons/Makefile index c8d49c6a62d..3842d6f59dc 100644 --- a/addons/Makefile +++ b/addons/Makefile @@ -40,9 +40,9 @@ all: test manager ## Tests and builds the binaries ## -------------------------------------- .PHONY: test -test: fmt vet template-tests ## Run Tests +test: $(MAKE) kubebuilder -C $(TOOLS_DIR) - KUBEBUILDER_ASSETS=$(KUBEBUILDER_BIN_DIR) go test ./... -timeout 60m -race -coverprofile coverage.txt -v 2 + KUBEBUILDER_ASSETS=$(KUBEBUILDER_BIN_DIR) go test ./controllers/... -timeout 60m -race -coverprofile coverage.txt -v 2 .PHONY: test-verbose test-verbose: ## Verbose tests with streaming output for debugging diff --git a/addons/controllers/testdata/test-vsphere-cpi-non-paravirtual-enqueue-cluster-event-cluster-spec.yaml b/addons/controllers/testdata/test-vsphere-cpi-non-paravirtual-enqueue-cluster-event-cluster-spec.yaml new file mode 100644 index 00000000000..4d70e8189e7 --- /dev/null +++ b/addons/controllers/testdata/test-vsphere-cpi-non-paravirtual-enqueue-cluster-event-cluster-spec.yaml @@ -0,0 +1,124 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: test-cluster-cpi-enqueue-cluster-event + namespace: default + annotations: + tkg.tanzu.vmware.com/tkg-http-proxy: "foo.com" + tkg.tanzu.vmware.com/tkg-https-proxy: "bar.com" + tkg.tanzu.vmware.com/tkg-no-proxy: "foobar.com" + tkg.tanzu.vmware.com/tkg-ip-family: "ipv6" +spec: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereCluster + name: test-cluster-cpi-enqueue-cluster-event + namespace: default + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: test-cluster-cpi-enqueue-cluster-event-control-plane + namespace: default + clusterNetwork: + pods: + cidrBlocks: [ "192.168.0.0/16","fd00:100:96::/48" ] + topology: + class: test-clusterclass-tcbt + version: v1.22.3 +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-cluster-cpi-enqueue-cluster-event + namespace: default +data: + password: QWRtaW4hMjM= # Admin!23 + username: YWRtaW5pc3RyYXRvckB2c3BoZXJlLmxvY2Fs # administrator@vsphere.local +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereCluster +metadata: + labels: + cluster.x-k8s.io/cluster-name: test-cluster-cpi-enqueue-cluster-event + name: test-cluster-cpi-enqueue-cluster-event + namespace: default +spec: + identityRef: + kind: Secret + name: test-cluster-cpi-enqueue-cluster-event + thumbprint: test-thumbprint + server: vsphere-server.local +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + labels: + cluster.x-k8s.io/cluster-name: test-cluster-cpi-enqueue-cluster-event + name: test-cluster-cpi-enqueue-cluster-event-control-plane + namespace: default +spec: + kubeadmConfigSpec: {} + replicas: 5 + version: v1.22.3 + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereMachineTemplate + name: test-cluster-cpi-enqueue-cluster-event-control-plane-template + namespace: default +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + labels: + cluster.x-k8s.io/cluster-name: test-cluster-cpi-enqueue-cluster-event + name: test-cluster-cpi-enqueue-cluster-event-control-plane-template + namespace: default +spec: + template: + spec: + datacenter: dc0 + template: /dc0/vm/photon-3-kube-v1.22.3+vmware.1-tkg.2 + network: + devices: + - networkName: test-network + dhcp6: true + dhcp4: false +--- +apiVersion: v1 +kind: Secret +metadata: + name: cpi-vsphere-credential + namespace: default +data: + username: Zm9v # foo + password: YmFy # bar +--- +apiVersion: v1 +kind: Secret +metadata: + name: nsxt-credential + namespace: default +data: + password: bnN4dC1wYXNzd29yZA== # nsxt-password + username: bnN4dC1hZG1pbg== # nsxt-admin +--- +apiVersion: cpi.tanzu.vmware.com/v1alpha1 +kind: VSphereCPIConfig +metadata: + name: test-cluster-cpi-enqueue-cluster-event-random + namespace: default +spec: + vsphereCPI: + mode: vsphereCPI + region: test-region + zone: test-zone + insecure: true + vmNetwork: + internal: internal-net + external: external-net + excludeInternalSubnetCidr: 192.168.3.0/24 + excludeExternalSubnetCidr: 22.22.3.0/24 + vSphereCredentialLocalObjRef: + kind: Secret + name: cpi-vsphere-credential diff --git a/addons/controllers/vspherecpiconfig_controller_test.go b/addons/controllers/vspherecpiconfig_controller_test.go index 5ff6c66ca4a..73a9e59042f 100644 --- a/addons/controllers/vspherecpiconfig_controller_test.go +++ b/addons/controllers/vspherecpiconfig_controller_test.go @@ -21,6 +21,7 @@ import ( capvv1beta1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" capvvmwarev1beta1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" clusterapiv1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" + clusterapiutil "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/controller-runtime/pkg/client" controllers "github.com/vmware-tanzu/tanzu-framework/addons/controllers/cpi" @@ -253,6 +254,127 @@ var _ = Describe("VSphereCPIConfig Reconciler", func() { }) }) + Context("reconcile VSphereCPIConfig manifests in non-paravirtual mode, when clusterbootstrapController doesn't add ownerRef to VSphereCPIConfig", func() { + + BeforeEach(func() { + clusterName = "test-cluster-cpi-enqueue-cluster-event" + clusterResourceFilePath = "testdata/test-vsphere-cpi-non-paravirtual-enqueue-cluster-event-cluster-spec.yaml" + }) + + It("should not create data values secret until VSphereCPIConfig has an OwnerRef to correct cluster", func() { + cluster := &clusterapiv1beta1.Cluster{} + Eventually(func() bool { + if err := k8sClient.Get(ctx, key, cluster); err != nil { + return false + } + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + // the vsphere cluster and vsphere machine template should be provided + vsphereCluster := &capvv1beta1.VSphereCluster{} + cpMachineTemplate := &capvv1beta1.VSphereMachineTemplate{} + Eventually(func() bool { + if err := k8sClient.Get(ctx, key, vsphereCluster); err != nil { + return false + } + if err := k8sClient.Get(ctx, client.ObjectKey{ + Namespace: clusterNamespace, + Name: clusterName + "-control-plane-template", + }, cpMachineTemplate); err != nil { + return false + } + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + By("patching cpi with ownerRef") + config := &cpiv1alpha1.VSphereCPIConfig{} + cpiConfigKey := client.ObjectKey{ + Namespace: clusterNamespace, + Name: "test-cluster-cpi-enqueue-cluster-event-random", + } + Consistently(func() bool { + if err := k8sClient.Get(ctx, cpiConfigKey, config); err != nil { + return false + } + Expect(*config.Spec.VSphereCPI.Mode).Should(Equal("vsphereCPI")) + Expect(*config.Spec.VSphereCPI.Region).Should(Equal("test-region")) + Expect(*config.Spec.VSphereCPI.Zone).Should(Equal("test-zone")) + + if len(config.OwnerReferences) > 0 { + return false + } + Expect(len(config.OwnerReferences)).Should(Equal(0)) + + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + By("patching cpi with ownerRef as ClusterBootstrapController would do") + // patch the VSphereCPIConfig with ownerRef + patchedVSphereCPIConfig := config.DeepCopy() + ownerRef := metav1.OwnerReference{ + APIVersion: clusterapiv1beta1.GroupVersion.String(), + Kind: cluster.Kind, + Name: cluster.Name, + UID: cluster.UID, + } + + ownerRef.Kind = "Cluster" + patchedVSphereCPIConfig.OwnerReferences = clusterapiutil.EnsureOwnerRef(patchedVSphereCPIConfig.OwnerReferences, ownerRef) + Expect(k8sClient.Patch(ctx, patchedVSphereCPIConfig, client.MergeFrom(config))).ShouldNot(HaveOccurred()) + + // the data values secret should be generated + secret := &v1.Secret{} + Eventually(func() bool { + secretKey := client.ObjectKey{ + Namespace: clusterNamespace, + Name: fmt.Sprintf("%s-%s-data-values", clusterName, constants.CPIAddonName), + } + if err := k8sClient.Get(ctx, secretKey, secret); err != nil { + return false + } + secretData := string(secret.Data["values.yaml"]) + Expect(len(secretData)).ShouldNot(BeZero()) + Expect(strings.Contains(secretData, "vsphereCPI:")).Should(BeTrue()) + Expect(strings.Contains(secretData, "mode: vsphereCPI")).Should(BeTrue()) + Expect(strings.Contains(secretData, "datacenter: dc0")).Should(BeTrue()) + Expect(strings.Contains(secretData, "region: test-region")).Should(BeTrue()) + Expect(strings.Contains(secretData, "zone: test-zone")).Should(BeTrue()) + Expect(strings.Contains(secretData, "insecureFlag: true")).Should(BeTrue()) + Expect(strings.Contains(secretData, "ipFamily: ipv6")).Should(BeTrue()) + Expect(strings.Contains(secretData, "vmInternalNetwork: internal-net")).Should(BeTrue()) + Expect(strings.Contains(secretData, "vmExternalNetwork: external-net")).Should(BeTrue()) + Expect(strings.Contains(secretData, "vmExcludeInternalNetworkSubnetCidr: 192.168.3.0/24")).Should(BeTrue()) + Expect(strings.Contains(secretData, "vmExcludeExternalNetworkSubnetCidr: 22.22.3.0/24")).Should(BeTrue()) + Expect(strings.Contains(secretData, "tlsThumbprint: test-thumbprint")).Should(BeTrue()) + Expect(strings.Contains(secretData, "server: vsphere-server.local")).Should(BeTrue()) + Expect(strings.Contains(secretData, "username: foo")).Should(BeTrue()) + Expect(strings.Contains(secretData, "password: bar")).Should(BeTrue()) + + Expect(strings.Contains(secretData, "http_proxy: foo.com")).Should(BeTrue()) + Expect(strings.Contains(secretData, "https_proxy: bar.com")).Should(BeTrue()) + Expect(strings.Contains(secretData, "no_proxy: foobar.com")).Should(BeTrue()) + + //assert that there are no paravirt datavalue keys + Expect(strings.Contains(secretData, "clusterAPIVersion:")).Should(BeFalse()) + Expect(strings.Contains(secretData, "clusterKind:")).Should(BeFalse()) + Expect(strings.Contains(secretData, "clusterName:")).Should(BeFalse()) + Expect(strings.Contains(secretData, "supervisorMasterEndpointIP:")).Should(BeFalse()) + Expect(strings.Contains(secretData, "supervisorMasterPort:")).Should(BeFalse()) + + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + // eventually the secret ref to the data values should be updated + Eventually(func() bool { + if err := k8sClient.Get(ctx, key, config); err != nil { + return false + } + Expect(config.Status.SecretRef).To(Equal(fmt.Sprintf("%s-%s-data-values", clusterName, constants.CPIAddonName))) + return true + }) + }) + }) + Context("reconcile VSphereCPIConfig manifests in non-paravirtual mode, with multi-tenancy enabled", func() { identity := &capvv1beta1.VSphereClusterIdentity{}