From de302f08ecb2717f49a06dcbcf667fd4670de16b Mon Sep 17 00:00:00 2001 From: Chenrui Li Date: Thu, 19 May 2022 15:23:56 +0800 Subject: [PATCH] Update SkipCNIBinaries param to CalicoConfig --- .../v1alpha1/calicoconfig/default.yaml | 12 +++ .../v1alpha1/calicoconfig.yaml | 12 +++ .../controllers/calico/calicoconfig_utils.go | 8 +- .../calicoconfig_controller_test.go | 100 ++++++++++++++++-- .../{test-calico.yaml => test-calico-1.yaml} | 8 +- .../controllers/testdata/test-calico-2.yaml | 23 ++++ apis/cni/v1alpha1/calicoconfig_types.go | 26 +++-- .../cni.tanzu.vmware.com_calicoconfigs.yaml | 31 +++--- 8 files changed, 182 insertions(+), 38 deletions(-) rename addons/controllers/testdata/{test-calico.yaml => test-calico-1.yaml} (75%) create mode 100644 addons/controllers/testdata/test-calico-2.yaml diff --git a/addons/config/expected/cni.tanzu.vmware.com/v1alpha1/calicoconfig/default.yaml b/addons/config/expected/cni.tanzu.vmware.com/v1alpha1/calicoconfig/default.yaml index 0e117ddc90d..4915409c2e2 100644 --- a/addons/config/expected/cni.tanzu.vmware.com/v1alpha1/calicoconfig/default.yaml +++ b/addons/config/expected/cni.tanzu.vmware.com/v1alpha1/calicoconfig/default.yaml @@ -7,3 +7,15 @@ spec: calico: config: vethMTU: 0 + skipCNIBinaries: true +--- +apiVersion: cni.tanzu.vmware.com/v1alpha1 +kind: CalicoConfig +metadata: + name: v1.23.3---vmware.1-tkg.1-docker + namespace: tkg-system +spec: + calico: + config: + vethMTU: 0 + skipCNIBinaries: false diff --git a/addons/config/templates/cni.tanzu.vmware.com/v1alpha1/calicoconfig.yaml b/addons/config/templates/cni.tanzu.vmware.com/v1alpha1/calicoconfig.yaml index 3b0babab635..8b613578d7b 100644 --- a/addons/config/templates/cni.tanzu.vmware.com/v1alpha1/calicoconfig.yaml +++ b/addons/config/templates/cni.tanzu.vmware.com/v1alpha1/calicoconfig.yaml @@ -9,3 +9,15 @@ spec: calico: config: vethMTU: 0 + skipCNIBinaries: true +--- +apiVersion: cni.tanzu.vmware.com/v1alpha1 +kind: CalicoConfig +metadata: + name: #@ "{}-docker".format(data.values.TKR_VERSION) + namespace: #@ data.values.GLOBAL_NAMESPACE +spec: + calico: + config: + vethMTU: 0 + skipCNIBinaries: false diff --git a/addons/controllers/calico/calicoconfig_utils.go b/addons/controllers/calico/calicoconfig_utils.go index 47464ed4823..f7da3f2aa13 100644 --- a/addons/controllers/calico/calicoconfig_utils.go +++ b/addons/controllers/calico/calicoconfig_utils.go @@ -16,7 +16,7 @@ import ( cniv1alpha1 "github.com/vmware-tanzu/tanzu-framework/apis/cni/v1alpha1" ) -// calicoConfigSpec defines the desired state of CalicoConfig +// calicoConfigSpec defines the desired state of CalicoConfig. type calicoConfigSpec struct { InfraProvider string `yaml:"infraProvider"` IPFamily string `yaml:"ipFamily,omitempty"` @@ -28,8 +28,9 @@ type calico struct { } type config struct { - VethMTU string `yaml:"vethMTU,omitempty"` - ClusterCIDR string `yaml:"clusterCIDR"` + VethMTU string `yaml:"vethMTU,omitempty"` + ClusterCIDR string `yaml:"clusterCIDR"` + SkipCNIBinaries bool `yaml:"skipCNIBinaries"` } func mapCalicoConfigSpec(cluster *clusterapiv1beta1.Cluster, config *cniv1alpha1.CalicoConfig) (*calicoConfigSpec, error) { @@ -37,6 +38,7 @@ func mapCalicoConfigSpec(cluster *clusterapiv1beta1.Cluster, config *cniv1alpha1 configSpec := &calicoConfigSpec{} configSpec.Calico.Config.VethMTU = strconv.FormatInt(config.Spec.Calico.Config.VethMTU, 10) + configSpec.Calico.Config.SkipCNIBinaries = config.Spec.Calico.Config.SkipCNIBinaries // Derive InfraProvider from the cluster configSpec.InfraProvider, err = util.GetInfraProvider(cluster) diff --git a/addons/controllers/calicoconfig_controller_test.go b/addons/controllers/calicoconfig_controller_test.go index 65538b4ad10..3adb43b1e1f 100644 --- a/addons/controllers/calicoconfig_controller_test.go +++ b/addons/controllers/calicoconfig_controller_test.go @@ -19,16 +19,19 @@ import ( cniv1alpha1 "github.com/vmware-tanzu/tanzu-framework/apis/cni/v1alpha1" ) -const testCluster = "test-cluster-calico" +const ( + testClusterCalico1 = "test-cluster-calico-1" + testClusterCalico2 = "test-cluster-calico-2" + testDataCalico1 = "testdata/test-calico-1.yaml" + testDataCalico2 = "testdata/test-calico-2.yaml" +) var _ = Describe("CalicoConfig Reconciler and Webhooks", func() { var ( - clusterName string + clusterName string + clusterResourceFilePath string ) - const ( - clusterResourceFilePath = "testdata/test-calico.yaml" - ) JustBeforeEach(func() { // Create the admission webhooks f, err := os.Open(cniWebhookManifestFile) @@ -66,15 +69,16 @@ var _ = Describe("CalicoConfig Reconciler and Webhooks", func() { f.Close() }) - Context("reconcile CalicoConfig for management cluster", func() { + Context("reconcile default CalicoConfig for management cluster on dual-stack CIDR", func() { BeforeEach(func() { - clusterName = testCluster + clusterName = testClusterCalico1 + clusterResourceFilePath = testDataCalico1 }) It("Should reconcile CalicoConfig and create data values secret for CalicoConfig on management cluster", func() { key := client.ObjectKey{ Namespace: "default", - Name: testCluster, + Name: testClusterCalico1, } cluster := &clusterapiv1beta1.Cluster{} @@ -93,13 +97,14 @@ var _ = Describe("CalicoConfig Reconciler and Webhooks", func() { // check spec values Expect(config.Spec.Calico.Config.VethMTU).Should(Equal(int64(0))) + Expect(config.Spec.Calico.Config.SkipCNIBinaries).Should(BeTrue()) // check owner reference if len(config.OwnerReferences) == 0 { return false } Expect(len(config.OwnerReferences)).Should(Equal(1)) - Expect(config.OwnerReferences[0].Name).Should(Equal(testCluster)) + Expect(config.OwnerReferences[0].Name).Should(Equal(testClusterCalico1)) return true }, waitTimeout, pollingInterval).Should(BeTrue()) @@ -121,6 +126,83 @@ var _ = Describe("CalicoConfig Reconciler and Webhooks", func() { Expect(strings.Contains(secretData, "ipFamily: ipv4,ipv6")).Should(BeTrue()) Expect(strings.Contains(secretData, "clusterCIDR: 192.168.0.0/16,fd00:100:96::/48")).Should(BeTrue()) Expect(strings.Contains(secretData, "vethMTU: \"0\"")).Should(BeTrue()) + Expect(strings.Contains(secretData, "skipCNIBinaries: true")).Should(BeTrue()) + + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + Eventually(func() bool { + config := &cniv1alpha1.CalicoConfig{} + err := k8sClient.Get(ctx, key, config) + if err != nil { + return false + } + // Check status.secretName after reconciliation + Expect(config.Status.SecretRef).Should(Equal(fmt.Sprintf("%s-%s-data-values", clusterName, constants.CalicoAddonName))) + + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + }) + }) + + Context("reconcile mtu customized and cni binaries installation skipped CalicoConfig for management cluster on ipv4 CIDR", func() { + BeforeEach(func() { + clusterName = testClusterCalico2 + clusterResourceFilePath = testDataCalico2 + }) + + It("Should reconcile CalicoConfig and create data values secret for CalicoConfig on management cluster", func() { + key := client.ObjectKey{ + Namespace: "default", + Name: testClusterCalico2, + } + + cluster := &clusterapiv1beta1.Cluster{} + Eventually(func() bool { + if err := k8sClient.Get(ctx, key, cluster); err != nil { + return false + } + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + config := &cniv1alpha1.CalicoConfig{} + Eventually(func() bool { + if err := k8sClient.Get(ctx, key, config); err != nil { + return false + } + + // check spec values + Expect(config.Spec.Calico.Config.VethMTU).Should(Equal(int64(1420))) + Expect(config.Spec.Calico.Config.SkipCNIBinaries).Should(BeFalse()) + + // check owner reference + if len(config.OwnerReferences) == 0 { + return false + } + Expect(len(config.OwnerReferences)).Should(Equal(1)) + Expect(config.OwnerReferences[0].Name).Should(Equal(testClusterCalico2)) + + return true + }, waitTimeout, pollingInterval).Should(BeTrue()) + + Eventually(func() bool { + secretKey := client.ObjectKey{ + Namespace: "default", + Name: fmt.Sprintf("%s-%s-data-values", clusterName, constants.CalicoAddonName), + } + secret := &v1.Secret{} + if err := k8sClient.Get(ctx, secretKey, secret); err != nil { + return false + } + + // check data values secret contents + Expect(secret.Type).Should(Equal(v1.SecretTypeOpaque)) + secretData := string(secret.Data["values.yaml"]) + Expect(strings.Contains(secretData, "infraProvider: docker")).Should(BeTrue()) + Expect(strings.Contains(secretData, "ipFamily: ipv4")).Should(BeTrue()) + Expect(strings.Contains(secretData, "clusterCIDR: 192.168.0.0/16")).Should(BeTrue()) + Expect(strings.Contains(secretData, "vethMTU: \"1420\"")).Should(BeTrue()) + Expect(strings.Contains(secretData, "skipCNIBinaries: false")).Should(BeTrue()) return true }, waitTimeout, pollingInterval).Should(BeTrue()) diff --git a/addons/controllers/testdata/test-calico.yaml b/addons/controllers/testdata/test-calico-1.yaml similarity index 75% rename from addons/controllers/testdata/test-calico.yaml rename to addons/controllers/testdata/test-calico-1.yaml index 35fcc2e395b..2c9dfd13126 100644 --- a/addons/controllers/testdata/test-calico.yaml +++ b/addons/controllers/testdata/test-calico-1.yaml @@ -2,7 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: - name: test-cluster-calico + name: test-cluster-calico-1 namespace: default spec: infrastructureRef: @@ -14,12 +14,10 @@ spec: apiVersion: cni.tanzu.vmware.com/v1alpha1 kind: CalicoConfig metadata: - name: test-cluster-calico + name: test-cluster-calico-1 namespace: default spec: - infraProvider: vsphere - ipFamily: ipv4 - clusterCIDR: "" calico: config: vethMTU: 0 + skipCNIBinaries: true diff --git a/addons/controllers/testdata/test-calico-2.yaml b/addons/controllers/testdata/test-calico-2.yaml new file mode 100644 index 00000000000..b050d879492 --- /dev/null +++ b/addons/controllers/testdata/test-calico-2.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: test-cluster-calico-2 + namespace: default +spec: + infrastructureRef: + kind: DockerCluster + clusterNetwork: + pods: + cidrBlocks: [ "192.168.0.0/16"] +--- +apiVersion: cni.tanzu.vmware.com/v1alpha1 +kind: CalicoConfig +metadata: + name: test-cluster-calico-2 + namespace: default +spec: + calico: + config: + vethMTU: 1420 + skipCNIBinaries: false diff --git a/apis/cni/v1alpha1/calicoconfig_types.go b/apis/cni/v1alpha1/calicoconfig_types.go index 6e5d98e3546..8295693e039 100644 --- a/apis/cni/v1alpha1/calicoconfig_types.go +++ b/apis/cni/v1alpha1/calicoconfig_types.go @@ -7,26 +7,35 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// CalicoConfigSpec defines the desired state of CalicoConfig +// CalicoConfigSpec defines the desired state of CalicoConfig. type CalicoConfigSpec struct { Calico Calico `json:"calico,omitempty"` } +// Calico stores the configurations for Calico. type Calico struct { Config CalicoConfigDataValue `json:"config,omitempty"` } +// CalicoConfigDataValue contains the specific configurations for the Calico package. type CalicoConfigDataValue struct { - // Maximum transmission unit setting. "0" as default means MTU will be auto detected - //+ kubebuilder:validation:Optional + // VethMTU defines maximum transmission unit. "0" as default means MTU will be auto detected. + //+kubebuilder:validation:Optional //+kubebuilder:validation:Minimum=0 //+kubebuilder:default:=0 VethMTU int64 `json:"vethMTU,omitempty"` + + // SkipCNIBinaries allows to skip the cni plugin binaries installation. + // Default to false. Set to true for providers who already installed + // cni plugin binaries in their OVAs and do not want Calico to overwrite them. + //+kubebuilder:validation:Optional + //+kubebuilder:default:=false + SkipCNIBinaries bool `json:"skipCNIBinaries,omitempty"` } -// CalicoConfigStatus defines the observed state of CalicoConfig +// CalicoConfigStatus defines the observed state of CalicoConfig. type CalicoConfigStatus struct { - // Name of the data value secret created by calico controller + // SecretRef is the name of the data value secret created by calico controller. //+ kubebuilder:validation:Optional SecretRef string `json:"secretRef,omitempty"` } @@ -34,11 +43,10 @@ type CalicoConfigStatus struct { //+kubebuilder:object:root=true //+kubebuilder:subresource:status //+kubebuilder:resource:path=calicoconfigs,shortName=calicoconf,scope=Namespaced -//+kubebuilder:printcolumn:name="Namespace",type="string",JSONPath=".spec.cni.namespace",description="The namespace in which calico is deployed" -//+kubebuilder:printcolumn:name="VethMTU",type="string",JSONPath=".spec.cni.calico.config.vethMTU",description="Maximum transmission unit setting. '0' as default means MTU will be auto detected" +//+kubebuilder:printcolumn:name="VethMTU",type="integer",JSONPath=".spec.calico.config.vethMTU",description="Maximum transmission unit setting. '0' as default means MTU will be auto detected" //+kubebuilder:printcolumn:name="SecretRef",type="string",JSONPath=".status.secretRef",description="Name of the Calico data values secret" -// CalicoConfig is the Schema for the calicoconfigs API +// CalicoConfig is the Schema for the calicoconfigs API. type CalicoConfig struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -49,7 +57,7 @@ type CalicoConfig struct { //+kubebuilder:object:root=true -// CalicoConfigList contains a list of CalicoConfig +// CalicoConfigList contains a list of CalicoConfig. type CalicoConfigList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/config/crd/bases/cni.tanzu.vmware.com_calicoconfigs.yaml b/config/crd/bases/cni.tanzu.vmware.com_calicoconfigs.yaml index b95f0efe07e..0685a240f3a 100644 --- a/config/crd/bases/cni.tanzu.vmware.com_calicoconfigs.yaml +++ b/config/crd/bases/cni.tanzu.vmware.com_calicoconfigs.yaml @@ -18,15 +18,11 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: - - description: The namespace in which calico is deployed - jsonPath: .spec.cni.namespace - name: Namespace - type: string - description: Maximum transmission unit setting. '0' as default means MTU will be auto detected - jsonPath: .spec.cni.calico.config.vethMTU + jsonPath: .spec.calico.config.vethMTU name: VethMTU - type: string + type: integer - description: Name of the Calico data values secret jsonPath: .status.secretRef name: SecretRef @@ -34,7 +30,7 @@ spec: name: v1alpha1 schema: openAPIV3Schema: - description: CalicoConfig is the Schema for the calicoconfigs API + description: CalicoConfig is the Schema for the calicoconfigs API. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -49,16 +45,26 @@ spec: metadata: type: object spec: - description: CalicoConfigSpec defines the desired state of CalicoConfig + description: CalicoConfigSpec defines the desired state of CalicoConfig. properties: calico: + description: Calico stores the configurations for Calico. properties: config: + description: CalicoConfigDataValue contains the specific configurations + for the Calico package. properties: + skipCNIBinaries: + default: false + description: SkipCNIBinaries allows to skip the cni plugin + binaries installation. Default to false. Set to true for + providers who already installed cni plugin binaries in their + OVAs and do not want Calico to overwrite them. + type: boolean vethMTU: default: 0 - description: Maximum transmission unit setting. "0" as default - means MTU will be auto detected + description: VethMTU defines maximum transmission unit. "0" + as default means MTU will be auto detected. format: int64 minimum: 0 type: integer @@ -66,10 +72,11 @@ spec: type: object type: object status: - description: CalicoConfigStatus defines the observed state of CalicoConfig + description: CalicoConfigStatus defines the observed state of CalicoConfig. properties: secretRef: - description: Name of the data value secret created by calico controller + description: SecretRef is the name of the data value secret created + by calico controller. type: string type: object required: