From 35aecbe43e3af509a9d4b92dfec9491d25f877af Mon Sep 17 00:00:00 2001 From: Peter Ochodo Date: Tue, 7 Jun 2022 10:15:32 -0700 Subject: [PATCH] propagates csi feature states from supervisor cluster to vsphere-pv-csi config used by workload clusters (#2493) --- .../csi/vspherecsiconfig_constants.go | 20 ++++-- .../csi/vspherecsiconfig_controller.go | 69 ++++++++++++++++--- .../csi/vspherecsiconfig_datavalues.go | 11 +-- .../controllers/csi/vspherecsiconfig_utils.go | 22 ++++-- .../test-vsphere-csi-paravirtual.yaml | 11 +++ .../testdata/vmware-csi-system-ns.yaml | 5 ++ .../vspherecsiconfig_controller_test.go | 23 +++++-- addons/test/testutil/test_helpers.go | 27 ++++++++ 8 files changed, 159 insertions(+), 29 deletions(-) create mode 100644 addons/controllers/testdata/vmware-csi-system-ns.yaml diff --git a/addons/controllers/csi/vspherecsiconfig_constants.go b/addons/controllers/csi/vspherecsiconfig_constants.go index d184056640..a71095eb66 100644 --- a/addons/controllers/csi/vspherecsiconfig_constants.go +++ b/addons/controllers/csi/vspherecsiconfig_constants.go @@ -26,10 +26,18 @@ const ( ) const ( - VSphereCSINamespace = "kube-system" - VSphereCSIProvisionTimeout = "300s" - VSphereCSIAttachTimeout = "300s" - VSphereCSIResizerTimeout = "300s" - VSphereCSIMinDeploymentReplicas = 1 - VSphereCSIMaxDeploymentReplicas = 3 + VSphereCSINamespace = "kube-system" + VSphereCSIProvisionTimeout = "300s" + VSphereCSIAttachTimeout = "300s" + VSphereCSIResizerTimeout = "300s" + VSphereCSIMinDeploymentReplicas = 1 + VSphereCSIMaxDeploymentReplicas = 3 + VSphereCSIFeatureStateNamespace = VSphereSystemCSINamepace + VSphereCSIFeatureStateConfigMapName = "csi-feature-states" +) + +const ( + VSphereSystemCSINamepace = "vmware-system-csi" + DefaultSupervisorMasterEndpointHostname = "supervisor.default.svc" + DefaultSupervisorMasterPort = 6443 ) diff --git a/addons/controllers/csi/vspherecsiconfig_controller.go b/addons/controllers/csi/vspherecsiconfig_controller.go index 4ede1cee5b..fb0a587f4d 100644 --- a/addons/controllers/csi/vspherecsiconfig_controller.go +++ b/addons/controllers/csi/vspherecsiconfig_controller.go @@ -26,7 +26,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants" "github.com/vmware-tanzu/tanzu-framework/addons/pkg/util" @@ -73,10 +77,46 @@ var providerServiceAccountRBACRules = []rbacv1.PolicyRule{ }, } +// SetupWithManager sets up the controller with the Manager. +func (r *VSphereCSIConfigReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, + options controller.Options) error { + + c, err := ctrl.NewControllerManagedBy(mgr). + For(&csiv1alpha1.VSphereCSIConfig{}). + WithOptions(options). + Build(r) + if err != nil { + return errors.Wrap(err, "failed to setup vspherecsiconfig controller") + } + + fsPredicates := predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return isFeatureStatesConfigMap(e.Object) + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return isFeatureStatesConfigMap(e.ObjectNew) && + e.ObjectOld.GetResourceVersion() != e.ObjectNew.GetResourceVersion() + }, + // Delete is not expected to occur + } + + if err = c.Watch(&source.Kind{Type: &v1.ConfigMap{}}, + handler.EnqueueRequestsFromMapFunc(r.ConfigMapToVSphereCSIConfig), + fsPredicates); err != nil { + return errors.Wrapf(err, + "Failed to watch for ConfigMap '%s/%s' while setting vspherecsiconfig controller", + VSphereCSIFeatureStateNamespace, + VSphereCSIFeatureStateConfigMapName) + } + + return nil +} + //+kubebuilder:rbac:groups=csi.tanzu.vmware.com,resources=vspherecsiconfigs,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=csi.tanzu.vmware.com,resources=vspherecsiconfigs/status,verbs=get;update;patch //+kubebuilder:rbac:groups=csi.tanzu.vmware.com,resources=vspherecsiconfigs/finalizers,verbs=update //+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch //+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch //+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=kubeadmcontrolplanes,verbs=get //+kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=providerserviceaccounts,verbs=get;create;list;watch;update;patch @@ -107,16 +147,6 @@ func (r *VSphereCSIConfigReconciler) Reconcile(ctx context.Context, req ctrl.Req return r.reconcileVSphereCSIConfig(ctx, vcsiConfig, cluster) } -// SetupWithManager sets up the controller with the Manager. -func (r *VSphereCSIConfigReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager, - options controller.Options) error { - - return ctrl.NewControllerManagedBy(mgr). - For(&csiv1alpha1.VSphereCSIConfig{}). - WithOptions(options). - Complete(r) -} - func (r *VSphereCSIConfigReconciler) reconcileVSphereCSIConfig(ctx context.Context, csiCfg *csiv1alpha1.VSphereCSIConfig, cluster *clusterapiv1beta1.Cluster) (result ctrl.Result, retErr error) { @@ -273,3 +303,22 @@ func (r *VSphereCSIConfigReconciler) getVsphereCluster(ctx context.Context, } return &vsphereClusters.Items[0], nil } + +func (r *VSphereCSIConfigReconciler) ConfigMapToVSphereCSIConfig(o client.Object) []ctrl.Request { + configs := &csiv1alpha1.VSphereCSIConfigList{} + _ = r.List(context.Background(), configs) + requests := []ctrl.Request{} + for i := 0; i < len(configs.Items); i++ { + if configs.Items[i].Spec.VSphereCSI.Mode == VSphereCSIParavirtualMode { + requests = append(requests, + ctrl.Request{NamespacedName: client.ObjectKey{Namespace: configs.Items[i].Namespace, + Name: configs.Items[i].Name}}) + } + } + return requests +} + +func isFeatureStatesConfigMap(o metav1.Object) bool { + return o.GetNamespace() == VSphereCSIFeatureStateNamespace && + o.GetName() == VSphereCSIFeatureStateConfigMapName +} diff --git a/addons/controllers/csi/vspherecsiconfig_datavalues.go b/addons/controllers/csi/vspherecsiconfig_datavalues.go index 66641612c9..ba52c59b5e 100644 --- a/addons/controllers/csi/vspherecsiconfig_datavalues.go +++ b/addons/controllers/csi/vspherecsiconfig_datavalues.go @@ -10,11 +10,12 @@ type DataValues struct { } type DataValuesVSpherePVCSI struct { - ClusterName string `yaml:"cluster_name"` - ClusterUID string `yaml:"cluster_uid"` - Namespace string `yaml:"namespace"` - SupervisorMasterEndpointHostname string `yaml:"supervisor_master_endpoint_hostname"` - SupervisorMasterPort int32 `yaml:"supervisor_master_port"` + ClusterName string `yaml:"cluster_name"` + ClusterUID string `yaml:"cluster_uid"` + Namespace string `yaml:"namespace"` + SupervisorMasterEndpointHostname string `yaml:"supervisor_master_endpoint_hostname"` + SupervisorMasterPort int32 `yaml:"supervisor_master_port"` + FeatureStates map[string]string `yaml:"feature_states,omitempty"` } type DataValuesVSphereCSI struct { diff --git a/addons/controllers/csi/vspherecsiconfig_utils.go b/addons/controllers/csi/vspherecsiconfig_utils.go index 216df71352..6663be6461 100644 --- a/addons/controllers/csi/vspherecsiconfig_utils.go +++ b/addons/controllers/csi/vspherecsiconfig_utils.go @@ -43,7 +43,7 @@ func (r *VSphereCSIConfigReconciler) mapVSphereCSIConfigToDataValues(ctx context vcsiConfig.Spec.VSphereCSI.Mode, VSphereCSIParavirtualMode, VSphereCSINonParavirtualMode) } -func (r *VSphereCSIConfigReconciler) mapVSphereCSIConfigToDataValuesParavirtual(_ context.Context, +func (r *VSphereCSIConfigReconciler) mapVSphereCSIConfigToDataValuesParavirtual(ctx context.Context, cluster *clusterapiv1beta1.Cluster) (*DataValues, error) { dvs := &DataValues{} @@ -51,9 +51,23 @@ func (r *VSphereCSIConfigReconciler) mapVSphereCSIConfigToDataValuesParavirtual( dvs.VSpherePVCSI.ClusterName = cluster.Name dvs.VSpherePVCSI.ClusterUID = string(cluster.UID) // default values from https://github.com/vmware-tanzu/community-edition/blob/main/addons/packages/vsphere-pv-csi/2.4.1/bundle/config/values.yaml - dvs.VSpherePVCSI.Namespace = "vmware-system-csi" - dvs.VSpherePVCSI.SupervisorMasterEndpointHostname = "supervisor.default.svc" - dvs.VSpherePVCSI.SupervisorMasterPort = 6443 + dvs.VSpherePVCSI.Namespace = VSphereSystemCSINamepace + dvs.VSpherePVCSI.SupervisorMasterEndpointHostname = DefaultSupervisorMasterEndpointHostname + dvs.VSpherePVCSI.SupervisorMasterPort = DefaultSupervisorMasterPort + dvs.VSpherePVCSI.FeatureStates = map[string]string{} + featureStatesCM := &v1.ConfigMap{} + key := types.NamespacedName{Namespace: VSphereCSIFeatureStateNamespace, Name: VSphereCSIFeatureStateConfigMapName} + if err := r.Get(ctx, key, featureStatesCM); err != nil { + if !apierrors.IsNotFound(err) { + return nil, errors.Errorf("Error reading configmap '%s/%s': %v", key.Namespace, key.Name, err) + } + dvs.VSpherePVCSI.FeatureStates = nil + } + if dvs.VSpherePVCSI.FeatureStates != nil { + for k, v := range featureStatesCM.Data { + dvs.VSpherePVCSI.FeatureStates[k] = v + } + } return dvs, nil } diff --git a/addons/controllers/testdata/test-vsphere-csi-paravirtual.yaml b/addons/controllers/testdata/test-vsphere-csi-paravirtual.yaml index 39cb4802d6..2ff415b6f3 100644 --- a/addons/controllers/testdata/test-vsphere-csi-paravirtual.yaml +++ b/addons/controllers/testdata/test-vsphere-csi-paravirtual.yaml @@ -1,3 +1,14 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: csi-feature-states + namespace: vmware-system-csi +data: + state1 : "value1" + state2 : "value2" + state3 : "value3" +--- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: diff --git a/addons/controllers/testdata/vmware-csi-system-ns.yaml b/addons/controllers/testdata/vmware-csi-system-ns.yaml new file mode 100644 index 0000000000..e48cf0646d --- /dev/null +++ b/addons/controllers/testdata/vmware-csi-system-ns.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: vmware-system-csi diff --git a/addons/controllers/vspherecsiconfig_controller_test.go b/addons/controllers/vspherecsiconfig_controller_test.go index adc2016e94..0f81f576dd 100644 --- a/addons/controllers/vspherecsiconfig_controller_test.go +++ b/addons/controllers/vspherecsiconfig_controller_test.go @@ -28,10 +28,11 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() { ) var ( - key client.ObjectKey - clusterName string - clusterResourceFilePath string - vsphereClusterName string + key client.ObjectKey + clusterName string + clusterResourceFilePath string + vsphereClusterName string + enduringResourcesFilePath string ) JustBeforeEach(func() { @@ -40,6 +41,13 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() { Namespace: clusterNamespace, Name: clusterName, } + if enduringResourcesFilePath != "" { + fers, err := os.Open(enduringResourcesFilePath) + Expect(err).ToNot(HaveOccurred()) + defer fers.Close() + err = testutil.EnsureResources(fers, cfg, dynamicClient) + Expect(err).ToNot(HaveOccurred()) + } f, err := os.Open(clusterResourceFilePath) Expect(err).ToNot(HaveOccurred()) defer f.Close() @@ -69,6 +77,7 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() { clusterName = testClusterCsiName clusterResourceFilePath = "testdata/test-vsphere-csi-non-paravirtual.yaml" vsphereClusterName = "test-cluster-pv-csi-kl5tl" + enduringResourcesFilePath = "" }) It("Should reconcile VSphereCSIConfig and create data values secret for VSphereCSIConfig on management cluster", func() { @@ -175,6 +184,7 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() { BeforeEach(func() { clusterName = "test-cluster-csi-minimal" clusterResourceFilePath = "testdata/test-vsphere-csi-non-paravirtual-minimal.yaml" + enduringResourcesFilePath = "" }) It("Should reconcile VSphereCSIConfig and create data values secret for VSphereCSIConfig", func() { @@ -260,6 +270,7 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() { BeforeEach(func() { clusterName = "test-cluster-pv-csi" clusterResourceFilePath = "testdata/test-vsphere-csi-paravirtual.yaml" + enduringResourcesFilePath = "testdata/vmware-csi-system-ns.yaml" }) It("Should reconcile VSphereCSIConfig and create data values secret for VSphereCSIConfig on management cluster", func() { // the data values secret should be generated @@ -282,6 +293,10 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() { Expect(strings.Contains(secretData, "namespace: vmware-system-csi")).Should(BeTrue()) Expect(strings.Contains(secretData, "supervisor_master_endpoint_hostname: supervisor.default.svc")).Should(BeTrue()) Expect(strings.Contains(secretData, "supervisor_master_port: 6443")).Should(BeTrue()) + Expect(strings.Contains(secretData, "feature_states:")).Should(BeTrue()) + Expect(strings.Contains(secretData, "state1: value1")).Should(BeTrue()) + Expect(strings.Contains(secretData, "state2: value2")).Should(BeTrue()) + Expect(strings.Contains(secretData, "state3: value3")).Should(BeTrue()) return true }, waitTimeout, pollingInterval).Should(BeTrue()) diff --git a/addons/test/testutil/test_helpers.go b/addons/test/testutil/test_helpers.go index f2211de79e..ad9f77fdcd 100644 --- a/addons/test/testutil/test_helpers.go +++ b/addons/test/testutil/test_helpers.go @@ -155,6 +155,33 @@ func DeleteResources(f *os.File, cfg *rest.Config, dynamicClient dynamic.Interfa return nil } +// EnsureResources verifies that resources exist, creating it if necessary +func EnsureResources(f *os.File, cfg *rest.Config, dynamicClient dynamic.Interface) error { + decoder, mapper, err := parseObjects(f, cfg) + if err != nil { + return err + } + + for { + resource, unstructuredObj, err := getResource(decoder, mapper, dynamicClient) + if err != nil { + if err == io.EOF { + break + } else { + return err + } + } + _, err = resource.Get(context.Background(), unstructuredObj.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + // create it + if _, err := resource.Create(context.Background(), unstructuredObj, metav1.CreateOptions{}); err != nil { + return err + } + } + } + return nil +} + // CreateKubeconfigSecret create a secret with kubeconfig token for the cluster provided by client func CreateKubeconfigSecret(cfg *rest.Config, clusterName, namespace string, crClient client.Client) error { clusters := make(map[string]*clientcmdapi.Cluster)