propagates csi feature states from supervisor cluster to vsphere-pv-c…

…si config used by workload clusters (vmware-tanzu#2493)

addons/controllers/csi/vspherecsiconfig_constants.go

@@ -26,10 +26,18 @@ const (
 )
 
 const (
-	VSphereCSINamespace             = "kube-system"
-	VSphereCSIProvisionTimeout      = "300s"
-	VSphereCSIAttachTimeout         = "300s"
-	VSphereCSIResizerTimeout        = "300s"
-	VSphereCSIMinDeploymentReplicas = 1
-	VSphereCSIMaxDeploymentReplicas = 3
+	VSphereCSINamespace                 = "kube-system"
+	VSphereCSIProvisionTimeout          = "300s"
+	VSphereCSIAttachTimeout             = "300s"
+	VSphereCSIResizerTimeout            = "300s"
+	VSphereCSIMinDeploymentReplicas     = 1
+	VSphereCSIMaxDeploymentReplicas     = 3
+	VSphereCSIFeatureStateNamespace     = VSphereSystemCSINamepace
+	VSphereCSIFeatureStateConfigMapName = "csi-feature-states"
+)
+
+const (
+	VSphereSystemCSINamepace                = "vmware-system-csi"
+	DefaultSupervisorMasterEndpointHostname = "supervisor.default.svc"
+	DefaultSupervisorMasterPort             = 6443
 )

addons/controllers/csi/vspherecsiconfig_controller.go

@@ -26,7 +26,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/constants"
 	"github.com/vmware-tanzu/tanzu-framework/addons/pkg/util"
@@ -73,10 +77,46 @@ var providerServiceAccountRBACRules = []rbacv1.PolicyRule{
 	},
 }
 
+// SetupWithManager sets up the controller with the Manager.
+func (r *VSphereCSIConfigReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager,
+	options controller.Options) error {
+
+	c, err := ctrl.NewControllerManagedBy(mgr).
+		For(&csiv1alpha1.VSphereCSIConfig{}).
+		WithOptions(options).
+		Build(r)
+	if err != nil {
+		return errors.Wrap(err, "failed to setup vspherecsiconfig controller")
+	}
+
+	fsPredicates := predicate.Funcs{
+		CreateFunc: func(e event.CreateEvent) bool {
+			return isFeatureStatesConfigMap(e.Object)
+		},
+		UpdateFunc: func(e event.UpdateEvent) bool {
+			return isFeatureStatesConfigMap(e.ObjectNew) &&
+				e.ObjectOld.GetResourceVersion() != e.ObjectNew.GetResourceVersion()
+		},
+		// Delete is not expected to occur
+	}
+
+	if err = c.Watch(&source.Kind{Type: &v1.ConfigMap{}},
+		handler.EnqueueRequestsFromMapFunc(r.ConfigMapToVSphereCSIConfig),
+		fsPredicates); err != nil {
+		return errors.Wrapf(err,
+			"Failed to watch for ConfigMap '%s/%s' while setting vspherecsiconfig controller",
+			VSphereCSIFeatureStateNamespace,
+			VSphereCSIFeatureStateConfigMapName)
+	}
+
+	return nil
+}
+
 //+kubebuilder:rbac:groups=csi.tanzu.vmware.com,resources=vspherecsiconfigs,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=csi.tanzu.vmware.com,resources=vspherecsiconfigs/status,verbs=get;update;patch
 //+kubebuilder:rbac:groups=csi.tanzu.vmware.com,resources=vspherecsiconfigs/finalizers,verbs=update
 //+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch
 //+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch
 //+kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=kubeadmcontrolplanes,verbs=get
 //+kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=providerserviceaccounts,verbs=get;create;list;watch;update;patch
@@ -107,16 +147,6 @@ func (r *VSphereCSIConfigReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	return r.reconcileVSphereCSIConfig(ctx, vcsiConfig, cluster)
 }
 
-// SetupWithManager sets up the controller with the Manager.
-func (r *VSphereCSIConfigReconciler) SetupWithManager(_ context.Context, mgr ctrl.Manager,
-	options controller.Options) error {
-
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&csiv1alpha1.VSphereCSIConfig{}).
-		WithOptions(options).
-		Complete(r)
-}
-
 func (r *VSphereCSIConfigReconciler) reconcileVSphereCSIConfig(ctx context.Context,
 	csiCfg *csiv1alpha1.VSphereCSIConfig,
 	cluster *clusterapiv1beta1.Cluster) (result ctrl.Result, retErr error) {
@@ -273,3 +303,22 @@ func (r *VSphereCSIConfigReconciler) getVsphereCluster(ctx context.Context,
 	}
 	return &vsphereClusters.Items[0], nil
 }
+
+func (r *VSphereCSIConfigReconciler) ConfigMapToVSphereCSIConfig(o client.Object) []ctrl.Request {
+	configs := &csiv1alpha1.VSphereCSIConfigList{}
+	_ = r.List(context.Background(), configs)
+	requests := []ctrl.Request{}
+	for i := 0; i < len(configs.Items); i++ {
+		if configs.Items[i].Spec.VSphereCSI.Mode == VSphereCSIParavirtualMode {
+			requests = append(requests,
+				ctrl.Request{NamespacedName: client.ObjectKey{Namespace: configs.Items[i].Namespace,
+					Name: configs.Items[i].Name}})
+		}
+	}
+	return requests
+}
+
+func isFeatureStatesConfigMap(o metav1.Object) bool {
+	return o.GetNamespace() == VSphereCSIFeatureStateNamespace &&
+		o.GetName() == VSphereCSIFeatureStateConfigMapName
+}

addons/controllers/csi/vspherecsiconfig_datavalues.go

@@ -10,11 +10,12 @@ type DataValues struct {
 }
 
 type DataValuesVSpherePVCSI struct {
-	ClusterName                      string `yaml:"cluster_name"`
-	ClusterUID                       string `yaml:"cluster_uid"`
-	Namespace                        string `yaml:"namespace"`
-	SupervisorMasterEndpointHostname string `yaml:"supervisor_master_endpoint_hostname"`
-	SupervisorMasterPort             int32  `yaml:"supervisor_master_port"`
+	ClusterName                      string            `yaml:"cluster_name"`
+	ClusterUID                       string            `yaml:"cluster_uid"`
+	Namespace                        string            `yaml:"namespace"`
+	SupervisorMasterEndpointHostname string            `yaml:"supervisor_master_endpoint_hostname"`
+	SupervisorMasterPort             int32             `yaml:"supervisor_master_port"`
+	FeatureStates                    map[string]string `yaml:"feature_states,omitempty"`
 }
 
 type DataValuesVSphereCSI struct {

addons/controllers/csi/vspherecsiconfig_utils.go

@@ -43,17 +43,31 @@ func (r *VSphereCSIConfigReconciler) mapVSphereCSIConfigToDataValues(ctx context
 		vcsiConfig.Spec.VSphereCSI.Mode, VSphereCSIParavirtualMode, VSphereCSINonParavirtualMode)
 }
 
-func (r *VSphereCSIConfigReconciler) mapVSphereCSIConfigToDataValuesParavirtual(_ context.Context,
+func (r *VSphereCSIConfigReconciler) mapVSphereCSIConfigToDataValuesParavirtual(ctx context.Context,
 	cluster *clusterapiv1beta1.Cluster) (*DataValues, error) {
 
 	dvs := &DataValues{}
 	dvs.VSpherePVCSI = &DataValuesVSpherePVCSI{}
 	dvs.VSpherePVCSI.ClusterName = cluster.Name
 	dvs.VSpherePVCSI.ClusterUID = string(cluster.UID)
 	// default values from https://github.com/vmware-tanzu/community-edition/blob/main/addons/packages/vsphere-pv-csi/2.4.1/bundle/config/values.yaml
-	dvs.VSpherePVCSI.Namespace = "vmware-system-csi"
-	dvs.VSpherePVCSI.SupervisorMasterEndpointHostname = "supervisor.default.svc"
-	dvs.VSpherePVCSI.SupervisorMasterPort = 6443
+	dvs.VSpherePVCSI.Namespace = VSphereSystemCSINamepace
+	dvs.VSpherePVCSI.SupervisorMasterEndpointHostname = DefaultSupervisorMasterEndpointHostname
+	dvs.VSpherePVCSI.SupervisorMasterPort = DefaultSupervisorMasterPort
+	dvs.VSpherePVCSI.FeatureStates = map[string]string{}
+	featureStatesCM := &v1.ConfigMap{}
+	key := types.NamespacedName{Namespace: VSphereCSIFeatureStateNamespace, Name: VSphereCSIFeatureStateConfigMapName}
+	if err := r.Get(ctx, key, featureStatesCM); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return nil, errors.Errorf("Error reading configmap '%s/%s': %v", key.Namespace, key.Name, err)
+		}
+		dvs.VSpherePVCSI.FeatureStates = nil
+	}
+	if dvs.VSpherePVCSI.FeatureStates != nil {
+		for k, v := range featureStatesCM.Data {
+			dvs.VSpherePVCSI.FeatureStates[k] = v
+		}
+	}
 
 	return dvs, nil
 }

addons/controllers/testdata/test-vsphere-csi-paravirtual.yaml

@@ -1,3 +1,14 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: csi-feature-states
+  namespace: vmware-system-csi
+data:
+  state1 : "value1"
+  state2 : "value2"
+  state3 : "value3"
+---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:

addons/controllers/testdata/vmware-csi-system-ns.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: vmware-system-csi

addons/controllers/vspherecsiconfig_controller_test.go

@@ -28,10 +28,11 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() {
 	)
 
 	var (
-		key                     client.ObjectKey
-		clusterName             string
-		clusterResourceFilePath string
-		vsphereClusterName      string
+		key                       client.ObjectKey
+		clusterName               string
+		clusterResourceFilePath   string
+		vsphereClusterName        string
+		enduringResourcesFilePath string
 	)
 
 	JustBeforeEach(func() {
@@ -40,6 +41,13 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() {
 			Namespace: clusterNamespace,
 			Name:      clusterName,
 		}
+		if enduringResourcesFilePath != "" {
+			fers, err := os.Open(enduringResourcesFilePath)
+			Expect(err).ToNot(HaveOccurred())
+			defer fers.Close()
+			err = testutil.EnsureResources(fers, cfg, dynamicClient)
+			Expect(err).ToNot(HaveOccurred())
+		}
 		f, err := os.Open(clusterResourceFilePath)
 		Expect(err).ToNot(HaveOccurred())
 		defer f.Close()
@@ -69,6 +77,7 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() {
 				clusterName = testClusterCsiName
 				clusterResourceFilePath = "testdata/test-vsphere-csi-non-paravirtual.yaml"
 				vsphereClusterName = "test-cluster-pv-csi-kl5tl"
+				enduringResourcesFilePath = ""
 			})
 
 			It("Should reconcile VSphereCSIConfig and create data values secret for VSphereCSIConfig on management cluster", func() {
@@ -175,6 +184,7 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() {
 			BeforeEach(func() {
 				clusterName = "test-cluster-csi-minimal"
 				clusterResourceFilePath = "testdata/test-vsphere-csi-non-paravirtual-minimal.yaml"
+				enduringResourcesFilePath = ""
 			})
 
 			It("Should reconcile VSphereCSIConfig and create data values secret for VSphereCSIConfig", func() {
@@ -260,6 +270,7 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() {
 		BeforeEach(func() {
 			clusterName = "test-cluster-pv-csi"
 			clusterResourceFilePath = "testdata/test-vsphere-csi-paravirtual.yaml"
+			enduringResourcesFilePath = "testdata/vmware-csi-system-ns.yaml"
 		})
 		It("Should reconcile VSphereCSIConfig and create data values secret for VSphereCSIConfig on management cluster", func() {
 			// the data values secret should be generated
@@ -282,6 +293,10 @@ var _ = Describe("VSphereCSIConfig Reconciler", func() {
 				Expect(strings.Contains(secretData, "namespace: vmware-system-csi")).Should(BeTrue())
 				Expect(strings.Contains(secretData, "supervisor_master_endpoint_hostname: supervisor.default.svc")).Should(BeTrue())
 				Expect(strings.Contains(secretData, "supervisor_master_port: 6443")).Should(BeTrue())
+				Expect(strings.Contains(secretData, "feature_states:")).Should(BeTrue())
+				Expect(strings.Contains(secretData, "state1: value1")).Should(BeTrue())
+				Expect(strings.Contains(secretData, "state2: value2")).Should(BeTrue())
+				Expect(strings.Contains(secretData, "state3: value3")).Should(BeTrue())
 
 				return true
 			}, waitTimeout, pollingInterval).Should(BeTrue())

addons/test/testutil/test_helpers.go

@@ -155,6 +155,33 @@ func DeleteResources(f *os.File, cfg *rest.Config, dynamicClient dynamic.Interfa
 	return nil
 }
 
+// EnsureResources verifies that resources exist, creating it if necessary
+func EnsureResources(f *os.File, cfg *rest.Config, dynamicClient dynamic.Interface) error {
+	decoder, mapper, err := parseObjects(f, cfg)
+	if err != nil {
+		return err
+	}
+
+	for {
+		resource, unstructuredObj, err := getResource(decoder, mapper, dynamicClient)
+		if err != nil {
+			if err == io.EOF {
+				break
+			} else {
+				return err
+			}
+		}
+		_, err = resource.Get(context.Background(), unstructuredObj.GetName(), metav1.GetOptions{})
+		if apierrors.IsNotFound(err) {
+			// create it
+			if _, err := resource.Create(context.Background(), unstructuredObj, metav1.CreateOptions{}); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 // CreateKubeconfigSecret create a secret with kubeconfig token for the cluster provided by client
 func CreateKubeconfigSecret(cfg *rest.Config, clusterName, namespace string, crClient client.Client) error {
 	clusters := make(map[string]*clientcmdapi.Cluster)