diff --git a/packages/tkg-clusterclass-vsphere/bundle/config/upstream/base.yaml b/packages/tkg-clusterclass-vsphere/bundle/config/upstream/base.yaml index 924d49c8c98..7f2b33fbfb8 100644 --- a/packages/tkg-clusterclass-vsphere/bundle/config/upstream/base.yaml +++ b/packages/tkg-clusterclass-vsphere/bundle/config/upstream/base.yaml @@ -293,6 +293,22 @@ spec: ipv6Primary: type: boolean default: false + addressesFromPools: + type: array + items: + type: object + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - apiGroup + - kind + - name + default: [] default: ipv6Primary: false - name: proxy @@ -772,8 +788,16 @@ spec: - {{ . }} {{- end }} {{- end }} - {{ if list "IPv4" "DualStack" | has .builtin.cluster.network.ipFamily -}} dhcp4: true {{- end }} - {{ if list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily -}} dhcp6: true {{- end }} + {{ if list "IPv4" "DualStack" | has .builtin.cluster.network.ipFamily | and (empty .network.addressesFromPools) -}} dhcp4: true {{- end }} + {{ if list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily | and (empty .network.addressesFromPools) -}} dhcp6: true {{- end }} + {{ if .network.addressesFromPools -}} + addressesFromPools: + {{- range .network.addressesFromPools }} + - apiGroup: {{ .apiGroup }} + kind: {{ .kind }} + name: {{ .name }} + {{- end }} + {{- end }} - selector: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate @@ -801,8 +825,16 @@ spec: - {{ . }} {{- end }} {{- end }} - {{ if list "IPv4" "DualStack" | has .builtin.cluster.network.ipFamily -}} dhcp4: true {{- end }} - {{ if list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily -}} dhcp6: true {{- end }} + {{ if list "IPv4" "DualStack" | has .builtin.cluster.network.ipFamily | and (empty .network.addressesFromPools) -}} dhcp4: true {{- end }} + {{ if list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily | and (empty .network.addressesFromPools) -}} dhcp6: true {{- end }} + {{ if .network.addressesFromPools -}} + addressesFromPools: + {{- range .network.addressesFromPools }} + - apiGroup: {{ .apiGroup }} + kind: {{ .kind }} + name: {{ .name }} + {{- end }} + {{- end }} - name: ipv6localhost enabledIf: '{{ list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily }}' definitions: diff --git a/providers/config.yaml b/providers/config.yaml index a1b36295d48..231769191cc 100644 --- a/providers/config.yaml +++ b/providers/config.yaml @@ -23,6 +23,9 @@ providers: - name: docker url: providers/infrastructure-docker/v1.2.4/infrastructure-components.yaml type: InfrastructureProvider +- name: ipam-in-cluster + url: providers/infrastructure-ipam-in-cluster/v0.1.0/ipam-components.yaml + type: InfrastructureProvider cert-manager: url: providers/cert-manager/v1.9.1/cert-manager.yaml version: "v1.9.1" diff --git a/providers/config_default.yaml b/providers/config_default.yaml index 37fff11486d..6372bd16647 100644 --- a/providers/config_default.yaml +++ b/providers/config_default.yaml @@ -453,6 +453,9 @@ TKG_PROXY_CA_CERT: "" #! IP Family setting TKG_IP_FAMILY: +#! IPAM settings +NODE_IPAM_IP_POOL_NAME: "" + #! Configure cloud provider permissions for TMC enablement. Only affects AWS at present. DISABLE_TMC_CLOUD_PERMISSIONS: false diff --git a/providers/infrastructure-ipam-in-cluster/v0.1.0/ipam-components.yaml b/providers/infrastructure-ipam-in-cluster/v0.1.0/ipam-components.yaml new file mode 100644 index 00000000000..256e97dfe17 --- /dev/null +++ b/providers/infrastructure-ipam-in-cluster/v0.1.0/ipam-components.yaml @@ -0,0 +1,506 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + control-plane: controller-manager + name: caip-in-cluster-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: inclusterippools.ipam.cluster.x-k8s.io +spec: + group: ipam.cluster.x-k8s.io + names: + kind: InClusterIPPool + listKind: InClusterIPPoolList + plural: inclusterippools + singular: inclusterippool + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Subnet to allocate IPs from + jsonPath: .spec.subnet + name: Subnet + type: string + - description: First address of the range to allocate from + jsonPath: .spec.first + name: First + type: string + - description: Last address of the range to allocate from + jsonPath: .spec.last + name: Last + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: InClusterIPPool is the Schema for the inclusterippools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InClusterIPPoolSpec defines the desired state of InClusterIPPool. + properties: + end: + description: Last is the last address that can be assigned. Must come + after first and needs to fit into a common subnet. If unset, the + second last address of subnet will be used. + type: string + gateway: + description: Gateway + type: string + prefix: + description: Prefix is the network prefix to use. If unset the prefix + from the subnet will be used. + maximum: 128 + type: integer + start: + description: First is the first address that can be assigned. If unset, + the second address of subnet will be used. + type: string + subnet: + description: Subnet is the subnet to assign IP addresses from. Can + be omitted if first, last and prefix are set. + type: string + type: object + status: + description: InClusterIPPoolStatus defines the observed state of InClusterIPPool. + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-controller-manager + namespace: caip-in-cluster-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-leader-election-role + namespace: caip-in-cluster-system +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-manager-role +rules: +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - inclusterippools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - inclusterippools/finalizers + verbs: + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - inclusterippools/status + verbs: + - get + - patch + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddressclaims + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddressclaims/status + - ipaddresses/finalizers + verbs: + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddressclaims/status + - ipaddresses/status + verbs: + - get + - patch + - update +- apiGroups: + - ipam.cluster.x-k8s.io + resources: + - ipaddresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-leader-election-rolebinding + namespace: caip-in-cluster-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: caip-in-cluster-leader-election-role +subjects: +- kind: ServiceAccount + name: caip-in-cluster-controller-manager + namespace: caip-in-cluster-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: caip-in-cluster-manager-role +subjects: +- kind: ServiceAccount + name: caip-in-cluster-controller-manager + namespace: caip-in-cluster-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: caip-in-cluster-proxy-role +subjects: +- kind: ServiceAccount + name: caip-in-cluster-controller-manager + namespace: caip-in-cluster-system +--- +apiVersion: v1 +data: + controller_manager_config.yaml: | + apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 + kind: ControllerManagerConfig + health: + healthProbeBindAddress: :8081 + metrics: + bindAddress: 127.0.0.1:8080 + webhook: + port: 9443 + leaderElection: + leaderElect: true + resourceName: 7bb7acb4.ipam.cluster.x-k8s.io +kind: ConfigMap +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-manager-config + namespace: caip-in-cluster-system +--- +apiVersion: v1 +kind: Service +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + control-plane: controller-manager + name: caip-in-cluster-controller-manager-metrics-service + namespace: caip-in-cluster-system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + cluster.x-k8s.io/provider: ipam-in-cluster + control-plane: controller-manager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-webhook-service + namespace: caip-in-cluster-system +spec: + ports: + - port: 443 + targetPort: webhook-server + selector: + cluster.x-k8s.io/provider: ipam-in-cluster +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + control-plane: controller-manager + name: caip-in-cluster-controller-manager + namespace: caip-in-cluster-system +spec: + replicas: 1 + selector: + matchLabels: + cluster.x-k8s.io/provider: ipam-in-cluster + control-plane: controller-manager + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + control-plane: controller-manager + spec: + containers: + - args: + - --leader-elect + command: + - /manager + image: registry.tkg.vmware.run/cluster-api/cluster-api-ipam-provider-in-cluster:${CAPI_IPAM_PROVIDER_IN_CLUSTER_IMAGE_TAG} + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + serviceAccountName: caip-in-cluster-controller-manager + terminationGracePeriodSeconds: 10 + volumes: + - name: cert + secret: + secretName: caip-in-cluster-webhook-service-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-serving-cert + namespace: caip-in-cluster-system +spec: + dnsNames: + - caip-in-cluster-webhook-service.caip-in-cluster-system.svc + - caip-in-cluster-webhook-service.caip-in-cluster-system.svc.cluster.local + issuerRef: + kind: Issuer + name: caip-in-cluster-selfsigned-issuer + secretName: caip-in-cluster-webhook-service-cert + subject: + organizations: + - k8s-sig-cluster-lifecycle +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-selfsigned-issuer + namespace: caip-in-cluster-system +spec: + selfSigned: {} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: caip-in-cluster-system/caip-in-cluster-serving-cert + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: caip-in-cluster-webhook-service + namespace: caip-in-cluster-system + path: /mutate-ipam-cluster-x-k8s-io-v1alpha1-inclusterippool + failurePolicy: Fail + matchPolicy: Equivalent + name: default.inclusterippool.ipam.cluster.x-k8s.io + rules: + - apiGroups: + - ipam.cluster.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - inclusterippools + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: caip-in-cluster-system/caip-in-cluster-serving-cert + labels: + cluster.x-k8s.io/provider: ipam-in-cluster + name: caip-in-cluster-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: caip-in-cluster-webhook-service + namespace: caip-in-cluster-system + path: /validate-ipam-cluster-x-k8s-io-v1alpha1-inclusterippool + failurePolicy: Fail + matchPolicy: Equivalent + name: validation.inclusterippool.ipam.cluster.x-k8s.io + rules: + - apiGroups: + - ipam.cluster.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - inclusterippools + sideEffects: None diff --git a/providers/infrastructure-ipam-in-cluster/v0.1.0/metadata.yaml b/providers/infrastructure-ipam-in-cluster/v0.1.0/metadata.yaml new file mode 100644 index 00000000000..11facb297c1 --- /dev/null +++ b/providers/infrastructure-ipam-in-cluster/v0.1.0/metadata.yaml @@ -0,0 +1,6 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 0 + minor: 1 + contract: v1beta1 diff --git a/providers/infrastructure-vsphere/v1.4.1/cconly/base.yaml b/providers/infrastructure-vsphere/v1.4.1/cconly/base.yaml index 924d49c8c98..7f2b33fbfb8 100644 --- a/providers/infrastructure-vsphere/v1.4.1/cconly/base.yaml +++ b/providers/infrastructure-vsphere/v1.4.1/cconly/base.yaml @@ -293,6 +293,22 @@ spec: ipv6Primary: type: boolean default: false + addressesFromPools: + type: array + items: + type: object + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - apiGroup + - kind + - name + default: [] default: ipv6Primary: false - name: proxy @@ -772,8 +788,16 @@ spec: - {{ . }} {{- end }} {{- end }} - {{ if list "IPv4" "DualStack" | has .builtin.cluster.network.ipFamily -}} dhcp4: true {{- end }} - {{ if list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily -}} dhcp6: true {{- end }} + {{ if list "IPv4" "DualStack" | has .builtin.cluster.network.ipFamily | and (empty .network.addressesFromPools) -}} dhcp4: true {{- end }} + {{ if list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily | and (empty .network.addressesFromPools) -}} dhcp6: true {{- end }} + {{ if .network.addressesFromPools -}} + addressesFromPools: + {{- range .network.addressesFromPools }} + - apiGroup: {{ .apiGroup }} + kind: {{ .kind }} + name: {{ .name }} + {{- end }} + {{- end }} - selector: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate @@ -801,8 +825,16 @@ spec: - {{ . }} {{- end }} {{- end }} - {{ if list "IPv4" "DualStack" | has .builtin.cluster.network.ipFamily -}} dhcp4: true {{- end }} - {{ if list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily -}} dhcp6: true {{- end }} + {{ if list "IPv4" "DualStack" | has .builtin.cluster.network.ipFamily | and (empty .network.addressesFromPools) -}} dhcp4: true {{- end }} + {{ if list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily | and (empty .network.addressesFromPools) -}} dhcp6: true {{- end }} + {{ if .network.addressesFromPools -}} + addressesFromPools: + {{- range .network.addressesFromPools }} + - apiGroup: {{ .apiGroup }} + kind: {{ .kind }} + name: {{ .name }} + {{- end }} + {{- end }} - name: ipv6localhost enabledIf: '{{ list "IPv6" "DualStack" | has .builtin.cluster.network.ipFamily }}' definitions: diff --git a/providers/ytt/lib/config_variable_association.star b/providers/ytt/lib/config_variable_association.star index 06ae0d3c6f7..6ecc7a4ab46 100644 --- a/providers/ytt/lib/config_variable_association.star +++ b/providers/ytt/lib/config_variable_association.star @@ -70,6 +70,8 @@ return { "NSXT_SECRET_NAME": ["vsphere"], "NSXT_SECRET_NAMESPACE": ["vsphere"], +"NODE_IPAM_IP_POOL_NAME": ["vsphere"], + "AWS_REGION": ["aws"], "AWS_NODE_AZ": ["aws"], "AWS_NODE_AZ_1": ["aws"], diff --git a/providers/yttcc/lib/config_variable_association.star b/providers/yttcc/lib/config_variable_association.star index ac1bfdf30ce..b0394c1ca4d 100644 --- a/providers/yttcc/lib/config_variable_association.star +++ b/providers/yttcc/lib/config_variable_association.star @@ -72,6 +72,8 @@ return { "NSXT_SECRET_NAME": ["vsphere"], "NSXT_SECRET_NAMESPACE": ["vsphere"], +"NODE_IPAM_IP_POOL_NAME": ["vsphere"], + "AWS_REGION": ["aws"], "AWS_NODE_AZ": ["aws"], "AWS_NODE_AZ_1": ["aws"], @@ -324,6 +326,14 @@ def get_cluster_variables(): network["ipv6Primary"] = True end + if data.values["NODE_IPAM_IP_POOL_NAME"] != "": + network["addressesFromPools"] = [{ + "apiGroup": "ipam.cluster.x-k8s.io", + "kind": "InClusterIPPool", + "name": data.values["NODE_IPAM_IP_POOL_NAME"], + }] + end + if network != {}: vars["network"] = network end diff --git a/tkg/client/client.go b/tkg/client/client.go index 38697879317..79845f73a51 100644 --- a/tkg/client/client.go +++ b/tkg/client/client.go @@ -70,6 +70,7 @@ type InitRegionOptions struct { BootstrapProvider string InfrastructureProvider string ControlPlaneProvider string + IPAMProvider string Namespace string CniType string VsphereControlPlaneEndpoint string diff --git a/tkg/client/init.go b/tkg/client/init.go index 15e83f4027d..dc55a2d6936 100644 --- a/tkg/client/init.go +++ b/tkg/client/init.go @@ -317,6 +317,14 @@ func (c *TkgClient) InitRegion(options *InitRegionOptions) error { //nolint:funl } } + if config.IsFeatureActivated(constants.FeatureFlagManagementClusterDeployInClusterIPAMProvider) { + ipamProvider, err := c.tkgConfigUpdaterClient.CheckInfrastructureVersion("ipam-in-cluster") + if err != nil { + return err + } + options.IPAMProvider = ipamProvider + } + err = bootStrapClusterClient.WaitForClusterInitialized(options.ClusterName, targetClusterNamespace) if err != nil { return errors.Wrap(err, "error waiting for cluster to be provisioned (this may take a few minutes)") @@ -584,9 +592,14 @@ func (c *TkgClient) teardownKindCluster(clusterName, kubeconfig string, useExist // InitializeProviders initializes providers func (c *TkgClient) InitializeProviders(options *InitRegionOptions, clusterClient clusterclient.Client, kubeconfigPath string) error { + infrastructureProviders := []string{options.InfrastructureProvider} + if options.IPAMProvider != "" { + infrastructureProviders = append(infrastructureProviders, options.IPAMProvider) + } + clusterctlClientInitOptions := clusterctl.InitOptions{ Kubeconfig: clusterctl.Kubeconfig{Path: kubeconfigPath}, - InfrastructureProviders: []string{options.InfrastructureProvider}, + InfrastructureProviders: infrastructureProviders, ControlPlaneProviders: []string{options.ControlPlaneProvider}, BootstrapProviders: []string{options.BootstrapProvider}, CoreProvider: options.CoreProvider, @@ -652,6 +665,7 @@ func (c *TkgClient) configureImageTagsForProviderInstallation() error { configImageTag(constants.ConfigVariableInternalCAPAManagerImageTag, "cluster_api_aws", "capaControllerImage") configImageTag(constants.ConfigVariableInternalCAPVManagerImageTag, "cluster_api_vsphere", "capvControllerImage") configImageTag(constants.ConfigVariableInternalCAPZManagerImageTag, "cluster-api-provider-azure", "capzControllerImage") + configImageTag(constants.ConfigVariableInternalCAPIIPAMProviderInClusterImageTag, "cluster-api-ipam-provider-in-cluster", "capiIPAMProviderInClusterControllerImage") configImageTag(constants.ConfigVariableInternalNMIImageTag, "aad-pod-identity", "nmiImage") return nil diff --git a/tkg/clusterclient/clusterclient.go b/tkg/clusterclient/clusterclient.go index cff96966e1b..651fc5b7ad0 100644 --- a/tkg/clusterclient/clusterclient.go +++ b/tkg/clusterclient/clusterclient.go @@ -1729,6 +1729,14 @@ func (c *client) GetRegionalClusterDefaultProviderName(providerType clusterctlv1 names.Insert(providers.Items[i].ProviderName) } } + + // At time of writing, clusterctl has no notion of IPAM providers, so the + // the IPAM provider is masquerading as a plain old Infrastructure Provider + // Upcoming versions of clusterctl have the ability to install IPAM providers. + // TODO: When clusterctl is bumped to a version with IPAM providers, this delete + // can be removed. + names.Delete("ipam-in-cluster") + // If there is only one provider, this is the default if names.Len() == 1 { return names.List()[0], nil diff --git a/tkg/constants/clusterclass_mapping.go b/tkg/constants/clusterclass_mapping.go index 468a428138e..e364edc6928 100644 --- a/tkg/constants/clusterclass_mapping.go +++ b/tkg/constants/clusterclass_mapping.go @@ -195,6 +195,9 @@ var ClusterAttributesToLegacyVariablesMapAzure = map[string]string{ // spec.topology.variables.* mapped as per config_variable_association.star:get_vsphere_vars() // other attributes mapped as per infrastructure-vsphere/v*.*.*/yttcc/overlay.yaml var ClusterAttributesToLegacyVariablesMapVsphere = map[string]string{ + "spec.topology.variables.network.addressesFromPools.apiGroup": "", + "spec.topology.variables.network.addressesFromPools.kind": "", + "spec.topology.variables.network.addressesFromPools.name": ConfigVariableNodeIPAMIPPoolName, // NODE_IPAM_IP_POOL_NAME "spec.topology.variables.apiServerEndpoint": ConfigVariableVsphereControlPlaneEndpoint, // VSPHERE_CONTROL_PLANE_ENDPOINT "spec.topology.variables.vipNetworkInterface": ConfigVariableVipNetworkInterface, // VIP_NETWORK_INTERFACE diff --git a/tkg/constants/config_variables.go b/tkg/constants/config_variables.go index 80ecaa372ce..9d9d80577cb 100644 --- a/tkg/constants/config_variables.go +++ b/tkg/constants/config_variables.go @@ -204,6 +204,8 @@ const ( ConfigVariableIPFamily = "TKG_IP_FAMILY" TKGIPV6Primary = "TKG_IPV6_PRIMARY" + ConfigVariableNodeIPAMIPPoolName = "NODE_IPAM_IP_POOL_NAME" + ConfigVariableControlPlaneNodeNameservers = "CONTROL_PLANE_NODE_NAMESERVERS" ConfigVariableWorkerNodeNameservers = "WORKER_NODE_NAMESERVERS" @@ -231,15 +233,16 @@ const ( ConfigVariableOIDCIdentiryProviderClientSecret = "OIDC_IDENTITY_PROVIDER_CLIENT_SECRET" //nolint:gosec // Config variables for image tags used for provider installation - ConfigVariableInternalKubeRBACProxyImageTag = "KUBE_RBAC_PROXY_IMAGE_TAG" - ConfigVariableInternalCABPKControllerImageTag = "CABPK_CONTROLLER_IMAGE_TAG" - ConfigVariableInternalCAPIControllerImageTag = "CAPI_CONTROLLER_IMAGE_TAG" - ConfigVariableInternalKCPControllerImageTag = "KCP_CONTROLLER_IMAGE_TAG" - ConfigVariableInternalCAPDManagerImageTag = "CAPD_CONTROLLER_IMAGE_TAG" - ConfigVariableInternalCAPAManagerImageTag = "CAPA_CONTROLLER_IMAGE_TAG" - ConfigVariableInternalCAPVManagerImageTag = "CAPV_CONTROLLER_IMAGE_TAG" - ConfigVariableInternalCAPZManagerImageTag = "CAPZ_CONTROLLER_IMAGE_TAG" - ConfigVariableInternalNMIImageTag = "NMI_IMAGE_TAG" + ConfigVariableInternalKubeRBACProxyImageTag = "KUBE_RBAC_PROXY_IMAGE_TAG" + ConfigVariableInternalCABPKControllerImageTag = "CABPK_CONTROLLER_IMAGE_TAG" + ConfigVariableInternalCAPIControllerImageTag = "CAPI_CONTROLLER_IMAGE_TAG" + ConfigVariableInternalKCPControllerImageTag = "KCP_CONTROLLER_IMAGE_TAG" + ConfigVariableInternalCAPDManagerImageTag = "CAPD_CONTROLLER_IMAGE_TAG" + ConfigVariableInternalCAPAManagerImageTag = "CAPA_CONTROLLER_IMAGE_TAG" + ConfigVariableInternalCAPVManagerImageTag = "CAPV_CONTROLLER_IMAGE_TAG" + ConfigVariableInternalCAPZManagerImageTag = "CAPZ_CONTROLLER_IMAGE_TAG" + ConfigVariableInternalCAPIIPAMProviderInClusterImageTag = "CAPI_IPAM_PROVIDER_IN_CLUSTER_IMAGE_TAG" + ConfigVariableInternalNMIImageTag = "NMI_IMAGE_TAG" // Other variables related to provider installation ConfigVariableClusterTopology = "CLUSTER_TOPOLOGY" diff --git a/tkg/constants/featureflags.go b/tkg/constants/featureflags.go index 9c4a9d86a67..573cf24a29b 100644 --- a/tkg/constants/featureflags.go +++ b/tkg/constants/featureflags.go @@ -38,4 +38,8 @@ const ( // FeatureFlagSingleNodeClusters is to enable Single Node Cluster deployment via tanzu CLI. // Setting the feature flag to true will allow the creation of Single Node Clusters. FeatureFlagSingleNodeClusters = "features.cluster.single-node-clusters" + // FeatureFlagManagementClusterDeployInClusterIPAMProvider feature flag + // determines whether to apply the In-Cluster IPAM provider to the + // management cluster. + FeatureFlagManagementClusterDeployInClusterIPAMProvider = "features.management-cluster.deploy-in-cluster-ipam-provider-beta" ) diff --git a/tkg/fakes/config/cluster_vsphere.yaml b/tkg/fakes/config/cluster_vsphere.yaml index b2627cca9a5..8e717071c6d 100644 --- a/tkg/fakes/config/cluster_vsphere.yaml +++ b/tkg/fakes/config/cluster_vsphere.yaml @@ -25,6 +25,10 @@ spec: - name: network value: ipv6Primary: false #TKG_IP_FAMILY (if true expect TKG_IP_FAMILY="ipv6,ipv4") + addressesFromPools: + - apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + name: inclusterpool - name: proxy value: #TKG_HTTP_PROXY_ENABLED (under this if any value is not null then set TKG_HTTP_PROXY_ENABLED) httpProxy: http://10.0.200.100 #TKG_HTTP_PROXY diff --git a/tkg/tkgctl/create_cluster_test.go b/tkg/tkgctl/create_cluster_test.go index 9ccb3ac7cc2..caea0091d53 100644 --- a/tkg/tkgctl/create_cluster_test.go +++ b/tkg/tkgctl/create_cluster_test.go @@ -492,6 +492,9 @@ var _ = Describe("Unit tests for - (Vsphere) - cluster_vsphere.yaml as input fil mappedVal, _ = ctl.TKGConfigReaderWriter().Get(constants.ConfigVariableVsphereAz2) Expect("us-east-1c").To(Equal(fmt.Sprintf("%v", mappedVal))) + // check value for ".network.addressesFromPools": NODE_IPAM_IP_POOL_NAME + mappedVal, _ = ctl.TKGConfigReaderWriter().Get(constants.ConfigVariableNodeIPAMIPPoolName) + Expect("inclusterpool").To(Equal(fmt.Sprintf("%v", mappedVal))) }) }) })