Skip to content

Commit

Permalink
container: add support for kubelet read only port
Browse files Browse the repository at this point in the history 
- Add `no_enable_insecure_kubelet_readonly_port` to google_container_cluster
- Allow setting `insecure_kubelet_readonly_port_enabled` for
  `container_node_pool`

https://cloud.google.com/kubernetes-engine/docs/how-to/disable-kubelet-readonly-port

Fixes hashicorp/terraform-provider-google#15208
  • Loading branch information
@wyardley
wyardley committed Aug 21, 2024
1 parent 33e0634 commit dd2cc22
Show file tree
Hide file tree
Showing 9 changed files with 545 additions and 36 deletions.
63 changes: 55 additions & 8 deletions mmv1/third_party/terraform/services/container/go/node_config.go.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,18 @@ func schemaContainerdConfig() *schema.Schema {
}
}

// Note: this is a bool within the API / go library, but implementing as an
// enum internally to make it easier to accept API level defaults.
func schemaInsecureKubeletReadonlyPortEnabled() *schema.Schema {
return &schema.Schema{
Type: schema.TypeString,
Optional: true,
Computed: true,
Description: "Controls whether the kubelet read-only port is enabled. It is strongly recommended to set this to FALSE.",
ValidateFunc: validation.StringInSlice([]string{"FALSE","TRUE"}, false),
}
}

func schemaLoggingVariant() *schema.Schema {
return &schema.Schema{
Type: schema.TypeString,
Expand Down Expand Up @@ -601,6 +613,13 @@ func schemaNodeConfig() *schema.Schema {
Optional: true,
Description: `Set the CPU CFS quota period value 'cpu.cfs_period_us'.`,
},
"insecure_kubelet_readonly_port_enabled": {
Type: schema.TypeString,
Optional: true,
Computed: true,
Description: `Enable or disable Kubelet read only port.`,
ValidateFunc: validation.StringInSlice([]string{"FALSE","TRUE"}, false),
},
"pod_pids_limit": {
Type: schema.TypeInt,
Optional: true,
Expand Down Expand Up @@ -773,14 +792,20 @@ func schemaNodeConfig() *schema.Schema {
}

func expandNodeConfigDefaults(configured interface{}) *container.NodeConfigDefaults {
configs := configured.([]interface{})
configs := configured.([]interface{})
if len(configs) == 0 || configs[0] == nil {
return nil
}
config := configs[0].(map[string]interface{})

nodeConfigDefaults := &container.NodeConfigDefaults{}
nodeConfigDefaults.ContainerdConfig = expandContainerdConfig(config["containerd_config"])
if v, ok := config["insecure_kubelet_readonly_port_enabled"]; ok {
nodeConfigDefaults.NodeKubeletConfig = &container.NodeKubeletConfig{
InsecureKubeletReadonlyPortEnabled: expandInsecureKubeletReadonlyPortEnabled(v),
ForceSendFields: []string{"InsecureKubeletReadonlyPortEnabled"},
}
}
if variant, ok := config["logging_variant"]; ok {
nodeConfigDefaults.LoggingConfig = &container.NodePoolLoggingConfig{
VariantConfig: &container.LoggingVariantConfig{
Expand All @@ -789,14 +814,14 @@ func expandNodeConfigDefaults(configured interface{}) *container.NodeConfigDefau
}
}
{{- if ne $.TargetVersionName "ga" }}
if v, ok := config["gcfs_config"]; ok && len(v.([]interface{})) > 0 {
gcfsConfig := v.([]interface{})[0].(map[string]interface{})
if v, ok := config["gcfs_config"]; ok && len(v.([]interface{})) > 0 {
gcfsConfig := v.([]interface{})[0].(map[string]interface{})
nodeConfigDefaults.GcfsConfig = &container.GcfsConfig{
Enabled: gcfsConfig["enabled"].(bool),
}
}
{{- end }}
return nodeConfigDefaults
return nodeConfigDefaults
}

func expandNodeConfig(v interface{}) *container.NodeConfig {
Expand Down Expand Up @@ -1134,6 +1159,13 @@ func expandWorkloadMetadataConfig(v interface{}) *container.WorkloadMetadataConf
return wmc
}

func expandInsecureKubeletReadonlyPortEnabled(v interface{}) bool {
if v == "TRUE" {
return true
}
return false
}

func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig {
if v == nil {
return nil
Expand All @@ -1154,6 +1186,10 @@ func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig {
if cpuCfsQuotaPeriod, ok := cfg["cpu_cfs_quota_period"]; ok {
kConfig.CpuCfsQuotaPeriod = cpuCfsQuotaPeriod.(string)
}
if insecureKubeletReadonlyPortEnabled, ok := cfg["insecure_kubelet_readonly_port_enabled"]; ok {
kConfig.InsecureKubeletReadonlyPortEnabled = expandInsecureKubeletReadonlyPortEnabled(insecureKubeletReadonlyPortEnabled)
kConfig.ForceSendFields = append(kConfig.ForceSendFields, "InsecureKubeletReadonlyPortEnabled")
}
if podPidsLimit, ok := cfg["pod_pids_limit"]; ok {
kConfig.PodPidsLimit = int64(podPidsLimit.(int))
}
Expand Down Expand Up @@ -1362,6 +1398,8 @@ func flattenNodeConfigDefaults(c *container.NodeConfigDefaults) []map[string]int

result[0]["containerd_config"] = flattenContainerdConfig(c.ContainerdConfig)

result[0]["insecure_kubelet_readonly_port_enabled"] = flattenInsecureKubeletReadonlyPortEnabled(c.NodeKubeletConfig)

result[0]["logging_variant"] = flattenLoggingVariant(c.LoggingConfig)

{{ if ne $.TargetVersionName `ga` -}}
Expand Down Expand Up @@ -1549,6 +1587,14 @@ func flattenSecondaryBootDisks(c []*container.SecondaryBootDisk) []map[string]in
return result
}

func flattenInsecureKubeletReadonlyPortEnabled(c *container.NodeKubeletConfig) string {
// Convert bool from the API to the enum values used internally
if c != nil && c.InsecureKubeletReadonlyPortEnabled {
return "TRUE"
}
return "FALSE"
}

func flattenLoggingVariant(c *container.NodePoolLoggingConfig) string {
variant := "DEFAULT"
if c != nil && c.VariantConfig != nil && c.VariantConfig.Variant != "" {
Expand Down Expand Up @@ -1698,10 +1744,11 @@ func flattenKubeletConfig(c *container.NodeKubeletConfig) []map[string]interface
result := []map[string]interface{}{}
if c != nil {
result = append(result, map[string]interface{}{
"cpu_cfs_quota": c.CpuCfsQuota,
"cpu_cfs_quota_period": c.CpuCfsQuotaPeriod,
"cpu_manager_policy": c.CpuManagerPolicy,
"pod_pids_limit": c.PodPidsLimit,
"cpu_cfs_quota": c.CpuCfsQuota,
"cpu_cfs_quota_period": c.CpuCfsQuotaPeriod,
"cpu_manager_policy": c.CpuManagerPolicy,
"insecure_kubelet_readonly_port_enabled": flattenInsecureKubeletReadonlyPortEnabled(c),
"pod_pids_limit": c.PodPidsLimit,
})
}
return result
Expand Down
23 changes: 23 additions & 0 deletions mmv1/third_party/terraform/services/container/go/resource_container_cluster.go.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ func clusterSchemaNodePoolDefaults() *schema.Schema {
{{- if ne $.TargetVersionName "ga" }}
"gcfs_config": schemaGcfsConfig(false),
{{- end }}
"insecure_kubelet_readonly_port_enabled": schemaInsecureKubeletReadonlyPortEnabled(),
"logging_variant": schemaLoggingVariant(),
},
},
Expand Down Expand Up @@ -4252,6 +4253,28 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
}
}

if d.HasChange("node_pool_defaults") && d.HasChange("node_pool_defaults.0.node_config_defaults.0.insecure_kubelet_readonly_port_enabled") {
if v, ok := d.GetOk("node_pool_defaults.0.node_config_defaults.0.insecure_kubelet_readonly_port_enabled"); ok {
insecureKubeletReadonlyPortEnabled := v.(string)
req := &container.UpdateClusterRequest{
Update: &container.ClusterUpdate{
DesiredNodeKubeletConfig: &container.NodeKubeletConfig{
InsecureKubeletReadonlyPortEnabled: expandInsecureKubeletReadonlyPortEnabled(insecureKubeletReadonlyPortEnabled),
ForceSendFields: []string{"InsecureKubeletReadonlyPortEnabled"},
},
},
}

updateF := updateFunc(req, "updating GKE cluster desired node pool logging configuration defaults.")
// Call update serially.
if err := transport_tpg.LockedCall(lockKey, updateF); err != nil {
return err
}

log.Printf("[INFO] GKE cluster %s node pool insecure_kubelet_readonly_port_enabled default has been updated", d.Id())
}
}

if d.HasChange("node_pool_defaults") && d.HasChange("node_pool_defaults.0.node_config_defaults.0.logging_variant") {
if v, ok := d.GetOk("node_pool_defaults.0.node_config_defaults.0.logging_variant"); ok {
loggingVariant := v.(string)
Expand Down
177 changes: 177 additions & 0 deletions mmv1/third_party/terraform/services/container/go/resource_container_cluster_test.go.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1518,6 +1518,107 @@ func TestAccContainerCluster_withNodeConfig(t *testing.T) {
})
}

func TestAccContainerCluster_withInsecureKubeletReadonlyPortEnabledInNodeConfig(t *testing.T) {
t.Parallel()
clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster")
subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName)

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledInNodeConfig(clusterName, networkName, subnetworkName, "TRUE"),
},
{
ResourceName: "google_container_cluster.with_insecure_kubelet_readonly_port_enabled_in_node_config",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
},
})
}

func TestAccContainerCluster_withInsecureKubeletReadonlyPortEnabledInNodePool(t *testing.T) {
t.Parallel()
clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
nodePoolName := fmt.Sprintf("tf-test-nodepool-%s", acctest.RandString(t, 10))
networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster")
subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName)

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
Steps: []resource.TestStep{
{
Config: testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledInNodePool(clusterName, nodePoolName, networkName, subnetworkName, "TRUE"),
},
{
ResourceName: "google_container_cluster.with_insecure_kubelet_readonly_port_enabled_in_node_pool",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
},
})
}

func TestAccContainerCluster_withInsecureKubeletReadonlyPortEnabledUpdates(t *testing.T) {
t.Parallel()
clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster")
subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName)

acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
Steps: []resource.TestStep{
// Test API default (no value set in config) first
{
Config: testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledNodePoolUpdateBaseline(clusterName, networkName, subnetworkName),
},
{
ResourceName: "google_container_cluster.with_insecure_kubelet_readonly_port_enabled_node_pool_update",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
{
Config: testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledNodePoolUpdate(clusterName, networkName, subnetworkName, "TRUE"),
},
{
ResourceName: "google_container_cluster.with_insecure_kubelet_readonly_port_enabled_node_pool_update",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
{
Config: testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledNodePoolUpdate(clusterName, networkName, subnetworkName, "FALSE"),
},
{
ResourceName: "google_container_cluster.with_insecure_kubelet_readonly_port_enabled_node_pool_update",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
{
Config: testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledNodePoolUpdate(clusterName, networkName, subnetworkName, "TRUE"),
},
{
ResourceName: "google_container_cluster.with_insecure_kubelet_readonly_port_enabled_node_pool_update",
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"deletion_protection"},
},
},
})
}

func TestAccContainerCluster_withLoggingVariantInNodeConfig(t *testing.T) {
t.Parallel()
clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
Expand Down Expand Up @@ -6421,6 +6522,82 @@ resource "google_container_cluster" "with_node_config" {
`, clusterName, networkName, subnetworkName)
}

func testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledInNodeConfig(clusterName, networkName, subnetworkName, insecureKubeletReadonlyPortEnabled string) string {
return fmt.Sprintf(`
resource "google_container_cluster" "with_insecure_kubelet_readonly_port_enabled_in_node_config" {
name = "%s"
location = "us-central1-f"
initial_node_count = 1

node_config {
kubelet_config {
cpu_manager_policy = "static"
insecure_kubelet_readonly_port_enabled = "%s"
}
}
deletion_protection = false
network = "%s"
subnetwork = "%s"
}
`, clusterName, insecureKubeletReadonlyPortEnabled, networkName, subnetworkName)
}

func testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledInNodePool(clusterName, nodePoolName, networkName, subnetworkName, insecureKubeletReadonlyPortEnabled string) string {
return fmt.Sprintf(`
resource "google_container_cluster" "with_insecure_kubelet_readonly_port_enabled_in_node_pool" {
name = "%s"
location = "us-central1-f"

node_pool {
name = "%s"
initial_node_count = 1
node_config {
kubelet_config {
cpu_manager_policy = "static"
insecure_kubelet_readonly_port_enabled = "%s"
}
}
}
deletion_protection = false
network = "%s"
subnetwork = "%s"
}
`, clusterName, nodePoolName, insecureKubeletReadonlyPortEnabled, networkName, subnetworkName)
}

func testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledNodePoolUpdateBaseline(clusterName, networkName, subnetworkName string) string {
return fmt.Sprintf(`
resource "google_container_cluster" "with_insecure_kubelet_readonly_port_enabled_node_pool_update" {
name = "%s"
location = "us-central1-f"
initial_node_count = 1

deletion_protection = false
network = "%s"
subnetwork = "%s"
}
`, clusterName, networkName, subnetworkName)
}

func testAccContainerCluster_withInsecureKubeletReadonlyPortEnabledNodePoolUpdate(clusterName, networkName, subnetworkName, insecureKubeletReadonlyPortEnabled string) string {
return fmt.Sprintf(`
resource "google_container_cluster" "with_insecure_kubelet_readonly_port_enabled_node_pool_update" {
name = "%s"
location = "us-central1-f"
initial_node_count = 1

node_pool_defaults {
node_config_defaults {
insecure_kubelet_readonly_port_enabled = "%s"
}
}
deletion_protection = false
network = "%s"
subnetwork = "%s"
}
`, clusterName, insecureKubeletReadonlyPortEnabled, networkName, subnetworkName)
}

func testAccContainerCluster_withLoggingVariantInNodeConfig(clusterName, loggingVariant, networkName, subnetworkName string) string {
return fmt.Sprintf(`
resource "google_container_cluster" "with_logging_variant_in_node_config" {
Expand Down
Loading

0 comments on commit dd2cc22

Please # to comment.