From 4bab1654b1ba3f544c758492576c365fcc5c3284 Mon Sep 17 00:00:00 2001 From: VeraQin Date: Sat, 4 Jan 2025 05:42:27 +0000 Subject: [PATCH] Add support for new flags in node_kubelet_config. The new flags include: * container_log_max_size * container_log_max_files * image_gc_low_threshold_percent * image_gc_high_threshold_percent * image_minimum_gc_age * image_maximum_gc_age * allowed_unsafe_sysctls Change-Id: If4be953da244d4c2b7a58a3d0cb958720a1eb609 --- .../services/container/node_config.go.tmpl | 69 ++++++++++++++++++- .../resource_container_node_pool_test.go.tmpl | 32 +++++++-- .../docs/r/container_cluster.html.markdown | 18 +++++ 3 files changed, 113 insertions(+), 6 deletions(-) diff --git a/mmv1/third_party/terraform/services/container/node_config.go.tmpl b/mmv1/third_party/terraform/services/container/node_config.go.tmpl index 1b38a8643bcb..914dce7b7244 100644 --- a/mmv1/third_party/terraform/services/container/node_config.go.tmpl +++ b/mmv1/third_party/terraform/services/container/node_config.go.tmpl @@ -625,10 +625,45 @@ func schemaNodeConfig() *schema.Schema { Optional: true, Description: `Controls the maximum number of processes allowed to run in a pod.`, }, + "container_log_max_size": { + Type: schema.TypeString, + Optional: true, + Description: `Defines the maximum size of the container log file before it is rotated.`, + }, + "container_log_max_files": { + Type: schema.TypeInt, + Optional: true, + Description: `Defines the maximum number of container log files that can be present for a container.`, + }, + "image_gc_low_threshold_percent": { + Type: schema.TypeInt, + Optional: true, + Description: `Defines the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.`, + }, + "image_gc_high_threshold_percent": { + Type: schema.TypeInt, + Optional: true, + Description: `Defines the percent of disk usage after which image garbage collection is always run.`, + }, + "image_minimum_gc_age": { + Type: schema.TypeString, + Optional: true, + Description: `Defines the minimum age for an unused image before it is garbage collected.`, + }, + "image_maximum_gc_age": { + Type: schema.TypeString, + Optional: true, + Description: `Defines the maximum age an image can be unused before it is garbage collected.`, + }, + "allowed_unsafe_sysctls": { + Type: schema.TypeList, + Optional: true, + Description: `Defines a comma-separated allowlist of unsafe sysctls or sysctl patterns which can be set on the Pods.`, + Elem: &schema.Schema{Type: schema.TypeString}, + }, }, }, }, - "linux_node_config": { Type: schema.TypeList, Optional: true, @@ -1277,6 +1312,31 @@ func expandKubeletConfig(v interface{}) *container.NodeKubeletConfig { if podPidsLimit, ok := cfg["pod_pids_limit"]; ok { kConfig.PodPidsLimit = int64(podPidsLimit.(int)) } + if containerLogMaxSize, ok := cfg["container_log_max_size"]; ok { + kConfig.ContainerLogMaxSize = containerLogMaxSize.(string) + } + if containerLogMaxFiles, ok := cfg["container_log_max_files"]; ok { + kConfig.ContainerLogMaxFiles = int64(containerLogMaxFiles.(int)) + } + if imageGcLowThresholdPercent, ok := cfg["image_gc_low_threshold_percent"]; ok { + kConfig.ImageGcLowThresholdPercent = int64(imageGcLowThresholdPercent.(int)) + } + if imageGcHighThresholdPercent, ok := cfg["image_gc_high_threshold_percent"]; ok { + kConfig.ImageGcHighThresholdPercent = int64(imageGcHighThresholdPercent.(int)) + } + if imageMinimumGcAge, ok := cfg["image_minimum_gc_age"]; ok { + kConfig.ImageMinimumGcAge = imageMinimumGcAge.(string) + } + if imageMaximumGcAge, ok := cfg["image_maximum_gc_age"]; ok { + kConfig.ImageMaximumGcAge = imageMaximumGcAge.(string) + } + if allowedUnsafeSysctls, ok := cfg["allowed_unsafe_sysctls"]; ok { + sysctls := allowedUnsafeSysctls.([]interface{}) + kConfig.allowedUnsafeSysctls = make([]string, len(sysctls)) + for i, s := range sysctls { + kConfig.AllowedUnsafeSysctls[i] = s.(string) + } + } return kConfig } @@ -1866,6 +1926,13 @@ func flattenKubeletConfig(c *container.NodeKubeletConfig) []map[string]interface "cpu_manager_policy": c.CpuManagerPolicy, "insecure_kubelet_readonly_port_enabled": flattenInsecureKubeletReadonlyPortEnabled(c), "pod_pids_limit": c.PodPidsLimit, + "container_log_max_size": c.ContainerLogMaxSize, + "container_log_max_files": c.ContainerLogMaxFiles, + "image_gc_low_threshold_percent": c.ImageGcLowThresholdPercent, + "image_gc_high_threshold_percent": c.ImageGcHighThresholdPercent, + "image_minimum_gc_age": c.ImageMinimumGcAge, + "image_maximum_gc_age": c.ImageMaximumGcAge, + "allowed_unsafe_sysctls": c.AllowedUnsafeSysctls, }) } return result diff --git a/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl b/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl index fe36360a2a68..7baa9fda3eaf 100644 --- a/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl +++ b/mmv1/third_party/terraform/services/container/resource_container_node_pool_test.go.tmpl @@ -527,7 +527,7 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) { CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "static", "100ms", networkName, subnetworkName, "TRUE", true, 2048), + Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "static", "100ms", networkName, subnetworkName, "TRUE", "100Mi", "1m", "10m", true, 2048, 10, 10, 85), ConfigPlanChecks: resource.ConfigPlanChecks{ PreApply: []plancheck.PlanCheck{ acctest.ExpectNoDelete(), @@ -540,6 +540,20 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) { "node_config.0.kubelet_config.0.insecure_kubelet_readonly_port_enabled", "TRUE"), resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config", "node_config.0.kubelet_config.0.pod_pids_limit", "2048"), + resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config", + "node_config.0.kubelet_config.0.container_log_max_size", "100Mi"), + resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config", + "node_config.0.kubelet_config.0.container_log_max_files", "10"), + resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config", + "node_config.0.kubelet_config.0.image_gc_low_threshold_percent", "10"), + resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config", + "node_config.0.kubelet_config.0.image_gc_high_threshold_percent", "85"), + resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config", + "node_config.0.kubelet_config.0.image_minimum_gc_age", "1m"), + resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config", + "node_config.0.kubelet_config.0.image_maximum_gc_age", "10m"), + resource.TestCheckResourceAttr("google_container_node_pool.with_kubelet_config", + "node_config.0.kubelet_config.0.allowed_unsafe_sysctls.0", "kernel.shm*"), ), }, { @@ -548,7 +562,7 @@ func TestAccContainerNodePool_withKubeletConfig(t *testing.T) { ImportStateVerify: true, }, { - Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "", "", networkName, subnetworkName, "FALSE", false, 1024), + Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "", "", networkName, subnetworkName, "FALSE", "500Mi", "30s", "", false, 1024, 5, 50, 80), ConfigPlanChecks: resource.ConfigPlanChecks{ PreApply: []plancheck.PlanCheck{ acctest.ExpectNoDelete(), @@ -586,7 +600,7 @@ func TestAccContainerNodePool_withInvalidKubeletCpuManagerPolicy(t *testing.T) { CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "dontexist", "100us", networkName, subnetworkName,"TRUE", false, 1024), + Config: testAccContainerNodePool_withKubeletConfig(cluster, np, "dontexist", "100us", networkName, subnetworkName,"TRUE", "", "", "", false, 1024, 2, 70, 75), ExpectError: regexp.MustCompile(`.*to be one of \["?static"? "?none"? "?"?\].*`), }, }, @@ -3162,7 +3176,7 @@ resource "google_container_node_pool" "with_sandbox_config" { } {{- end }} -func testAccContainerNodePool_withKubeletConfig(cluster, np, policy, period, networkName, subnetworkName, insecureKubeletReadonlyPortEnabled string, quota bool, podPidsLimit int) string { +func testAccContainerNodePool_withKubeletConfig(cluster, np, policy, period, networkName, subnetworkName, insecureKubeletReadonlyPortEnabled, containerLogMaxSize, imageMinimumGcAge, imageMaximumGcAge string, quota bool, podPidsLimit, containerLogMaxFiles, imageGcLowThresholdPercent, imageGcHighThresholdPercent int) string { return fmt.Sprintf(` data "google_container_engine_versions" "central1a" { location = "us-central1-a" @@ -3193,6 +3207,13 @@ resource "google_container_node_pool" "with_kubelet_config" { cpu_cfs_quota_period = %q insecure_kubelet_readonly_port_enabled = "%s" pod_pids_limit = %d + container_log_max_size = %q + container_log_max_files = %d + image_gc_low_threshold_percent = %d + image_gc_high_threshold_percent = %d + image_minimum_gc_age = %q + image_maximum_gc_age = %q + allowed_unsafe_sysctls = ["kernel.shm*", "kernel.msg*", "kernel.sem", "fs.mqueue.*", "net.*"] } oauth_scopes = [ "https://www.googleapis.com/auth/logging.write", @@ -3201,7 +3222,7 @@ resource "google_container_node_pool" "with_kubelet_config" { logging_variant = "DEFAULT" } } -`, cluster, networkName, subnetworkName, np, policy, quota, period, insecureKubeletReadonlyPortEnabled, podPidsLimit) +`, cluster, networkName, subnetworkName, np, policy, quota, period, insecureKubeletReadonlyPortEnabled, podPidsLimit, containerLogMaxSize, containerLogMaxFiles, imageGcLowThresholdPercent, imageGcHighThresholdPercent, imageMinimumGcAge, imageMaximumGcAge) } func testAccContainerNodePool_withLinuxNodeConfig(cluster, np, tcpMem, networkName, subnetworkName string) string { @@ -3223,6 +3244,7 @@ func testAccContainerNodePool_withLinuxNodeConfig(cluster, np, tcpMem, networkNa "net.ipv4.tcp_rmem" = "%s" "net.ipv4.tcp_wmem" = "%s" "net.ipv4.tcp_tw_reuse" = 1 + "kernel.shmmni" = 8192 } } `, tcpMem, tcpMem) diff --git a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown index b69680f24eca..a1b78c2ba017 100644 --- a/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown +++ b/mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown @@ -1342,6 +1342,24 @@ such as `"300ms"`. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", * `pod_pids_limit` - (Optional) Controls the maximum number of processes allowed to run in a pod. The value must be greater than or equal to 1024 and less than 4194304. +* `container_log_max_size` - (Optional) Defines the maximum size of the + container log file before it is rotated. Specified as a positive number and a + unit suffix, such as `"100Ki"`, `"10Mi"`. Valid units are "Ki", "Mi", "Gi". + The value must be between `"10Mi"` and `"500Mi"`, inclusive. And the total container log size + (`container_log_max_size` * `container_log_max_files`) cannot exceed 1% of the total storage of the node. + +* `container_log_max_files` - (Optional) Defines the maximum number of container log files that can be present for a container. The integer must be between 2 and 10, inclusive. + +* `image_gc_low_threshold_percent` - (Optional) Defines the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. The integer must be between 10 and 85, inclusive. + +* `image_gc_high_threshold_percent` - (Optional) Defines the percent of disk usage after which image garbage collection is always run. The integer must be between 10 and 85, inclusive. + +* `image_minimum_gc_age` - (Optional) Defines the minimum age for an unused image before it is garbage collected. Specified as a sequence of decimal numbers, each with optional fraction and a unit suffix, such as `"300s"`, `"1.5m"`. The value cannot be greater than "2m". + +* `image_maximum_gc_age` - (Optional) Defines the maximum age an image can be unused before it is garbage collected. Specified as a sequence of decimal numbers, each with optional fraction and a unit suffix, such as `"300s"`, "1.5m"`, and `"2h45m"`. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". The value must be a positive duration. + +* `allowed_unsafe_sysctls` - (Optional) Defines a comma-separated allowlist of unsafe sysctls or sysctl patterns which can be set on the Pods. The allowed sysctl groups are `kernel.shm*`, `kernel.msg*`, `kernel.sem`, `fs.mqueue.*`, and `net.*`. + The `linux_node_config` block supports: * `sysctls` - (Optional) The Linux kernel parameters to be applied to the nodes