From 24c5f709169e6eb2d85465f3ee8cf3bdb52434c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9a=20Grondier?= Date: Wed, 3 Apr 2024 16:39:20 +0200 Subject: [PATCH] SKS nodepool: allow specifying kubelet image gc parameters on creation --- cmd/sks_create.go | 82 +++++++++++-------- cmd/sks_nodepool_add.go | 43 ++++++---- cmd/sks_nodepool_show.go | 57 +++++++++---- go.mod | 2 +- go.sum | 4 +- .../exoscale/egoscale/v2/sks_nodepool.go | 43 +++++++++- vendor/modules.txt | 2 +- 7 files changed, 158 insertions(+), 75 deletions(-) diff --git a/cmd/sks_create.go b/cmd/sks_create.go index 47221d71f..2d6e495c9 100644 --- a/cmd/sks_create.go +++ b/cmd/sks_create.go @@ -30,36 +30,39 @@ type sksCreateCmd struct { Name string `cli-arg:"#" cli-usage:"NAME"` - AutoUpgrade bool `cli-usage:"enable automatic upgrading of the SKS cluster control plane Kubernetes version"` - CNI string `cli-usage:"CNI plugin to deploy. e.g. 'calico', or 'cilium'"` - Description string `cli-usage:"SKS cluster description"` - KubernetesVersion string `cli-usage:"SKS cluster control plane Kubernetes version"` - Labels map[string]string `cli-flag:"label" cli-usage:"SKS cluster label (format: key=value)"` - NoCNI bool `cli-usage:"do not deploy a default Container Network Interface plugin in the cluster control plane"` - NoExoscaleCCM bool `cli-usage:"do not deploy the Exoscale Cloud Controller Manager in the cluster control plane"` - NoMetricsServer bool `cli-usage:"do not deploy the Kubernetes Metrics Server in the cluster control plane"` - ExoscaleCSI bool `cli-usage:"deploy the Exoscale Container Storage Interface on worker nodes"` - NodepoolAntiAffinityGroups []string `cli-flag:"nodepool-anti-affinity-group" cli-usage:"default Nodepool Anti-Affinity Group NAME|ID (can be specified multiple times)"` - NodepoolDeployTarget string `cli-usage:"default Nodepool Deploy Target NAME|ID"` - NodepoolDescription string `cli-usage:"default Nodepool description"` - NodepoolDiskSize int64 `cli-usage:"default Nodepool Compute instances disk size"` - NodepoolInstancePrefix string `cli-usage:"string to prefix default Nodepool member names with"` - NodepoolInstanceType string `cli-usage:"default Nodepool Compute instances type"` - NodepoolLabels map[string]string `cli-flag:"nodepool-label" cli-usage:"default Nodepool label (format: key=value)"` - NodepoolName string `cli-usage:"default Nodepool name"` - NodepoolPrivateNetworks []string `cli-flag:"nodepool-private-network" cli-usage:"default Nodepool Private Network NAME|ID (can be specified multiple times)"` - NodepoolSecurityGroups []string `cli-flag:"nodepool-security-group" cli-usage:"default Nodepool Security Group NAME|ID (can be specified multiple times)"` - NodepoolSize int64 `cli-usage:"default Nodepool size. If 0, no default Nodepool will be added to the cluster."` - NodepoolTaints []string `cli-flag:"nodepool-taint" cli-usage:"Kubernetes taint to apply to default Nodepool Nodes (format: KEY=VALUE:EFFECT, can be specified multiple times)"` - OIDCClientID string `cli-flag:"oidc-client-id" cli-usage:"OpenID client ID"` - OIDCGroupsClaim string `cli-flag:"oidc-groups-claim" cli-usage:"OpenID JWT claim to use as the user's group"` - OIDCGroupsPrefix string `cli-flag:"oidc-groups-prefix" cli-usage:"OpenID prefix prepended to group claims"` - OIDCIssuerURL string `cli-flag:"oidc-issuer-url" cli-usage:"OpenID provider URL"` - OIDCRequiredClaim map[string]string `cli-flag:"oidc-required-claim" cli-usage:"OpenID token required claim (format: key=value)"` - OIDCUsernameClaim string `cli-flag:"oidc-username-claim" cli-usage:"OpenID JWT claim to use as the user name"` - OIDCUsernamePrefix string `cli-flag:"oidc-username-prefix" cli-usage:"OpenID prefix prepended to username claims"` - ServiceLevel string `cli-usage:"SKS cluster control plane service level (starter|pro)"` - Zone string `cli-short:"z" cli-usage:"SKS cluster zone"` + AutoUpgrade bool `cli-usage:"enable automatic upgrading of the SKS cluster control plane Kubernetes version"` + CNI string `cli-usage:"CNI plugin to deploy. e.g. 'calico', or 'cilium'"` + Description string `cli-usage:"SKS cluster description"` + KubernetesVersion string `cli-usage:"SKS cluster control plane Kubernetes version"` + Labels map[string]string `cli-flag:"label" cli-usage:"SKS cluster label (format: key=value)"` + NoCNI bool `cli-usage:"do not deploy a default Container Network Interface plugin in the cluster control plane"` + NoExoscaleCCM bool `cli-usage:"do not deploy the Exoscale Cloud Controller Manager in the cluster control plane"` + NoMetricsServer bool `cli-usage:"do not deploy the Kubernetes Metrics Server in the cluster control plane"` + ExoscaleCSI bool `cli-usage:"deploy the Exoscale Container Storage Interface on worker nodes"` + NodepoolAntiAffinityGroups []string `cli-flag:"nodepool-anti-affinity-group" cli-usage:"default Nodepool Anti-Affinity Group NAME|ID (can be specified multiple times)"` + NodepoolDeployTarget string `cli-usage:"default Nodepool Deploy Target NAME|ID"` + NodepoolDescription string `cli-usage:"default Nodepool description"` + NodepoolDiskSize int64 `cli-usage:"default Nodepool Compute instances disk size"` + NodepoolImageGcLowThreshold int64 `cli-flag:"nodepool-image-gc-low-threshold" cli-usage:"default Nodepool the percent of disk usage after which image garbage collection is never run"` + NodepoolImageGcHighThreshold int64 `cli-flag:"nodepool-image-gc-high-threshold" cli-usage:"default Nodepool the percent of disk usage after which image garbage collection is always run"` + NodepoolImageGcMinAge string `cli-flag:"nodepool-image-gc-min-age" cli-usage:"default Nodepool maximum age an image can be unused before it is garbage collected"` + NodepoolInstancePrefix string `cli-usage:"string to prefix default Nodepool member names with"` + NodepoolInstanceType string `cli-usage:"default Nodepool Compute instances type"` + NodepoolLabels map[string]string `cli-flag:"nodepool-label" cli-usage:"default Nodepool label (format: key=value)"` + NodepoolName string `cli-usage:"default Nodepool name"` + NodepoolPrivateNetworks []string `cli-flag:"nodepool-private-network" cli-usage:"default Nodepool Private Network NAME|ID (can be specified multiple times)"` + NodepoolSecurityGroups []string `cli-flag:"nodepool-security-group" cli-usage:"default Nodepool Security Group NAME|ID (can be specified multiple times)"` + NodepoolSize int64 `cli-usage:"default Nodepool size. If 0, no default Nodepool will be added to the cluster."` + NodepoolTaints []string `cli-flag:"nodepool-taint" cli-usage:"Kubernetes taint to apply to default Nodepool Nodes (format: KEY=VALUE:EFFECT, can be specified multiple times)"` + OIDCClientID string `cli-flag:"oidc-client-id" cli-usage:"OpenID client ID"` + OIDCGroupsClaim string `cli-flag:"oidc-groups-claim" cli-usage:"OpenID JWT claim to use as the user's group"` + OIDCGroupsPrefix string `cli-flag:"oidc-groups-prefix" cli-usage:"OpenID prefix prepended to group claims"` + OIDCIssuerURL string `cli-flag:"oidc-issuer-url" cli-usage:"OpenID provider URL"` + OIDCRequiredClaim map[string]string `cli-flag:"oidc-required-claim" cli-usage:"OpenID token required claim (format: key=value)"` + OIDCUsernameClaim string `cli-flag:"oidc-username-claim" cli-usage:"OpenID JWT claim to use as the user name"` + OIDCUsernamePrefix string `cli-flag:"oidc-username-prefix" cli-usage:"OpenID prefix prepended to username claims"` + ServiceLevel string `cli-usage:"SKS cluster control plane service level (starter|pro)"` + Zone string `cli-short:"z" cli-usage:"SKS cluster zone"` } func (c *sksCreateCmd) cmdAliases() []string { return gCreateAlias } @@ -188,6 +191,11 @@ func (c *sksCreateCmd) cmdRun(_ *cobra.Command, _ []string) error { //nolint:goc return c.Name }()), Size: &c.NodepoolSize, + KubeletImageGc: &egoscale.SKSNodepoolKubeletImageGc{ + MinAge: &c.NodepoolImageGcMinAge, + LowThreshold: &c.NodepoolImageGcLowThreshold, + HighThreshold: &c.NodepoolImageGcHighThreshold, + }, } if l := len(c.NodepoolAntiAffinityGroups); l > 0 { @@ -275,12 +283,14 @@ func init() { cobra.CheckErr(registerCLICommand(sksCmd, &sksCreateCmd{ cliCommandSettings: defaultCLICmdSettings(), - CNI: defaultSKSClusterCNI, - KubernetesVersion: "latest", - NodepoolDiskSize: 50, - NodepoolInstanceType: fmt.Sprintf("%s.%s", defaultInstanceTypeFamily, defaultInstanceType), - - ServiceLevel: defaultSKSClusterServiceLevel, + CNI: defaultSKSClusterCNI, + KubernetesVersion: "latest", + NodepoolDiskSize: 50, + NodepoolInstanceType: fmt.Sprintf("%s.%s", defaultInstanceTypeFamily, defaultInstanceType), + NodepoolImageGcLowThreshold: 80, + NodepoolImageGcHighThreshold: 85, + NodepoolImageGcMinAge: "2m", + ServiceLevel: defaultSKSClusterServiceLevel, })) } diff --git a/cmd/sks_nodepool_add.go b/cmd/sks_nodepool_add.go index 9361ff08d..9017f2e30 100644 --- a/cmd/sks_nodepool_add.go +++ b/cmd/sks_nodepool_add.go @@ -23,19 +23,22 @@ type sksNodepoolAddCmd struct { Cluster string `cli-arg:"#" cli-usage:"CLUSTER-NAME|ID"` Name string `cli-arg:"#" cli-usage:"NODEPOOL-NAME"` - AntiAffinityGroups []string `cli-flag:"anti-affinity-group" cli-usage:"Nodepool Anti-Affinity Group NAME|ID (can be specified multiple times)"` - DeployTarget string `cli-usage:"Nodepool Deploy Target NAME|ID"` - Description string `cli-usage:"Nodepool description"` - DiskSize int64 `cli-usage:"Nodepool Compute instances disk size"` - InstancePrefix string `cli-usage:"string to prefix Nodepool member names with"` - InstanceType string `cli-usage:"Nodepool Compute instances type"` - Labels []string `cli-flag:"label" cli-usage:"Nodepool label (format: key=value)"` - PrivateNetworks []string `cli-flag:"private-network" cli-usage:"Nodepool Private Network NAME|ID (can be specified multiple times)"` - SecurityGroups []string `cli-flag:"security-group" cli-usage:"Nodepool Security Group NAME|ID (can be specified multiple times)"` - Size int64 `cli-usage:"Nodepool size"` - StorageLvm bool `cli-usage:"Create nodes with non-standard partitioning for persistent storage"` - Taints []string `cli-flag:"taint" cli-usage:"Kubernetes taint to apply to Nodepool Nodes (format: KEY=VALUE:EFFECT, can be specified multiple times)"` - Zone string `cli-short:"z" cli-usage:"SKS cluster zone"` + AntiAffinityGroups []string `cli-flag:"anti-affinity-group" cli-usage:"Nodepool Anti-Affinity Group NAME|ID (can be specified multiple times)"` + DeployTarget string `cli-usage:"Nodepool Deploy Target NAME|ID"` + Description string `cli-usage:"Nodepool description"` + DiskSize int64 `cli-usage:"Nodepool Compute instances disk size"` + ImageGcLowThreshold int64 `cli-flag:"image-gc-low-threshold" cli-usage:"the percent of disk usage after which image garbage collection is never run"` + ImageGcHighThreshold int64 `cli-flag:"image-gc-high-threshold" cli-usage:"the percent of disk usage after which image garbage collection is always run"` + ImageGcMinAge string `cli-flag:"image-gc-min-age" cli-usage:"maximum age an image can be unused before it is garbage collected"` + InstancePrefix string `cli-usage:"string to prefix Nodepool member names with"` + InstanceType string `cli-usage:"Nodepool Compute instances type"` + Labels []string `cli-flag:"label" cli-usage:"Nodepool label (format: key=value)"` + PrivateNetworks []string `cli-flag:"private-network" cli-usage:"Nodepool Private Network NAME|ID (can be specified multiple times)"` + SecurityGroups []string `cli-flag:"security-group" cli-usage:"Nodepool Security Group NAME|ID (can be specified multiple times)"` + Size int64 `cli-usage:"Nodepool size"` + StorageLvm bool `cli-usage:"Create nodes with non-standard partitioning for persistent storage"` + Taints []string `cli-flag:"taint" cli-usage:"Kubernetes taint to apply to Nodepool Nodes (format: KEY=VALUE:EFFECT, can be specified multiple times)"` + Zone string `cli-short:"z" cli-usage:"SKS cluster zone"` } func (c *sksNodepoolAddCmd) cmdAliases() []string { return nil } @@ -61,6 +64,11 @@ func (c *sksNodepoolAddCmd) cmdRun(_ *cobra.Command, _ []string) error { InstancePrefix: utils.NonEmptyStringPtr(c.InstancePrefix), Name: &c.Name, Size: &c.Size, + KubeletImageGc: &egoscale.SKSNodepoolKubeletImageGc{ + MinAge: &c.ImageGcMinAge, + LowThreshold: &c.ImageGcLowThreshold, + HighThreshold: &c.ImageGcHighThreshold, + }, } ctx := exoapi.WithEndpoint(gContext, exoapi.NewReqEndpoint(account.CurrentAccount.Environment, c.Zone)) @@ -180,8 +188,11 @@ func init() { cobra.CheckErr(registerCLICommand(sksNodepoolCmd, &sksNodepoolAddCmd{ cliCommandSettings: defaultCLICmdSettings(), - Size: 2, - InstanceType: fmt.Sprintf("%s.%s", defaultInstanceTypeFamily, defaultInstanceType), - DiskSize: 50, + Size: 2, + InstanceType: fmt.Sprintf("%s.%s", defaultInstanceTypeFamily, defaultInstanceType), + DiskSize: 50, + ImageGcLowThreshold: 80, + ImageGcHighThreshold: 85, + ImageGcMinAge: "2m", })) } diff --git a/cmd/sks_nodepool_show.go b/cmd/sks_nodepool_show.go index cee9cc2a5..869ded9e3 100644 --- a/cmd/sks_nodepool_show.go +++ b/cmd/sks_nodepool_show.go @@ -16,24 +16,27 @@ import ( ) type sksNodepoolShowOutput struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - CreationDate string `json:"creation_date"` - InstancePoolID string `json:"instance_pool_id"` - InstancePrefix string `json:"instance_prefix"` - InstanceType string `json:"instance_type"` - Template string `json:"template"` - DiskSize int64 `json:"disk_size"` - AntiAffinityGroups []string `json:"anti_affinity_groups"` - SecurityGroups []string `json:"security_groups"` - PrivateNetworks []string `json:"private_networks"` - Version string `json:"version"` - Size int64 `json:"size"` - State string `json:"state"` - Taints []string `json:"taints"` - Labels map[string]string `json:"labels"` - AddOns []string `json:"addons"` + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + CreationDate string `json:"creation_date"` + InstancePoolID string `json:"instance_pool_id"` + InstancePrefix string `json:"instance_prefix"` + InstanceType string `json:"instance_type"` + Template string `json:"template"` + DiskSize int64 `json:"disk_size"` + AntiAffinityGroups []string `json:"anti_affinity_groups"` + SecurityGroups []string `json:"security_groups"` + PrivateNetworks []string `json:"private_networks"` + Version string `json:"version"` + Size int64 `json:"size"` + State string `json:"state"` + Taints []string `json:"taints"` + Labels map[string]string `json:"labels"` + AddOns []string `json:"addons"` + ImageGCMin string `json:"image_gc_min_age"` + ImageGcLowThreshold int64 `json:"image_gc_low_threshold"` + ImageGcHighThreshold int64 `json:"image_gc_high_threshold"` } func (o *sksNodepoolShowOutput) Type() string { return "SKS Nodepool" } @@ -125,6 +128,24 @@ func (c *sksNodepoolShowCmd) cmdRun(_ *cobra.Command, _ []string) error { } return }(), + ImageGCMin: func() (s string) { + if nodepool.KubeletImageGc != nil && nodepool.KubeletImageGc.MinAge != nil { + return *nodepool.KubeletImageGc.MinAge + } + return "2m" + }(), + ImageGcLowThreshold: func() (s int64) { + if nodepool.KubeletImageGc != nil && nodepool.KubeletImageGc.LowThreshold != nil { + return *nodepool.KubeletImageGc.LowThreshold + } + return 80 + }(), + ImageGcHighThreshold: func() (s int64) { + if nodepool.KubeletImageGc != nil && nodepool.KubeletImageGc.HighThreshold != nil { + return *nodepool.KubeletImageGc.HighThreshold + } + return 85 + }(), Version: *nodepool.Version, } diff --git a/go.mod b/go.mod index a89d70742..bcf2a22be 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.2.0 github.com/aws/smithy-go v1.1.0 github.com/dustin/go-humanize v1.0.1 - github.com/exoscale/egoscale v0.102.4-0.20240223092311-76285ea0504f + github.com/exoscale/egoscale v0.102.4-0.20240327105104-a8a8cb197ac6 github.com/exoscale/openapi-cli-generator v1.1.0 github.com/fatih/camelcase v1.0.0 github.com/google/uuid v1.4.0 diff --git a/go.sum b/go.sum index 5bc2670d3..1e7826ec3 100644 --- a/go.sum +++ b/go.sum @@ -195,8 +195,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= -github.com/exoscale/egoscale v0.102.4-0.20240223092311-76285ea0504f h1:Ivacu1126KBSLhXvXST6H0ftMjehMV637fL/nb222LM= -github.com/exoscale/egoscale v0.102.4-0.20240223092311-76285ea0504f/go.mod h1:sFBCvHJx/h6u8Z5igoIcgbPs+wltBbEJrXA2xp05KoU= +github.com/exoscale/egoscale v0.102.4-0.20240327105104-a8a8cb197ac6 h1:UQwsohHjLp5mkcB+vZ7UuYrTi0FsBrQvgYD0guyNcSo= +github.com/exoscale/egoscale v0.102.4-0.20240327105104-a8a8cb197ac6/go.mod h1:sFBCvHJx/h6u8Z5igoIcgbPs+wltBbEJrXA2xp05KoU= github.com/exoscale/openapi-cli-generator v1.1.0 h1:fYjmPqHR5vxlOBrbvde7eo7bISNQIFxsGn4A5/acwKA= github.com/exoscale/openapi-cli-generator v1.1.0/go.mod h1:TZBnbT7f3hJ5ImyUphJwRM+X5xF/zCQZ6o8a42gQeTs= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= diff --git a/vendor/github.com/exoscale/egoscale/v2/sks_nodepool.go b/vendor/github.com/exoscale/egoscale/v2/sks_nodepool.go index fc5f83570..8445466f0 100644 --- a/vendor/github.com/exoscale/egoscale/v2/sks_nodepool.go +++ b/vendor/github.com/exoscale/egoscale/v2/sks_nodepool.go @@ -22,6 +22,30 @@ func sksNodepoolTaintFromAPI(t *oapi.SksNodepoolTaint) *SKSNodepoolTaint { } } +type SKSNodepoolKubeletImageGc struct { + MinAge *string + HighThreshold *int64 + LowThreshold *int64 +} + +func sksNodepoolKubeletImageGcFromAPI(gc *oapi.KubeletImageGc) *SKSNodepoolKubeletImageGc { + r := &SKSNodepoolKubeletImageGc{} + + if gc != nil { + if gc.MinAge != nil { + r.MinAge = gc.MinAge + } + if gc.HighThreshold != nil { + r.HighThreshold = gc.HighThreshold + } + if gc.LowThreshold != nil { + r.LowThreshold = gc.LowThreshold + } + } + + return r +} + // SKSNodepool represents an SKS Nodepool. type SKSNodepool struct { AddOns *[]string @@ -40,6 +64,7 @@ type SKSNodepool struct { SecurityGroupIDs *[]string Size *int64 `req-for:"create"` State *string + KubeletImageGc *SKSNodepoolKubeletImageGc Taints *map[string]*SKSNodepoolTaint TemplateID *string Version *string @@ -81,6 +106,7 @@ func sksNodepoolFromAPI(n *oapi.SksNodepool) *SKSNodepool { InstancePoolID: n.InstancePool.Id, InstancePrefix: n.InstancePrefix, InstanceTypeID: n.InstanceType.Id, + KubeletImageGc: sksNodepoolKubeletImageGcFromAPI(n.KubeletImageGc), Labels: func() (v *map[string]string) { if n.Labels != nil && len(n.Labels.AdditionalProperties) > 0 { v = &n.Labels.AdditionalProperties @@ -176,6 +202,22 @@ func (c *Client) CreateSKSNodepool( DiskSize: *nodepool.DiskSize, InstancePrefix: nodepool.InstancePrefix, InstanceType: oapi.InstanceType{Id: nodepool.InstanceTypeID}, + KubeletImageGc: func() (v *oapi.KubeletImageGc) { + v = &oapi.KubeletImageGc{} + + if nodepool.KubeletImageGc != nil { + if nodepool.KubeletImageGc.MinAge != nil { + v.MinAge = nodepool.KubeletImageGc.MinAge + } + if nodepool.KubeletImageGc.HighThreshold != nil { + v.HighThreshold = nodepool.KubeletImageGc.HighThreshold + } + if nodepool.KubeletImageGc.LowThreshold != nil { + v.LowThreshold = nodepool.KubeletImageGc.LowThreshold + } + } + return + }(), Labels: func() (v *oapi.Labels) { if nodepool.Labels != nil { v = &oapi.Labels{AdditionalProperties: *nodepool.Labels} @@ -240,7 +282,6 @@ func (c *Client) CreateSKSNodepool( if err != nil { return nil, fmt.Errorf("unable to retrieve Nodepool: %s", err) } - return sksNodepoolFromAPI(nodepoolRes.JSON200), nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 40981bc2b..4e2db297f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -214,7 +214,7 @@ github.com/dlclark/regexp2/syntax # github.com/dustin/go-humanize v1.0.1 ## explicit; go 1.16 github.com/dustin/go-humanize -# github.com/exoscale/egoscale v0.102.4-0.20240223092311-76285ea0504f +# github.com/exoscale/egoscale v0.102.4-0.20240327105104-a8a8cb197ac6 ## explicit; go 1.20 github.com/exoscale/egoscale/v2 github.com/exoscale/egoscale/v2/api