From e7bd1bd9ea6a947a02a8b809f9cd9ab29905aabd Mon Sep 17 00:00:00 2001 From: DONG BEIQING Date: Mon, 22 Jan 2024 16:43:46 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Configurable=20qos=20resources=20fo?= =?UTF-8?q?r=20deployments=20managed=20by=20clustermanager=20and=20kluster?= =?UTF-8?q?let=20(#316)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * configurable qos resources for clustermanager and klusterlet deployments Signed-off-by: Dong Beiqing <350758787@qq.com> * move ResourceRequirement to a separate file Signed-off-by: Dong Beiqing <350758787@qq.com> --------- Signed-off-by: Dong Beiqing <350758787@qq.com> --- ...cluster-management.io_klusterlets.crd.yaml | 40 +++++++++++- ...cluster-management.io_klusterlets.crd.yaml | 55 +++++++++++++++- ...ter-management.io_clustermanagers.crd.yaml | 63 +++++++++++++++++++ operator/v1/funcs_clustermanager.go | 5 ++ operator/v1/funcs_klusterlet.go | 5 ++ operator/v1/type_resourcerequirement.go | 28 +++++++++ operator/v1/types_clustermanager.go | 5 ++ operator/v1/types_klusterlet.go | 23 ++----- operator/v1/zz_generated.deepcopy.go | 12 +++- .../v1/zz_generated.swagger_doc_generated.go | 11 +--- 10 files changed, 216 insertions(+), 31 deletions(-) create mode 100644 operator/v1/funcs_clustermanager.go create mode 100644 operator/v1/funcs_klusterlet.go create mode 100644 operator/v1/type_resourcerequirement.go diff --git a/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml index 9ccbdae5d..071da4342 100644 --- a/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml +++ b/crdsv1beta1/0001_00_operator.open-cluster-management.io_klusterlets.crd.yaml @@ -156,15 +156,53 @@ spec: description: RegistrationImagePullSpec represents the desired image configuration of registration agent. quay.io/open-cluster-management.io/registration:latest will be used if unspecified. type: string resourceRequirement: - description: ResourceRequirement specify QoS classes of klusterlet deployment + description: ResourceRequirement specify QoS classes of deployments managed by klusterlet. It applies to all the containers in the deployments. type: object properties: + resourceRequirements: + description: ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement + type: object + properties: + claims: + description: "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. \n This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. \n This field is immutable. It can only be set for containers." + type: array + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + type: object + required: + - name + properties: + name: + description: Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container. + type: string + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + requests: + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + additionalProperties: + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true type: type: string default: Default enum: - Default - BestEffort + - ResourceRequirement workConfiguration: description: WorkConfiguration contains the configuration of work type: object diff --git a/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml b/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml index df8d8324d..ad6e6434b 100644 --- a/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml +++ b/operator/v1/0000_00_operator.open-cluster-management.io_klusterlets.crd.yaml @@ -244,14 +244,65 @@ spec: will be used if unspecified. type: string resourceRequirement: - description: ResourceRequirement specify QoS classes of klusterlet - deployment + description: ResourceRequirement specify QoS classes of deployments + managed by klusterlet. It applies to all the containers in the deployments. properties: + resourceRequirements: + description: ResourceRequirements defines resource requests and + limits when Type is ResourceQosClassResourceRequirement + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object type: default: Default enum: - Default - BestEffort + - ResourceRequirement type: string type: object workConfiguration: diff --git a/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml b/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml index b57e95f5f..8cc24234b 100644 --- a/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml +++ b/operator/v1/0000_01_operator.open-cluster-management.io_clustermanagers.crd.yaml @@ -259,6 +259,69 @@ spec: description: RegistrationImagePullSpec represents the desired image of registration controller/webhook installed on hub. type: string + resourceRequirement: + description: ResourceRequirement specify QoS classes of deployments + managed by clustermanager. It applies to all the containers in the + deployments. + properties: + resourceRequirements: + description: ResourceRequirements defines resource requests and + limits when Type is ResourceQosClassResourceRequirement + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable. It can only be + set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in + pod.spec.resourceClaims of the Pod where this field + is used. It makes that resource available inside a + container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. Requests cannot exceed + Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: + default: Default + enum: + - Default + - BestEffort + - ResourceRequirement + type: string + type: object workConfiguration: description: WorkConfiguration contains the configuration of work properties: diff --git a/operator/v1/funcs_clustermanager.go b/operator/v1/funcs_clustermanager.go new file mode 100644 index 000000000..8ed923b8c --- /dev/null +++ b/operator/v1/funcs_clustermanager.go @@ -0,0 +1,5 @@ +package v1 + +func (cm *ClusterManager) GetResourceRequirement() *ResourceRequirement { + return cm.Spec.ResourceRequirement +} diff --git a/operator/v1/funcs_klusterlet.go b/operator/v1/funcs_klusterlet.go new file mode 100644 index 000000000..3bd56472f --- /dev/null +++ b/operator/v1/funcs_klusterlet.go @@ -0,0 +1,5 @@ +package v1 + +func (k *Klusterlet) GetResourceRequirement() *ResourceRequirement { + return k.Spec.ResourceRequirement +} diff --git a/operator/v1/type_resourcerequirement.go b/operator/v1/type_resourcerequirement.go new file mode 100644 index 000000000..e2f9ba6b3 --- /dev/null +++ b/operator/v1/type_resourcerequirement.go @@ -0,0 +1,28 @@ +package v1 + +import corev1 "k8s.io/api/core/v1" + +type ResourceRequirementAcquirer interface { + GetResourceRequirement() *ResourceRequirement +} + +// ResourceRequirement allow user override the default pod QoS classes +type ResourceRequirement struct { + // +kubebuilder:validation:Enum=Default;BestEffort;ResourceRequirement + // +kubebuilder:default:=Default + Type ResourceQosClass `json:"type"` + // ResourceRequirements defines resource requests and limits when Type is ResourceQosClassResourceRequirement + // +optional + ResourceRequirements *corev1.ResourceRequirements `json:"resourceRequirements,omitempty"` +} + +type ResourceQosClass string + +const ( + // Default use resource setting in the template file (with requests but no limits in the resources) + ResourceQosClassDefault ResourceQosClass = "Default" + // If all containers in the pod don't set resource request and limits, the pod is treated as BestEffort. + ResourceQosClassBestEffort ResourceQosClass = "BestEffort" + // Configurable resource requirements with requests and limits + ResourceQosClassResourceRequirement ResourceQosClass = "ResourceRequirement" +) diff --git a/operator/v1/types_clustermanager.go b/operator/v1/types_clustermanager.go index ed97ffd4c..33b0804f9 100644 --- a/operator/v1/types_clustermanager.go +++ b/operator/v1/types_clustermanager.go @@ -70,6 +70,11 @@ type ClusterManagerSpec struct { // AddOnManagerConfiguration contains the configuration of addon manager // +optional AddOnManagerConfiguration *AddOnManagerConfiguration `json:"addOnManagerConfiguration,omitempty"` + + // ResourceRequirement specify QoS classes of deployments managed by clustermanager. + // It applies to all the containers in the deployments. + // +optional + ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"` } // NodePlacement describes node scheduling configuration for the pods. diff --git a/operator/v1/types_klusterlet.go b/operator/v1/types_klusterlet.go index 2ae4bef9b..bef65984f 100644 --- a/operator/v1/types_klusterlet.go +++ b/operator/v1/types_klusterlet.go @@ -1,6 +1,8 @@ package v1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) // +genclient // +genclient:nonNamespaced @@ -87,27 +89,12 @@ type KlusterletSpec struct { // +optional HubApiServerHostAlias *HubApiServerHostAlias `json:"hubApiServerHostAlias,omitempty"` - // ResourceRequirement specify QoS classes of klusterlet deployment + // ResourceRequirement specify QoS classes of deployments managed by klusterlet. + // It applies to all the containers in the deployments. // +optional ResourceRequirement *ResourceRequirement `json:"resourceRequirement,omitempty"` } -type ResourceQosClass string - -const ( - // Default use resource setting in the template file - ResourceQosClassDefault ResourceQosClass = "Default" - // If all containers in the pod don't set resource request and limits, the pod is treated as BestEffort. - ResourceQosClassBestEffort ResourceQosClass = "BestEffort" -) - -// ResourceRequirement allow user override the default pod QoS classes -type ResourceRequirement struct { - // +kubebuilder:validation:Enum=Default;BestEffort - // +kubebuilder:default:=Default - Type ResourceQosClass `json:"type"` -} - // ServerURL represents the apiserver url and ca bundle that is accessible externally type ServerURL struct { // URL is the url of apiserver endpoint of the managed cluster. diff --git a/operator/v1/zz_generated.deepcopy.go b/operator/v1/zz_generated.deepcopy.go index 0de27326b..aaddf5b36 100644 --- a/operator/v1/zz_generated.deepcopy.go +++ b/operator/v1/zz_generated.deepcopy.go @@ -134,6 +134,11 @@ func (in *ClusterManagerSpec) DeepCopyInto(out *ClusterManagerSpec) { *out = new(AddOnManagerConfiguration) (*in).DeepCopyInto(*out) } + if in.ResourceRequirement != nil { + in, out := &in.ResourceRequirement, &out.ResourceRequirement + *out = new(ResourceRequirement) + (*in).DeepCopyInto(*out) + } return } @@ -353,7 +358,7 @@ func (in *KlusterletSpec) DeepCopyInto(out *KlusterletSpec) { if in.ResourceRequirement != nil { in, out := &in.ResourceRequirement, &out.ResourceRequirement *out = new(ResourceRequirement) - **out = **in + (*in).DeepCopyInto(*out) } return } @@ -504,6 +509,11 @@ func (in *RelatedResourceMeta) DeepCopy() *RelatedResourceMeta { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceRequirement) DeepCopyInto(out *ResourceRequirement) { *out = *in + if in.ResourceRequirements != nil { + in, out := &in.ResourceRequirements, &out.ResourceRequirements + *out = new(corev1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } return } diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go index 330df7997..8d8a14f20 100644 --- a/operator/v1/zz_generated.swagger_doc_generated.go +++ b/operator/v1/zz_generated.swagger_doc_generated.go @@ -60,6 +60,7 @@ var map_ClusterManagerSpec = map[string]string{ "registrationConfiguration": "RegistrationConfiguration contains the configuration of registration", "workConfiguration": "WorkConfiguration contains the configuration of work", "addOnManagerConfiguration": "AddOnManagerConfiguration contains the configuration of addon manager", + "resourceRequirement": "ResourceRequirement specify QoS classes of deployments managed by clustermanager. It applies to all the containers in the deployments.", } func (ClusterManagerSpec) SwaggerDoc() map[string]string { @@ -213,7 +214,7 @@ var map_KlusterletSpec = map[string]string{ "registrationConfiguration": "RegistrationConfiguration contains the configuration of registration", "workConfiguration": "WorkConfiguration contains the configuration of work", "hubApiServerHostAlias": "HubApiServerHostAlias contains the host alias for hub api server. registration-agent and work-agent will use it to communicate with hub api server.", - "resourceRequirement": "ResourceRequirement specify QoS classes of klusterlet deployment", + "resourceRequirement": "ResourceRequirement specify QoS classes of deployments managed by klusterlet. It applies to all the containers in the deployments.", } func (KlusterletSpec) SwaggerDoc() map[string]string { @@ -244,14 +245,6 @@ func (RegistrationConfiguration) SwaggerDoc() map[string]string { return map_RegistrationConfiguration } -var map_ResourceRequirement = map[string]string{ - "": "ResourceRequirement allow user override the default pod QoS classes", -} - -func (ResourceRequirement) SwaggerDoc() map[string]string { - return map_ResourceRequirement -} - var map_ServerURL = map[string]string{ "": "ServerURL represents the apiserver url and ca bundle that is accessible externally", "url": "URL is the url of apiserver endpoint of the managed cluster.",