diff --git a/api/v1/coroot_types.go b/api/v1/coroot_types.go index 23d8206..37d4681 100644 --- a/api/v1/coroot_types.go +++ b/api/v1/coroot_types.go @@ -3,6 +3,7 @@ package v1 import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -44,69 +45,110 @@ type NodeAgentSpec struct { UpdateStrategy appsv1.DaemonSetUpdateStrategy `json:"update_strategy,omitempty"` Affinity *corev1.Affinity `json:"affinity,omitempty"` Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` Env []corev1.EnvVar `json:"env,omitempty"` } type ClusterAgentSpec struct { Version string `json:"version,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - Resources corev1.ResourceRequirements `json:"resources,omitempty"` - Env []corev1.EnvVar `json:"env,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` + Env []corev1.EnvVar `json:"env,omitempty"` } type PrometheusSpec struct { - Affinity *corev1.Affinity `json:"affinity,omitempty"` - Storage StorageSpec `json:"storage,omitempty"` - Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + Storage StorageSpec `json:"storage,omitempty"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` } type ClickhouseSpec struct { Shards int `json:"shards,omitempty"` Replicas int `json:"replicas,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - Storage StorageSpec `json:"storage,omitempty"` - Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + Storage StorageSpec `json:"storage,omitempty"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` Keeper ClickhouseKeeperSpec `json:"keeper,omitempty"` } type ClickhouseKeeperSpec struct { - Affinity *corev1.Affinity `json:"affinity,omitempty"` - Storage StorageSpec `json:"storage,omitempty"` - Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + Storage StorageSpec `json:"storage,omitempty"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` } type ExternalClickhouseSpec struct { - Address string `json:"address,omitempty"` - User string `json:"user,omitempty"` - Password string `json:"password,omitempty"` - Database string `json:"database,omitempty"` + Address string `json:"address,omitempty"` + User string `json:"user,omitempty"` + Database string `json:"database,omitempty"` + Password string `json:"password,omitempty"` + PasswordSecret *corev1.SecretKeySelector `json:"passwordSecret,omitempty"` } type PostgresSpec struct { - ConnectionString string `json:"connectionString,omitempty"` + Host string `json:"host,omitempty"` + Port int32 `json:"port,omitempty"` + User string `json:"user,omitempty"` + Database string `json:"database,omitempty"` + Password string `json:"password,omitempty"` + PasswordSecret *corev1.SecretKeySelector `json:"passwordSecret,omitempty"` + Params map[string]string `json:"params,omitempty"` +} + +type IngressSpec struct { + ClassName *string `json:"className,omitempty"` + Host string `json:"host,omitempty"` + Path string `json:"path,omitempty"` + TLS *networkingv1.IngressTLS `json:"tls,omitempty"` +} + +type ProjectSpec struct { + // +kubebuilder:validation:Required + Name string `json:"name,omitempty"` + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinItems=1 + ApiKeys []ApiKeySpec `json:"apiKeys,omitempty"` +} + +type ApiKeySpec struct { + // +kubebuilder:validation:Required + Key string `json:"key,omitempty"` + Description string `json:"description,omitempty"` } type CorootSpec struct { - ApiKey string `json:"apiKey,omitempty"` MetricsRefreshInterval metav1.Duration `json:"metricsRefreshInterval,omitempty"` CacheTTL metav1.Duration `json:"cacheTTL,omitempty"` AuthAnonymousRole string `json:"authAnonymousRole,omitempty"` AuthBootstrapAdminPassword string `json:"authBootstrapAdminPassword,omitempty"` + Projects []ProjectSpec `json:"projects,omitempty"` Env []corev1.EnvVar `json:"env,omitempty"` CommunityEdition CommunityEditionSpec `json:"communityEdition,omitempty"` EnterpriseEdition *EnterpriseEditionSpec `json:"enterpriseEdition,omitempty"` AgentsOnly *AgentsOnlySpec `json:"agentsOnly,omitempty"` - Replicas int `json:"replicas,omitempty"` - Service ServiceSpec `json:"service,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - Storage StorageSpec `json:"storage,omitempty"` - Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Replicas int `json:"replicas,omitempty"` + Service ServiceSpec `json:"service,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + Storage StorageSpec `json:"storage,omitempty"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` + ApiKey string `json:"apiKey,omitempty"` NodeAgent NodeAgentSpec `json:"nodeAgent,omitempty"` ClusterAgent ClusterAgentSpec `json:"clusterAgent,omitempty"` @@ -116,6 +158,8 @@ type CorootSpec struct { ExternalClickhouse *ExternalClickhouseSpec `json:"externalClickhouse,omitempty"` Postgres *PostgresSpec `json:"postgres,omitempty"` + + Ingress *IngressSpec `json:"ingress,omitempty"` } type CorootStatus struct { // TODO diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index d8a1ec3..869c2dc 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1 import ( corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -25,6 +26,21 @@ func (in *AgentsOnlySpec) DeepCopy() *AgentsOnlySpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApiKeySpec) DeepCopyInto(out *ApiKeySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApiKeySpec. +func (in *ApiKeySpec) DeepCopy() *ApiKeySpec { + if in == nil { + return nil + } + out := new(ApiKeySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClickhouseKeeperSpec) DeepCopyInto(out *ClickhouseKeeperSpec) { *out = *in @@ -35,6 +51,20 @@ func (in *ClickhouseKeeperSpec) DeepCopyInto(out *ClickhouseKeeperSpec) { } in.Storage.DeepCopyInto(&out.Storage) in.Resources.DeepCopyInto(&out.Resources) + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClickhouseKeeperSpec. @@ -57,6 +87,20 @@ func (in *ClickhouseSpec) DeepCopyInto(out *ClickhouseSpec) { } in.Storage.DeepCopyInto(&out.Storage) in.Resources.DeepCopyInto(&out.Resources) + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } in.Keeper.DeepCopyInto(&out.Keeper) } @@ -79,6 +123,20 @@ func (in *ClusterAgentSpec) DeepCopyInto(out *ClusterAgentSpec) { (*in).DeepCopyInto(*out) } in.Resources.DeepCopyInto(&out.Resources) + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Env != nil { in, out := &in.Env, &out.Env *out = make([]corev1.EnvVar, len(*in)) @@ -177,6 +235,13 @@ func (in *CorootSpec) DeepCopyInto(out *CorootSpec) { *out = *in out.MetricsRefreshInterval = in.MetricsRefreshInterval out.CacheTTL = in.CacheTTL + if in.Projects != nil { + in, out := &in.Projects, &out.Projects + *out = make([]ProjectSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.Env != nil { in, out := &in.Env, &out.Env *out = make([]corev1.EnvVar, len(*in)) @@ -203,6 +268,20 @@ func (in *CorootSpec) DeepCopyInto(out *CorootSpec) { } in.Storage.DeepCopyInto(&out.Storage) in.Resources.DeepCopyInto(&out.Resources) + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } in.NodeAgent.DeepCopyInto(&out.NodeAgent) in.ClusterAgent.DeepCopyInto(&out.ClusterAgent) in.Prometheus.DeepCopyInto(&out.Prometheus) @@ -210,12 +289,17 @@ func (in *CorootSpec) DeepCopyInto(out *CorootSpec) { if in.ExternalClickhouse != nil { in, out := &in.ExternalClickhouse, &out.ExternalClickhouse *out = new(ExternalClickhouseSpec) - **out = **in + (*in).DeepCopyInto(*out) } if in.Postgres != nil { in, out := &in.Postgres, &out.Postgres *out = new(PostgresSpec) - **out = **in + (*in).DeepCopyInto(*out) + } + if in.Ingress != nil { + in, out := &in.Ingress, &out.Ingress + *out = new(IngressSpec) + (*in).DeepCopyInto(*out) } } @@ -269,6 +353,11 @@ func (in *EnterpriseEditionSpec) DeepCopy() *EnterpriseEditionSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExternalClickhouseSpec) DeepCopyInto(out *ExternalClickhouseSpec) { *out = *in + if in.PasswordSecret != nil { + in, out := &in.PasswordSecret, &out.PasswordSecret + *out = new(corev1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalClickhouseSpec. @@ -281,6 +370,31 @@ func (in *ExternalClickhouseSpec) DeepCopy() *ExternalClickhouseSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IngressSpec) DeepCopyInto(out *IngressSpec) { + *out = *in + if in.ClassName != nil { + in, out := &in.ClassName, &out.ClassName + *out = new(string) + **out = **in + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(networkingv1.IngressTLS) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressSpec. +func (in *IngressSpec) DeepCopy() *IngressSpec { + if in == nil { + return nil + } + out := new(IngressSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeAgentSpec) DeepCopyInto(out *NodeAgentSpec) { *out = *in @@ -291,6 +405,20 @@ func (in *NodeAgentSpec) DeepCopyInto(out *NodeAgentSpec) { (*in).DeepCopyInto(*out) } in.Resources.DeepCopyInto(&out.Resources) + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Env != nil { in, out := &in.Env, &out.Env *out = make([]corev1.EnvVar, len(*in)) @@ -313,6 +441,18 @@ func (in *NodeAgentSpec) DeepCopy() *NodeAgentSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { *out = *in + if in.PasswordSecret != nil { + in, out := &in.PasswordSecret, &out.PasswordSecret + *out = new(corev1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.Params != nil { + in, out := &in.Params, &out.Params + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PostgresSpec. @@ -325,6 +465,26 @@ func (in *PostgresSpec) DeepCopy() *PostgresSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProjectSpec) DeepCopyInto(out *ProjectSpec) { + *out = *in + if in.ApiKeys != nil { + in, out := &in.ApiKeys, &out.ApiKeys + *out = make([]ApiKeySpec, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProjectSpec. +func (in *ProjectSpec) DeepCopy() *ProjectSpec { + if in == nil { + return nil + } + out := new(ProjectSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) { *out = *in @@ -335,6 +495,20 @@ func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) { } in.Storage.DeepCopyInto(&out.Storage) in.Resources.DeepCopyInto(&out.Resources) + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusSpec. diff --git a/config/crd/coroot.com_coroots.yaml b/config/crd/coroot.com_coroots.yaml index 30cb2d5..bb22fde 100644 --- a/config/crd/coroot.com_coroots.yaml +++ b/config/crd/coroot.com_coroots.yaml @@ -2833,6 +2833,10 @@ spec: x-kubernetes-list-type: atomic type: object type: object + podAnnotations: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -2904,6 +2908,48 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + podAnnotations: + additionalProperties: + type: string type: object replicas: type: integer @@ -2980,6 +3026,44 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array type: object clusterAgent: properties: @@ -4029,6 +4113,10 @@ spec: - name type: object type: array + podAnnotations: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -4089,6 +4177,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array version: type: string type: object @@ -4229,9 +4355,65 @@ spec: type: string password: type: string + passwordSecret: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic user: type: string type: object + ingress: + properties: + className: + type: string + host: + type: string + path: + type: string + tls: + description: IngressTLS describes the transport layer security + associated with an ingress. + properties: + hosts: + description: |- + hosts is a list of hosts included in the TLS certificate. The values in + this list must match the name/s used in the tlsSecret. Defaults to the + wildcard host setting for the loadbalancer controller fulfilling this + Ingress, if left unspecified. + items: + type: string + type: array + x-kubernetes-list-type: atomic + secretName: + description: |- + secretName is the name of the secret used to terminate TLS traffic on + port 443. Field is left optional to allow TLS routing based on SNI + hostname alone. If the SNI host in a listener conflicts with the "Host" + header field used by an IngressRule, the SNI host is used for termination + and value of the "Host" header is used for routing. + type: string + type: object + type: object metricsRefreshInterval: type: string nodeAgent: @@ -5282,6 +5464,10 @@ spec: - name type: object type: array + podAnnotations: + additionalProperties: + type: string + type: object priorityClassName: type: string resources: @@ -5344,6 +5530,44 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array update_strategy: description: DaemonSetUpdateStrategy is a struct used to control the update strategy for a DaemonSet. @@ -5405,11 +5629,74 @@ spec: version: type: string type: object + podAnnotations: + additionalProperties: + type: string + type: object postgres: properties: - connectionString: + database: + type: string + host: + type: string + params: + additionalProperties: + type: string + type: object + password: + type: string + passwordSecret: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + port: + format: int32 + type: integer + user: type: string type: object + projects: + items: + properties: + apiKeys: + items: + properties: + description: + type: string + key: + type: string + required: + - key + type: object + minItems: 1 + type: array + name: + type: string + required: + - apiKeys + - name + type: object + type: array prometheus: properties: affinity: @@ -6339,6 +6626,10 @@ spec: x-kubernetes-list-type: atomic type: object type: object + podAnnotations: + additionalProperties: + type: string + type: object resources: description: ResourceRequirements describes the compute resource requirements. @@ -6410,6 +6701,44 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array type: object replicas: type: integer @@ -6496,6 +6825,44 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array type: object status: properties: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index cb5f59e..4e2b76c 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -100,6 +100,18 @@ rules: - get - patch - update +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - rbac.authorization.k8s.io resources: diff --git a/controller/clickhouse.go b/controller/clickhouse.go index da66511..3230113 100644 --- a/controller/clickhouse.go +++ b/controller/clickhouse.go @@ -25,7 +25,7 @@ func (r *CorootReconciler) clickhouseSecret(cr *corootv1.Coroot) *corev1.Secret Namespace: cr.Namespace, Labels: ls, }, - Data: map[string][]byte{"password": []byte(GeneratePassword(16))}, + Data: map[string][]byte{"password": []byte(RandomString(16))}, } return s } @@ -174,12 +174,14 @@ func (r *CorootReconciler) clickhouseStatefulSets(cr *corootv1.Coroot) []*appsv1 }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: ls, + Labels: ls, + Annotations: cr.Spec.Clickhouse.PodAnnotations, }, Spec: corev1.PodSpec{ ServiceAccountName: cr.Name + "-clickhouse", SecurityContext: nonRootSecurityContext, Affinity: cr.Spec.Clickhouse.Affinity, + Tolerations: cr.Spec.Clickhouse.Tolerations, InitContainers: []corev1.Container{ { Image: UBIMinimalImage, @@ -321,29 +323,29 @@ var clickhouseConfigTemplate = template.Must(template.New("").Parse(` - {{ range $shard := .Shards }} + {{- range $shard := .Shards }} true - {{ range $replica := $.Replicas }} + {{- range $replica := $.Replicas }} {{$.Name}}-clickhouse-shard-{{$shard}}-{{$replica}}.{{$.Name}}-clickhouse-headless.{{$.Namespace}} 9000 default - {{ end }} + {{- end }} - {{ end }} + {{- end }} - {{ range $keeper := .Keepers }} + {{- range $keeper := .Keepers }} {{$.Name}}-clickhouse-keeper-{{$keeper}}.{{$.Name}}-clickhouse-keeper-headless.{{$.Namespace}} 9181 - {{ end }} + {{- end }} diff --git a/controller/clickhouse_keeper.go b/controller/clickhouse_keeper.go index ebdf614..0e45aad 100644 --- a/controller/clickhouse_keeper.go +++ b/controller/clickhouse_keeper.go @@ -101,12 +101,14 @@ func (r *CorootReconciler) clickhouseKeeperStatefulSet(cr *corootv1.Coroot) *app }}, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: ls, + Labels: ls, + Annotations: cr.Spec.Clickhouse.Keeper.PodAnnotations, }, Spec: corev1.PodSpec{ ServiceAccountName: cr.Name + "-clickhouse-keeper", SecurityContext: nonRootSecurityContext, Affinity: cr.Spec.Clickhouse.Keeper.Affinity, + Tolerations: cr.Spec.Clickhouse.Keeper.Tolerations, InitContainers: []corev1.Container{ { Image: UBIMinimalImage, @@ -207,13 +209,13 @@ var clickhouseKeeperConfigTemplate = template.Must(template.New("").Parse(` - {{ range $id := .Ids }} + {{- range $id := .Ids }} {{$id}} {{$.Name}}-clickhouse-keeper-{{$id}}.{{$.Name}}-clickhouse-keeper-headless.{{$.Namespace}} 9234 - {{ end }} + {{- end }} diff --git a/controller/cluster_agent.go b/controller/cluster_agent.go index fcbacc7..9b8636d 100644 --- a/controller/cluster_agent.go +++ b/controller/cluster_agent.go @@ -101,12 +101,14 @@ func (r *CorootReconciler) clusterAgentDeployment(cr *corootv1.Coroot) *appsv1.D }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: ls, + Labels: ls, + Annotations: cr.Spec.ClusterAgent.PodAnnotations, }, Spec: corev1.PodSpec{ ServiceAccountName: cr.Name + "-cluster-agent", SecurityContext: nonRootSecurityContext, Affinity: cr.Spec.ClusterAgent.Affinity, + Tolerations: cr.Spec.ClusterAgent.Tolerations, Containers: []corev1.Container{ { Image: r.getAppImage(cr, AppClusterAgent), diff --git a/controller/controller.go b/controller/controller.go index 9ed634a..e405b04 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -7,6 +7,7 @@ import ( "golang.org/x/exp/maps" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -72,6 +73,7 @@ func NewCorootReconciler(mgr ctrl.Manager) *CorootReconciler { // +kubebuilder:rbac:groups=apps,resources=deployments;replicasets;daemonsets;statefulsets;cronjobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses;volumeattachments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles;clusterrolebindings;roles;rolebindings,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=use @@ -117,8 +119,8 @@ func (r *CorootReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, nil } - if cr.Spec.Replicas > 1 && (cr.Spec.Postgres == nil || cr.Spec.Postgres.ConnectionString == "") { - logger.Error(fmt.Errorf("Postgres.ConnectionString is empty"), "Coroot requires Postgres to run multiple replicas (will run only one replica)") + if cr.Spec.Replicas > 1 && cr.Spec.Postgres == nil { + logger.Error(fmt.Errorf("postgres not configured"), "Coroot requires Postgres to run multiple replicas (will run only one replica)") cr.Spec.Replicas = 1 } r.CreateOrUpdateServiceAccount(ctx, cr, "coroot", sccNonroot) @@ -131,6 +133,7 @@ func (r *CorootReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr _ = r.Delete(ctx, r.corootDeployment(cr)) r.deploymentDeleted = true } + r.CreateOrUpdateIngress(ctx, cr, r.corootIngress(cr), cr.Spec.Ingress == nil) r.CreateOrUpdateServiceAccount(ctx, cr, "prometheus", sccNonroot) r.CreateOrUpdatePVC(ctx, cr, r.prometheusPVC(cr)) @@ -163,8 +166,15 @@ func (r *CorootReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, nil } -func (r *CorootReconciler) CreateOrUpdate(ctx context.Context, cr *corootv1.Coroot, obj client.Object, f controllerutil.MutateFn) { - logger := ctrl.Log.WithValues("type", fmt.Sprintf("%T", obj), "name", obj.GetName(), "namespace", obj.GetNamespace()) +func (r *CorootReconciler) CreateOrUpdate(ctx context.Context, cr *corootv1.Coroot, obj client.Object, delete bool, f controllerutil.MutateFn) { + logger := ctrl.Log.WithValues("namespace", obj.GetNamespace(), "name", obj.GetName(), "type", fmt.Sprintf("%T", obj)) + if delete { + err := r.Delete(ctx, obj) + if err == nil { + logger.Info("deleted") + } + return + } _ = ctrl.SetControllerReference(cr, obj, r.Scheme) errMsg := "failed to create or update" if f == nil { @@ -182,28 +192,28 @@ func (r *CorootReconciler) CreateOrUpdate(ctx context.Context, cr *corootv1.Coro } func (r *CorootReconciler) CreateSecret(ctx context.Context, cr *corootv1.Coroot, s *corev1.Secret) { - r.CreateOrUpdate(ctx, cr, s, nil) + r.CreateOrUpdate(ctx, cr, s, false, nil) } func (r *CorootReconciler) CreateOrUpdateDeployment(ctx context.Context, cr *corootv1.Coroot, d *appsv1.Deployment) { spec := d.Spec - r.CreateOrUpdate(ctx, cr, d, func() error { - return Merge(&d.Spec, spec) + r.CreateOrUpdate(ctx, cr, d, false, func() error { + return MergeSpecs(d, &d.Spec, spec) }) } func (r *CorootReconciler) CreateOrUpdateDaemonSet(ctx context.Context, cr *corootv1.Coroot, ds *appsv1.DaemonSet) { spec := ds.Spec - r.CreateOrUpdate(ctx, cr, ds, func() error { - return Merge(&ds.Spec, spec) + r.CreateOrUpdate(ctx, cr, ds, false, func() error { + return MergeSpecs(ds, &ds.Spec, spec) }) } func (r *CorootReconciler) CreateOrUpdateStatefulSet(ctx context.Context, cr *corootv1.Coroot, ss *appsv1.StatefulSet) { spec := ss.Spec - r.CreateOrUpdate(ctx, cr, ss, func() error { + r.CreateOrUpdate(ctx, cr, ss, false, func() error { volumeClaimTemplates := ss.Spec.VolumeClaimTemplates[:] - err := Merge(&ss.Spec, spec) + err := MergeSpecs(ss, &ss.Spec, spec) ss.Spec.VolumeClaimTemplates = volumeClaimTemplates return err }) @@ -211,15 +221,15 @@ func (r *CorootReconciler) CreateOrUpdateStatefulSet(ctx context.Context, cr *co func (r *CorootReconciler) CreateOrUpdatePVC(ctx context.Context, cr *corootv1.Coroot, pvc *corev1.PersistentVolumeClaim) { spec := pvc.Spec - r.CreateOrUpdate(ctx, cr, pvc, func() error { - return Merge(&pvc.Spec, spec) + r.CreateOrUpdate(ctx, cr, pvc, false, func() error { + return MergeSpecs(pvc, &pvc.Spec, spec) }) } func (r *CorootReconciler) CreateOrUpdateService(ctx context.Context, cr *corootv1.Coroot, s *corev1.Service) { spec := s.Spec - r.CreateOrUpdate(ctx, cr, s, func() error { - err := Merge(&s.Spec, spec) + r.CreateOrUpdate(ctx, cr, s, false, func() error { + err := MergeSpecs(s, &s.Spec, spec) s.Spec.Ports = spec.Ports return err }) @@ -231,13 +241,13 @@ func (r *CorootReconciler) CreateOrUpdateServiceAccount(ctx context.Context, cr Namespace: cr.Namespace, Labels: Labels(cr, component), }} - r.CreateOrUpdate(ctx, cr, sa, nil) - r.CreateOrUpdate(ctx, cr, r.openshiftSCCRoleBinding(cr, component, scc), nil) + r.CreateOrUpdate(ctx, cr, sa, false, nil) + r.CreateOrUpdate(ctx, cr, r.openshiftSCCRoleBinding(cr, component, scc), false, nil) } func (r *CorootReconciler) CreateOrUpdateRole(ctx context.Context, cr *corootv1.Coroot, role *rbacv1.Role) { rules := role.Rules - r.CreateOrUpdate(ctx, cr, role, func() error { + r.CreateOrUpdate(ctx, cr, role, false, func() error { role.Rules = rules return nil }) @@ -245,14 +255,21 @@ func (r *CorootReconciler) CreateOrUpdateRole(ctx context.Context, cr *corootv1. func (r *CorootReconciler) CreateOrUpdateClusterRole(ctx context.Context, cr *corootv1.Coroot, role *rbacv1.ClusterRole) { rules := role.Rules - r.CreateOrUpdate(ctx, cr, role, func() error { + r.CreateOrUpdate(ctx, cr, role, false, func() error { role.Rules = rules return nil }) } func (r *CorootReconciler) CreateOrUpdateClusterRoleBinding(ctx context.Context, cr *corootv1.Coroot, b *rbacv1.ClusterRoleBinding) { - r.CreateOrUpdate(ctx, cr, b, nil) + r.CreateOrUpdate(ctx, cr, b, false, nil) +} + +func (r *CorootReconciler) CreateOrUpdateIngress(ctx context.Context, cr *corootv1.Coroot, i *networkingv1.Ingress, delete bool) { + spec := i.Spec + r.CreateOrUpdate(ctx, cr, i, delete, func() error { + return MergeSpecs(i, &i.Spec, spec) + }) } func (r *CorootReconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -267,6 +284,7 @@ func (r *CorootReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&rbacv1.ClusterRoleBinding{}). Owns(&corev1.PersistentVolumeClaim{}). Owns(&corev1.Secret{}). + Owns(&networkingv1.Ingress{}). Complete(r) } diff --git a/controller/coroot.go b/controller/coroot.go index 516d915..ea1532e 100644 --- a/controller/coroot.go +++ b/controller/coroot.go @@ -1,7 +1,12 @@ package controller import ( + "bytes" "fmt" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/utils/ptr" + "strings" + "text/template" corootv1 "github.io/coroot/operator/api/v1" appsv1 "k8s.io/api/apps/v1" @@ -78,6 +83,50 @@ func (r *CorootReconciler) corootPVCs(cr *corootv1.Coroot) []*corev1.PersistentV return res } +func (r *CorootReconciler) corootIngress(cr *corootv1.Coroot) *networkingv1.Ingress { + ls := Labels(cr, "ingress") + i := &networkingv1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name, + Namespace: cr.Namespace, + Labels: ls, + }, + } + if cr.Spec.Ingress == nil { + return i + } + path := cr.Spec.Ingress.Path + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + i.Spec = networkingv1.IngressSpec{ + IngressClassName: cr.Spec.Ingress.ClassName, + Rules: []networkingv1.IngressRule{{ + Host: cr.Spec.Ingress.Host, + IngressRuleValue: networkingv1.IngressRuleValue{ + HTTP: &networkingv1.HTTPIngressRuleValue{ + Paths: []networkingv1.HTTPIngressPath{{ + Path: path, + PathType: ptr.To(networkingv1.PathTypePrefix), + Backend: networkingv1.IngressBackend{ + Service: &networkingv1.IngressServiceBackend{ + Name: fmt.Sprintf("%s-coroot", cr.Name), + Port: networkingv1.ServiceBackendPort{ + Name: "http", + }, + }, + }, + }}, + }, + }, + }}, + } + if cr.Spec.Ingress.TLS != nil { + i.Spec.TLS = append(i.Spec.TLS, *cr.Spec.Ingress.TLS) + } + return i +} + func (r *CorootReconciler) corootDeployment(cr *corootv1.Coroot) *appsv1.Deployment { ls := Labels(cr, "coroot") d := &appsv1.Deployment{ @@ -131,13 +180,19 @@ func (r *CorootReconciler) corootStatefulSet(cr *corootv1.Coroot) *appsv1.Statef image = r.getAppImage(cr, AppCorootCE) } - if cr.Spec.ExternalClickhouse != nil { + if ec := cr.Spec.ExternalClickhouse; ec != nil { env = append(env, - corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_ADDRESS", Value: cr.Spec.ExternalClickhouse.Address}, - corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_USER", Value: cr.Spec.ExternalClickhouse.User}, - corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_PASSWORD", Value: cr.Spec.ExternalClickhouse.Password}, - corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_INITIAL_DATABASE", Value: cr.Spec.ExternalClickhouse.Database}, + corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_ADDRESS", Value: ec.Address}, + corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_USER", Value: ec.User}, + corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_INITIAL_DATABASE", Value: ec.Database}, ) + password := corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_PASSWORD"} + if ec.PasswordSecret != nil { + password.ValueFrom = &corev1.EnvVarSource{SecretKeyRef: ec.PasswordSecret} + } else { + password.Value = ec.Password + } + env = append(env, password) } else { env = append(env, corev1.EnvVar{ @@ -146,19 +201,24 @@ func (r *CorootReconciler) corootStatefulSet(cr *corootv1.Coroot) *appsv1.Statef }, corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_USER", Value: "default"}, corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_PASSWORD", ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-clickhouse", cr.Name), - }, - Key: "password", - }, - }}, + SecretKeyRef: secretKeySelector(fmt.Sprintf("%s-clickhouse", cr.Name), "password")}}, corev1.EnvVar{Name: "GLOBAL_CLICKHOUSE_INITIAL_DATABASE", Value: "default"}, ) } - if cr.Spec.Postgres != nil && cr.Spec.Postgres.ConnectionString != "" { - env = append(env, corev1.EnvVar{Name: "PG_CONNECTION_STRING", Value: cr.Spec.Postgres.ConnectionString}) + if p := cr.Spec.Postgres; p != nil { + password := corev1.EnvVar{Name: "PG_PASSWORD"} + if p.PasswordSecret != nil { + password.ValueFrom = &corev1.EnvVarSource{SecretKeyRef: p.PasswordSecret} + } else { + password.Value = p.Password + } + env = append(env, password) + env = append(env, corev1.EnvVar{Name: "PG_CONNECTION_STRING", Value: postgresConnectionString(*p, "PG_PASSWORD")}) + } + + if cr.Spec.Ingress != nil && cr.Spec.Ingress.Path != "" { + env = append(env, corev1.EnvVar{Name: "URL_BASE_PATH", Value: cr.Spec.Ingress.Path}) } replicas := int32(cr.Spec.Replicas) @@ -179,17 +239,29 @@ func (r *CorootReconciler) corootStatefulSet(cr *corootv1.Coroot) *appsv1.Statef }}, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: ls, + Labels: ls, + Annotations: cr.Spec.PodAnnotations, }, Spec: corev1.PodSpec{ ServiceAccountName: cr.Name + "-coroot", SecurityContext: nonRootSecurityContext, Affinity: cr.Spec.Affinity, + Tolerations: cr.Spec.Tolerations, + InitContainers: []corev1.Container{ + { + Image: UBIMinimalImage, + Name: "config", + Command: []string{"/bin/sh", "-c"}, + Args: []string{corootConfigCmd("/config/config.yaml", cr)}, + VolumeMounts: []corev1.VolumeMount{{Name: "config", MountPath: "/config"}}, + }, + }, Containers: []corev1.Container{ { Image: image, Name: "coroot", Args: []string{ + "--config=/config/config.yaml", "--listen=:8080", "--data-dir=/data", }, @@ -198,6 +270,7 @@ func (r *CorootReconciler) corootStatefulSet(cr *corootv1.Coroot) *appsv1.Statef {Name: "http", ContainerPort: 8080, Protocol: corev1.ProtocolTCP}, }, VolumeMounts: []corev1.VolumeMount{ + {Name: "config", MountPath: "/config"}, {Name: "data", MountPath: "/data"}, }, Resources: cr.Spec.Resources, @@ -208,9 +281,35 @@ func (r *CorootReconciler) corootStatefulSet(cr *corootv1.Coroot) *appsv1.Statef }, }, }, + Volumes: []corev1.Volume{ + { + Name: "config", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, }, }, } return ss } + +func corootConfigCmd(filename string, cr *corootv1.Coroot) string { + var out bytes.Buffer + _ = corootConfigTemplate.Execute(&out, cr.Spec) + return "cat < " + filename + out.String() + "EOF" +} + +var corootConfigTemplate = template.Must(template.New("").Parse(` +projects: +{{- range $project := .Projects }} +- name: {{ $project.Name }} + api_keys: + {{- range $key := $project.ApiKeys }} + - key: {{ $key.Key }} + description: {{ $key.Description }} + {{- end }} +{{- end }} +`)) diff --git a/controller/node_agent.go b/controller/node_agent.go index b440fdd..54b612f 100644 --- a/controller/node_agent.go +++ b/controller/node_agent.go @@ -51,6 +51,11 @@ func (r *CorootReconciler) nodeAgentDaemonSet(cr *corootv1.Coroot) *appsv1.Daemo } } + tolerations := cr.Spec.NodeAgent.Tolerations + if len(tolerations) == 0 { + tolerations = []corev1.Toleration{{Operator: corev1.TolerationOpExists}} + } + ds.Spec = appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: ls, @@ -58,12 +63,13 @@ func (r *CorootReconciler) nodeAgentDaemonSet(cr *corootv1.Coroot) *appsv1.Daemo UpdateStrategy: cr.Spec.NodeAgent.UpdateStrategy, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: ls, + Labels: ls, + Annotations: cr.Spec.NodeAgent.PodAnnotations, }, Spec: corev1.PodSpec{ ServiceAccountName: cr.Name + "-node-agent", HostPID: true, - Tolerations: []corev1.Toleration{{Operator: corev1.TolerationOpExists}}, + Tolerations: tolerations, PriorityClassName: cr.Spec.NodeAgent.PriorityClassName, Affinity: cr.Spec.NodeAgent.Affinity, Containers: []corev1.Container{ diff --git a/controller/postgres.go b/controller/postgres.go new file mode 100644 index 0000000..b82f296 --- /dev/null +++ b/controller/postgres.go @@ -0,0 +1,33 @@ +package controller + +import ( + "fmt" + corootv1 "github.io/coroot/operator/api/v1" + "sort" + "strings" +) + +func postgresConnectionString(p corootv1.PostgresSpec, passwordEnvVar string) string { + kv := p.Params + if kv == nil { + kv = map[string]string{} + } + kv["host"] = p.Host + if p.Port > 0 { + kv["port"] = fmt.Sprintf("%d", p.Port) + } + kv["user"] = p.User + kv["password"] = fmt.Sprintf("$(%s)", passwordEnvVar) + kv["dbname"] = p.Database + if kv["sslmode"] == "" { + kv["sslmode"] = "disable" + } + var kvs []string + for k, v := range kv { + if v != "" { + kvs = append(kvs, fmt.Sprintf("%s='%s'", k, v)) + } + } + sort.Strings(kvs) + return strings.Join(kvs, " ") +} diff --git a/controller/prometheus.go b/controller/prometheus.go index 02c8e79..3e20f70 100644 --- a/controller/prometheus.go +++ b/controller/prometheus.go @@ -85,12 +85,14 @@ func (r *CorootReconciler) prometheusDeployment(cr *corootv1.Coroot) *appsv1.Dep }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: ls, + Labels: ls, + Annotations: cr.Spec.Prometheus.PodAnnotations, }, Spec: corev1.PodSpec{ ServiceAccountName: cr.Name + "-prometheus", SecurityContext: nonRootSecurityContext, Affinity: cr.Spec.Prometheus.Affinity, + Tolerations: cr.Spec.Prometheus.Tolerations, Containers: []corev1.Container{ { Image: PrometheusImage, diff --git a/controller/utils.go b/controller/utils.go index a39a581..0aa03c7 100644 --- a/controller/utils.go +++ b/controller/utils.go @@ -4,57 +4,81 @@ import ( "crypto/rand" "encoding/json" "fmt" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/strategicpatch" "math/big" "reflect" + "sigs.k8s.io/controller-runtime/pkg/client" ) -func GeneratePassword(length int) string { - charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" +const ( + LastAppliedAnnotation = "operator.coroot.com/last-applied-configuration" + RandomStringCharset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" +) + +func RandomString(length int) string { res := make([]byte, length) for i := range res { - randomIndex, _ := rand.Int(rand.Reader, big.NewInt(int64(len(charset)))) - res[i] = charset[randomIndex.Int64()] + randomIndex, _ := rand.Int(rand.Reader, big.NewInt(int64(len(RandomStringCharset)))) + res[i] = RandomStringCharset[randomIndex.Int64()] } return string(res) } -// Merge merges `overrides` into `base` using the SMP (structural merge patch) approach. -// - It intentionally does not remove fields present in base but missing from overrides -// - It merges slices only if the `patchStrategy:"merge"` tag is present and the `patchMergeKey` identifies the unique field -func Merge[T any](base *T, overrides T) error { - baseBytes, err := json.Marshal(base) - if err != nil { - return fmt.Errorf("failed to marshal base: %w", err) +func MergeSpecs[T any](obj client.Object, currentSpec *T, targetSpec T) error { + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} } - overrideBytes, err := json.Marshal(overrides) + current, err := json.Marshal(currentSpec) if err != nil { - return fmt.Errorf("failed to marshal overrides: %w", err) + return fmt.Errorf("failed to marshal current: %w", err) } + target, err := json.Marshal(targetSpec) + if err != nil { + return fmt.Errorf("failed to marshal target: %w", err) + } + + original := []byte(annotations[LastAppliedAnnotation]) + annotations[LastAppliedAnnotation] = string(target) + obj.SetAnnotations(annotations) - patchMeta, err := strategicpatch.NewPatchMetaFromStruct(base) + patchMeta, err := strategicpatch.NewPatchMetaFromStruct(currentSpec) if err != nil { return fmt.Errorf("failed to produce patch meta from struct: %w", err) } - patch, err := strategicpatch.CreateThreeWayMergePatch(overrideBytes, overrideBytes, baseBytes, patchMeta, true) + patch, err := strategicpatch.CreateThreeWayMergePatch(original, target, current, patchMeta, true) if err != nil { return fmt.Errorf("failed to create three way merge patch: %w", err) } - merged, err := strategicpatch.StrategicMergePatchUsingLookupPatchMeta(baseBytes, patch, patchMeta) + if string(patch) == "{}" { + return nil + } + + merged, err := strategicpatch.StrategicMergePatchUsingLookupPatchMeta(current, patch, patchMeta) if err != nil { return fmt.Errorf("failed to apply patch: %w", err) } - valueOfBase := reflect.Indirect(reflect.ValueOf(base)) - into := reflect.New(valueOfBase.Type()) + valueOfCurrent := reflect.Indirect(reflect.ValueOf(currentSpec)) + into := reflect.New(valueOfCurrent.Type()) if err = json.Unmarshal(merged, into.Interface()); err != nil { return fmt.Errorf("failed to unmarshal merged data: %w", err) } - if !valueOfBase.CanSet() { - return fmt.Errorf("unable to set unmarshalled value into base object") + if !valueOfCurrent.CanSet() { + return fmt.Errorf("unable to set unmarshalled value into current object") } - valueOfBase.Set(reflect.Indirect(into)) + valueOfCurrent.Set(reflect.Indirect(into)) return nil } + +func secretKeySelector(name, key string) *corev1.SecretKeySelector { + return &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name, + }, + Key: key, + } +} diff --git a/controller/versions.go b/controller/versions.go index 94bc557..a455863 100644 --- a/controller/versions.go +++ b/controller/versions.go @@ -39,6 +39,9 @@ func (r *CorootReconciler) getAppImage(cr *corootv1.Coroot, app App) string { return "latest" } } + if strings.Contains(v, ":") { + return v + } return fmt.Sprintf("ghcr.io/coroot/%s:%s", app, v) }