Merge branch 'main' into feat/clean-kubernetes-labels

* main: (42 commits) Bump appVersion to 0.8.2 Feat issue #98: merging jobs and bootstrap section Fix metastore volume indentation Bump to v0.6.4 Fix helm chart comments Reduce indentation by 2 Bump version to 0.6.2 Introduce component specific initContainers, volumes and volumeMounts Use nodeSelector, affinity, tolerations from bootstrap config values. This ensures pods generated from k8s Job have these settings propagated. Allow command and args to be overridden via values files. This helps in providing custom config files generated from templates as necessary Add possibility to specify services type per service fix: readme missing newlines fix: readme suggested changes fix: add back indexer & search settings example fix: readme depreciation notice chore: document the breaking change and change the versioning accordingly fix: add QW_METASTORE_URI doc chore: add configMapRef example in values.yaml fix: add suggested change Add artifacthub metadata file feat: add requested changes fix: quickwit secret leak in configmap ... Signed-off-by: Nicolas Lamirault <[email protected]>
quickwit-oss · Sep 4, 2024 · 2067857 · 2067857
2 parents 20c674e + 44056e3
commit 2067857
Show file tree

Hide file tree

Showing 17 changed files with 467 additions and 127 deletions.
diff --git a/README.md b/README.md
@@ -28,3 +28,33 @@ The way storage config is defined changed and you have to update your helm value
 The changes are:
 - the `config.s3` and `config.azure_blob` values are no more supported. You now have to use the storage config as defined in the [docs](https://quickwit.io/docs/configuration/storage-config).
 - the keys of secrets have changed: `s3.secret_key` is replaced by `storage.s3.secret_access_key` and `azure_blob.access_key` is replace by `storage.azure.access_key`.
+
+## Upgrade helm chart from 0.5.0 to 0.6.0
+
+The way the `config` value works has changed in 0.6.0. It is now copied "as is"
+to the Quickwit nodes' configurations. In particular:
+
+- the `config.postgres` section does not support the following attributes
+  anymore
+```
+host: ""
+port: 5432
+database: metastore
+username: quickwit
+assword: ""
+```
+Configure `QW_METASTORE_URI` in `extraEnvFrom` instead (see
+[documentation](https://quickwit.io/docs/configuration/metastore-config) for
+more details).
+
+- the seed configuration has moved from `config` to a dedicated attribute. The
+  changes are:
+  - the `config.indexes` field is moved to `seed.indexes`
+  - the `config.sources` field is moved to `seed.sources`
+
+## Upgrade helm chart from 0.6.0 to 0.7.0
+
+The `jobs` and `bootstrap` sections got merged in 0.7.0:
+
+* `jobs.sources` section is now replaced by `bootstrap.sources`
+* `jobs.indexes` section is now replaced by `bootstrap.indexes`
diff --git a/artifacthub-repo.yml b/artifacthub-repo.yml
@@ -0,0 +1,7 @@
+repositoryID: a70501fa-55bf-4218-9cc8-89d8f64520f8
+owners:
+  - name: fmassot
+    email: [email protected]
+  - name: guilload
+    email: [email protected]
+
diff --git a/charts/quickwit/Chart.yaml b/charts/quickwit/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: quickwit
 description: Sub-second search & analytics engine on cloud storage.
 type: application
-version: 0.5.15
-appVersion: "v0.8.1"
+version: 0.8.0
+appVersion: "v0.8.2"
 keywords:
   - quickwit
   - search

diff --git a/charts/quickwit/templates/_helpers.tpl b/charts/quickwit/templates/_helpers.tpl
@@ -48,8 +48,7 @@ helm.sh/chart: {{ include "quickwit.chart" . }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
-app.kubernetes.io/part-of: quickwit
-{{ include "quickwit.additionalLabels" . }}
+{{- include "quickwit.additionalLabels" . }}
 {{- end }}
 
 {{/*
@@ -144,66 +143,19 @@ Quickwit environment
     fieldRef:
       fieldPath: status.podIP
 - name: QW_CONFIG
-  value: node.yaml
+  value: {{ .Values.configLocation }}
 - name: QW_CLUSTER_ID
   value: {{ .Release.Namespace }}-{{ include "quickwit.fullname" . }}
-{{- if ((.Values.config.storage).s3).access_key_id }}
-- name: AWS_ACCESS_KEY_ID
-  value: {{ .Values.config.storage.s3.access_key_id }}
-{{- end }}
-{{- if ((.Values.config.storage).s3).secret_access_key }}
-- name: AWS_SECRET_ACCESS_KEY
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "quickwit.fullname" $ }}
-      key: storage.s3.secret_access_key
-{{- end }}
-{{- if ((.Values.config.storage).azure).account }}
-- name: QW_AZURE_STORAGE_ACCOUNT
-  value: {{ .Values.config.storage.azure.account }}
-{{- end }}
-{{- if ((.Values.config.storage).azure).access_key }}
-- name: QW_AZURE_STORAGE_ACCESS_KEY
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "quickwit.fullname" $ }}
-      key: storage.azure.access_key
-{{- end }}
 - name: QW_NODE_ID
   value: "$(POD_NAME)"
 - name: QW_PEER_SEEDS
   value: {{ include "quickwit.fullname" . }}-headless
 - name: QW_ADVERTISE_ADDRESS
   value: "$(POD_IP)"
+- name: QW_CLUSTER_ENDPOINT
+  value: http://{{ include "quickwit.fullname" $ }}-metastore.{{ $.Release.Namespace }}.svc.cluster.local:7280
 {{- range $key, $value := .Values.environment }}
 - name: "{{ $key }}"
   value: "{{ $value }}"
 {{- end }}
 {{- end }}
-
-{{/*
-Quickwit metastore environment
-*/}}
-{{- define "quickwit.metastore.environment" -}}
-{{ include "quickwit.environment" . }}
-{{- if .Values.config.metastore_uri }}
-- name: QW_METASTORE_URI
-  value: {{ .Values.config.metastore_uri }}
-{{- else if .Values.config.postgres }}
-- name: POSTGRES_HOST
-  value: {{ required "A valid config.postgres.host is required!" .Values.config.postgres.host }}
-- name: POSTGRES_PORT
-  value: {{ .Values.config.postgres.port | default 5432 | quote }}
-- name: POSTGRES_DATABASE
-  value: {{ .Values.config.postgres.database | default "metastore" }}
-- name: POSTGRES_USERNAME
-  value: {{ .Values.config.postgres.username | default "quickwit" }}
-- name: POSTGRES_PASSWORD
-  valueFrom:
-    secretKeyRef:
-      name: {{ include "quickwit.fullname" . }}
-      key: postgres.password
-- name: QW_METASTORE_URI
-  value: "postgres://$(POSTGRES_USERNAME):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST):$(POSTGRES_PORT)/$(POSTGRES_DATABASE)"
-{{- end }}
-{{- end }}
diff --git a/charts/quickwit/templates/configmap-bootstrap.yaml b/charts/quickwit/templates/configmap-bootstrap.yaml
@@ -1,4 +1,4 @@
-{{- if and (or .Values.config.sources .Values.config.indexes) .Values.bootstrap.enabled -}}
+{{- if and (or .Values.seed.sources .Values.seed.indexes) .Values.bootstrap.enabled -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -8,11 +8,11 @@ metadata:
     {{- include "quickwit.selectorLabels" . | nindent 4 }}
     app.kubernetes.io/component: config
 data:
-  {{- range .Values.config.indexes }}
+  {{- range .Values.seed.indexes }}
   {{ .index_id }}.yaml: |-
     {{- toYaml . | nindent 4 }}
   {{- end }}
-  {{- range .Values.config.sources }}
+  {{- range .Values.seed.sources }}
   {{ .source.source_id }}.yaml: |-
     {{- toYaml .source | nindent 4 }}
   {{- end }}

diff --git a/charts/quickwit/templates/configmap.yaml b/charts/quickwit/templates/configmap.yaml
@@ -8,33 +8,4 @@ metadata:
     app.kubernetes.io/component: config
 data:
   node.yaml: |-
-    version: 0.8
-    listen_address: 0.0.0.0
-    gossip_listen_port: 7282
-    data_dir: /quickwit/qwdata
-    default_index_root_uri: {{ .Values.config.default_index_root_uri }}
-    {{- with .Values.config.indexer }}
-    indexer:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- with .Values.config.storage }}
-    storage:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- with .Values.config.ingest_api }}
-    ingest_api:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- with .Values.config.searcher }}
-    searcher:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- with .Values.config.jaeger }}
-    jaeger:
-      {{- toYaml . | nindent 6 }}
-    {{- end }}
-    {{- if (.Values.config.postgres).max_num_connections }}
-    metastore:
-      postgres:
-        max_num_connections: {{ .Values.config.postgres.max_num_connections }}
-    {{- end }}
+    {{- toYaml .Values.config | nindent 4 }}
diff --git a/charts/quickwit/templates/control-plane-deployment.yaml b/charts/quickwit/templates/control-plane-deployment.yaml
@@ -40,19 +40,36 @@ spec:
       serviceAccountName: {{ include "quickwit.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.control_plane.initContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{ end }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          {{- if $.Values.control_plane.args }}
+          args: {{- toYaml $.Values.control_plane.args | nindent 10 }}
+          {{- else }}
           args: ["run", "--service", "control_plane"]
+          {{- end }}
           env:
             {{- include "quickwit.environment" . | nindent 12 }}
             {{- range $key, $value := .Values.control_plane.extraEnv }}
             - name: "{{ $key }}"
               value: "{{ $value }}"
             {{- end }}
+          {{- if or (.Values.environmentFrom) (.Values.control_plane.extraEnvFrom) }}
+          envFrom:
+          {{- with .Values.environmentFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.control_plane.extraEnvFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- end }}
           ports:
             {{- include "quickwit.ports" . | nindent 12 }}
           startupProbe:
@@ -71,6 +88,9 @@ spec:
             - name: {{ .name }}
               mountPath: {{ .mountPath }}
             {{- end }}
+            {{- with .Values.indexer.extraVolumeMounts }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
           resources:
             {{- toYaml .Values.control_plane.resources | nindent 14 }}
       volumes:
@@ -87,6 +107,9 @@ spec:
           configMap:
             name: {{ .name }}
         {{- end }}
+        {{- with .Values.indexer.extraVolumes }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- with .Values.control_plane.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

diff --git a/charts/quickwit/templates/indexer-statefulset.yaml b/charts/quickwit/templates/indexer-statefulset.yaml
@@ -43,19 +43,36 @@ spec:
       serviceAccountName: {{ include "quickwit.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.indexer.initContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{ end }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          {{- if $.Values.indexer.args }}
+          args: {{- toYaml $.Values.indexer.args | nindent 10 }}
+          {{- else }}
+          args: ["run", "--service", "indexer"]
+          {{- end }}
           env:
             {{- include "quickwit.environment" . | nindent 12 }}
             {{- range $key, $value := .Values.indexer.extraEnv }}
             - name: "{{ $key }}"
               value: "{{ $value }}"
             {{- end }}
-          args: ["run", "--service", "indexer"]
+          {{- if or (.Values.environmentFrom) (.Values.indexer.extraEnvFrom) }}
+          envFrom:
+          {{- with .Values.environmentFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.indexer.extraEnvFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- end }}
           ports:
             {{- include "quickwit.ports" . | nindent 12 }}
           startupProbe:
@@ -74,6 +91,9 @@ spec:
             - name: {{ .name }}
               mountPath: {{ .mountPath }}
             {{- end }}
+            {{- with .Values.indexer.extraVolumeMounts }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
           resources:
             {{- toYaml .Values.indexer.resources | nindent 12 }}
       volumes:
@@ -92,6 +112,9 @@ spec:
           configMap:
             name: {{ .name }}
         {{- end }}
+        {{- with .Values.indexer.extraVolumes }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- with .Values.indexer.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

diff --git a/charts/quickwit/templates/janitor-deployment.yaml b/charts/quickwit/templates/janitor-deployment.yaml
@@ -40,19 +40,36 @@ spec:
       serviceAccountName: {{ include "quickwit.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.janitor.initContainers }}
+      initContainers:
+        {{ toYaml . | nindent 8 }}
+      {{ end }}
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          {{- if $.Values.janitor.args }}
+          args: {{- toYaml $.Values.janitor.args | nindent 10 }}
+          {{- else }}
           args: ["run", "--service", "janitor"]
+          {{- end }}
           env:
             {{- include "quickwit.environment" . | nindent 12 }}
             {{- range $key, $value := .Values.janitor.extraEnv }}
             - name: "{{ $key }}"
               value: "{{ $value }}"
             {{- end }}
+          {{- if or (.Values.environmentFrom) (.Values.janitor.extraEnvFrom) }}
+          envFrom:
+          {{- with .Values.environmentFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.janitor.extraEnvFrom }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- end }}
           ports:
             {{- include "quickwit.ports" . | nindent 12 }}
           startupProbe:
@@ -71,6 +88,9 @@ spec:
             - name: {{ .name }}
               mountPath: {{ .mountPath }}
             {{- end }}
+            {{- with .Values.indexer.extraVolumeMounts }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
           resources:
             {{- toYaml .Values.janitor.resources | nindent 14 }}
       volumes:
@@ -87,6 +107,9 @@ spec:
           configMap:
             name: {{ .name }}
         {{- end }}
+        {{- with .Values.indexer.extraVolumes }}
+          {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- with .Values.janitor.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}