feat(helm): allow configuring number of threads for Dask workers (#877)

Following the integration of Dask into REANA and discussions with several REANA users, we identified the need to make the number of threads configurable. This commit introduces two variables to be configured by admins. `REANA_DASK_CLUSTER_DEFAULT_SINGLE_WORKER_THREADS` defines the default number of threads per worker when not specified in `reana.yaml` `REANA_DASK_CLUSTER_MAX_SINGLE_WORKER_THREADS` sets the upper limit for user-defined values in `reana.yaml`. Closes #874
reanahub · Feb 20, 2025 · 3e3dc24 · 3e3dc24
1 parent 251a172
commit 3e3dc24
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 0 deletions.
diff --git a/helm/reana/README.md b/helm/reana/README.md
@@ -79,6 +79,8 @@ collisions.
 | `dask.cluster_max_number_of_workers`                                                                      | The maximum number of Dask workers that users can ask for                                                                                                                                                                                                                                                                                                                                                                            | 20                                                                                                     |
 | `dask.cluster_default_single_worker_memory`                                                               | The amount of memory used by default by a single Dask worker                                                                                                                                                                                                                                                                                                                                                                         | "2Gi"                                                                                                  |
 | `dask.cluster_max_single_worker_memory`                                                                   | The maximum amount of memory that users can ask for the single Dask worker                                                                                                                                                                                                                                                                                                                                                           | "8Gi"                                                                                                  |
+| `dask.cluster_default_single_worker_threads`                                                              | The number of threads used by default by a single Dask worker                                                                                                                                                                                                                                                                                                                                                                        | 4                                                                                                      |
+| `dask.cluster_max_single_worker_threads`                                                                  | The maximum number of threads that users can ask for the single Dask worker                                                                                                                                                                                                                                                                                                                                                          | 8                                                                                                      |
 | `db_env_config.REANA_DB_HOST`                                                                             | Environment variable to connect to external databases                                                                                                                                                                                                                                                                                                                                                                                | `<chart-release-name>-db`                                                                              |
 | `db_env_config.REANA_DB_NAME`                                                                             | Environment variable to connect to external databases                                                                                                                                                                                                                                                                                                                                                                                | reana                                                                                                  |
 | `db_env_config.REANA_DB_PORT`                                                                             | Environment variable to connect to external databases                                                                                                                                                                                                                                                                                                                                                                                | "5432"                                                                                                 |

diff --git a/helm/reana/templates/reana-server.yaml b/helm/reana/templates/reana-server.yaml
@@ -115,6 +115,10 @@ spec:
             value: {{ .Values.dask.cluster_default_single_worker_memory | default "2Gi" }}
           - name: REANA_DASK_CLUSTER_MAX_SINGLE_WORKER_MEMORY
             value: {{ .Values.dask.cluster_max_single_worker_memory | default "8Gi" }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_SINGLE_WORKER_THREADS
+            value: !!str {{ .Values.dask.cluster_default_single_worker_threads | default 4 }}
+          - name: REANA_DASK_CLUSTER_MAX_SINGLE_WORKER_THREADS
+            value: !!str {{ .Values.dask.cluster_max_single_worker_threads | default 8 }}
           {{- end }}
           - name: WORKSPACE_PATHS
             value: {{ .Values.workspaces.paths | toJson | quote }}

diff --git a/helm/reana/templates/reana-workflow-controller.yaml b/helm/reana/templates/reana-workflow-controller.yaml
@@ -196,6 +196,8 @@ spec:
             value: {{ .Values.dask.cluster_default_single_worker_memory | default "2Gi" }}
           - name: REANA_DASK_CLUSTER_MAX_SINGLE_WORKER_MEMORY
             value: {{ .Values.dask.cluster_max_single_worker_memory | default "8Gi" }}
+          - name: REANA_DASK_CLUSTER_DEFAULT_SINGLE_WORKER_THREADS
+            value: !!str {{ .Values.dask.cluster_default_single_worker_threads | default 4 }}
           {{- end }}
           {{- if .Values.reana_hostname }}
           - name: REANA_HOSTNAME

diff --git a/helm/reana/values.yaml b/helm/reana/values.yaml
@@ -196,6 +196,8 @@ dask:
   cluster_max_number_of_workers: 20
   cluster_default_single_worker_memory: "2Gi"
   cluster_max_single_worker_memory: "8Gi"
+  cluster_default_single_worker_threads: 4
+  cluster_max_single_worker_threads: 8
 
 pgbouncer:
   enabled: false