feat(config): set maximum limit for number of Dask workers (#842)

reanahub · Nov 11, 2024 · 4ffcecb · 4ffcecb
1 parent debb354
commit 4ffcecb
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 0 deletions.
diff --git a/helm/reana/README.md b/helm/reana/README.md
@@ -72,6 +72,7 @@ This Helm automatically prefixes all names using the release name to avoid colli
 | `dask.autoscaler_enabled`                                | Enable Dask autoscaler | true |
 | `dask.cluster_max_memory_limit`                          | The maximum memory limit for Dask clusters created by users | "16Gi" |
 | `dask.cluster_default_number_of_workers`                 | The number of Dask workers created by default | 2 |
+| `dask.cluster_max_number_of_workers`                     | The maximum number of Dask workers that users can ask for | 20 |
 | `dask.cluster_default_single_worker_memory`              | The amount of memory used by default by a single Dask worker | "2Gi" |
 | `dask.cluster_max_single_worker_memory`                  | The maximum amount of memory that users can ask for the single Dask worker | "8Gi" |
 | `db_env_config.REANA_DB_HOST`                            | Environment variable to connect to external databases                                | `<chart-release-name>-db`                       |

diff --git a/helm/reana/templates/reana-server.yaml b/helm/reana/templates/reana-server.yaml
@@ -107,6 +107,8 @@ spec:
             value: {{ .Values.dask.cluster_max_memory_limit | default "16Gi" }}
           - name: REANA_DASK_CLUSTER_DEFAULT_NUMBER_OF_WORKERS
             value: !!str {{ .Values.dask.cluster_default_number_of_workers | default 2 }}
+          - name: REANA_DASK_CLUSTER_MAX_NUMBER_OF_WORKERS
+            value: !!str {{ .Values.dask.cluster_max_number_of_workers | default 20 }}
           - name: REANA_DASK_CLUSTER_DEFAULT_SINGLE_WORKER_MEMORY
             value: {{ .Values.dask.cluster_default_single_worker_memory | default "2Gi" }}
           - name: REANA_DASK_CLUSTER_MAX_SINGLE_WORKER_MEMORY

diff --git a/helm/reana/values.yaml b/helm/reana/values.yaml
@@ -183,6 +183,7 @@ dask:
   autoscaler_enabled: true
   cluster_max_memory_limit: "16Gi"
   cluster_default_number_of_workers: 2
+  cluster_max_number_of_workers: 20
   cluster_default_single_worker_memory: "2Gi"
   cluster_max_single_worker_memory: "8Gi"