From 7b472fe5761acafc488f2493b0e44e6963d3c16d Mon Sep 17 00:00:00 2001 From: Marco Donadoni Date: Wed, 31 Jul 2024 17:35:21 +0200 Subject: [PATCH] feat(helm): add support for PgBouncer (#818) Closes reanahub/reana#819 --- helm/configurations/values-dev.yaml | 3 + helm/reana/README.md | 6 ++ helm/reana/templates/cronjobs.yaml | 40 ++++----- helm/reana/templates/database-config.yaml | 17 ++++ helm/reana/templates/pgbouncer.yaml | 88 +++++++++++++++++++ helm/reana/templates/reana-db.yaml | 7 -- helm/reana/templates/reana-server.yaml | 20 ++--- .../templates/reana-workflow-controller.yaml | 20 ++--- helm/reana/templates/secrets.yaml | 13 +++ helm/reana/values.yaml | 8 ++ 10 files changed, 167 insertions(+), 55 deletions(-) create mode 100644 helm/reana/templates/database-config.yaml create mode 100644 helm/reana/templates/pgbouncer.yaml diff --git a/helm/configurations/values-dev.yaml b/helm/configurations/values-dev.yaml index 9526e068..861a9126 100644 --- a/helm/configurations/values-dev.yaml +++ b/helm/configurations/values-dev.yaml @@ -26,3 +26,6 @@ components: image: docker.io/reanahub/reana-message-broker reana_ui: image: docker.io/reanahub/reana-ui + +pgbouncer: + enabled: true diff --git a/helm/reana/README.md b/helm/reana/README.md index e5f504a3..0ccf74cf 100644 --- a/helm/reana/README.md +++ b/helm/reana/README.md @@ -98,6 +98,12 @@ This Helm automatically prefixes all names using the release name to avoid colli | `reana_hostname` | REANA hostname (e.g. reana.example.org) | None | | `namespace_runtime` | Namespace in which the REANA runtime pods (workflow engines, jobs etc...) will run | `.Release.Namespace` | | `naming_scheme` | REANA component naming scheme | None | +| `pgbouncer.enabled` | Instantiate PgBouncer inside the cluster to pool database connections | false | +| `pgbouncer.image` | [PgBouncer image](https://hub.docker.com/r/bitnami/pgbouncer/) to use | `bitnami/pgbouncer:1.23.1` | +| `pgbouncer.pool_mode` | Pool mode to use (session, transaction, statement) | transaction | +| `pgbouncer.max_client_conn` | Maximum number of client connections allowed | 1000 | +| `pgbouncer.max_db_connections` | Maximum number of server connections allowed | 100 | +| `pgbouncer.environment` | Additional PgBouncer environment variables | `{}` | | `secrets.cern.sso.CERN_CONSUMER_KEY` | CERN SSO consumer key | None | | `secrets.cern.sso.CERN_CONSUMER_SECRET` | **[Do not use in production, use secrets instead]** CERN SSO consumer secret | None | | `secrets.database.password` | **[Do not use in production, use secrets instead]** PostgreSQL database password | None | diff --git a/helm/reana/templates/cronjobs.yaml b/helm/reana/templates/cronjobs.yaml index be6c2640..234b27ef 100644 --- a/helm/reana/templates/cronjobs.yaml +++ b/helm/reana/templates/cronjobs.yaml @@ -30,19 +30,18 @@ spec: tty: true stdin: true {{- end }} + envFrom: + - configMapRef: + name: {{ include "reana.prefix" . }}-database-config env: {{- if .Values.reana_hostname }} - name: REANA_HOSTNAME value: {{ .Values.reana_hostname }} {{- end }} - {{- range $key, $value := .Values.db_env_config }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} {{- if .Values.debug.enabled }} - name: FLASK_ENV value: "development" - {{- else }} + {{- end }} - name: REANA_DB_USERNAME valueFrom: secretKeyRef: @@ -53,7 +52,6 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password - {{- end }} - name: REANA_NOTIFICATIONS_ENABLED value: "{{ .Values.notifications.enabled }}" - name: REANA_EMAIL_RECEIVER @@ -154,6 +152,9 @@ spec: tty: true stdin: true {{- end }} + envFrom: + - configMapRef: + name: {{ include "reana.prefix" . }}-database-config env: - name: REANA_PERIODIC_RESOURCE_QUOTA_UPDATE_POLICY value: "true" @@ -161,14 +162,10 @@ spec: - name: REANA_HOSTNAME value: {{ .Values.reana_hostname }} {{- end }} - {{- range $key, $value := .Values.db_env_config }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} {{- if .Values.debug.enabled }} - name: FLASK_ENV value: "development" - {{- else }} + {{- end }} - name: REANA_DB_USERNAME valueFrom: secretKeyRef: @@ -179,7 +176,6 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password - {{- end }} - name: REANA_COMPONENT_PREFIX value: {{ include "reana.prefix" . }} - name: REANA_INFRASTRUCTURE_KUBERNETES_NAMESPACE @@ -239,19 +235,18 @@ spec: tty: true stdin: true {{- end }} + envFrom: + - configMapRef: + name: {{ include "reana.prefix" . }}-database-config env: {{- if .Values.reana_hostname }} - name: REANA_HOSTNAME value: {{ .Values.reana_hostname }} {{- end }} - {{- range $key, $value := .Values.db_env_config }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} {{- if .Values.debug.enabled }} - name: FLASK_ENV value: "development" - {{- else }} + {{- end }} - name: REANA_DB_USERNAME valueFrom: secretKeyRef: @@ -262,7 +257,6 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password - {{- end }} - name: REANA_COMPONENT_PREFIX value: {{ include "reana.prefix" . }} - name: REANA_ADMIN_ACCESS_TOKEN @@ -333,19 +327,18 @@ spec: tty: true stdin: true {{- end }} + envFrom: + - configMapRef: + name: {{ include "reana.prefix" . }}-database-config env: {{- if .Values.reana_hostname }} - name: REANA_HOSTNAME value: {{ .Values.reana_hostname }} {{- end }} - {{- range $key, $value := .Values.db_env_config }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} {{- if .Values.debug.enabled }} - name: FLASK_ENV value: "development" - {{- else }} + {{- end }} - name: REANA_DB_USERNAME valueFrom: secretKeyRef: @@ -356,7 +349,6 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password - {{- end }} - name: REANA_COMPONENT_PREFIX value: {{ include "reana.prefix" . }} - name: REANA_ADMIN_ACCESS_TOKEN diff --git a/helm/reana/templates/database-config.yaml b/helm/reana/templates/database-config.yaml new file mode 100644 index 00000000..d7df78c7 --- /dev/null +++ b/helm/reana/templates/database-config.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "reana.prefix" . }}-database-config + namespace: {{ .Release.Namespace }} +data: + REANA_DB_NAME: {{ .Values.db_env_config.REANA_DB_NAME | quote }} + {{- if .Values.pgbouncer.enabled }} + REANA_DB_HOST: {{ include "reana.prefix" . }}-pgbouncer + REANA_DB_PORT: "6432" + {{- else if .Values.components.reana_db.enabled }} + REANA_DB_HOST: {{ include "reana.prefix" . }}-db + REANA_DB_PORT: "5432" + {{- else }} + REANA_DB_HOST: {{ .Values.db_env_config.REANA_DB_HOST | quote }} + REANA_DB_PORT: {{ .Values.db_env_config.REANA_DB_PORT | quote }} + {{- end }} diff --git a/helm/reana/templates/pgbouncer.yaml b/helm/reana/templates/pgbouncer.yaml new file mode 100644 index 00000000..c43b1dfd --- /dev/null +++ b/helm/reana/templates/pgbouncer.yaml @@ -0,0 +1,88 @@ +{{- if .Values.pgbouncer.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "reana.prefix" . }}-pgbouncer + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + app: {{ include "reana.prefix" . }}-pgbouncer + ports: + - port: 6432 + targetPort: 6432 + protocol: TCP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "reana.prefix" . }}-pgbouncer-conf + namespace: {{ .Release.Namespace }} +data: + PGBOUNCER_POOL_MODE: {{ .Values.pgbouncer.pool_mode | quote }} + PGBOUNCER_MAX_CLIENT_CONN: {{ .Values.pgbouncer.max_client_conn | quote }} + PGBOUNCER_MAX_DB_CONNECTIONS: {{ .Values.pgbouncer.max_db_connections | quote }} + PGBOUNCER_DEFAULT_POOL_SIZE: {{ .Values.pgbouncer.max_db_connections | quote }} + {{- if .Values.components.reana_db.enabled }} + PGBOUNCER_DATABASE: reana + POSTGRESQL_HOST: {{ include "reana.prefix" . }}-db + POSTGRESQL_PORT: "5432" + {{- else }} + PGBOUNCER_DATABASE: {{ .Values.db_env_config.REANA_DB_NAME | quote }} + POSTGRESQL_HOST: {{ .Values.db_env_config.REANA_DB_HOST | quote }} + POSTGRESQL_PORT: {{ .Values.db_env_config.REANA_DB_PORT | quote }} + {{- end }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "reana.prefix" . }}-pgbouncer + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ if .Values.maintenance.enabled -}} 0 {{- else -}} 1 {{- end }} + selector: + matchLabels: + app: {{ include "reana.prefix" . }}-pgbouncer + template: + metadata: + labels: + app: {{ include "reana.prefix" . }}-pgbouncer + spec: + containers: + - name: pgbouncer + image: {{ .Values.pgbouncer.image | quote }} + ports: + - containerPort: 6432 + envFrom: + - configMapRef: + name: {{ include "reana.prefix" . }}-pgbouncer-conf + env: + - name: POSTGRESQL_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "reana.prefix" . }}-db-secrets + key: user + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "reana.prefix" . }}-db-secrets + key: password + - name: PGBOUNCER_USERLIST + valueFrom: + secretKeyRef: + name: {{ include "reana.prefix" . }}-pgbouncer-secrets + key: userlist + {{- range $key, $value := .Values.pgbouncer.environment }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- if .Values.node_label_infrastructuredb }} + {{- $full_label := split "=" .Values.node_label_infrastructuredb }} + nodeSelector: + {{ $full_label._0 }}: {{ $full_label._1 }} + {{- else if .Values.node_label_infrastructure }} + {{- $full_label := split "=" .Values.node_label_infrastructure }} + nodeSelector: + {{ $full_label._0 }}: {{ $full_label._1 }} + {{- end }} +{{- end }} diff --git a/helm/reana/templates/reana-db.yaml b/helm/reana/templates/reana-db.yaml index 90121d72..e253822d 100644 --- a/helm/reana/templates/reana-db.yaml +++ b/helm/reana/templates/reana-db.yaml @@ -59,12 +59,6 @@ spec: value: "Europe/Zurich" - name: POSTGRES_DB value: reana - {{- if not .Values.debug.enabled }} - - name: POSTGRES_USER - value: reana - - name: POSTGRES_PASSWORD - value: reana - {{- else }} - name: POSTGRES_USER valueFrom: secretKeyRef: @@ -75,7 +69,6 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password - {{- end }} volumeMounts: - mountPath: /var/lib/postgresql/data subPath: db diff --git a/helm/reana/templates/reana-server.yaml b/helm/reana/templates/reana-server.yaml index 82d4449e..b460f73e 100644 --- a/helm/reana/templates/reana-server.yaml +++ b/helm/reana/templates/reana-server.yaml @@ -67,6 +67,9 @@ spec: mountPath: '/var/reana/uwsgi' - name: reana-config mountPath: '/var/reana/config' + envFrom: + - configMapRef: + name: {{ include "reana.prefix" . }}-database-config env: - name: REANA_COMPONENT_PREFIX value: {{ include "reana.prefix" . }} @@ -100,10 +103,6 @@ spec: - name: REANA_WORKFLOW_TERMINATION_QUOTA_UPDATE_POLICY value: {{ tpl .Values.quota.workflow_termination_update_policy . | default "null" }} {{- end }} - {{- range $key, $value := .Values.db_env_config }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} {{- range $key, $value := .Values.components.reana_server.environment }} - name: {{ $key }} value: {{ $value | quote }} @@ -169,7 +168,7 @@ spec: value: "" - name: GIT_SSL_NO_VERIFY value: "true" - {{- else }} + {{- end }} - name: REANA_DB_USERNAME valueFrom: secretKeyRef: @@ -180,7 +179,6 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password - {{- end }} - name: REANA_NOTIFICATIONS_ENABLED value: "{{ .Values.notifications.enabled }}" {{- if .Values.notifications.enabled }} @@ -223,6 +221,9 @@ spec: {{- end }} - mountPath: {{ .Values.shared_storage.shared_volume_mount_path }} name: reana-shared-volume + envFrom: + - configMapRef: + name: {{ include "reana.prefix" . }}-database-config env: - name: REANA_COMPONENT_PREFIX value: {{ include "reana.prefix" . }} @@ -230,10 +231,6 @@ spec: value: {{ .Release.Namespace }} - name: REANA_RUNTIME_KUBERNETES_NAMESPACE value: {{ .Values.namespace_runtime | default .Release.Namespace }} - {{- range $key, $value := .Values.db_env_config }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} {{- range $key, $value := .Values.components.reana_server.environment }} - name: {{ $key }} value: {{ $value | quote }} @@ -256,7 +253,7 @@ spec: value: "" - name: GIT_SSL_NO_VERIFY value: "true" - {{- else }} + {{- end }} - name: REANA_DB_USERNAME valueFrom: secretKeyRef: @@ -267,7 +264,6 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password - {{- end }} {{- if .Values.login }} - name: LOGIN_PROVIDERS_CONFIGS value: {{ .Values.login | toJson | quote }} diff --git a/helm/reana/templates/reana-workflow-controller.yaml b/helm/reana/templates/reana-workflow-controller.yaml index 273da6fc..b7f06884 100644 --- a/helm/reana/templates/reana-workflow-controller.yaml +++ b/helm/reana/templates/reana-workflow-controller.yaml @@ -77,6 +77,9 @@ spec: mountPath: {{ $workspace_path._1 }} {{- end }} {{- end }} + envFrom: + - configMapRef: + name: {{ include "reana.prefix" . }}-database-config env: - name: REANA_COMPONENT_PREFIX value: {{ include "reana.prefix" . }} @@ -106,10 +109,6 @@ spec: {{- end }} - name: WORKSPACE_PATHS value: {{ .Values.workspaces.paths | toJson | quote }} - {{- range $key, $value := .Values.db_env_config }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} {{- range $key, $value := .Values.components.reana_workflow_controller.environment }} - name: {{ $key }} value: {{ $value | quote }} @@ -200,7 +199,7 @@ spec: value: "" - name: GIT_SSL_NO_VERIFY value: "true" - {{- else }} + {{- end }} - name: REANA_DB_USERNAME valueFrom: secretKeyRef: @@ -211,7 +210,6 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password - {{ end }} - name: job-status-consumer image: {{ .Values.components.reana_workflow_controller.image }} imagePullPolicy: {{ .Values.components.reana_workflow_controller.imagePullPolicy }} @@ -223,6 +221,9 @@ spec: {{- end }} - mountPath: {{ .Values.shared_storage.shared_volume_mount_path }} name: reana-shared-volume + envFrom: + - configMapRef: + name: {{ include "reana.prefix" . }}-database-config env: - name: REANA_COMPONENT_PREFIX value: {{ include "reana.prefix" . }} @@ -234,10 +235,6 @@ spec: - name: REANA_WORKFLOW_TERMINATION_QUOTA_UPDATE_POLICY value: {{ tpl .Values.quota.workflow_termination_update_policy . | default "null" }} {{- end }} - {{- range $key, $value := .Values.db_env_config }} - - name: {{ $key }} - value: {{ $value | quote }} - {{- end }} {{- range $key, $value := .Values.components.reana_workflow_controller.environment }} - name: {{ $key }} value: {{ $value | quote }} @@ -258,7 +255,7 @@ spec: value: "" - name: GIT_SSL_NO_VERIFY value: "true" - {{- else }} + {{- end }} - name: REANA_DB_USERNAME valueFrom: secretKeyRef: @@ -269,7 +266,6 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password - {{ end }} - name: REANA_GITLAB_HOST valueFrom: secretKeyRef: diff --git a/helm/reana/templates/secrets.yaml b/helm/reana/templates/secrets.yaml index 2e165571..92c689d0 100644 --- a/helm/reana/templates/secrets.yaml +++ b/helm/reana/templates/secrets.yaml @@ -10,6 +10,19 @@ type: Opaque data: user: {{ .Values.secrets.database.user | default "reana" | b64enc }} password: {{ .Values.secrets.database.password | default "reana" | b64enc }} +{{- if .Values.pgbouncer.enabled }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "reana.prefix" . }}-pgbouncer-secrets + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/resource-policy": keep +type: Opaque +data: + userlist: {{ printf "%s %s" (.Values.secrets.database.user | default "reana" | quote) (.Values.secrets.database.password | default "reana" | quote) | b64enc}} +{{- end }} --- apiVersion: v1 kind: Secret diff --git a/helm/reana/values.yaml b/helm/reana/values.yaml index 5e8cbf8c..08283775 100644 --- a/helm/reana/values.yaml +++ b/helm/reana/values.yaml @@ -173,6 +173,14 @@ traefik: enabled: true isDefaultClass: true +pgbouncer: + enabled: false + image: docker.io/bitnami/pgbouncer:1.23.1 + pool_mode: transaction + max_client_conn: 1000 + max_db_connections: 100 + environment: {} + # Quota quota: enabled: true