Improve StreamFlow on Kubernetes

This commit heavily refactors the StreamFlow Helm chart to simplify its
deployment on top of Kubernetes clusters.

In addition, this commit adds a `networkPolicy` flag to control the
behaviour of CWL `DockerRequirement` objects into Kubernetes `Pod`
items. Normally, the CWL `NetworkAccess` requirement is enforced through
Kubernetes `NetworkPolicy` objects. However, `NetworkPolicy` objects
regulate the network security inside a cluster, and giving the
StreamFlow `Pod` permissions to create/delete them may result in
unwanted security flaws. The `networkPolicy` option can be set to
`False` to ignore the CWL `NetworkAccess` enforcement in such cases.

.github/workflows/ci-tests.yaml

@@ -157,7 +157,7 @@ jobs:
           python -m pip install -r docs/requirements.txt
       - name: "Build documentation and check for consistency"
         env:
-          CHECKSUM: "221232eb10bca5a16895e6ed4bb8fc959044a804bfc045a4751f2f17238b9c93"
+          CHECKSUM: "1f7be9cd7f0f23aae63f6657e22a8b008b73ccca786af77dcf7975cc2422fdc0"
         run: |
           cd docs
           HASH="$(make checksum | tail -n1)"

helm/chart/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: streamflow
-description: A Helm chart for StreamFlow
+description: A Helm chart for the StreamFlow workflow management system
 type: application
 version: 0.2.0
-appVersion: latest
+appVersion: 0.2.0.dev11

helm/chart/templates/_helpers.tpl

@@ -1,6 +1,6 @@
 {{/* vim: set filetype=mustache: */}}
 {{/*
-Expand the name of the chart.
+Expand the name of the chart
 */}}
 {{- define "streamflow.name" -}}
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
@@ -9,7 +9,7 @@ Expand the name of the chart.
 {{/*
 Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
+If release name contains chart name it will be used as a full name
 */}}
 {{- define "streamflow.fullname" -}}
 {{- if .Values.fullnameOverride -}}
@@ -25,12 +25,49 @@ If release name contains chart name it will be used as a full name.
 {{- end -}}
 
 {{/*
-Create chart name and version as used by the chart label.
+Create chart name and version as used by the chart label
 */}}
 {{- define "streamflow.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Return the proper StreamFlow image name
+*/}}
+{{- define "streamflow.image" -}}
+{{- $registryName := default .Values.image.registry -}}
+{{- $repositoryName := .Values.image.repository -}}
+{{- $separator := ":" -}}
+{{- $termination := default .Chart.AppVersion .Values.image.tag | toString -}}
+
+{{- if not .Values.image.tag }}
+  {{- if .Chart }}
+    {{- $termination = .Chart.AppVersion | toString -}}
+  {{- end -}}
+{{- end -}}
+{{- if .Values.image.digest }}
+    {{- $separator = "@" -}}
+    {{- $termination = .Values.image.digest | toString -}}
+{{- end -}}
+{{- if $registryName }}
+    {{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}}
+{{- else -}}
+    {{- printf "%s%s%s"  $repositoryName $separator $termination -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper Docker Image Registry Secret Names evaluating values as templates
+*/}}
+{{- define "streamflow.imagePullSecrets" -}}
+{{- if (not (empty .Values.image.pullSecrets)) -}}
+imagePullSecrets:
+  {{- range .Values.image.pullSecrets | uniq }}
+  - name: {{ . }}
+  {{- end }}
+{{- end }}
+{{- end }}
+
 {{/*
 Common labels
 */}}

helm/chart/templates/configmap.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "streamflow.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "streamflow.labels" . | nindent 4 }}
+data:
+  streamflow.yml: |-
+    version: v1.0
+    workflows:
+      {{ .Values.streamflow.workflow.name | default uuidv4 }}:
+        type: {{ .Values.streamflow.workflow.type }}
+        {{- if .Values.streamflow.workflow.bindings }}
+        {{- with .Values.streamflow.workflow.bindings }}
+        bindings:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- end }}
+        config:
+          {{- if eq .Values.streamflow.workflow.type "cwl" }}
+          file: {{ required "CWL processfile is mandatory" .Values.streamflow.workflow.cwl.processfile }}
+          {{- if .Values.streamflow.workflow.cwl.jobfile }}
+          settings: {{ .Values.streamflow.workflow.cwl.jobfile }}
+          {{- end }}
+          docker:
+            - step: /
+              deployment:
+                type: kubernetes
+                config:
+                  inCluster: true
+                  networkPolicy: {{ .Values.streamflow.workflow.cwl.restrictNetworkAccess }}
+          {{- end }}
+    {{- if .Values.streamflow.config }}
+    {{- toYaml .Values.streamflow.config | nindent 4 }}
+    {{- end }}

helm/chart/templates/job.yaml

@@ -13,32 +13,67 @@ spec:
       labels:
         {{- include "streamflow.selectorLabels" . | nindent 8 }}
     spec:
-    {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-    {{- end }}
       serviceAccountName: {{ include "streamflow.serviceAccountName" . }}
+      {{- include "streamflow.imagePullSecrets" . | nindent 6 }}
+      {{- if .Values.podSecurityContext.enabled }}
       securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+        {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
       containers:
-        - name: {{ .Chart.Name }}
+        - name: {{ include "streamflow.fullname" . }}
+          {{- if .Values.containerSecurityContext.enabled }}
           securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}"
+            {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
+          image: {{ include "streamflow.image" . }}
+          {{- if .Values.command }}
+          command: {{ .Values.command }}
+          {{- end }}
+          {{- if .Values.args }}
           args: {{ .Values.args }}
+          {{- end }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          {{- if .Values.resources }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: streamflow-config
+              mountPath: /streamflow/results/streamflow.yml
+              subPath: streamflow.yml
+            - name: streamflow-metadata
+              mountPath: /.streamflow
+            - name: streamflow-outdir
+              mountPath: /tmp/streamflow
+            - name: streamflow-workdir
+              mountPath: /streamflow/results
+      {{ if .Values.restartPolicy }}
       restartPolicy: {{ .Values.restartPolicy }}
+      {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-    {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-    {{- end }}
-    {{- with .Values.tolerations }}
+      {{- with .Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
-    {{- end }}
+      {{- end }}
+      volumes:
+        - name: streamflow-metadata
+          {{- if .Values.persistence.metadata }}
+          {{ toYaml .Values.persistence.metadata | nindent 10}}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: streamflow-outdir
+          {{- if .Values.persistence.outdir }}
+          {{ toYaml .Values.persistence.outdir | nindent 10}}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}
+        - name: streamflow-workdir
+          {{- if .Values.persistence.workdir }}
+          {{ toYaml .Values.persistence.workdir | nindent 10}}
+          {{- else }}
+          emptyDir: {}
+          {{- end }}

helm/chart/templates/role.yaml

@@ -0,0 +1,34 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "streamflow.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "streamflow.labels" . | nindent 4 }}
+rules:
+- verbs:
+    - get
+    - watch
+    - list
+    - create
+    - delete
+  apiGroups:
+    - ''
+  resources:
+    - pods
+    - pods/exec
+{{- if eq .Values.streamflow.workflow.type "cwl" }}
+{{- if .Values.streamflow.workflow.restrictNetworkAccess }}
+- verbs:
+    - get
+    - list
+    - create
+    - delete
+  apiGroups:
+    - networking.k8s.io
+  resources:
+    - networkpolicies
+{{- end }}
+{{- end }}
+{{- end }}

helm/chart/templates/rolebinding.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "streamflow.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "streamflow.labels" . | nindent 4 }}
+roleRef:
+  kind: Role
+  name: {{ include "streamflow.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "streamflow.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}

helm/chart/templates/serviceaccount.yaml

@@ -9,4 +9,5 @@ metadata:
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
 {{- end -}}