Improve StreamFlow on Kubernetes

This commit heavily refactors the StreamFlow Helm chart to simplify its deployment on top of Kubernetes clusters. In addition, this commit adds a `networkPolicy` flag to control the behaviour of CWL `DockerRequirement` objects into Kubernetes `Pod` items. Normally, the CWL `NetworkAccess` requirement is enforced through Kubernetes `NetworkPolicy` objects. However, `NetworkPolicy` objects regulate the network security inside a cluster, and giving the StreamFlow `Pod` permissions to create/delete them may result in unwanted security flaws. The `networkPolicy` option can be set to `False` to ignore the CWL `NetworkAccess` enforcement in such cases.
alpha-unito · Feb 9, 2025 · 5eef4be · 5eef4be
1 parent e86d21c
commit 5eef4be
Show file tree

Hide file tree

Showing 10 changed files with 255 additions and 26 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.13-slim AS builder
+FROM python:3.13-alpine3.21 AS builder
 ARG HELM_VERSION
 
 ENV VIRTUAL_ENV="/opt/streamflow"
@@ -14,8 +14,18 @@ COPY ./requirements.txt           \
 COPY ./docs/requirements.txt /build/docs
 COPY ./streamflow /build/streamflow
 
-RUN apt update -y \
-    && apt install -y --no-install-recommends curl \
+RUN apk --no-cache add \
+        bash \
+        cargo \
+        curl \
+        g++ \
+        libffi-dev \
+        libxml2-dev \
+        libxslt-dev \
+        make \
+        musl-dev \
+        openssl \
+        openssl-dev \
     && curl -fsSL \
           --retry 5 \
           --retry-max-time 60 \
@@ -28,7 +38,7 @@ RUN apt update -y \
     && python -m venv ${VIRTUAL_ENV} \
     && pip install .
 
-FROM python:3.13-slim
+FROM python:3.13-alpine3.21
 LABEL maintainer="[email protected]"
 
 ENV VIRTUAL_ENV="/opt/streamflow"
@@ -37,9 +47,11 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 COPY --from=builder /usr/local/bin/helm /usr/local/bin/helm
 
-RUN apt update -y \
-    && apt install -y --no-install-recommends nodejs \
-    && rm -rf /var/lib/apt/lists/* \
+RUN apk --no-cache add \
+        libxml2 \
+        libxslt \
+        nodejs \
+        openssl \
     && mkdir -p /streamflow/results
 
 WORKDIR /streamflow/results

diff --git a/helm/chart/Chart.yaml b/helm/chart/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: streamflow
-description: A Helm chart for StreamFlow
+description: A Helm chart for the StreamFlow workflow management system
 type: application
 version: 0.2.0
-appVersion: latest
+appVersion: 0.2.0.dev11
diff --git a/helm/chart/templates/_helpers.tpl b/helm/chart/templates/_helpers.tpl
@@ -1,6 +1,6 @@
 {{/* vim: set filetype=mustache: */}}
 {{/*
-Expand the name of the chart.
+Expand the name of the chart
 */}}
 {{- define "streamflow.name" -}}
 {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
@@ -9,7 +9,7 @@ Expand the name of the chart.
 {{/*
 Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
+If release name contains chart name it will be used as a full name
 */}}
 {{- define "streamflow.fullname" -}}
 {{- if .Values.fullnameOverride -}}
@@ -25,12 +25,49 @@ If release name contains chart name it will be used as a full name.
 {{- end -}}
 
 {{/*
-Create chart name and version as used by the chart label.
+Create chart name and version as used by the chart label
 */}}
 {{- define "streamflow.chart" -}}
 {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Return the proper StreamFlow image name
+*/}}
+{{- define "streamflow.image" -}}
+{{- $registryName := default .Values.image.registry -}}
+{{- $repositoryName := .Values.image.repository -}}
+{{- $separator := ":" -}}
+{{- $termination := default .Chart.AppVersion .Values.image.tag | toString -}}
+
+{{- if not .Values.image.tag }}
+  {{- if .Chart }}
+    {{- $termination = .Chart.AppVersion | toString -}}
+  {{- end -}}
+{{- end -}}
+{{- if .Values.image.digest }}
+    {{- $separator = "@" -}}
+    {{- $termination = .Values.image.digest | toString -}}
+{{- end -}}
+{{- if $registryName }}
+    {{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}}
+{{- else -}}
+    {{- printf "%s%s%s"  $repositoryName $separator $termination -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper Docker Image Registry Secret Names evaluating values as templates
+*/}}
+{{- define "streamflow.imagePullSecrets" -}}
+{{- if (not (empty .Values.image.pullSecrets)) -}}
+imagePullSecrets:
+  {{- range .Values.image.pullSecrets | uniq }}
+  - name: {{ . }}
+  {{- end }}
+{{- end }}
+{{- end }}
+
 {{/*
 Common labels
 */}}

diff --git a/helm/chart/templates/role.yaml b/helm/chart/templates/role.yaml
@@ -0,0 +1,32 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "streamflow.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "streamflow.labels" . | nindent 4 }}
+rules:
+- verbs:
+    - get
+    - watch
+    - list
+    - create
+    - delete
+  apiGroups:
+    - ''
+  resources:
+    - pods
+    - pods/exec
+{{- if .Values.rbac.networkPolicies }}
+- verbs:
+    - get
+    - list
+    - create
+    - delete
+  apiGroups:
+    - networking.k8s.io
+  resources:
+    - networkpolicies
+{{- end }}
+{{- end }}
diff --git a/helm/chart/templates/rolebinding.yaml b/helm/chart/templates/rolebinding.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "streamflow.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "streamflow.labels" . | nindent 4 }}
+roleRef:
+  kind: Role
+  name: {{ include "streamflow.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "streamflow.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/helm/chart/values.yaml b/helm/chart/values.yaml
@@ -1,28 +1,150 @@
-replicaCount: 1
+## String to partially override streamflow.fullname template (will maintain the release name)
+##
+nameOverride: ""
+
+## String to fully override streamflow.fullname template
+##
+fullnameOverride: ""
 
+## StreamFlow image version
+## ref: https://hub.docker.com/r/alphaunito/streamflow/tags/
+##
 image:
+  ## @param image.registry StreamFlow image registry
+  ##
+  registry: docker.io
+  ## @param image.repository StreamFlow image repository
+  ##
   repository: alphaunito/streamflow
-  pullPolicy: Always
+  ## @skip image.tag StreamFlow image tag (immutable tags are recommended)
+  ##
+  tag: ""
+  ## @param image.digest StreamFlow image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag
+  ##
+  digest: ""
+  ## @param image.pullPolicy StreamFLow image pull
+  ## ref: https://kubernetes.io/docs/concepts/containers/images/#pre-pulled-images
+  ##
+  pullPolicy: IfNotPresent
+  ## @param image.pullSecrets Specify image pull secrets
+  ## Secrets must be manually created in the namespace.
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ##
+  pullSecrets: []
 
-args: ["streamflow", "version"]
-restartPolicy: OnFailure
-imagePullSecrets: []
-nameOverride: ""
-fullnameOverride: ""
+## Override default container command
+##
+command: []
 
+## Override default container args
+##
+args: []
+
+## Container Security Context
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+##
+containerSecurityContext:
+  ## @param containerSecurityContext.enabled Enable security context
+  ##
+  enabled: true
+  ## @param containerSecurityContext.seLinuxOptions Set SELinux options in StreamFlow container
+  ##
+  seLinuxOptions: {}
+  ## @param containerSecurityContext.runAsUser Set user in StreamFlow container
+  ##
+  runAsUser: 1001
+  ## @param containerSecurityContext.runAsGroup Set group in StreamFlow container
+  ##
+  runAsGroup: 1001
+  ## @param containerSecurityContext.runAsNonRoot Require StreamFlow container to run as non-root user
+  ##
+  runAsNonRoot: true
+  ## @param containerSecurityContext.privileged Runs the StreamFlow container as privileged
+  ##
+  privileged: false
+  ## @param containerSecurityContext.readOnlyRootFilesystem Mounts the StreamFlow container's root filesystem as read-only
+  ##
+  readOnlyRootFilesystem: true
+  ## @param containerSecurityContext.allowPrivilegeEscalation ontrols whether a process can gain more privileges than its parent process in the StreamFlow container
+  ##
+  allowPrivilegeEscalation: false
+  ## @param containerSecurityContext.capabilities Controls processes' privileges in the StreamFlow container
+  ##
+  capabilities:
+    drop: ["ALL"]
+  ## @param containerSecurityContext.seccompProfile Filter processes' system calls in the StreamFlow container
+  ##
+  seccompProfile:
+    type: "RuntimeDefault"
+
+## Pod Security Context
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+##
+podSecurityContext:
+  ## @param podSecurityContext.enabled Enable security context
+  ##
+  enabled: true
+  ## @param podSecurityContext.fsGroupChangePolicy Set filesystem group change policy
+  ##
+  fsGroupChangePolicy: Always
+  ## @param podSecurityContext.sysctls Set kernel settings using the sysctl interface
+  ##
+  sysctls: []
+  ## @param podSecurityContext.supplementalGroups Set filesystem extra groups
+  ##
+  supplementalGroups: []
+  ## @param podSecurityContext.fsGroup Group ID for the StreamFlow pod
+  ##
+  fsGroup: 1001
+
+## Service account for StreamFlow to use.
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+##
 serviceAccount:
+  ## @param serviceAccount.create Enable creation of ServiceAccount for StreamFlow pod
+  ##
   create: true
+  ## @param serviceAccount.name The name of the ServiceAccount to use.
+  ## If not set and create is true, a name is generated using the streamflow.fullname template
+  ##
+  name: ""
+  ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created
+  ## Can be set to false if pods using this serviceAccount do not need to use K8s API
+  ##
+  automountServiceAccountToken: true
+  ## @param serviceAccount.annotations Additional custom annotations for the ServiceAccount
+  ##
   annotations: {}
-  name:
 
-podSecurityContext: {}
+## Creates role for ServiceAccount
+## ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#service-account-permissions
+##
+rbac:
+  ## @param rbac.create Create Role and RoleBinding (required for StreamFlow to instantiate other Pods in the cluster)
+  ##
+  create: true
+  ## @param rbac.networkPolicies Enable the StreamFlow Pod to manage NetworkPolicy objects
+  ##
+  networkPolicies: false
+  ## @param rbac.rules Custom RBAC rules to set
+  ##
+  rules: []
 
-securityContext: {}
+## Configure the Restart Policy for the StreamFlow container
+## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy
+##
+restartPolicy: ""
 
+## Set requests and limits for different resources (e.g., CPU or memory) for the StreamFlow container
+##
 resources: {}
 
+## Node labels for StreamFlow pod assignment
+## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/
+##
 nodeSelector: {}
 
+## Tolerations for StreamFlow pod assignment
+## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+##
 tolerations: []
-
-affinity: {}
diff --git a/streamflow/cwl/requirement/docker/kubernetes.py b/streamflow/cwl/requirement/docker/kubernetes.py
@@ -22,6 +22,7 @@ def __init__(
         kubeContext: str | None = None,
         maxConcurrentConnections: int = 4096,
         namespace: str | None = None,
+        networkPolicy: bool = False,
         locationsCacheSize: int | None = None,
         locationsCacheTTL: int | None = None,
         transferBufferSize: int = (2**25) - 1,
@@ -45,6 +46,7 @@ def __init__(
         self.kubeContext: str | None = kubeContext
         self.maxConcurrentConnections: int = maxConcurrentConnections
         self.namespace: str | None = namespace
+        self.networkPolicy: bool = networkPolicy
         self.locationsCacheSize: int | None = locationsCacheSize
         self.locationsCacheTTL: int | None = locationsCacheTTL
         self.transferBufferSize: int = transferBufferSize
@@ -73,6 +75,7 @@ def get_target(
                 name=name,
                 image=image,
                 network_access=network_access,
+                network_policy=self.networkPolicy,
                 output_directory=output_directory,
             ).dump(f.name)
             return Target(

diff --git a/streamflow/cwl/requirement/docker/schemas/kubernetes.jinja2 b/streamflow/cwl/requirement/docker/schemas/kubernetes.jinja2
@@ -26,7 +26,7 @@ spec:
     - name: {{ name }}-workdir
       emptyDir: {}
     {% endif %}
-{% if not network_access %}
+{% if network_policy and not network_access %}
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy

diff --git a/streamflow/cwl/requirement/docker/schemas/kubernetes.json b/streamflow/cwl/requirement/docker/schemas/kubernetes.json
@@ -45,6 +45,11 @@
       "description": "Available locations cache TTL (in seconds). When such cache expires, the connector performs a new request to check locations availability",
       "default": 10
     },
+    "networkPolicy": {
+      "type": "boolean",
+      "description": "Use a NetworkPolicy object to explicitly limit containers' network access when the NetworkAccess requirement is set to false",
+      "default": false
+    },
     "timeout": {
       "type": "integer",
       "description": "Time (in seconds) to wait for any individual Kubernetes operation",

diff --git a/tests/cwl-conformance/streamflow-kubernetes.yml b/tests/cwl-conformance/streamflow-kubernetes.yml
@@ -6,7 +6,8 @@ workflows:
         - step: /
           deployment:
             type: kubernetes
-            config: {}
+            config:
+              networkPolicy: true
 database:
   type: default
   config: