6789 git-sync volume support

.gitignore

@@ -17,3 +17,4 @@ kubeconfig
 letsencrypt*
 repo_state.log
 repository
+test*.yaml

docs/git-clone.d2

@@ -0,0 +1,49 @@
+Git repository -> The Internet
+The Internet -> Airflow Deployment.git-clone pod.git-clone container
+
+The Internet: {shape: cloud}
+
+Airflow Deployment: {
+  git-repo pvc: {
+    shape: cylinder
+    label: /git RWO pvc
+  }
+
+  git-clone pod: {
+    git-clone container: {
+      description: |md
+        - pulls from external repo infrequently
+        - stores repo in /git/reponame
+      |
+    }
+  }
+
+  worker pod: {
+    worker container: {
+      description: |md
+        - mounts /git/reponame/dags as airflow dags dir
+      |
+    }
+  }
+
+  scheduler pod: {
+    scheduler container: {
+      description: |md
+        - mounts /git/reponame/dags as airflow dags dir
+      |
+    }
+  }
+
+  kpo pod: {
+    kpo container: {
+      description: |md
+        - mounts /git/reponame/dags as airflow dags dir
+      |
+    }
+  }
+
+  git-clone pod.git-clone container -> git-repo pvc
+  git-repo pvc -> worker pod.worker container
+  git-repo pvc -> scheduler pod.scheduler container
+  git-repo pvc -> kpo pod.kpo container
+}

templates/git-sync-relay/git-sync-relay-deployment.yaml

@@ -34,7 +34,12 @@ spec:
       securityContext: {{ toYaml .Values.gitSyncRelay.securityContext | nindent 8 }}
       volumes:
         - name: git-repo-contents
+        {{- if eq .Values.gitSyncRelay.repoShareMode "shared_volume" }}
+          persistentVolumeClaim:
+            claimName: git-repo-contents
+        {{- else }}
           emptyDir: {}
+        {{- end }}
         {{- if .Values.gitSyncRelay.repo.sshPrivateKeySecretName }}
         - name: git-secret
           secret:
@@ -110,8 +115,8 @@ spec:
           {{- if .Values.gitSyncRelay.gitSync.readinessProbe  }}
           readinessProbe: {{ tpl (toYaml .Values.gitSyncRelay.gitSync.readinessProbe) . | nindent 12 }}
           {{- end }}
-          resources:
-            {{- toYaml .Values.gitSyncRelay.gitSync.resources | nindent 12 }}
+          resources: {{- toYaml .Values.gitSyncRelay.gitSync.resources | nindent 12 }}
+        {{- if eq .Values.gitSyncRelay.repoShareMode "git_daemon" }}
         - name: git-daemon
           image: "{{ .Values.gitSyncRelay.images.gitDaemon.repository }}:{{ .Values.gitSyncRelay.images.gitDaemon.tag }}"
           volumeMounts:
@@ -136,6 +141,8 @@ spec:
           readinessProbe: {{ tpl (toYaml .Values.gitSyncRelay.gitDaemon.readinessProbe) . | nindent 12 }}
           {{- end }}
           resources: {{- toYaml .Values.gitSyncRelay.gitDaemon.resources | nindent 12 }}
+        {{- end }}
+
       {{- if .Values.loggingSidecar.enabled }}
         {{- include "sidecar_container_spec" . | nindent 8 }}
       {{- end }}

templates/git-sync-relay/git-sync-relay-pvc.yaml

@@ -0,0 +1,22 @@
+########################
+## git-sync-relay pvc ##
+########################
+{{- if and .Values.gitSyncRelay.enabled ( eq .Values.gitSyncRelay.repoShareMode "shared_volume") }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: git-repo-contents
+  labels:
+    component: git-sync-relay
+    tier: airflow
+    release: {{ .Release.Name }}
+    chart: "{{ .Release.Name }}-{{ .Chart.Version }}"
+    heritage: {{ .Release.Service }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.gitSyncRelay.volumeSync.volumeSize }}
+  storageClassName: {{ .Values.gitSyncRelay.volumeSync.storageClassName }}
+{{- end }}

tests/__init__.py

@@ -8,3 +8,11 @@
 metadata = yaml.safe_load((Path(git_root_dir) / "metadata.yaml").read_text())
 # replace all patch versions with 0 so we end up with ['a.b.0', 'x.y.0']
 supported_k8s_versions = [".".join(x.split(".")[:-1] + ["0"]) for x in metadata["test_k8s_versions"]]
+
+
+def container_env_to_dict(container):
+    """Return a dict of a k8s container env structure.
+
+    This only works with serializable env structures, and will error if the env is not serializable.
+    """
+    return {env["name"]: env["value"] for env in container["env"]}

tests/chart/helm_template_generator.py

@@ -17,6 +17,7 @@
 
 import json
 import os
+import shlex
 import subprocess
 import sys
 from functools import cache
@@ -86,9 +87,7 @@ def render_chart(
     namespace: str | None = None,
     validate_objects: bool = True,
 ):
-    """
-    Render a helm chart into dictionaries. For helm chart testing only.
-    """
+    """Render a helm chart into dictionaries."""
     values = values or {}
     chart_dir = chart_dir or sys.path[0]
     with NamedTemporaryFile(delete=not DEBUG) as tmp_file:  # export DEBUG=true to keep
@@ -110,16 +109,19 @@ def render_chart(
         if show_only:
             if isinstance(show_only, str):
                 show_only = [show_only]
-            for i in show_only:
-                command.extend(["--show-only", i])
+            for file in show_only:
+                command.extend(["--show-only", str(file)])
+
+        if DEBUG:
+            print(f"helm command:\n  {shlex.join(command)}")
+
         try:
-            templates = subprocess.check_output(command, stderr=subprocess.PIPE)
-            if not templates:
+            manifests = subprocess.check_output(command, stderr=subprocess.PIPE)
+            if not manifests:
                 return None
         except subprocess.CalledProcessError as error:
             if DEBUG:
                 print("ERROR: subprocess.CalledProcessError:")
-                print(f"helm command: {' '.join(command)}")
                 print(f"Values file contents:\n{'-' * 21}\n{yaml.dump(values)}{'-' * 21}")
                 print(f"{error.output=}\n{error.stderr=}")
 
@@ -129,15 +131,20 @@ def render_chart(
                         + "usually means there is a helm value that needs to be set to render "
                         + "the content of the chart.\n"
                         + "command: "
-                        + " ".join(command)
+                        + shlex.join(command)
                     )
             raise
-        k8s_objects = yaml.full_load_all(templates)
-        k8s_objects: list = [k8s_object for k8s_object in k8s_objects if k8s_object]
-        if validate_objects:
-            for k8s_object in k8s_objects:
-                validate_k8s_object(k8s_object, kube_version=kube_version)
-        return k8s_objects
+        return load_and_validate_k8s_manifests(manifests, validate_objects=validate_objects, kube_version=kube_version)
+
+
+def load_and_validate_k8s_manifests(manifests: str, validate_objects: bool = True, kube_version: str = default_version):
+    """Load k8s objecdts from yaml into python, optionally validating them. yaml can contain multiple documents."""
+    k8s_objects = [k8s_object for k8s_object in yaml.full_load_all(manifests) if k8s_object]
+
+    if validate_objects:
+        for k8s_object in k8s_objects:
+            validate_k8s_object(k8s_object, kube_version=kube_version)
+    return k8s_objects
 
 
 def prepare_k8s_lookup_dict(k8s_objects) -> dict[tuple[str, str], dict[str, Any]]:

tests/chart/test_git_sync_relay_deployment.py

@@ -1,6 +1,6 @@
 import pytest
 
-from tests import supported_k8s_versions
+from tests import container_env_to_dict, git_root_dir, supported_k8s_versions
 from tests.chart.helm_template_generator import render_chart
 
 from . import get_containers_by_name
@@ -15,17 +15,20 @@ def test_gsr_deployment_default(self, kube_version):
         """Test that no git-sync-relay templates are rendered by default."""
         docs = render_chart(
             kube_version=kube_version,
-            show_only="templates/git-sync-relay/git-sync-relay-deployment.yaml",
+            show_only=[x.relative_to(git_root_dir) for x in (git_root_dir / "templates/git-sync-relay").glob("*")],
         )
         assert len(docs) == 0
 
-    def test_gsr_deployment_gsr_enabled(self, kube_version):
-        """Test that a valid deployment is rendered when git-sync-relay is enabled."""
+    def test_gsr_deployment_gsr_enabled_with_defaults(self, kube_version):
+        """Test that a valid deployment is rendered when git-sync-relay is enabled with defaults."""
         values = {"gitSyncRelay": {"enabled": True}}
 
         docs = render_chart(
             kube_version=kube_version,
-            show_only="templates/git-sync-relay/git-sync-relay-deployment.yaml",
+            show_only=[
+                "templates/git-sync-relay/git-sync-relay-deployment.yaml",
+                "templates/git-sync-relay/git-sync-relay-pvc.yaml",
+            ],
             values=values,
         )
         assert len(docs) == 1
@@ -39,6 +42,37 @@ def test_gsr_deployment_gsr_enabled(self, kube_version):
         assert c_by_name["git-daemon"]["image"].startswith("quay.io/astronomer/ap-git-daemon:")
         assert c_by_name["git-daemon"]["livenessProbe"]
 
+    def test_gsr_deployment_gsr_repo_share_mode_volume(self, kube_version):
+        """Test that a valid deployment is rendered when git-sync-relay is enabled with repoShareMode=shared_volume."""
+        values = {
+            "gitSyncRelay": {
+                "enabled": True,
+                "repoShareMode": "shared_volume",
+                "volumeSync": {"storageClassName": "my-usb-thumb-drive"},
+            }
+        }
+
+        docs = render_chart(
+            kube_version=kube_version,
+            show_only=[
+                "templates/git-sync-relay/git-sync-relay-deployment.yaml",
+                "templates/git-sync-relay/git-sync-relay-pvc.yaml",
+            ],
+            values=values,
+        )
+        assert len(docs) == 2
+        deployment, pvc = docs if docs[0]["kind"] == "Deployment" else docs[::-1]
+        assert deployment["kind"] == "Deployment"
+        assert deployment["apiVersion"] == "apps/v1"
+        assert deployment["metadata"]["name"] == "release-name-git-sync-relay"
+        assert pvc["kind"] == "PersistentVolumeClaim"
+        assert pvc["kind"] == "PersistentVolumeClaim"
+        assert pvc["spec"]["storageClassName"] == "my-usb-thumb-drive"
+        c_by_name = get_containers_by_name(deployment)
+        assert not c_by_name.get("git-daemon")
+        assert len(c_by_name) == 1
+        assert c_by_name["git-sync"]["image"].startswith("quay.io/astronomer/ap-git-sync-relay:")
+
     def test_gsr_deployment_with_ssh_credentials_and_known_hosts(self, kube_version):
         """Test that a valid deployment is rendered when enabling git-sync with ssh credentials and known hosts and other custom configs."""
         values = {
@@ -96,20 +130,17 @@ def test_gsr_deployment_with_ssh_credentials_and_known_hosts(self, kube_version)
             },
             {"name": "git-repo-contents", "mountPath": "/git"},
         ]
-        assert c_by_name["git-sync"]["env"] == [
-            {"name": "GIT_SYNC_ROOT", "value": "/git"},
-            {"name": "GIT_SYNC_REPO", "value": "not-the-default-url"},
-            {"name": "GIT_SYNC_BRANCH", "value": "not-the-default-branch"},
-            {"name": "GIT_SYNC_DEPTH", "value": "22"},
-            {"name": "GIT_SYNC_WAIT", "value": "333"},
-            {"name": "GIT_SYNC_SSH", "value": "true"},
-            {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"},
-            {"name": "GIT_KNOWN_HOSTS", "value": "true"},
-            {
-                "name": "GIT_SSH_KNOWN_HOSTS_FILE",
-                "value": "/etc/git-secret/known_hosts",
-            },
-        ]
+        assert container_env_to_dict(c_by_name["git-sync"]) == {
+            "GIT_SYNC_ROOT": "/git",
+            "GIT_SYNC_REPO": "not-the-default-url",
+            "GIT_SYNC_BRANCH": "not-the-default-branch",
+            "GIT_SYNC_DEPTH": "22",
+            "GIT_SYNC_WAIT": "333",
+            "GIT_SYNC_SSH": "true",
+            "GIT_SSH_KEY_FILE": "/etc/git-secret/ssh",
+            "GIT_KNOWN_HOSTS": "true",
+            "GIT_SSH_KNOWN_HOSTS_FILE": "/etc/git-secret/known_hosts",
+        }
         assert c_by_name["git-daemon"]["livenessProbe"]
 
     def test_gsr_deployment_without_ssh_credentials_and_known_hosts(self, kube_version):
@@ -151,13 +182,13 @@ def test_gsr_deployment_without_ssh_credentials_and_known_hosts(self, kube_versi
         assert c_by_name["git-sync"]["volumeMounts"] == [
             {"name": "git-repo-contents", "mountPath": "/git"},
         ]
-        assert c_by_name["git-sync"]["env"] == [
-            {"name": "GIT_SYNC_ROOT", "value": "/git"},
-            {"name": "GIT_SYNC_REPO", "value": "not-the-default-url"},
-            {"name": "GIT_SYNC_BRANCH", "value": "not-the-default-branch"},
-            {"name": "GIT_SYNC_DEPTH", "value": "22"},
-            {"name": "GIT_SYNC_WAIT", "value": "333"},
-        ]
+        assert container_env_to_dict(c_by_name["git-sync"]) == {
+            "GIT_SYNC_ROOT": "/git",
+            "GIT_SYNC_REPO": "not-the-default-url",
+            "GIT_SYNC_BRANCH": "not-the-default-branch",
+            "GIT_SYNC_DEPTH": "22",
+            "GIT_SYNC_WAIT": "333",
+        }
         assert c_by_name["git-daemon"]["livenessProbe"]
 
     def test_gsr_deployment_with_resource_overrides(self, kube_version):