Add python stages

pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
   "ruamel-yaml",
   "svgutils",
   "tomli; python_version<='3.10'",
+  "uv",
 ]
 
 optional-dependencies.dev = [ "hatch", "pre-commit" ]

src/dso/cli/_create.py

@@ -14,7 +14,9 @@
 # list of stage template with description - can be later populated also from external directories
 STAGE_TEMPLATES = {
     "bash": "Execute a simple bash snippet or call an external script (e.g. nextflow)",
-    "quarto": "Generate a report using quarto",
+    "quarto_r": "Generate a quarto report using R (qmd file)",
+    "quarto_py": "Generate a quarto report using Python (qmd file)",
+    "quarto_ipynb": "Generate a quarto report using Python (ipynb file)",
 }
 # Create help text for CLI listing all templates
 STAGE_TEMPLATE_TEXT = "\n".join(f" * __{name}__: {description}" for name, description in STAGE_TEMPLATES.items())

src/dso/templates/init/default/.pre-commit-config.yaml

@@ -24,6 +24,13 @@ repos:
   #       args: [--fix, --exit-non-zero-on-fix]
   #     - id: ruff-format
   #       types_or: [python, pyi, jupyter]
+
+  # for ipynb files in `src` directories: we never want to commit any output as rendered output files
+  # are tracked by dvc
+  - repo: https://github.com/kynan/nbstripout
+    rev: "0.8.1"
+    hooks:
+      - id: nbstripout
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.6.0
     hooks:

src/dso/templates/stage/quarto_ipynb/dvc.yaml

@@ -0,0 +1,11 @@
+stages:
+  {{ stage_name }}:
+    params:
+      - dso.quarto
+    deps:
+      - src/{{ stage_name }}.qmd
+    outs:
+      - output
+      - report/{{ stage_name }}.html
+    cmd:
+      - uv run dso exec quarto .

src/dso/templates/stage/quarto_ipynb/src/{{ stage_name }}.ipynb

@@ -0,0 +1,64 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# | label: load_libraries\n",
+    "\n",
+    "import pandas as pd\n",
+    "\n",
+    "from dso import read_params, stage_here"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Load the stage-specific 'params.yaml' config using the `read_params(..)` function. This function specifically loads\n",
+    "only the stage-dependent parameters that are defined in the 'params' section of the 'dvc.yaml' file."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# | label: read_params\n",
+    "\n",
+    "params = read_params(\"{{ stage_path }}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To locate your files relative to the stage path use `stage_here(..)`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# | label: obtain_files_relative_to_stage_dir\n",
+    "\n",
+    "# e.g.\n",
+    "samplesheet = pd.read_csv(stage_here(params[\"samplesheet\"]))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

src/dso/templates/stage/quarto_py/.gitignore

@@ -0,0 +1,7 @@
+/log
+/tmp
+/temp
+/input/*
+!/input/*.dvc
+/output/*
+/report/*

src/dso/templates/stage/quarto_py/README.md

@@ -0,0 +1,3 @@
+# {{ stage_name }}
+
+{{ stage_description }}

src/dso/templates/stage/quarto_py/dvc.yaml

@@ -0,0 +1,11 @@
+stages:
+  {{ stage_name }}:
+    params:
+      - dso.quarto
+    deps:
+      - src/{{ stage_name }}.qmd
+    outs:
+      - output
+      - report/{{ stage_name }}.html
+    cmd:
+      - uv run dso exec quarto .

src/dso/templates/stage/quarto_py/src/.gitignore

@@ -0,0 +1,6 @@
+/.quarto/
+/*.html
+# temporary quarto files
+*.rmarkdown
+/_quarto.yml
+/*_files/

src/dso/templates/stage/quarto_py/src/{{ stage_name }}.qmd

@@ -0,0 +1,25 @@
+```{python}
+# | label: load_libraries
+
+from dso import read_params, stage_here
+import pandas as pd
+```
+
+Load the stage-specific 'params.yaml' config using the `read_params(..)` function. This function specifically loads
+only the stage-dependent parameters that are defined in the 'params' section of the 'dvc.yaml' file.
+
+```{python}
+#| label: read_params
+
+params = read_params("{{ stage_path }}")
+```
+
+
+To locate your files relative to the stage path use `stage_here(..)`.
+
+```{python}
+# | label: obtain_files_relative_to_stage_dir
+
+# e.g.
+samplesheet = pd.read_csv(stage_here(params["samplesheet"]))
+```

src/dso/templates/stage/quarto_r/.gitignore

@@ -0,0 +1,7 @@
+/log
+/tmp
+/temp
+/input/*
+!/input/*.dvc
+/output/*
+/report/*

src/dso/templates/stage/quarto_r/README.md

@@ -0,0 +1,3 @@
+# {{ stage_name }}
+
+{{ stage_description }}

src/dso/templates/stage/quarto_r/src/.gitignore

@@ -0,0 +1,6 @@
+/.quarto/
+/*.html
+# temporary quarto files
+*.rmarkdown
+/_quarto.yml
+/*_files/

tests/conftest.py

@@ -31,7 +31,7 @@ def quarto_stage(dso_project) -> Path:
     runner = CliRunner()
     stage_name = "quarto_stage"
     chdir(dso_project)
-    runner.invoke(dso_create, ["stage", stage_name, "--template", "quarto", "--description", "a quarto stage"])
+    runner.invoke(dso_create, ["stage", stage_name, "--template", "quarto_r", "--description", "a quarto stage"])
     with (Path(stage_name) / "src" / f"{stage_name}.qmd").open("w") as f:
         f.write(
             dedent(

tests/test_create.py

@@ -7,7 +7,7 @@
 from dso.cli import dso_create
 
 
-@pytest.mark.parametrize("template", ["bash", "quarto"])
+@pytest.mark.parametrize("template", ["bash", "quarto_r", "quarto_py", "quarto_ipynb"])
 def test_create_stage(dso_project, template):
     runner = CliRunner()
     chdir(dso_project)