diff --git a/docs/conf.py b/docs/conf.py index 541355b..919dea8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,7 +54,8 @@ # Add any Sphinx extension module names here, as strings. # They can be extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - "myst_nb", + "myst_parser", + "sphinx_design", "sphinx_copybutton", "sphinx.ext.autodoc", "sphinx.ext.intersphinx", diff --git a/docs/getting_started.md b/docs/getting_started.md index 2208534..38ac3ad 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -52,6 +52,31 @@ For more information about project and stage templates, click [here](user_guide/ ## Configuration files +DSO proposes to put all parameters for your analysis (e.g. paths to input or output files, thresholds, etc) +in YAML-base configuration files. You can add a `params.in.yaml` at any level of a dso project (_project_, _folder_ or _stage_). + +By running + +```bash +dso compile-config +``` + +`params.in.yaml` files will be _compiled_ into `params.yaml` files. Compilation offers the following advantages: + +- _inheritance_: All variables defined in `params.in.yaml` files in any parent directory will be included. +- _templating_: Variables can be composed using [jinja2 syntax](https://jinja.palletsprojects.com/en/stable/templates/#variables), e.g. `foo: "{{ bar }}_version2"`. +- _path resolving_: Paths will be always relative to each compiled `params.yaml` file, no matter where they were defined. + +```{eval-rst} +.. image:: img/dso-yaml-inherit.png + :width: 80% + +``` + +

+ +For more details, please refer to [Configuration files](user_guide/params_files.md). + ## Implementing a stage A stage is a single step in your analysis and usually generates some kind of output data from input data. The input data can also be supplied by previous stages. To create a stage, use the `dso create stage` command and select either the _bash_ or _quarto_ template as a starting-point. diff --git a/docs/user_guide/params_files.md b/docs/user_guide/params_files.md index 4790387..a023ef0 100644 --- a/docs/user_guide/params_files.md +++ b/docs/user_guide/params_files.md @@ -2,10 +2,11 @@ YAML-based config files in a _project_, _folder_, or _stage_ serve as a single point of truth for all input files, output files or parameters. For this purpose, configurations can be defined at each level of your project in a `params.in.yaml` file. -Using `dso compile-config` the `params.in.yaml` files are compiled into `params.yaml` with two features: +Using `dso compile-config` the `params.in.yaml` files are compiled into `params.yaml` with the following features: - _inheritance_: All variables defined in `params.in.yaml` files in any parent directory will be included. - _templating_: Variables can be composed using [jinja2 syntax](https://jinja.palletsprojects.com/en/stable/templates/#variables), e.g. `foo: "{{ bar }}_version2"`. +- _path resolving_: Paths will be always relative to each compiled `params.yaml` file, no matter where they were defined. Therefore, you only need to [read in](#accessing-stage-config) a single `params.yaml` file in each stage. @@ -49,7 +50,7 @@ This allows variable to be composed using [jinja2 syntax](https://jinja.palletsp To ensure that, despite inheritance, paths are always relative to each compiled `params.yaml` file, relative paths need to be preceded with `!path`, e.g.: -``` +```yaml samplesheet: !path "01_preprocessing/input/samplesheet.txt" ``` @@ -63,13 +64,9 @@ how to work with relative paths in Python/R scripts see [python usage](../python Let's consider a project which has the following two `params.in.yaml` files at the project root and in a stage subfolder. - - - - - - - - - -
/params.in.yaml/stage/params.in.yaml
+::::{grid} 1 1 2 2 + +:::{grid-item-card} `/params.in.yaml` ```yaml thresholds: @@ -87,8 +84,9 @@ exclude_samples: - sample_6 ``` - +::: + +:::{grid-item-card} `/stage/params.in.yaml` ```yaml thresholds: @@ -103,19 +101,14 @@ exclude_samples: - sample_42 ``` -
+::: +:::: This results in the following **compiled `params.yaml` files**: - - - - - - - - - -
/params.yaml/stage/params.yaml
+::::{grid} 1 1 2 2 + +:::{grid-item-card} `/params.yaml` ```yaml thresholds: @@ -133,8 +126,8 @@ exclude_samples: - sample_6 ``` - +::: +:::{grid-item-card} `/stage/params.yaml` ```yaml thresholds: @@ -155,9 +148,8 @@ exclude_samples: samplesheet: 01_preprocessing/input/samplesheet.txt ``` -
+::: +:::: ## Accessing stage config diff --git a/pyproject.toml b/pyproject.toml index 5d84127..331cacd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,11 +48,12 @@ optional-dependencies.doc = [ "ipykernel", "ipython", "matplotlib", - "myst-nb>=1.1", + "myst-parser", "sphinx>=4", "sphinx-autodoc-typehints", "sphinx-book-theme>=1", "sphinx-copybutton", + "sphinx-design", "sphinxcontrib-bibtex>=1", "sphinxext-opengraph", ]