From fd0016c65c0a6510539d11f776714c9886ba02ab Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 20 Jan 2025 15:32:39 +0100 Subject: [PATCH 01/33] bump version to 3.2.0dev --- .gitpod.yml | 2 +- CHANGELOG.md | 12 ++++++++++++ setup.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index db31d01bed..d5948695bf 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 1197a379b6..c5aa7f22fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # nf-core/tools: Changelog +## v3.2.0dev + +### Template + +### Linting + +### Modules + +### Subworkflows + +### General + ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] ### Template diff --git a/setup.py b/setup.py index fb1621adfc..4a142c20fd 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.1.2" +version = "3.2.0dev" with open("README.md") as f: readme = f.read() From c1692f5b886d15e09f3d343e6bac8336ef26acf4 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:52:52 +0100 Subject: [PATCH 02/33] fix(1929): more type-gating --- nf_core/components/components_command.py | 4 + nf_core/components/create.py | 96 +++++++------------ nf_core/components/nfcore_component.py | 29 +++--- nf_core/modules/lint/module_tests.py | 2 + nf_core/pipelines/lint/__init__.py | 3 + .../lint/local_component_structure.py | 31 ++++++ .../subworkflows/lint/subworkflow_tests.py | 3 +- tests/modules/test_create.py | 4 +- 8 files changed, 98 insertions(+), 74 deletions(-) create mode 100644 nf_core/pipelines/lint/local_component_structure.py diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index f25fb33a6f..f04bb7da6b 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -71,6 +71,10 @@ def get_local_components(self) -> List[str]: """ local_component_dir = Path(self.directory, self.component_type, "local") return [ + str(Path(directory).relative_to(local_component_dir)) + for directory, _, files in os.walk(local_component_dir) + if "main.nf" in files + ] + [ str(path.relative_to(local_component_dir)) for path in local_component_dir.iterdir() if path.suffix == ".nf" ] diff --git a/nf_core/components/create.py b/nf_core/components/create.py index c781905618..0a2c6aaff7 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -75,11 +75,11 @@ def create(self) -> bool: e.g bam_sort or bam_sort_samtools, respectively. If is a pipeline, this function creates a file called: - '/modules/local/tool.nf' + '/modules/local/tool/main.nf' OR - '/modules/local/tool_subtool.nf' + '/modules/local/tool/subtool/main.nf' OR for subworkflows - '/subworkflows/local/subworkflow_name.nf' + '/subworkflows/local/subworkflow_name/main.nf' If is a clone of nf-core/modules, it creates or modifies the following files: @@ -355,70 +355,46 @@ def _get_component_dirs(self) -> Dict[str, Path]: """ file_paths = {} if self.repo_type == "pipeline": - local_component_dir = Path(self.directory, self.component_type, "local") - # Check whether component file already exists - component_file = local_component_dir / f"{self.component_name}.nf" - if component_file.exists() and not self.force_overwrite: - raise UserWarning( - f"{self.component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" - ) - - if self.component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - if self.subtool and (local_component_dir / f"{self.component}.nf").exists(): - raise UserWarning( - f"Module '{self.component}' exists already, cannot make subtool '{self.component_name}'" - ) - - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob(f"{local_component_dir}/{self.component}_*.nf") - if not self.subtool and tool_glob: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" - ) - - # Set file paths - file_paths["main.nf"] = component_file + component_dir = Path(self.directory, self.component_type, "local", self.component_dir) elif self.repo_type == "modules": component_dir = Path(self.directory, self.component_type, self.org, self.component_dir) - # Check if module/subworkflow directories exist already - if component_dir.exists() and not self.force_overwrite and not self.migrate_pytest: - raise UserWarning( - f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" - ) + else: + raise ValueError("`repo_type` not set correctly") - if self.component_type == "modules": - # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = Path( - self.directory, - self.component_type, - self.org, - self.component, - "main.nf", + # Check if module/subworkflow directories exist already + if component_dir.exists() and not self.force_overwrite and not self.migrate_pytest: + raise UserWarning( + f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" + ) + + if self.component_type == "modules": + # If a subtool, check if there is a module called the base tool name already + parent_tool_main_nf = Path( + self.directory, + self.component_type, + self.org, + self.component, + "main.nf", + ) + if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest: + raise UserWarning( + f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) - if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest: - raise UserWarning( - f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" - ) - # If no subtool, check that there isn't already a tool/subtool - tool_glob = glob.glob( - f"{Path(self.directory, self.component_type, self.org, self.component)}/*/main.nf" + # If no subtool, check that there isn't already a tool/subtool + tool_glob = glob.glob(f"{Path(self.directory, self.component_type, self.org, self.component)}/*/main.nf") + if not self.subtool and tool_glob and not self.migrate_pytest: + raise UserWarning( + f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" ) - if not self.subtool and tool_glob and not self.migrate_pytest: - raise UserWarning( - f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{self.component_name}'" - ) - # Set file paths - # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths["main.nf"] = component_dir / "main.nf" - file_paths["meta.yml"] = component_dir / "meta.yml" - if self.component_type == "modules": - file_paths["environment.yml"] = component_dir / "environment.yml" - file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" - else: - raise ValueError("`repo_type` not set correctly") + # Set file paths + # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure + file_paths["main.nf"] = component_dir / "main.nf" + file_paths["meta.yml"] = component_dir / "meta.yml" + if self.component_type == "modules": + file_paths["environment.yml"] = component_dir / "environment.yml" + file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" return file_paths diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 81c0ba98e7..03ee51b18a 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -74,8 +74,8 @@ def __init__( repo_dir = self.component_dir.parts[:name_index][-1] self.org = repo_dir - self.nftest_testdir = Path(self.component_dir, "tests") - self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test") + self.nftest_testdir: Optional[Path] = Path(self.component_dir, "tests") + self.nftest_main_nf: Optional[Path] = Path(self.nftest_testdir, "main.nf.test") if self.repo_type == "pipeline": patch_fn = f"{self.component_name.replace('/', '-')}.diff" @@ -85,15 +85,22 @@ def __init__( self.patch_path = patch_path else: # The main file is just the local module - self.main_nf = self.component_dir - self.component_name = self.component_dir.stem - # These attributes are only used by nf-core modules - # so just initialize them to None - self.meta_yml = None - self.environment_yml = None - self.test_dir = None - self.test_yml = None - self.test_main_nf = None + if self.component_dir.is_dir(): + self.main_nf = Path(self.component_dir, "main.nf") + self.component_name = self.component_dir.stem + # These attributes are only required by nf-core modules + # so just set them to None if they don't exist + self.meta_yml = p if (p := Path(self.component_dir, "meta.yml")).exists() else None + self.environment_yml = p if (p := Path(self.component_dir, "environment.yml")).exists() else None + self.nftest_testdir = p if (p := Path(self.component_dir, "tests")).exists() else None + if self.nftest_testdir is not None: + self.nftest_main_nf = p if (p := Path(self.nftest_testdir, "main.nf.test")).exists() else None + else: + self.main_nf = self.component_dir + self.meta_yml = None + self.environment_yml = None + self.nftest_testdir = None + self.nftest_main_nf = None self.process_name: str = self._get_process_name() diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 6722c12129..8f5fc91425 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -22,6 +22,8 @@ def module_tests(_, module: NFCoreComponent): and contains a ``main.nf.test`` and a ``main.nf.test.snap`` """ + if module.nftest_testdir is None or module.nftest_main_nf is None: + raise ValueError() repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] test_dir = Path(module.base_dir, "tests", "modules", repo_dir, module.component_name) pytest_main_nf = Path(test_dir, "main.nf") diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index 154e38aea6..7f15b092cc 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -38,6 +38,7 @@ from .files_exist import files_exist from .files_unchanged import files_unchanged from .included_configs import included_configs +from .local_component_structure import local_component_structure from .merge_markers import merge_markers from .modules_json import modules_json from .modules_structure import modules_structure @@ -89,6 +90,7 @@ class PipelineLint(nf_core.utils.Pipeline): merge_markers = merge_markers modules_json = modules_json modules_structure = modules_structure + local_component_structure = local_component_structure multiqc_config = multiqc_config nextflow_config = nextflow_config nfcore_yml = nfcore_yml @@ -151,6 +153,7 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "local_component_structure", "base_config", "modules_config", "nfcore_yml", diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py new file mode 100644 index 0000000000..e5e1a00dee --- /dev/null +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -0,0 +1,31 @@ +import logging +from pathlib import Path + +log = logging.getLogger(__name__) + + +def local_component_structure(self): + """ + Check that the local modules and subworkflows directories in a pipeline have the correct format: + + .. code-block:: bash + + modules/local/TOOL/SUBTOOL + + Prior to nf-core/tools release 3.0.3 the directory structure allowed top-level `*.nf` files: + + .. code-block:: bash + + modules/local/modules/TOOL_SUBTOOL.nf + """ + warned = [] + for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): + warned.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") + for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): + warned.append(f"{nf_file.name} in subworkflows/local should be moved to a TOOL/SUBTOOL/main.nf structure") + + # If there are modules installed in the wrong location + passed = [] + if len(warned) == 0: + passed = ["modules directory structure is correct 'modules/nf-core/TOOL/SUBTOOL'"] + return {"passed": passed, "warned": warned, "failed": [], "ignored": []} diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 8e9e62430a..1caf73d00b 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -23,7 +23,8 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ - + if subworkflow.nftest_testdir is None or subworkflow.nftest_main_nf is None: + raise ValueError() repo_dir = subworkflow.component_dir.parts[ : subworkflow.component_dir.parts.index(subworkflow.component_name.split("/")[0]) ][-1] diff --git a/tests/modules/test_create.py b/tests/modules/test_create.py index 219f869997..4ae9f9b5c4 100644 --- a/tests/modules/test_create.py +++ b/tests/modules/test_create.py @@ -31,7 +31,7 @@ def test_modules_create_succeed(self): ) with requests_cache.disabled(): module_create.create() - assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore.nf")) + assert os.path.exists(os.path.join(self.pipeline_dir, "modules", "local", "trimgalore/main.nf")) def test_modules_create_fail_exists(self): """Fail at creating the same module twice""" @@ -46,7 +46,7 @@ def test_modules_create_fail_exists(self): with pytest.raises(UserWarning) as excinfo: with requests_cache.disabled(): module_create.create() - assert "Module file exists already" in str(excinfo.value) + assert "module directory exists:" in str(excinfo.value) def test_modules_create_nfcore_modules(self): """Create a module in nf-core/modules clone""" From 90978ead6f36252e192bfee77e4a756145710f13 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 28 Oct 2024 15:54:44 +0000 Subject: [PATCH 03/33] fix CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5aa7f22fe..331e2fcb56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ ### Modules +- feat(1929): Make local modules use remote structure ([#3256](https://github.com/nf-core/tools/pull/3256)) + ### Subworkflows ### General From a5bb0f0ed40244c7283fe57155acf6b3ebf0fb6f Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:03:36 +0100 Subject: [PATCH 04/33] fix(1929): update subworkflow tests and add lint test markdown --- .../_src/pipeline_lint_tests/local_component_structure.md | 5 +++++ tests/subworkflows/test_create.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 docs/api/_src/pipeline_lint_tests/local_component_structure.md diff --git a/docs/api/_src/pipeline_lint_tests/local_component_structure.md b/docs/api/_src/pipeline_lint_tests/local_component_structure.md new file mode 100644 index 0000000000..1884d862be --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/local_component_structure.md @@ -0,0 +1,5 @@ +# modules_structure + +```{eval-rst} +.. automethod:: nf_core.pipelines.lint.PipelineLint.local_component_structure +``` diff --git a/tests/subworkflows/test_create.py b/tests/subworkflows/test_create.py index 48cb482260..704a23772e 100644 --- a/tests/subworkflows/test_create.py +++ b/tests/subworkflows/test_create.py @@ -19,7 +19,7 @@ def test_subworkflows_create_succeed(self): self.pipeline_dir, "test_subworkflow_local", "@author", True ) subworkflow_create.create() - assert Path(self.pipeline_dir, "subworkflows", "local", "test_subworkflow_local.nf").exists() + assert Path(self.pipeline_dir, "subworkflows", "local", "test_subworkflow_local/main.nf").exists() def test_subworkflows_create_fail_exists(self): """Fail at creating the same subworkflow twice""" @@ -29,7 +29,7 @@ def test_subworkflows_create_fail_exists(self): subworkflow_create.create() with pytest.raises(UserWarning) as excinfo: subworkflow_create.create() - assert "Subworkflow file exists already" in str(excinfo.value) + assert "subworkflow directory exists" in str(excinfo.value) def test_subworkflows_create_nfcore_modules(self): """Create a subworkflow in nf-core/modules clone""" From 732c78635f237e3edfdc7b88050d469299b1f49f Mon Sep 17 00:00:00 2001 From: awgymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:47:50 +0100 Subject: [PATCH 05/33] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/pipelines/lint/local_component_structure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py index e5e1a00dee..61c1d45e29 100644 --- a/nf_core/pipelines/lint/local_component_structure.py +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -12,7 +12,7 @@ def local_component_structure(self): modules/local/TOOL/SUBTOOL - Prior to nf-core/tools release 3.0.3 the directory structure allowed top-level `*.nf` files: + Prior to nf-core/tools release 3.1.0 the directory structure allowed top-level `*.nf` files: .. code-block:: bash @@ -22,7 +22,7 @@ def local_component_structure(self): for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): warned.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): - warned.append(f"{nf_file.name} in subworkflows/local should be moved to a TOOL/SUBTOOL/main.nf structure") + warned.append(f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure") # If there are modules installed in the wrong location passed = [] From 7ba6f79f7c6800bf5fe5f3c71f9358c393a12e72 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:17:00 +0100 Subject: [PATCH 06/33] feat(1929): allow missing files when linting local modules --- CHANGELOG.md | 9 ++++++++ nf_core/components/lint/__init__.py | 4 ++++ nf_core/modules/lint/__init__.py | 18 +++++++++++++++ nf_core/modules/lint/environment_yml.py | 11 +++++++++- nf_core/modules/lint/meta_yml.py | 12 ++++++---- nf_core/modules/lint/module_tests.py | 29 ++++++++++++++++++++++--- 6 files changed, 75 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 331e2fcb56..c910b9cd02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,11 +96,16 @@ ### Modules - add a panel around diff previews when updating ([#3246](https://github.com/nf-core/tools/pull/3246)) +- Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) ### Subworkflows +<<<<<<< HEAD - Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) - Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) +======= +- Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) +>>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) ### General @@ -110,6 +115,7 @@ - use correct `--profile` options for `nf-core subworkflows test` ([#3233](https://github.com/nf-core/tools/pull/3233)) - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) +<<<<<<< HEAD - fix headers in api docs ([#3323](https://github.com/nf-core/tools/pull/3323)) - handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276)) - Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274)) @@ -132,6 +138,9 @@ - Update codecov/codecov-action action to v5 ([#3283](https://github.com/nf-core/tools/pull/3283)) - Update gitpod/workspace-base Docker digest to 12853f7 ([#3309](https://github.com/nf-core/tools/pull/3309)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.2 ([#3325](https://github.com/nf-core/tools/pull/3325)) +======= +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) +>>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 69740135a8..96b611cc20 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -162,6 +162,10 @@ def _set_registry(self, registry) -> None: self.registry = registry log.debug(f"Registry set to {self.registry}") + @property + def local_module_exclude_tests(self): + return ["module_version", "module_changes", "modules_patch"] + @staticmethod def get_all_module_lint_tests(is_pipeline): if is_pipeline: diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 49012cff40..1ce9902a31 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -234,13 +234,31 @@ def lint_module( # TODO: consider unifying modules and subworkflows lint_module() function and add it to the ComponentLint class # Only check the main script in case of a local module if local: + mod.get_inputs_from_main_nf() + mod.get_outputs_from_main_nf() + # Update meta.yml file if requested + if self.fix and mod.meta_yml is not None: + self.update_meta_yml_file(mod) + + for test_name in self.lint_tests: + if test_name in self.local_module_exclude_tests: + continue + if test_name == "main_nf": + getattr(self, test_name)(mod, fix_version, self.registry, progress_bar) + elif test_name in ["meta_yml", "environment_yml"]: + # Allow files to be missing for local + getattr(self, test_name)(mod, allow_missing=True) + """ self.main_nf(mod, fix_version, self.registry, progress_bar) + """ + self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] if not self.fail_warned: self.warned += warned else: self.failed += warned + self.failed += [LintResult(mod, *m) for m in mod.failed] # Otherwise run all the lint tests else: diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 4488b0befa..6571f07ae8 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) -def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: +def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_missing: bool = False) -> None: """ Lint an ``environment.yml`` file. @@ -23,6 +23,15 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) env_yml = None # load the environment.yml file if module.environment_yml is None: + if allow_missing: + module.warned.append( + ( + "environment_yml_exists", + "Module's `environment.yml` does not exist", + Path(module.component_dir, "environment.yml"), + ), + ) + return raise LintExceptionError("Module does not have an `environment.yml` file") try: with open(module.environment_yml) as fh: diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index d0268a40cc..76345bdcb3 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -13,7 +13,7 @@ log = logging.getLogger(__name__) -def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: +def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_missing: bool = False) -> None: """ Lint a ``meta.yml`` file @@ -42,7 +42,13 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None module (NFCoreComponent): The module to lint """ - + if module.meta_yml is None: + if allow_missing: + module.warned.append( + ("meta_yml_exists", "Module `meta.yml` does not exist", Path(module.component_dir, "meta.yml")) + ) + return + raise LintExceptionError("Module does not have a `meta.yml` file") # Check if we have a patch file, get original file in that case meta_yaml = read_meta_yml(module_lint_object, module) if module.is_patched and module_lint_object.modules_repo.repo_path is not None: @@ -57,8 +63,6 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None if lines is not None: yaml = ruamel.yaml.YAML() meta_yaml = yaml.safe_load("".join(lines)) - if module.meta_yml is None: - raise LintExceptionError("Module does not have a `meta.yml` file") if meta_yaml is None: module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) return diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 8f5fc91425..6826b2e743 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -9,12 +9,13 @@ import yaml +from nf_core.components.lint import LintExceptionError from nf_core.components.nfcore_component import NFCoreComponent log = logging.getLogger(__name__) -def module_tests(_, module: NFCoreComponent): +def module_tests(_, module: NFCoreComponent, allow_missing: bool = False): """ Lint the tests of a module in ``nf-core/modules`` @@ -22,8 +23,30 @@ def module_tests(_, module: NFCoreComponent): and contains a ``main.nf.test`` and a ``main.nf.test.snap`` """ - if module.nftest_testdir is None or module.nftest_main_nf is None: - raise ValueError() + if module.nftest_testdir is None: + if allow_missing: + module.warned.append( + ( + "test_dir_exists", + "nf-test directory is missing", + Path(module.component_dir, "tests"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + + if module.nftest_main_nf is None: + if allow_missing: + module.warned.append( + ( + "test_main_nf_exists", + "test `main.nf.test` does not exist", + Path(module.component_dir, "tests", "main.nf.test"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] test_dir = Path(module.base_dir, "tests", "modules", repo_dir, module.component_name) pytest_main_nf = Path(test_dir, "main.nf") From 8a1d44a09941e040b5b6c0a2961ccb97f00c9a53 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:35:33 +0100 Subject: [PATCH 07/33] feat(1929): local linting option now *only* lints local modules. Add tests --- nf_core/modules/lint/__init__.py | 7 ++- nf_core/subworkflows/lint/__init__.py | 3 +- nf_core/subworkflows/lint/meta_yml.py | 24 ++++++++--- .../subworkflows/lint/subworkflow_tests.py | 29 +++++++++++-- tests/modules/test_lint.py | 43 ++++++++++++++++++- tests/subworkflows/test_lint.py | 39 ++++++++++++++++- 6 files changed, 129 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 1ce9902a31..af057b67fc 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -170,7 +170,7 @@ def lint( self.lint_modules(local_modules, registry=registry, local=True, fix_version=fix_version) # Lint nf-core modules - if len(remote_modules) > 0: + if not local and len(remote_modules) > 0: self.lint_modules(remote_modules, registry=registry, local=False, fix_version=fix_version) if print_results: @@ -248,9 +248,8 @@ def lint_module( elif test_name in ["meta_yml", "environment_yml"]: # Allow files to be missing for local getattr(self, test_name)(mod, allow_missing=True) - """ - self.main_nf(mod, fix_version, self.registry, progress_bar) - """ + else: + getattr(self, test_name)(mod) self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index cedae62f11..ee46f3f3b2 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -152,7 +152,7 @@ def lint( self.lint_subworkflows(local_subworkflows, registry=registry, local=True) # Lint nf-core subworkflows - if len(remote_subworkflows) > 0: + if not local and len(remote_subworkflows) > 0: self.lint_subworkflows(remote_subworkflows, registry=registry, local=False) if print_results: @@ -208,6 +208,7 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): # Only check the main script in case of a local subworkflow if local: self.main_nf(swf) + self.meta_yml(swf, allow_missing=True) self.passed += [LintResult(swf, *s) for s in swf.passed] warned = [LintResult(swf, *m) for m in (swf.warned + swf.failed)] if not self.fail_warned: diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index be282bc453..262c80f948 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -6,11 +6,12 @@ import yaml import nf_core.components.components_utils +from nf_core.components.lint import LintExceptionError log = logging.getLogger(__name__) -def meta_yml(subworkflow_lint_object, subworkflow): +def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): """ Lint a ``meta.yml`` file @@ -28,6 +29,18 @@ def meta_yml(subworkflow_lint_object, subworkflow): """ # Read the meta.yml file + if subworkflow.meta_yml is None: + if allow_missing: + subworkflow.warned.append( + ( + "meta_yml_exists", + "Subworkflow `meta.yml` does not exist", + Path(subworkflow.component_dir, "meta.yml"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `meta.yml` file") + try: with open(subworkflow.meta_yml) as fh: meta_yaml = yaml.safe_load(fh) @@ -49,7 +62,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {e.path[0]}.{e.path[2]} are indented correctly." subworkflow.failed.append( ( "meta_yml_valid", @@ -96,10 +109,9 @@ def meta_yml(subworkflow_lint_object, subworkflow): ) # confirm that all included components in ``main.nf`` are specified in ``meta.yml`` - included_components = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) - included_components = ( - included_components[0] + included_components[1] - ) # join included modules and included subworkflows in a single list + included_components_ = nf_core.components.components_utils.get_components_to_install(subworkflow.component_dir) + included_components = included_components_[0] + included_components_[1] + # join included modules and included subworkflows in a single list if "components" in meta_yaml: meta_components = [x for x in meta_yaml["components"]] for component in set(included_components): diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 1caf73d00b..74b170416d 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -9,12 +9,13 @@ import yaml +from nf_core.components.lint import LintExceptionError from nf_core.components.nfcore_component import NFCoreComponent log = logging.getLogger(__name__) -def subworkflow_tests(_, subworkflow: NFCoreComponent): +def subworkflow_tests(_, subworkflow: NFCoreComponent, allow_missing: bool = False): """ Lint the tests of a subworkflow in ``nf-core/modules`` @@ -23,8 +24,30 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ - if subworkflow.nftest_testdir is None or subworkflow.nftest_main_nf is None: - raise ValueError() + if subworkflow.nftest_testdir is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_dir_exists", + "nf-test directory is missing", + Path(subworkflow.component_dir, "tests"), + ) + ) + return + raise LintExceptionError("Module does not have a `tests` dir") + + if subworkflow.nftest_main_nf is None: + if allow_missing: + subworkflow.warned.append( + ( + "test_main_nf_exists", + "test `main.nf.test` does not exist", + Path(subworkflow.component_dir, "tests", "main.nf.test"), + ) + ) + return + raise LintExceptionError("Subworkflow does not have a `tests` dir") + repo_dir = subworkflow.component_dir.parts[ : subworkflow.component_dir.parts.index(subworkflow.component_name.split("/")[0]) ][-1] diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 5372807987..37f992a7c9 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -1,4 +1,5 @@ import json +import shutil from pathlib import Path from typing import Union @@ -158,7 +159,7 @@ ] -class TestModulesCreate(TestModules): +class TestModulesLint(TestModules): def _setup_patch(self, pipeline_dir: Union[str, Path], modify_module: bool): install_obj = nf_core.modules.install.ModuleInstall( pipeline_dir, @@ -760,6 +761,46 @@ def test_modules_empty_file_in_stub_snapshot(self): with open(snap_file, "w") as fh: fh.write(content) + def test_modules_lint_local(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + def test_modules_lint_local_missing_files(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore") + local = Path(self.pipeline_dir, "modules", "local", "trimgalore") + shutil.move(installed, local) + Path(self.pipeline_dir, "modules", "local", "trimgalore", "environment.yml").unlink() + Path(self.pipeline_dir, "modules", "local", "trimgalore", "meta.yml").unlink() + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + warnings = [x.message for x in module_lint.warned] + assert "Module's `environment.yml` does not exist" in warnings + assert "Module `meta.yml` does not exist" in warnings + + def test_modules_lint_local_old_format(self): + assert self.mods_install.install("trimgalore") + installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore", "main.nf") + Path(self.pipeline_dir, "modules", "local").mkdir() + local = Path(self.pipeline_dir, "modules", "local", "trimgalore.nf") + shutil.copy(installed, local) + self.mods_remove.remove("trimgalore", force=True) + module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) + module_lint.lint(print_results=False, local=True, all_modules=True) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + # A skeleton object with the passed/warned/failed list attrs # Use this in place of a ModuleLint object to test behaviour of diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index d94b55b3d3..49c671c088 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -31,7 +31,6 @@ def test_subworkflows_lint_new_subworkflow(self): subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.nfcore_modules) subworkflow_lint.lint(print_results=True, all_subworkflows=True) assert len(subworkflow_lint.failed) == 0 - assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -397,3 +396,41 @@ def test_subworkflows_empty_file_in_stub_snapshot(self): # reset the file with open(snap_file, "w") as fh: fh.write(content) + + def test_subworkflows_lint_local(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + + def test_subworkflows_lint_local_missing_files(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2") + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") + shutil.move(installed, local) + Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2", "meta.yml").unlink() + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + warnings = [x.message for x in subworkflow_lint.warned] + assert "Subworkflow `meta.yml` does not exist" in warnings + + def test_subworkflows_lint_local_old_format(self): + assert self.subworkflow_install.install("fastq_align_bowtie2") + installed = Path(self.pipeline_dir, "subworkflows", "nf-core", "fastq_align_bowtie2", "main.nf") + Path(self.pipeline_dir, "subworkflows", "local").mkdir(exist_ok=True) + local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2.nf") + shutil.copy(installed, local) + self.subworkflow_remove.remove("fastq_align_bowtie2", force=True) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) + subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 From 6c9c1b146bb69d87a513cfc0ff9d4fb0f5490e84 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:46:36 +0100 Subject: [PATCH 08/33] feat(1929): don't ask for a module/subworkflow name to lint if linting local --- nf_core/modules/lint/__init__.py | 3 +-- nf_core/subworkflows/lint/__init__.py | 2 +- tests/modules/test_lint.py | 6 +++--- tests/subworkflows/test_lint.py | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index af057b67fc..5f64953116 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -117,7 +117,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for module or all - if module is None and not all_modules and len(self.all_remote_components) > 0: + if module is None and not (local or all_modules) and len(self.all_remote_components) > 0: questions = [ { "type": "list", @@ -257,7 +257,6 @@ def lint_module( self.warned += warned else: self.failed += warned - self.failed += [LintResult(mod, *m) for m in mod.failed] # Otherwise run all the lint tests else: diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index ee46f3f3b2..8538cc3bf8 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -99,7 +99,7 @@ def lint( """ # TODO: consider unifying modules and subworkflows lint() function and add it to the ComponentLint class # Prompt for subworkflow or all - if subworkflow is None and not all_subworkflows: + if subworkflow is None and not (local or all_subworkflows): questions = [ { "type": "list", diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 37f992a7c9..38597b44a0 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -767,7 +767,7 @@ def test_modules_lint_local(self): local = Path(self.pipeline_dir, "modules", "local", "trimgalore") shutil.move(installed, local) module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) - module_lint.lint(print_results=False, local=True, all_modules=True) + module_lint.lint(print_results=False, local=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -780,7 +780,7 @@ def test_modules_lint_local_missing_files(self): Path(self.pipeline_dir, "modules", "local", "trimgalore", "environment.yml").unlink() Path(self.pipeline_dir, "modules", "local", "trimgalore", "meta.yml").unlink() module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) - module_lint.lint(print_results=False, local=True, all_modules=True) + module_lint.lint(print_results=False, local=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -796,7 +796,7 @@ def test_modules_lint_local_old_format(self): shutil.copy(installed, local) self.mods_remove.remove("trimgalore", force=True) module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) - module_lint.lint(print_results=False, local=True, all_modules=True) + module_lint.lint(print_results=False, local=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 diff --git a/tests/subworkflows/test_lint.py b/tests/subworkflows/test_lint.py index 49c671c088..8f6ff353a1 100644 --- a/tests/subworkflows/test_lint.py +++ b/tests/subworkflows/test_lint.py @@ -403,7 +403,7 @@ def test_subworkflows_lint_local(self): local = Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2") shutil.move(installed, local) subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + subworkflow_lint.lint(print_results=False, local=True) assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -415,7 +415,7 @@ def test_subworkflows_lint_local_missing_files(self): shutil.move(installed, local) Path(self.pipeline_dir, "subworkflows", "local", "fastq_align_bowtie2", "meta.yml").unlink() subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + subworkflow_lint.lint(print_results=False, local=True) assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -430,7 +430,7 @@ def test_subworkflows_lint_local_old_format(self): shutil.copy(installed, local) self.subworkflow_remove.remove("fastq_align_bowtie2", force=True) subworkflow_lint = nf_core.subworkflows.SubworkflowLint(directory=self.pipeline_dir) - subworkflow_lint.lint(print_results=False, local=True, all_subworkflows=True) + subworkflow_lint.lint(print_results=False, local=True) assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 From 19df0447c06c7b1e8266ca60a706039e686aa667 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:59:47 +0100 Subject: [PATCH 09/33] feat(1929): add test for new pipeline lint check of local component dirs --- .../lint/local_component_structure.py | 23 +++++++++------ .../lint/test_local_component_structure.py | 28 +++++++++++++++++++ 2 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 tests/pipelines/lint/test_local_component_structure.py diff --git a/nf_core/pipelines/lint/local_component_structure.py b/nf_core/pipelines/lint/local_component_structure.py index 61c1d45e29..71f02ba545 100644 --- a/nf_core/pipelines/lint/local_component_structure.py +++ b/nf_core/pipelines/lint/local_component_structure.py @@ -18,14 +18,21 @@ def local_component_structure(self): modules/local/modules/TOOL_SUBTOOL.nf """ - warned = [] + warned_mods = [] for nf_file in Path(self.wf_path, "modules", "local").glob("*.nf"): - warned.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") - for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): - warned.append(f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure") - + warned_mods.append(f"{nf_file.name} in modules/local should be moved to a TOOL/SUBTOOL/main.nf structure") # If there are modules installed in the wrong location passed = [] - if len(warned) == 0: - passed = ["modules directory structure is correct 'modules/nf-core/TOOL/SUBTOOL'"] - return {"passed": passed, "warned": warned, "failed": [], "ignored": []} + if len(warned_mods) == 0: + passed = ["local modules directory structure is correct 'modules/local/TOOL/SUBTOOL'"] + + warned_swfs = [] + for nf_file in Path(self.wf_path, "subworkflows", "local").glob("*.nf"): + warned_swfs.append( + f"{nf_file.name} in subworkflows/local should be moved to a SUBWORKFLOW_NAME/main.nf structure" + ) + + if len(warned_swfs) == 0: + passed = ["local subworkflows directory structure is correct 'subworkflows/local/TOOL/SUBTOOL'"] + + return {"passed": passed, "warned": warned_mods + warned_swfs, "failed": [], "ignored": []} diff --git a/tests/pipelines/lint/test_local_component_structure.py b/tests/pipelines/lint/test_local_component_structure.py new file mode 100644 index 0000000000..93dc3174a3 --- /dev/null +++ b/tests/pipelines/lint/test_local_component_structure.py @@ -0,0 +1,28 @@ +from pathlib import Path + +import nf_core.pipelines.lint + +from ..test_lint import TestLint + + +class TestLintLocalComponentStructure(TestLint): + def setUp(self) -> None: + super().setUp() + self.new_pipeline = self._make_pipeline_copy() + + def test_local_component_structure(self): + local_modules = Path(self.new_pipeline, "modules", "local") + local_swf = Path(self.new_pipeline, "subworkflows", "local") + local_modules.mkdir(parents=True, exist_ok=True) + local_swf.mkdir(parents=True, exist_ok=True) + + (local_modules / "dummy_module.nf").touch() + (local_swf / "dummy_subworkflow.nf").touch() + + lint_obj = nf_core.pipelines.lint.PipelineLint(self.new_pipeline) + lint_obj._load() + + results = lint_obj.local_component_structure() + assert len(results.get("warned", [])) == 2 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 From 5d1d44b07d0de940b69cbbf8c404d0412df65585 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:04:31 +0100 Subject: [PATCH 10/33] feat(1929): run todo linting on local subworkflows --- nf_core/subworkflows/lint/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index 8538cc3bf8..c1be26d0d5 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -209,6 +209,7 @@ def lint_subworkflow(self, swf, progress_bar, registry, local=False): if local: self.main_nf(swf) self.meta_yml(swf, allow_missing=True) + self.subworkflow_todos(swf) self.passed += [LintResult(swf, *s) for s in swf.passed] warned = [LintResult(swf, *m) for m in (swf.warned + swf.failed)] if not self.fail_warned: From 5689ff6258ae5bc8f24397392b6e7f0ca87dd4d9 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:30:13 +1030 Subject: [PATCH 11/33] fix: Update to latest CHANGELOG --- CHANGELOG.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c910b9cd02..ad20861f8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,11 @@ ### Modules +<<<<<<< HEAD - feat(1929): Make local modules use remote structure ([#3256](https://github.com/nf-core/tools/pull/3256)) +======= +>>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ### Subworkflows ### General @@ -96,16 +99,20 @@ ### Modules - add a panel around diff previews when updating ([#3246](https://github.com/nf-core/tools/pull/3246)) -- Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) ### Subworkflows +<<<<<<< HEAD <<<<<<< HEAD - Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) - Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) ======= - Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) >>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) +======= +- Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) +- Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) +>>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ### General @@ -116,6 +123,9 @@ - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) - fix headers in api docs ([#3323](https://github.com/nf-core/tools/pull/3323)) - handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276)) - Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274)) @@ -138,9 +148,12 @@ - Update codecov/codecov-action action to v5 ([#3283](https://github.com/nf-core/tools/pull/3283)) - Update gitpod/workspace-base Docker digest to 12853f7 ([#3309](https://github.com/nf-core/tools/pull/3309)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.2 ([#3325](https://github.com/nf-core/tools/pull/3325)) +<<<<<<< HEAD ======= - Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) >>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) +======= +>>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] From 3f76e999aa09694a4d9b90e8f2a08e956883cb2c Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:39:53 +1030 Subject: [PATCH 12/33] fix: Clean up CHANGELOG.md --- CHANGELOG.md | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad20861f8c..592d80ec62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,12 @@ ### Modules -<<<<<<< HEAD -- feat(1929): Make local modules use remote structure ([#3256](https://github.com/nf-core/tools/pull/3256)) +- Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) -======= ->>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ### Subworkflows +- Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) + ### General ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] @@ -102,17 +101,8 @@ ### Subworkflows -<<<<<<< HEAD -<<<<<<< HEAD -- Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) -- Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) -======= -- Subworkflows created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) ->>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) -======= - Add `nf-core subworkflows patch` command ([#2861](https://github.com/nf-core/tools/pull/2861)) - Improve subworkflow nf-test migration warning ([#3298](https://github.com/nf-core/tools/pull/3298)) ->>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ### General @@ -122,10 +112,6 @@ - use correct `--profile` options for `nf-core subworkflows test` ([#3233](https://github.com/nf-core/tools/pull/3233)) - Update GitHub Actions ([#3237](https://github.com/nf-core/tools/pull/3237)) - add `--dir/-d` option to schema commands ([#3247](https://github.com/nf-core/tools/pull/3247)) -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) - fix headers in api docs ([#3323](https://github.com/nf-core/tools/pull/3323)) - handle new schema structure in `nf-core pipelines create-params-file` ([#3276](https://github.com/nf-core/tools/pull/3276)) - Update Gitpod image to use Miniforge instead of Miniconda([#3274](https://github.com/nf-core/tools/pull/3274)) @@ -148,12 +134,6 @@ - Update codecov/codecov-action action to v5 ([#3283](https://github.com/nf-core/tools/pull/3283)) - Update gitpod/workspace-base Docker digest to 12853f7 ([#3309](https://github.com/nf-core/tools/pull/3309)) - Update pre-commit hook astral-sh/ruff-pre-commit to v0.8.2 ([#3325](https://github.com/nf-core/tools/pull/3325)) -<<<<<<< HEAD -======= -- Update pre-commit hook astral-sh/ruff-pre-commit to v0.7.1 ([#3250](https://github.com/nf-core/tools/pull/3250)) ->>>>>>> 31cffdd5 (feat(1929): allow missing files when linting local modules) -======= ->>>>>>> cb7c0d7b (fix: Update to latest CHANGELOG) ## [v3.0.2 - Titanium Tapir Patch](https://github.com/nf-core/tools/releases/tag/3.0.2) - [2024-10-11] From 660c8a7801c80f66edda3778681849874ef0b71f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 21 Jan 2025 08:16:11 +0000 Subject: [PATCH 13/33] Update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3.1.2 --- .pre-commit-config.yaml | 2 +- nf_core/pipeline-template/.pre-commit-config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1827289b76..54eedaff9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - prettier@3.3.3 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "3.0.3" + rev: "3.1.2" hooks: - id: editorconfig-checker alias: ec diff --git a/nf_core/pipeline-template/.pre-commit-config.yaml b/nf_core/pipeline-template/.pre-commit-config.yaml index 9e9f0e1c4e..1dec865026 100644 --- a/nf_core/pipeline-template/.pre-commit-config.yaml +++ b/nf_core/pipeline-template/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: - prettier@3.2.5 - repo: https://github.com/editorconfig-checker/editorconfig-checker.python - rev: "3.0.3" + rev: "3.1.2" hooks: - id: editorconfig-checker alias: ec From 8d336624f200b7e6cf7588990fa992ca93359c78 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 08:16:57 +0000 Subject: [PATCH 14/33] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5aa7f22fe..dfa6701501 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### General +- Update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3.1.2 ([#3414](https://github.com/nf-core/tools/pull/3414)) + ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] ### Template From d1f6dc7f2d348be313e867066448e2720b02c19a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 21 Jan 2025 14:43:13 +0100 Subject: [PATCH 15/33] Fix 'process.shell' in nextflow.config --- nf_core/pipeline-template/nextflow.config | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 5bbe47d9d9..217b1c254d 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -238,14 +238,13 @@ env { } // Set bash options -process.shell = """\ -bash - -set -e # Exit if a tool returns a non-zero status/exit code -set -u # Treat unset variables and parameters as an error -set -o pipefail # Returns the status of the last command to exit with a non-zero status or zero if all successfully execute -set -C # No clobber - prevent output redirection from overwriting files. -""" +process.shell = [ + "bash", + "-C", // No clobber - prevent output redirection from overwriting files. + "-e", // Exit if a tool returns a non-zero status/exit code + "-u", // Treat unset variables and parameters as an error + "-o pipefail" // Returns the status of the last command to exit with a non-zero status or zero if all successfully execute +] // Disable process selector warnings by default. Use debug profile to enable warnings. nextflow.enable.configProcessNamesValidation = false From 702e00c6e6c3d7663e4865b469cf915c4ab92aaf Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 13:59:49 +0000 Subject: [PATCH 16/33] [automated] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5aa7f22fe..ecb4fcb935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### General +- Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) + ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] ### Template From bee05205db39bcb8e680e588402f9a3a74d3ca8d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 21 Jan 2025 23:00:16 +0100 Subject: [PATCH 17/33] Remove automated release tweets --- .../workflows/release-announcements.yml | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index e1b654d34b..820cc49695 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -27,39 +27,6 @@ jobs: ${{ steps.get_topics.outputs.topics }} #nfcore #openscience #nextflow #bioinformatics - send-tweet: - runs-on: ubuntu-latest - - steps: - - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5 - with: - python-version: "3.10" - - name: Install dependencies - run: pip install tweepy==4.14.0 - - name: Send tweet - shell: python - run: | - import os - import tweepy - - client = tweepy.Client( - access_token=os.getenv("TWITTER_ACCESS_TOKEN"), - access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"), - consumer_key=os.getenv("TWITTER_CONSUMER_KEY"), - consumer_secret=os.getenv("TWITTER_CONSUMER_SECRET"), - ) - tweet = os.getenv("TWEET") - client.create_tweet(text=tweet) - env: - TWEET: | - Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! - - Please see the changelog: ${{ github.event.release.html_url }} - TWITTER_CONSUMER_KEY: ${{ secrets.TWITTER_CONSUMER_KEY }} - TWITTER_CONSUMER_SECRET: ${{ secrets.TWITTER_CONSUMER_SECRET }} - TWITTER_ACCESS_TOKEN: ${{ secrets.TWITTER_ACCESS_TOKEN }} - TWITTER_ACCESS_TOKEN_SECRET: ${{ secrets.TWITTER_ACCESS_TOKEN_SECRET }} - bsky-post: runs-on: ubuntu-latest steps: From 078e4ee9e6edc435fc874976d5de5a922e1e9d52 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 22:01:35 +0000 Subject: [PATCH 18/33] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecb4fcb935..a09eb4e9fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ ### General - Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) +- Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] From 48ee89cfab8d739dfe22668a500923f025047a88 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 21 Jan 2025 22:09:51 +0000 Subject: [PATCH 19/33] [automated] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a09eb4e9fc..5045622b41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) + ### Linting ### Modules @@ -13,7 +15,6 @@ ### General - Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) -- Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] From 3dd2e00c6b42e25744287f7d5f3ec4b75446dba2 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Wed, 22 Jan 2025 20:17:02 +1030 Subject: [PATCH 20/33] fix: broken test. Need to assign component_dir to the parent dir if a local module is an nf file only --- nf_core/components/nfcore_component.py | 1 + tests/modules/test_lint.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 03ee51b18a..b04cb71e35 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -97,6 +97,7 @@ def __init__( self.nftest_main_nf = p if (p := Path(self.nftest_testdir, "main.nf.test")).exists() else None else: self.main_nf = self.component_dir + self.component_dir = self.component_dir.parent self.meta_yml = None self.environment_yml = None self.nftest_testdir = None diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 38597b44a0..b71c729d1c 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -789,11 +789,11 @@ def test_modules_lint_local_missing_files(self): assert "Module `meta.yml` does not exist" in warnings def test_modules_lint_local_old_format(self): + Path(self.pipeline_dir, "modules", "local").mkdir() assert self.mods_install.install("trimgalore") installed = Path(self.pipeline_dir, "modules", "nf-core", "trimgalore", "main.nf") - Path(self.pipeline_dir, "modules", "local").mkdir() local = Path(self.pipeline_dir, "modules", "local", "trimgalore.nf") - shutil.copy(installed, local) + shutil.move(installed, local) self.mods_remove.remove("trimgalore", force=True) module_lint = nf_core.modules.lint.ModuleLint(directory=self.pipeline_dir) module_lint.lint(print_results=False, local=True) From 262dea395dba2990dcfd7d607fd521153239c270 Mon Sep 17 00:00:00 2001 From: Arthur Gymer <24782660+awgymer@users.noreply.github.com> Date: Thu, 23 Jan 2025 08:08:25 +1030 Subject: [PATCH 21/33] fix: Add new lint test to lint config pydantic model --- nf_core/pipelines/lint/__init__.py | 2 +- nf_core/utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/pipelines/lint/__init__.py b/nf_core/pipelines/lint/__init__.py index 7f15b092cc..36f1bfca52 100644 --- a/nf_core/pipelines/lint/__init__.py +++ b/nf_core/pipelines/lint/__init__.py @@ -27,7 +27,7 @@ from nf_core import __version__ from nf_core.components.lint import ComponentLint from nf_core.pipelines.lint_utils import console -from nf_core.utils import NFCoreYamlConfig, NFCoreYamlLintConfig, strip_ansi_codes +from nf_core.utils import NFCoreYamlLintConfig, strip_ansi_codes from nf_core.utils import plural_s as _s from .actions_awsfulltest import actions_awsfulltest diff --git a/nf_core/utils.py b/nf_core/utils.py index 27334d473c..c0d7029e46 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -1244,6 +1244,8 @@ class NFCoreYamlLintConfig(BaseModel): """ Lint for version consistency """ included_configs: Optional[bool] = None """ Lint for included configs """ + local_component_structure: Optional[bool] = None + """ Lint local components use correct structure mirroring remote""" def __getitem__(self, item: str) -> Any: return getattr(self, item) From 583809cfacbbc60fd3bb14c9523af736e4b14b69 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 08:15:40 +0100 Subject: [PATCH 22/33] Update python:3.12-slim Docker digest to 123be56 (#3421) * Update python:3.12-slim Docker digest to 123be56 * [automated] Update CHANGELOG.md --------- Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: nf-core-bot --- CHANGELOG.md | 1 + Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5045622b41..e67dce371e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ### General - Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) +- Update python:3.12-slim Docker digest to 123be56 ([#3421](https://github.com/nf-core/tools/pull/3421)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] diff --git a/Dockerfile b/Dockerfile index f5a6796a27..d26381a066 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim@sha256:10f3aaab98db50cba827d3b33a91f39dc9ec2d02ca9b85cbc5008220d07b17f3 +FROM python:3.12-slim@sha256:123be5684f39d8476e64f47a5fddf38f5e9d839baff5c023c815ae5bdfae0df7 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ description="Docker image containing requirements for nf-core/tools" From 1283406814592c2a1b5e51bc3f819d948906678b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 21:14:52 +0000 Subject: [PATCH 23/33] Update pre-commit hook astral-sh/ruff-pre-commit to v0.9.3 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1827289b76..b902ab3ea0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.6 + rev: v0.9.3 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix From 52e8d4b90c9b8a99961f32514d541aeabb69ee6f Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 23 Jan 2025 22:25:38 +0000 Subject: [PATCH 24/33] [automated] Fix code linting --- nf_core/components/components_differ.py | 4 +--- nf_core/components/info.py | 4 ++-- nf_core/components/lint/__init__.py | 2 +- nf_core/components/remove.py | 2 +- nf_core/modules/lint/meta_yml.py | 2 +- nf_core/pipelines/download.py | 6 +++--- nf_core/pipelines/lint/multiqc_config.py | 2 +- nf_core/pipelines/lint/version_consistency.py | 2 +- nf_core/pipelines/params_file.py | 4 ++-- nf_core/pipelines/rocrate.py | 2 +- nf_core/pipelines/schema.py | 2 +- nf_core/pipelines/sync.py | 2 +- nf_core/subworkflows/lint/meta_yml.py | 2 +- tests/modules/test_lint.py | 18 +++++++++--------- tests/pipelines/test_create.py | 10 ++++++---- 15 files changed, 32 insertions(+), 32 deletions(-) diff --git a/nf_core/components/components_differ.py b/nf_core/components/components_differ.py index db51c1910d..18454471f4 100644 --- a/nf_core/components/components_differ.py +++ b/nf_core/components/components_differ.py @@ -271,9 +271,7 @@ def print_diff( console = Console(force_terminal=nf_core.utils.rich_force_colors()) if current_version is not None and new_version is not None: log.info( - f"Changes in component '{Path(repo_path, component)}' between" - f" ({current_version}) and" - f" ({new_version})" + f"Changes in component '{Path(repo_path, component)}' between ({current_version}) and ({new_version})" ) else: log.info(f"Changes in component '{Path(repo_path, component)}'") diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 31769785a1..4cf7dc946c 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -265,9 +265,9 @@ def generate_component_info_help(self): intro_text.append( Text.from_markup( ":globe_with_meridians: Repository: " - f"{ '[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" + f"{'[link={self.remote_location}]' if self.remote_location.startswith('http') else ''}" f"{self.remote_location}" - f"{'[/link]' if self.remote_location.startswith('http') else '' }" + f"{'[/link]' if self.remote_location.startswith('http') else ''}" "\n" ) ) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index 69740135a8..1e1dad9c34 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -102,7 +102,7 @@ def __init__( continue if isinstance(components, str): raise LookupError( - f"Error parsing modules.json: {components}. " f"Please check the file for errors or try again." + f"Error parsing modules.json: {components}. Please check the file for errors or try again." ) for org, comp in components: self.all_remote_components.append( diff --git a/nf_core/components/remove.py b/nf_core/components/remove.py index 37208629c0..316b8e7cb4 100644 --- a/nf_core/components/remove.py +++ b/nf_core/components/remove.py @@ -176,6 +176,6 @@ def remove(self, component, removed_by=None, removed_components=None, force=Fals f"Did not remove '{component}', because it was also manually installed. Only updated 'installed_by' entry in modules.json." ) log.info( - f"""Did not remove {self.component_type[:-1]} '{component}', because it was also installed by {', '.join(f"'{d}'" for d in installed_by)}. Only updated the 'installed_by' entry in modules.json.""" + f"""Did not remove {self.component_type[:-1]} '{component}', because it was also installed by {", ".join(f"'{d}'" for d in installed_by)}. Only updated the 'installed_by' entry in modules.json.""" ) return removed diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index d0268a40cc..328c15067f 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -78,7 +78,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {str(e.path[0])+'.'+str(e.path[2])} are indented correctly." + hint = f"\nCheck that the child entries of {str(e.path[0]) + '.' + str(e.path[2])} are indented correctly." if e.schema and isinstance(e.schema, dict) and "message" in e.schema: e.message = e.schema["message"] incorrect_value = meta_yaml diff --git a/nf_core/pipelines/download.py b/nf_core/pipelines/download.py index c0f6e8c2f3..3d371ca681 100644 --- a/nf_core/pipelines/download.py +++ b/nf_core/pipelines/download.py @@ -202,7 +202,7 @@ def download_workflow(self): raise DownloadError(e) from e summary_log = [ - f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'", + f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0] + ',[' + str(len(self.revision) - 2) + ' more revisions],' + self.revision[-1]}'", f"Use containers: '{self.container_system}'", ] if self.container_system: @@ -1908,8 +1908,8 @@ def __init__( log.error(self.error_type.message) log.info(self.error_type.helpmessage) - log.debug(f'Failed command:\n{" ".join(singularity_command)}') - log.debug(f'Singularity error messages:\n{"".join(error_msg)}') + log.debug(f"Failed command:\n{' '.join(singularity_command)}") + log.debug(f"Singularity error messages:\n{''.join(error_msg)}") raise self.error_type diff --git a/nf_core/pipelines/lint/multiqc_config.py b/nf_core/pipelines/lint/multiqc_config.py index fec5b518e3..49f632faa8 100644 --- a/nf_core/pipelines/lint/multiqc_config.py +++ b/nf_core/pipelines/lint/multiqc_config.py @@ -122,7 +122,7 @@ def multiqc_config(self) -> Dict[str, List[str]]: f"The expected comment is: \n" f"```{hint}``` \n" f"The current comment is: \n" - f"```{ mqc_yml['report_comment'].strip()}```" + f"```{mqc_yml['report_comment'].strip()}```" ) else: passed.append("`assets/multiqc_config.yml` contains a matching 'report_comment'.") diff --git a/nf_core/pipelines/lint/version_consistency.py b/nf_core/pipelines/lint/version_consistency.py index 2f9cead83c..e347e8e825 100644 --- a/nf_core/pipelines/lint/version_consistency.py +++ b/nf_core/pipelines/lint/version_consistency.py @@ -53,7 +53,7 @@ def version_consistency(self): # Check if they are consistent if len(set(versions.values())) != 1: failed.append( - "The versioning is not consistent between container, release tag " "and config. Found {}".format( + "The versioning is not consistent between container, release tag and config. Found {}".format( ", ".join([f"{k} = {v}" for k, v in versions.items()]) ) ) diff --git a/nf_core/pipelines/params_file.py b/nf_core/pipelines/params_file.py index 69326c142d..48e4dfe46c 100644 --- a/nf_core/pipelines/params_file.py +++ b/nf_core/pipelines/params_file.py @@ -19,7 +19,7 @@ "of nextflow run with the {pipeline_name} pipeline." ) -USAGE = "Uncomment lines with a single '#' if you want to pass the parameter " "to the pipeline." +USAGE = "Uncomment lines with a single '#' if you want to pass the parameter to the pipeline." H1_SEPERATOR = "## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" H2_SEPERATOR = "## ----------------------------------------------------------------------------" @@ -107,7 +107,7 @@ def get_pipeline(self) -> Optional[bool]: # Prompt for pipeline if not supplied if self.pipeline is None: launch_type = questionary.select( - "Generate parameter file for local pipeline " "or remote GitHub pipeline?", + "Generate parameter file for local pipeline or remote GitHub pipeline?", choices=["Remote pipeline", "Local path"], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() diff --git a/nf_core/pipelines/rocrate.py b/nf_core/pipelines/rocrate.py index b0f9611f1f..2bf46beede 100644 --- a/nf_core/pipelines/rocrate.py +++ b/nf_core/pipelines/rocrate.py @@ -212,7 +212,7 @@ def set_main_entity(self, main_entity_filename: str): else: url = self.version self.crate.mainEntity.append_to( - "url", f"https://nf-co.re/{self.crate.name.replace('nf-core/','')}/{url}/", compact=True + "url", f"https://nf-co.re/{self.crate.name.replace('nf-core/', '')}/{url}/", compact=True ) self.crate.mainEntity.append_to("version", self.version, compact=True) diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index b425ec64ed..47990b1064 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -770,7 +770,7 @@ def build_schema(self, pipeline_dir, no_prompts, web_only, url): if self.web_schema_build_web_url: log.info( "To save your work, open {}\n" - f"Click the blue 'Finished' button, copy the schema and paste into this file: { self.web_schema_build_web_url, self.schema_filename}" + f"Click the blue 'Finished' button, copy the schema and paste into this file: {self.web_schema_build_web_url, self.schema_filename}" ) return False diff --git a/nf_core/pipelines/sync.py b/nf_core/pipelines/sync.py index 14365da3f8..f0c8f4a321 100644 --- a/nf_core/pipelines/sync.py +++ b/nf_core/pipelines/sync.py @@ -345,7 +345,7 @@ def create_merge_base_branch(self): if merge_branch_format.match(branch) ] ) - new_branch = f"{self.original_merge_branch}-{max_branch+1}" + new_branch = f"{self.original_merge_branch}-{max_branch + 1}" log.info(f"Branch already existed: '{self.merge_branch}', creating branch '{new_branch}' instead.") self.merge_branch = new_branch diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index be282bc453..0de5ea626b 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -49,7 +49,7 @@ def meta_yml(subworkflow_lint_object, subworkflow): if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {e.path[0] + '.' + e.path[2]} are indented correctly." subworkflow.failed.append( ( "meta_yml_valid", diff --git a/tests/modules/test_lint.py b/tests/modules/test_lint.py index 5372807987..b6e816d37e 100644 --- a/tests/modules/test_lint.py +++ b/tests/modules/test_lint.py @@ -295,15 +295,15 @@ def test_modules_lint_check_url(self): if line.strip(): check_container_link_line(mocked_ModuleLint, line, registry="quay.io") - assert ( - len(mocked_ModuleLint.passed) == passed - ), f"{test}: Expected {passed} PASS, got {len(mocked_ModuleLint.passed)}." - assert ( - len(mocked_ModuleLint.warned) == warned - ), f"{test}: Expected {warned} WARN, got {len(mocked_ModuleLint.warned)}." - assert ( - len(mocked_ModuleLint.failed) == failed - ), f"{test}: Expected {failed} FAIL, got {len(mocked_ModuleLint.failed)}." + assert len(mocked_ModuleLint.passed) == passed, ( + f"{test}: Expected {passed} PASS, got {len(mocked_ModuleLint.passed)}." + ) + assert len(mocked_ModuleLint.warned) == warned, ( + f"{test}: Expected {warned} WARN, got {len(mocked_ModuleLint.warned)}." + ) + assert len(mocked_ModuleLint.failed) == failed, ( + f"{test}: Expected {failed} FAIL, got {len(mocked_ModuleLint.failed)}." + ) def test_modules_lint_update_meta_yml(self): """update the meta.yml of a module""" diff --git a/tests/pipelines/test_create.py b/tests/pipelines/test_create.py index f83cc274fc..7c0eebe7d6 100644 --- a/tests/pipelines/test_create.py +++ b/tests/pipelines/test_create.py @@ -158,9 +158,9 @@ def test_template_customisation_all_files_grouping(self): str_path = str((Path(root) / file).relative_to(PIPELINE_TEMPLATE)) if str_path not in base_required_files: try: - assert ( - str_path in all_skipped_files - ), f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + assert str_path in all_skipped_files, ( + f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + ) except AssertionError: if "/" in str_path: # Check if the parent directory is in the skipped files @@ -170,6 +170,8 @@ def test_template_customisation_all_files_grouping(self): if upper_dir in all_skipped_files: upper_dir_present = True break - assert upper_dir_present, f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + assert upper_dir_present, ( + f"Template file `{str_path}` not present in a group for pipeline customisation in `template_features.yml`." + ) else: raise From 3be4b603e9ff10e71deb0c7e165a6542d8c4538f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 09:39:54 +0100 Subject: [PATCH 25/33] chore(deps): update dawidd6/action-download-artifact action to v8 (#3424) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- nf_core/pipeline-template/.github/workflows/linting_comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 63b20bb311..b0b6755858 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7 + uses: dawidd6/action-download-artifact@20319c5641d495c8a52e688b7dc5fada6c3a9fbc # v8 with: workflow: linting.yml workflow_conclusion: completed From 61d971045728307963752d534ad3f6898ab49071 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Jan 2025 09:53:22 +0100 Subject: [PATCH 26/33] fix mypy error --- nf_core/modules/lint/meta_yml.py | 2 +- nf_core/subworkflows/lint/meta_yml.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 13eee2ef96..977ea819ae 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -62,7 +62,7 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent, allow_m ).get("meta.yml") if lines is not None: yaml = ruamel.yaml.YAML() - meta_yaml = yaml.safe_load("".join(lines)) + meta_yaml = yaml.load("".join(lines)) if meta_yaml is None: module.failed.append(("meta_yml_exists", "Module `meta.yml` does not exist", module.meta_yml)) return diff --git a/nf_core/subworkflows/lint/meta_yml.py b/nf_core/subworkflows/lint/meta_yml.py index 3c63bf7353..8a2120ed0a 100644 --- a/nf_core/subworkflows/lint/meta_yml.py +++ b/nf_core/subworkflows/lint/meta_yml.py @@ -3,7 +3,7 @@ from pathlib import Path import jsonschema.validators -import yaml +import ruamel.yaml import nf_core.components.components_utils from nf_core.components.lint import LintExceptionError @@ -43,7 +43,8 @@ def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): try: with open(subworkflow.meta_yml) as fh: - meta_yaml = yaml.safe_load(fh) + yaml = ruamel.yaml.YAML(typ="safe") + meta_yaml = yaml.load(fh) subworkflow.passed.append(("meta_yml_exists", "Subworkflow `meta.yml` exists", subworkflow.meta_yml)) except FileNotFoundError: subworkflow.failed.append(("meta_yml_exists", "Subworkflow `meta.yml` does not exist", subworkflow.meta_yml)) @@ -62,7 +63,7 @@ def meta_yml(subworkflow_lint_object, subworkflow, allow_missing: bool = False): if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0] + '.' + e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {str(e.path[0]) + '.' + str(e.path[2])} are indented correctly." subworkflow.failed.append( ( "meta_yml_valid", From f8b099c5c0db44ef2f54b01329ee149e4c6be8ae Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 27 Jan 2025 10:55:28 +0100 Subject: [PATCH 27/33] Separate -o pipefail into separate list elements --- nf_core/pipeline-template/nextflow.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 217b1c254d..d58ca72196 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -243,7 +243,8 @@ process.shell = [ "-C", // No clobber - prevent output redirection from overwriting files. "-e", // Exit if a tool returns a non-zero status/exit code "-u", // Treat unset variables and parameters as an error - "-o pipefail" // Returns the status of the last command to exit with a non-zero status or zero if all successfully execute + "-o", // Returns the status of the last command to exit.. + "pipefail" // ..with a non-zero status or zero if all successfully execute ] // Disable process selector warnings by default. Use debug profile to enable warnings. From 2d7e1a6cfea4549560468ccdeafb922e38cdbe09 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 27 Jan 2025 09:57:21 +0000 Subject: [PATCH 28/33] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00a8abd87b..0ae99a0057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) - Update python:3.12-slim Docker digest to 123be56 ([#3421](https://github.com/nf-core/tools/pull/3421)) +- `-o pipefail` newline ([#3425](https://github.com/nf-core/tools/pull/3425)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20] From 008dabba3ae0825786ff2d92958fcfde3a5db4d1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Jan 2025 11:46:39 +0100 Subject: [PATCH 29/33] update template components --- nf_core/pipeline-template/modules.json | 4 ++-- .../modules/nf-core/fastqc/main.nf | 20 ++++++++-------- .../modules/nf-core/fastqc/tests/tags.yml | 2 -- .../modules/nf-core/multiqc/environment.yml | 2 +- .../modules/nf-core/multiqc/main.nf | 4 ++-- .../nf-core/multiqc/tests/main.nf.test.snap | 24 +++++++++---------- 6 files changed, 27 insertions(+), 29 deletions(-) delete mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 7d2761d290..750826ffac 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -8,12 +8,12 @@ {%- if fastqc %} "fastqc": { "branch": "master", - "git_sha": "dc94b6ee04a05ddb9f7ae050712ff30a13149164", + "git_sha": "08108058ea36a63f141c25c4e75f9f872a5b2296", "installed_by": ["modules"] }{% endif %}{%- if multiqc %}{% if fastqc %},{% endif %} "multiqc": { "branch": "master", - "git_sha": "cf17ca47590cc578dfb47db1c2a44ef86f89976d", + "git_sha": "f0719ae309075ae4a291533883847c3f7c441dad", "installed_by": ["modules"] } {%- endif %} diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 752c3a10c6..033f4154a4 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -1,5 +1,5 @@ process FASTQC { - tag "$meta.id" + tag "${meta.id}" label 'process_medium' conda "${moduleDir}/environment.yml" @@ -19,30 +19,30 @@ process FASTQC { task.ext.when == null || task.ext.when script: - def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" // Make list of old name and new name pairs to use for renaming in the bash while loop def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } - def rename_to = old_new_pairs*.join(' ').join(' ') + def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ _old_name, new_name -> new_name }.join(' ') // The total amount of allocated RAM by FastQC is equal to the number of threads defined (--threads) time the amount of RAM defined (--memory) // https://github.com/s-andrews/FastQC/blob/1faeea0412093224d7f6a07f777fad60a5650795/fastqc#L211-L222 // Dividing the task.memory by task.cpu allows to stick to requested amount of RAM in the label - def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') / task.cpus + def memory_in_mb = task.memory ? task.memory.toUnit('MB').toFloat() / task.cpus : null // FastQC memory value allowed range (100 - 10000) def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) """ - printf "%s %s\\n" $rename_to | while read old_name new_name; do + printf "%s %s\\n" ${rename_to} | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name done fastqc \\ - $args \\ - --threads $task.cpus \\ - --memory $fastqc_memory \\ - $renamed_files + ${args} \\ + --threads ${task.cpus} \\ + --memory ${fastqc_memory} \\ + ${renamed_files} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml deleted file mode 100644 index 7834294ba0..0000000000 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -fastqc: - - modules/nf-core/fastqc/** diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml index 6f5b867b76..a27122ce1a 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml @@ -2,4 +2,4 @@ channels: - conda-forge - bioconda dependencies: - - bioconda::multiqc=1.25.1 + - bioconda::multiqc=1.27 diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index cc0643e1d5..58d9313c6e 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -3,8 +3,8 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.25.1--pyhdfd78af_0' : - 'biocontainers/multiqc:1.25.1--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.27--pyhdfd78af_0' : + 'biocontainers/multiqc:1.27--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap index 2fcbb5ff7d..7b7c132205 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + "versions.yml:md5,8f3b8c1cec5388cf2708be948c9fa42f" ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.4" }, - "timestamp": "2024-10-02T17:51:46.317523" + "timestamp": "2025-01-27T09:29:57.631982377" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + "versions.yml:md5,8f3b8c1cec5388cf2708be948c9fa42f" ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.4" }, - "timestamp": "2024-10-02T17:52:20.680978" + "timestamp": "2025-01-27T09:30:34.743726958" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,41f391dcedce7f93ca188f3a3ffa0916" + "versions.yml:md5,8f3b8c1cec5388cf2708be948c9fa42f" ] ], "meta": { - "nf-test": "0.9.0", - "nextflow": "24.04.4" + "nf-test": "0.9.2", + "nextflow": "24.10.4" }, - "timestamp": "2024-10-02T17:52:09.185842" + "timestamp": "2025-01-27T09:30:21.44383553" } } \ No newline at end of file From ae92258b7307308a806990d718604db39bb6b3c0 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Mon, 27 Jan 2025 10:48:50 +0000 Subject: [PATCH 30/33] [automated] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a21e3e05..8884ad042b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) +- Update template components ([#3426](https://github.com/nf-core/tools/pull/3426)) ### Linting From 92408c301396b1b39785537006bb046d3f530764 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Jan 2025 12:09:04 +0100 Subject: [PATCH 31/33] bump to v3.2.0 --- .gitpod.yml | 2 +- CHANGELOG.md | 4 +--- setup.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitpod.yml b/.gitpod.yml index d5948695bf..db31d01bed 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,4 +1,4 @@ -image: nfcore/gitpod:dev +image: nfcore/gitpod:latest tasks: - name: install current state of nf-core/tools and setup pre-commit command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 8884ad042b..6fcaface96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,12 @@ # nf-core/tools: Changelog -## v3.2.0dev +## [v3.2.0 - Pewter Pangolin](https://github.com/nf-core/tools/releases/tag/3.2.0) - [2025-01-27] ### Template - Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) - Update template components ([#3426](https://github.com/nf-core/tools/pull/3426)) -### Linting - ### Modules - Modules created in pipelines "local" dir now use the full template ([#3256](https://github.com/nf-core/tools/pull/3256)) diff --git a/setup.py b/setup.py index 4a142c20fd..d426478786 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "3.2.0dev" +version = "3.2.0" with open("README.md") as f: readme = f.read() From 0d9a4f13a4390d85674d93cc8e71adc3fab0d1af Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Jan 2025 12:31:43 +0100 Subject: [PATCH 32/33] template line break fixes --- nf_core/pipeline-template/docs/output.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/docs/output.md b/nf_core/pipeline-template/docs/output.md index a9be6620e1..d9bc3a188f 100644 --- a/nf_core/pipeline-template/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -15,7 +15,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d {% if fastqc -%} - [FastQC](#fastqc) - Raw read QC{% endif %} - {%- if multiqc -%} + {%- if multiqc %} - [MultiQC](#multiqc) - Aggregate report describing results and QC from the whole pipeline {%- endif %} - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution @@ -34,7 +34,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d [FastQC](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/) gives general quality metrics about your sequenced reads. It provides information about the quality score distribution across your reads, per base sequence content (%A/T/G/C), adapter contamination and overrepresented sequences. For further reading and documentation see the [FastQC help pages](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/). -{%- endif -%} +{%- endif %} {% if multiqc -%} @@ -53,7 +53,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d [MultiQC](http://multiqc.info) is a visualization tool that generates a single HTML report summarising all samples in your project. Most of the pipeline QC results are visualised in the report and further statistics are available in the report data directory. Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQC. The pipeline has special steps which also allow the software versions to be reported in the MultiQC output for future traceability. For more information about how to use MultiQC reports, see . -{%- endif -%} +{%- endif %} ### Pipeline information From 381a6b49c744bf9223222e4c7ce06102f0f0a350 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 27 Jan 2025 13:51:16 +0100 Subject: [PATCH 33/33] update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fcaface96..7527b374cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Remove automated release tweets ([#3419](https://github.com/nf-core/tools/pull/3419)) - Update template components ([#3426](https://github.com/nf-core/tools/pull/3426)) +- Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) and split into new lines ([#3425](https://github.com/nf-core/tools/pull/3425)) ### Modules @@ -18,9 +19,7 @@ ### General - Update pre-commit hook editorconfig-checker/editorconfig-checker.python to v3.1.2 ([#3414](https://github.com/nf-core/tools/pull/3414)) -- Fix `process.shell` in `nextflow.config` ([#3416](https://github.com/nf-core/tools/pull/3416)) - Update python:3.12-slim Docker digest to 123be56 ([#3421](https://github.com/nf-core/tools/pull/3421)) -- `-o pipefail` newline ([#3425](https://github.com/nf-core/tools/pull/3425)) ## [v3.1.2 - Brass Boxfish Patch](https://github.com/nf-core/tools/releases/tag/3.1.2) - [2025-01-20]