From 943affa80a7cd9efafe99bbca596784d088abb03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 19 Feb 2025 10:29:11 +0100 Subject: [PATCH 01/55] add cram support to fq2bam --- modules/nf-core/parabricks/fq2bam/main.nf | 23 +++-- .../parabricks/fq2bam/tests/main.nf.test | 97 ++++++++++++++++++- .../parabricks/fq2bam/tests/nextflow.config | 1 + 3 files changed, 111 insertions(+), 10 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index 0cd69048048..fe43f65de83 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -14,12 +14,14 @@ process PARABRICKS_FQ2BAM { path(known_sites) output: - tuple val(meta), path("*.bam") , emit: bam - tuple val(meta), path("*.bai") , emit: bai - tuple val(meta), path("*.table"), emit: bqsr_table , optional:true + tuple val(meta), path("*.bam") , emit: bam , optional:true + tuple val(meta), path("*.bai") , emit: bai , optional:true + tuple val(meta), path("*.cram") , emit: cram , optional:true + tuple val(meta), path("*.crai") , emit: crai , optional:true + tuple val(meta), path("*.table"), emit: bqsr_table , optional:true path("versions.yml") , emit: versions - path("qc_metrics") , emit: qc_metrics , optional:true - path("duplicate-metrics.txt") , emit: duplicate_metrics , optional:true + path("qc_metrics") , emit: qc_metrics , optional:true + path("duplicate-metrics.txt") , emit: duplicate_metrics, optional:true when: task.ext.when == null || task.ext.when @@ -30,8 +32,10 @@ process PARABRICKS_FQ2BAM { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' + def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def in_fq_command = meta.single_end ? "--in-se-fq $reads" : "--in-fq $reads" + def extension = args2.contains("--output-fmt bam") ? "bam" : "cram" def known_sites_command = known_sites ? known_sites.collect{"--knownSites $it"}.join(' ') : "" def known_sites_output = known_sites ? "--out-recal-file ${prefix}.table" : "" def interval_file_command = interval_file ? interval_file.collect{"--interval-file $it"}.join(' ') : "" @@ -44,7 +48,7 @@ process PARABRICKS_FQ2BAM { fq2bam \\ --ref \$INDEX \\ $in_fq_command \\ - --out-bam ${prefix}.bam \\ + --out-bam ${prefix}.${extension} \\ $known_sites_command \\ $known_sites_output \\ $interval_file_command \\ @@ -62,10 +66,13 @@ process PARABRICKS_FQ2BAM { if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } + def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def extension = args2.contains("--output-fmt bam") ? "bam" : "cram" + def extension_index = extension == "cram" ? ".crai" : ".bai" """ - touch ${prefix}.bam - touch ${prefix}.bam.bai + touch ${prefix}.${extension} + touch ${prefix}.${extension}.${extension_index} cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 0dec1c5f8c9..2483e808099 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -11,6 +11,8 @@ nextflow_process { tag "parabricks" tag "gpu" + config './nextflow.config' + setup { run("BWA_INDEX") { script "../../../bwa/index/main.nf" @@ -47,6 +49,7 @@ nextflow_process { // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing + module_args2 = '--output-fmt bam' } process { """ @@ -81,9 +84,13 @@ nextflow_process { } test("SRR389222 - fastq - se - stub") { + options '-stub' when { + params { + module_args2 = '--output-fmt bam' + } process { """ input[0] = Channel.of([ @@ -116,14 +123,13 @@ nextflow_process { test("sarscov2 - fastq - pe") { - config './nextflow.config' - when { params { module_args = '--low-memory' // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing + module_args2 = '--output-fmt bam' } process { """ @@ -164,6 +170,93 @@ nextflow_process { options '-stub' when { + params { + module_args2 = '--output-fmt bam' + } + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BWA_INDEX_PE.out.index + input[3] = [ [], [] ] + input[4] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out, + path(process.out.versions[0]).yaml + ).match() } + ) + } + + } + + test("sarscov2 - fastq - pe - cram") { + + when { + params { + module_args = '--low-memory' + // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 + // Parabricks’s fq2bam requires 24GB of memory. + // Using --low-memory for testing + module_args2 = '--output-fmt cram' + } + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ] + input[1] = Channel.of([ + [ id:'test' ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BWA_INDEX_PE.out.index + input[3] = [ [], [] ] + input[4] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + cram(process.out.cram[0][1]).getReadsMD5(), + file(process.out.crai[0][1]).name, + process.out.versions, + path(process.out.versions[0]).yaml + ).match() } + ) + } + + } + + test("sarscov2 - fastq - pe - stub") { + + options '-stub' + + when { + params { + module_args2 = '--output-fmt cram' + } process { """ input[0] = [ diff --git a/modules/nf-core/parabricks/fq2bam/tests/nextflow.config b/modules/nf-core/parabricks/fq2bam/tests/nextflow.config index 6f58b220deb..ebbb03b5004 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/nextflow.config +++ b/modules/nf-core/parabricks/fq2bam/tests/nextflow.config @@ -2,6 +2,7 @@ process { withName: 'PARABRICKS_FQ2BAM' { ext.args = params.module_args + ext.args2 = params.module_args2 } } From 488d592d78bb88ed16c452eb2b7d12fd702f12ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 19 Feb 2025 10:30:52 +0100 Subject: [PATCH 02/55] gpu --- tests/config/nf-test.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index d8350fafaea..8d9c06cdc59 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -23,7 +23,7 @@ profiles { } docker { docker.enabled = true - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64 --gpus all' } docker_self_hosted { docker.enabled = true From 5913394ed6934386910d20429464024aac15a2eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 19 Feb 2025 10:36:30 +0100 Subject: [PATCH 03/55] correct module args --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 2483e808099..3eed7c0713c 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -89,6 +89,7 @@ nextflow_process { when { params { + module_args = '' module_args2 = '--output-fmt bam' } process { @@ -171,6 +172,7 @@ nextflow_process { when { params { + module_args = '' module_args2 = '--output-fmt bam' } process { @@ -249,12 +251,13 @@ nextflow_process { } - test("sarscov2 - fastq - pe - stub") { + test("sarscov2 - fastq - pe - cram - stub") { options '-stub' when { params { + module_args = '' module_args2 = '--output-fmt cram' } process { From ee7a5cb55e6a40347d8a23e1124ead072c3e18a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 19 Feb 2025 10:52:43 +0100 Subject: [PATCH 04/55] update config --- tests/config/nf-test.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 8d9c06cdc59..193de24b278 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -83,3 +83,8 @@ includeConfig 'test_data.config' manifest { nextflowVersion = '!>=24.10.2' } + +config { + testsDir "tests" + workDir ".nf-test" +} \ No newline at end of file From a825dd75a0f7cd443658844637a0aa6775f52116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 19 Feb 2025 10:55:06 +0100 Subject: [PATCH 05/55] update config --- tests/config/nf-test.config | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 193de24b278..f25064f8779 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -84,7 +84,5 @@ manifest { nextflowVersion = '!>=24.10.2' } -config { - testsDir "tests" - workDir ".nf-test" -} \ No newline at end of file +testsDir "tests" +workDir ".nf-test" From 6468d50985561bebe7f973ae8450c077cbe71e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 19 Feb 2025 10:56:40 +0100 Subject: [PATCH 06/55] update config --- tests/config/nf-test.config | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index f25064f8779..8d9c06cdc59 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -83,6 +83,3 @@ includeConfig 'test_data.config' manifest { nextflowVersion = '!>=24.10.2' } - -testsDir "tests" -workDir ".nf-test" From b5f7b77a80dbc74ab5b1fc848c96f583faa2dfaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 19 Feb 2025 12:53:12 +0100 Subject: [PATCH 07/55] update conf --- tests/config/nf-test.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 8d9c06cdc59..e0974825ee4 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -36,6 +36,7 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true + singularity.runOptions = '--nv' } podman { podman.enabled = true @@ -83,3 +84,4 @@ includeConfig 'test_data.config' manifest { nextflowVersion = '!>=24.10.2' } + From fdd5277be5f151bd3f12a83b75bce7d5b061aafc Mon Sep 17 00:00:00 2001 From: famosab Date: Wed, 19 Feb 2025 12:58:26 +0100 Subject: [PATCH 08/55] update snap --- .../parabricks/fq2bam/tests/main.nf.test.snap | 132 ++++++++++++++++-- 1 file changed, 122 insertions(+), 10 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap index 73d5ced7c50..3fce7ec3da3 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap @@ -17,19 +17,25 @@ "id": "test", "single_end": true }, - "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.bam..bai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "2": [ ], "3": [ - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], "4": [ ], "5": [ + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], + "6": [ + + ], + "7": [ ], "bai": [ @@ -38,7 +44,7 @@ "id": "test", "single_end": true }, - "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.bam..bai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "bam": [ @@ -52,6 +58,12 @@ ], "bqsr_table": [ + ], + "crai": [ + + ], + "cram": [ + ], "duplicate_metrics": [ @@ -71,9 +83,9 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.2" + "nextflow": "24.10.4" }, - "timestamp": "2024-12-16T12:16:33.055785098" + "timestamp": "2025-02-19T12:51:20.062636745" }, "sarscov2 - fastq - pe - stub": { "content": [ @@ -93,19 +105,25 @@ "id": "test", "single_end": false }, - "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.bam..bai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "2": [ ], "3": [ - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], "4": [ ], "5": [ + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], + "6": [ + + ], + "7": [ ], "bai": [ @@ -114,7 +132,7 @@ "id": "test", "single_end": false }, - "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.bam..bai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "bam": [ @@ -128,6 +146,12 @@ ], "bqsr_table": [ + ], + "crai": [ + + ], + "cram": [ + ], "duplicate_metrics": [ @@ -147,9 +171,9 @@ ], "meta": { "nf-test": "0.9.2", - "nextflow": "24.10.2" + "nextflow": "24.10.4" }, - "timestamp": "2024-12-16T12:16:48.158061416" + "timestamp": "2025-02-19T12:51:52.203323547" }, "sarscov2 - fastq - pe": { "content": [ @@ -170,6 +194,94 @@ }, "timestamp": "2024-12-16T12:25:23.63061876" }, + "sarscov2 - fastq - pe - cram - stub": { + "content": [ + { + "0": [ + + ], + "1": [ + + ], + "2": [ + [ + { + "id": "test", + "single_end": false + }, + "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + [ + { + "id": "test", + "single_end": false + }, + "test.cram..crai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "4": [ + + ], + "5": [ + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], + "6": [ + + ], + "7": [ + + ], + "bai": [ + + ], + "bam": [ + + ], + "bqsr_table": [ + + ], + "crai": [ + [ + { + "id": "test", + "single_end": false + }, + "test.cram..crai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "cram": [ + [ + { + "id": "test", + "single_end": false + }, + "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "duplicate_metrics": [ + + ], + "qc_metrics": [ + + ], + "versions": [ + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ] + }, + { + "PARABRICKS_FQ2BAM": { + "pbrun": "4.4.0-1" + } + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.4" + }, + "timestamp": "2025-02-19T12:52:24.02703983" + }, "SRR389222 - fastq - se": { "content": [ "3d5b94990c7fdf90a682edb5ee0f59de", From e8dc35cd881211fe8e6abceba2ddc3004c119d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 19 Feb 2025 12:59:35 +0100 Subject: [PATCH 09/55] change stub --- modules/nf-core/parabricks/fq2bam/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index fe43f65de83..af48b207d72 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -69,7 +69,7 @@ process PARABRICKS_FQ2BAM { def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def extension = args2.contains("--output-fmt bam") ? "bam" : "cram" - def extension_index = extension == "cram" ? ".crai" : ".bai" + def extension_index = extension == "cram" ? "crai" : "bai" """ touch ${prefix}.${extension} touch ${prefix}.${extension}.${extension_index} From 95a28e5ae5ae9ead4425743263be12e03f10d608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 19 Feb 2025 13:29:57 +0100 Subject: [PATCH 10/55] update meta --- modules/nf-core/parabricks/fq2bam/main.nf | 4 ++++ modules/nf-core/parabricks/fq2bam/meta.yml | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index af48b207d72..60c56178461 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -70,9 +70,13 @@ process PARABRICKS_FQ2BAM { def prefix = task.ext.prefix ?: "${meta.id}" def extension = args2.contains("--output-fmt bam") ? "bam" : "cram" def extension_index = extension == "cram" ? "crai" : "bai" + def known_sites_output = known_sites ? "touch ${prefix}.table" : "" """ touch ${prefix}.${extension} touch ${prefix}.${extension}.${extension_index} + $known_sites_output + mkdir qc_metrics + touch duplicate-metrics.txt cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/parabricks/fq2bam/meta.yml b/modules/nf-core/parabricks/fq2bam/meta.yml index 8b45cb16284..15490613f07 100644 --- a/modules/nf-core/parabricks/fq2bam/meta.yml +++ b/modules/nf-core/parabricks/fq2bam/meta.yml @@ -75,6 +75,26 @@ output: type: file description: index corresponding to sorted BAM file pattern: "*.bai" + - cram: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - "*.cram": + type: file + description: Sorted CRAM file + pattern: "*.cram" + - crai: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - "*.crai": + type: file + description: index corresponding to sorted CRAM file + pattern: "*.crai" - bqsr_table: - meta: type: map From 2fa57bfabcdd231e648f86c37ffac7c4a5e7f6bb Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 20 Feb 2025 09:52:19 +0100 Subject: [PATCH 11/55] update snap --- .../parabricks/fq2bam/tests/main.nf.test.snap | 54 +++++++++++-------- .../tests/main.nf.test.snap | 18 +++---- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap index 3fce7ec3da3..342630dc53f 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap @@ -17,7 +17,7 @@ "id": "test", "single_end": true }, - "test.bam..bai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "2": [ @@ -33,10 +33,12 @@ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ], "6": [ - + [ + + ] ], "7": [ - + "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "bai": [ [ @@ -44,7 +46,7 @@ "id": "test", "single_end": true }, - "test.bam..bai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "bam": [ @@ -66,10 +68,12 @@ ], "duplicate_metrics": [ - + "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "qc_metrics": [ - + [ + + ] ], "versions": [ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" @@ -85,7 +89,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-19T12:51:20.062636745" + "timestamp": "2025-02-20T09:45:49.3707083" }, "sarscov2 - fastq - pe - stub": { "content": [ @@ -105,7 +109,7 @@ "id": "test", "single_end": false }, - "test.bam..bai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "2": [ @@ -121,10 +125,12 @@ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ], "6": [ - + [ + + ] ], "7": [ - + "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "bai": [ [ @@ -132,7 +138,7 @@ "id": "test", "single_end": false }, - "test.bam..bai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "bam": [ @@ -154,10 +160,12 @@ ], "duplicate_metrics": [ - + "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "qc_metrics": [ - + [ + + ] ], "versions": [ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" @@ -173,7 +181,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-19T12:51:52.203323547" + "timestamp": "2025-02-20T09:46:52.337789008" }, "sarscov2 - fastq - pe": { "content": [ @@ -218,7 +226,7 @@ "id": "test", "single_end": false }, - "test.cram..crai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "4": [ @@ -228,10 +236,12 @@ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ], "6": [ - + [ + + ] ], "7": [ - + "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "bai": [ @@ -248,7 +258,7 @@ "id": "test", "single_end": false }, - "test.cram..crai:md5,d41d8cd98f00b204e9800998ecf8427e" + "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], "cram": [ @@ -261,10 +271,12 @@ ] ], "duplicate_metrics": [ - + "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" ], "qc_metrics": [ - + [ + + ] ], "versions": [ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" @@ -280,7 +292,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-19T12:52:24.02703983" + "timestamp": "2025-02-20T09:47:56.345022633" }, "SRR389222 - fastq - se": { "content": [ diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap index 1535c2619ed..7e648f2e5f9 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap @@ -2,39 +2,37 @@ "sarscov2 single-end [fastq_gz]": { "content": [ [ - "7e2bd786d964e42ddbc2ab0c9f340b09" + ], [ - "test.recal.bam.bai" + ], [ - "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6", - "versions.yml:md5,df165e28f025dad39d826caead132115" + "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-17T16:25:03.460025311" + "timestamp": "2025-02-20T09:48:46.795069344" }, "sarscov2 paired-end [fastq_gz]": { "content": [ [ - "73e8e89cda8fce1cf07bdebff0f793ec" + ], [ - "test.recal.bam.bai" + ], [ - "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6", - "versions.yml:md5,df165e28f025dad39d826caead132115" + "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6" ] ], "meta": { "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-17T16:26:01.468588642" + "timestamp": "2025-02-20T09:49:38.079623864" } } \ No newline at end of file From 961396d91eb1ba44345734a4b17d1ce458bf4ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 09:55:20 +0100 Subject: [PATCH 12/55] swap cram test to se --- .../parabricks/fq2bam/tests/main.nf.test | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 3eed7c0713c..2db2f59db61 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -207,7 +207,7 @@ nextflow_process { } - test("sarscov2 - fastq - pe - cram") { + test("sarscov2 - fastq - se - cram") { when { params { @@ -219,18 +219,17 @@ nextflow_process { } process { """ - input[0] = [ - [ id:'test', single_end:false ], // meta map - [ - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) - ] - ] + input[0] = Channel.of([ + [ id:'test', single_end:true ], // meta map + [ + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ] + ]) input[1] = Channel.of([ [ id:'test' ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) ]) - input[2] = BWA_INDEX_PE.out.index + input[2] = BWA_INDEX.out.index input[3] = [ [], [] ] input[4] = [] """ @@ -286,7 +285,8 @@ nextflow_process { { assert snapshot( process.out, path(process.out.versions[0]).yaml - ).match() } + ).match() + } ) } From 2f8a2962e31f98b74d691b80520c7f60435cc813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 10:07:35 +0100 Subject: [PATCH 13/55] expand stub --- modules/nf-core/parabricks/fq2bam/main.nf | 11 +++++++---- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 8 ++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index 60c56178461..0e1d28bdedd 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -20,8 +20,8 @@ process PARABRICKS_FQ2BAM { tuple val(meta), path("*.crai") , emit: crai , optional:true tuple val(meta), path("*.table"), emit: bqsr_table , optional:true path("versions.yml") , emit: versions - path("qc_metrics") , emit: qc_metrics , optional:true - path("duplicate-metrics.txt") , emit: duplicate_metrics, optional:true + path("*") , emit: qc_metrics , optional:true + path("*.txt") , emit: duplicate_metrics, optional:true when: task.ext.when == null || task.ext.when @@ -66,17 +66,20 @@ process PARABRICKS_FQ2BAM { if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } + def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def extension = args2.contains("--output-fmt bam") ? "bam" : "cram" def extension_index = extension == "cram" ? "crai" : "bai" def known_sites_output = known_sites ? "touch ${prefix}.table" : "" + def qc_metrics_output = args.contains("--out-qc-metrics-dir") ? "mkdir ${prefix}_qc_metrics" : "" + def duplicate_metrics_output = args.contains("--out-duplicate-metrics") ? "touch ${prefix}.duplicate-metrics.txt" : "" """ touch ${prefix}.${extension} touch ${prefix}.${extension}.${extension_index} $known_sites_output - mkdir qc_metrics - touch duplicate-metrics.txt + $qc_metrics_output + $duplicate_metrics_output cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 2db2f59db61..50cd86b398b 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -39,13 +39,13 @@ nextflow_process { } } - test("SRR389222 - fastq - se") { + test("SRR389222 - fastq - se - allfiles") { config './nextflow.config' when { params { - module_args = '--low-memory' + module_args = "--low-memory --out-recal-file ${meta.id}.table --out-duplicate-metrics ${meta.id}.duplicate-metrics.txt --out-qc-metrics-dir ${meta.id}_qc_metrics" // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing @@ -83,13 +83,13 @@ nextflow_process { } } - test("SRR389222 - fastq - se - stub") { + test("SRR389222 - fastq - se - allfiles - stub") { options '-stub' when { params { - module_args = '' + module_args = "--low-memory --out-recal-file ${meta.id}.table --out-duplicate-metrics ${meta.id}.duplicate-metrics.txt --out-qc-metrics-dir ${meta.id}_qc_metrics" module_args2 = '--output-fmt bam' } process { From 16abf3507e8dfb2da6ece6d118248549b04ad9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 10:12:34 +0100 Subject: [PATCH 14/55] fix linting --- modules/nf-core/parabricks/fq2bam/meta.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/meta.yml b/modules/nf-core/parabricks/fq2bam/meta.yml index 15490613f07..5030dea9649 100644 --- a/modules/nf-core/parabricks/fq2bam/meta.yml +++ b/modules/nf-core/parabricks/fq2bam/meta.yml @@ -112,12 +112,12 @@ output: description: File containing software versions pattern: "versions.yml" - qc_metrics: - - qc_metrics: + - "*": type: directory description: (optional) optional directory of qc metrics pattern: "qc_metrics" - duplicate_metrics: - - duplicate-metrics.txt: + - "*.txt": type: file description: (optional) metrics calculated from marking duplicates in the bam file From 94a878e488b57d3e9dac679d3d812bb2ea6ec179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 10:14:19 +0100 Subject: [PATCH 15/55] correct flag --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 50cd86b398b..c17687c5d3d 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -45,7 +45,7 @@ nextflow_process { when { params { - module_args = "--low-memory --out-recal-file ${meta.id}.table --out-duplicate-metrics ${meta.id}.duplicate-metrics.txt --out-qc-metrics-dir ${meta.id}_qc_metrics" + module_args = "--low-memory --out-duplicate-metrics ${meta.id}.duplicate-metrics.txt --out-qc-metrics-dir ${meta.id}_qc_metrics" // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing @@ -89,7 +89,7 @@ nextflow_process { when { params { - module_args = "--low-memory --out-recal-file ${meta.id}.table --out-duplicate-metrics ${meta.id}.duplicate-metrics.txt --out-qc-metrics-dir ${meta.id}_qc_metrics" + module_args = "--low-memory --out-duplicate-metrics ${meta.id}.duplicate-metrics.txt --out-qc-metrics-dir ${meta.id}_qc_metrics" module_args2 = '--output-fmt bam' } process { From 2ec044e67c463941848a69e20ebb181cd777369a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 10:20:14 +0100 Subject: [PATCH 16/55] remove args --- .../parabricks/fq2bam/tests/main.nf.test | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index c17687c5d3d..a52a27b07a9 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -39,13 +39,13 @@ nextflow_process { } } - test("SRR389222 - fastq - se - allfiles") { + test("SRR389222 - fastq - se") { config './nextflow.config' when { params { - module_args = "--low-memory --out-duplicate-metrics ${meta.id}.duplicate-metrics.txt --out-qc-metrics-dir ${meta.id}_qc_metrics" + module_args = "--low-memory" // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing @@ -83,13 +83,13 @@ nextflow_process { } } - test("SRR389222 - fastq - se - allfiles - stub") { + test("SRR389222 - fastq - se - stub") { options '-stub' when { params { - module_args = "--low-memory --out-duplicate-metrics ${meta.id}.duplicate-metrics.txt --out-qc-metrics-dir ${meta.id}_qc_metrics" + module_args = "--low-memory" module_args2 = '--output-fmt bam' } process { @@ -250,7 +250,7 @@ nextflow_process { } - test("sarscov2 - fastq - pe - cram - stub") { + test("sarscov2 - fastq - se - cram - stub") { options '-stub' @@ -261,18 +261,17 @@ nextflow_process { } process { """ - input[0] = [ - [ id:'test', single_end:false ], // meta map - [ - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) - ] - ] + input[0] = Channel.of([ + [ id:'test', single_end:true ], // meta map + [ + file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) + ] + ]) input[1] = Channel.of([ [ id:'test' ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) ]) - input[2] = BWA_INDEX_PE.out.index + input[2] = BWA_INDEX.out.index input[3] = [ [], [] ] input[4] = [] """ From 744f66b2671152cd6c758309ccd9be631fbd290a Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 20 Feb 2025 10:26:17 +0100 Subject: [PATCH 17/55] update snapshoz --- .../parabricks/fq2bam/tests/main.nf.test.snap | 138 ++++++++++++++++-- 1 file changed, 123 insertions(+), 15 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap index 342630dc53f..db70d7e2527 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap @@ -34,11 +34,13 @@ ], "6": [ [ - + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ] ], "7": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], "bai": [ [ @@ -68,11 +70,13 @@ ], "duplicate_metrics": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], "qc_metrics": [ [ - + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ] ], "versions": [ @@ -89,7 +93,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-20T09:45:49.3707083" + "timestamp": "2025-02-20T10:21:54.533223544" }, "sarscov2 - fastq - pe - stub": { "content": [ @@ -126,11 +130,13 @@ ], "6": [ [ - + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ] ], "7": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], "bai": [ [ @@ -160,11 +166,13 @@ ], "duplicate_metrics": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], "qc_metrics": [ [ - + "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ] ], "versions": [ @@ -181,7 +189,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-20T09:46:52.337789008" + "timestamp": "2025-02-20T10:09:23.092123799" }, "sarscov2 - fastq - pe": { "content": [ @@ -237,11 +245,13 @@ ], "6": [ [ - + "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e", + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ] ], "7": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + ], "bai": [ @@ -271,11 +281,109 @@ ] ], "duplicate_metrics": [ - "duplicate-metrics.txt:md5,d41d8cd98f00b204e9800998ecf8427e" + + ], + "qc_metrics": [ + [ + "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e", + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ] + ], + "versions": [ + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ] + }, + { + "PARABRICKS_FQ2BAM": { + "pbrun": "4.4.0-1" + } + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.4" + }, + "timestamp": "2025-02-20T10:10:28.426301855" + }, + "sarscov2 - fastq - se - cram - stub": { + "content": [ + { + "0": [ + + ], + "1": [ + + ], + "2": [ + [ + { + "id": "test", + "single_end": true + }, + "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "3": [ + [ + { + "id": "test", + "single_end": true + }, + "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "4": [ + + ], + "5": [ + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], + "6": [ + [ + "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e", + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ] + ], + "7": [ + + ], + "bai": [ + + ], + "bam": [ + + ], + "bqsr_table": [ + + ], + "crai": [ + [ + { + "id": "test", + "single_end": true + }, + "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "cram": [ + [ + { + "id": "test", + "single_end": true + }, + "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "duplicate_metrics": [ + ], "qc_metrics": [ [ - + "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e", + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ] ], "versions": [ @@ -292,7 +400,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-20T09:47:56.345022633" + "timestamp": "2025-02-20T10:24:03.653598546" }, "SRR389222 - fastq - se": { "content": [ From 3a5c9b26d4622d210b5d83a1760db101d107272d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 10:38:38 +0100 Subject: [PATCH 18/55] swap cram to PE data --- .../parabricks/fq2bam/tests/main.nf.test | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index a52a27b07a9..b37be249cf7 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -207,7 +207,7 @@ nextflow_process { } - test("sarscov2 - fastq - se - cram") { + test("sarscov2 - fastq - pe - cram") { when { params { @@ -219,17 +219,18 @@ nextflow_process { } process { """ - input[0] = Channel.of([ - [ id:'test', single_end:true ], // meta map - [ - file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) - ] - ]) + input[0] = [ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ] input[1] = Channel.of([ [ id:'test' ], // meta map - file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ]) - input[2] = BWA_INDEX.out.index + input[2] = BWA_INDEX_PE.out.index input[3] = [ [], [] ] input[4] = [] """ @@ -250,7 +251,7 @@ nextflow_process { } - test("sarscov2 - fastq - se - cram - stub") { + test("sarscov2 - fastq - pe - cram - stub") { options '-stub' @@ -261,17 +262,18 @@ nextflow_process { } process { """ - input[0] = Channel.of([ - [ id:'test', single_end:true ], // meta map - [ - file('https://github.com/nf-core/test-datasets/raw/methylseq/testdata/SRR389222_sub1.fastq.gz', checkIfExists: true) - ] - ]) + input[0] = [ + [ id:'test', single_end:false ], // meta map + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ] input[1] = Channel.of([ [ id:'test' ], // meta map - file('https://github.com/nf-core/test-datasets/raw/methylseq/reference/genome.fa', checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) ]) - input[2] = BWA_INDEX.out.index + input[2] = BWA_INDEX_PE.out.index input[3] = [ [], [] ] input[4] = [] """ From 35ed688fd616668fb2f8c5b79b87ced9aef678c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 10:59:22 +0100 Subject: [PATCH 19/55] swap cram to PE data --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index b37be249cf7..5627d578c7b 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -241,7 +241,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot( - cram(process.out.cram[0][1]).getReadsMD5(), + //cram(process.out.cram[0][1]).getReadsMD5(), file(process.out.crai[0][1]).name, process.out.versions, path(process.out.versions[0]).yaml From 1c4e08b2a576fade6c3efcd374f0209c1177cc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 14:21:21 +0100 Subject: [PATCH 20/55] new setup for cram test --- .../parabricks/fq2bam/tests/main.nf.test | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 5627d578c7b..0fd10499ff6 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -37,6 +37,18 @@ nextflow_process { """ } } + + run("BWA_INDEX", alias: 'BWA_INDEX_CRAM') { + script "../../../bwa/index/main.nf" + process { + """ + input[0] = [ + [id: 'test'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ] + """ + } + } } test("SRR389222 - fastq - se") { @@ -207,7 +219,7 @@ nextflow_process { } - test("sarscov2 - fastq - pe - cram") { + test("sarscov2 - fastq - se - cram") { when { params { @@ -220,17 +232,13 @@ nextflow_process { process { """ input[0] = [ - [ id:'test', single_end:false ], // meta map + [ id:'test', single_end:true ], // meta map [ - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] ] - input[1] = Channel.of([ - [ id:'test' ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ]) - input[2] = BWA_INDEX_PE.out.index + input[1] = [[id: 'test'],file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true)] + input[2] = BWA_INDEX_CRAM.out.index input[3] = [ [], [] ] input[4] = [] """ From 7f0748296d82022c452faf404ceb87c536bb3b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 14:41:16 +0100 Subject: [PATCH 21/55] swap sbwf to cram --- .../nf-core/fastq_align_parabricks/main.nf | 30 +++++++------------ .../fastq_align_parabricks/tests/main.nf.test | 8 ++--- .../tests/nextflow.config | 3 +- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_parabricks/main.nf b/subworkflows/nf-core/fastq_align_parabricks/main.nf index e3ad9c598a5..51a7d5e66d9 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/main.nf +++ b/subworkflows/nf-core/fastq_align_parabricks/main.nf @@ -15,8 +15,8 @@ workflow FASTQ_ALIGN_PARABRICKS { main: ch_versions = Channel.empty() - ch_bam = Channel.empty() - ch_bai = Channel.empty() + ch_cram = Channel.empty() + ch_crai = Channel.empty() ch_bqsr_table = Channel.empty() ch_qc_metrics = Channel.empty() ch_duplicate_metrics = Channel.empty() @@ -30,28 +30,18 @@ workflow FASTQ_ALIGN_PARABRICKS { ) // Collecting FQ2BAM outputs - ch_bam = ch_bam.mix(PARABRICKS_FQ2BAM.out.bam) - ch_bai = ch_bai.mix(PARABRICKS_FQ2BAM.out.bai) + ch_cram = ch_cram.mix(PARABRICKS_FQ2BAM.out.cram) + ch_crai = ch_crai.mix(PARABRICKS_FQ2BAM.out.crai) ch_qc_metrics = ch_qc_metrics.mix(PARABRICKS_FQ2BAM.out.qc_metrics) ch_bqsr_table = ch_bqsr_table.mix(PARABRICKS_FQ2BAM.out.bqsr_table) ch_duplicate_metrics = ch_duplicate_metrics.mix(PARABRICKS_FQ2BAM.out.duplicate_metrics) ch_versions = ch_versions.mix(PARABRICKS_FQ2BAM.out.versions) - // Apply BQSR - PARABRICKS_APPLYBQSR( - ch_bam, - ch_bai, - ch_bqsr_table.ifEmpty([]), - ch_interval_file, - ch_fasta - ) - ch_versions = ch_versions.mix(PARABRICKS_APPLYBQSR.out.versions) - emit: - bam = PARABRICKS_APPLYBQSR.out.bam // channel: [ [meta], bam ] - bai = PARABRICKS_APPLYBQSR.out.bai // channel: [ [meta], bai ] - qc_metrics = ch_qc_metrics // channel: [ [meta], qc_metrics ] - duplicate_metrics = ch_duplicate_metrics // channel: [ [meta], duplicate_metrics ] - bqsr_table = ch_bqsr_table // channel: [ [meta], bqsr_table ] - versions = ch_versions // channel: [ versions.yml ] + cram = ch_cram // channel: [ [meta], cram ] + crai = ch_crai // channel: [ [meta], crai ] + qc_metrics = ch_qc_metrics // channel: [ [meta], qc_metrics ] + duplicate_metrics = ch_duplicate_metrics // channel: [ [meta], duplicate_metrics ] + bqsr_table = ch_bqsr_table // channel: [ [meta], bqsr_table ] + versions = ch_versions // channel: [ versions.yml ] } diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test index 7f102f528a9..f4c34b86520 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test @@ -56,8 +56,8 @@ nextflow_workflow { assertAll( { assert workflow.success}, { assert snapshot( - workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, - workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.cram.collect { meta, cramfile -> cram(cramfile).getReadsMD5() }, + workflow.out.crai.collect { meta, crai -> file(crai).name }, workflow.out.versions ).match() } @@ -95,8 +95,8 @@ nextflow_workflow { assertAll( { assert workflow.success}, { assert snapshot( - workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, - workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.cram.collect { meta, cramfile -> cram(cramfile).getReadsMD5() }, + workflow.out.crai.collect { meta, crai -> file(crai).name }, workflow.out.versions ).match() } diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config b/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config index 335587c2bf2..c26523226ac 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config @@ -1,7 +1,8 @@ process { withName: 'PARABRICKS_FQ2BAM' { - ext.args = '--low-memory' + ext.args = '--low-memory' + ext.args2 = '--output-fmt cram' } // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 From f04bdd6b95f821fc59f0374e2261682b7db4186e Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 20 Feb 2025 14:46:18 +0100 Subject: [PATCH 22/55] update snap --- .../parabricks/fq2bam/tests/main.nf.test.snap | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap index db70d7e2527..90bd71179bc 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap @@ -210,6 +210,24 @@ }, "timestamp": "2024-12-16T12:25:23.63061876" }, + "sarscov2 - fastq - se - cram": { + "content": [ + "test.cram.crai", + [ + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], + { + "PARABRICKS_FQ2BAM": { + "pbrun": "4.4.0-1" + } + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.4" + }, + "timestamp": "2025-02-20T14:27:42.828104858" + }, "sarscov2 - fastq - pe - cram - stub": { "content": [ { From 83ccfce3d4769ecc8a30b06e2314367fe050f86c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 14:47:49 +0100 Subject: [PATCH 23/55] temp remove tag --- subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test index f4c34b86520..b663498e9f8 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test @@ -9,7 +9,7 @@ nextflow_workflow { tag "subworkflows_nfcore" tag "subworkflows/fastq_align_parabricks" tag "parabricks" - tag "parabricks/fq2bam" + tag "parabricks/applybqsr" tag "bwa" tag "bwa/index" From 4bf4b8dd17f5cb485ad20c5d0841147199c392c0 Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 20 Feb 2025 15:01:34 +0100 Subject: [PATCH 24/55] update snap --- .../parabricks/fq2bam/tests/main.nf.test.snap | 96 ------------------- 1 file changed, 96 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap index 90bd71179bc..6be3b193244 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap @@ -324,102 +324,6 @@ }, "timestamp": "2025-02-20T10:10:28.426301855" }, - "sarscov2 - fastq - se - cram - stub": { - "content": [ - { - "0": [ - - ], - "1": [ - - ], - "2": [ - [ - { - "id": "test", - "single_end": true - }, - "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "3": [ - [ - { - "id": "test", - "single_end": true - }, - "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "4": [ - - ], - "5": [ - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ], - "6": [ - [ - "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e", - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ] - ], - "7": [ - - ], - "bai": [ - - ], - "bam": [ - - ], - "bqsr_table": [ - - ], - "crai": [ - [ - { - "id": "test", - "single_end": true - }, - "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "cram": [ - [ - { - "id": "test", - "single_end": true - }, - "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], - "duplicate_metrics": [ - - ], - "qc_metrics": [ - [ - "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e", - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ] - ], - "versions": [ - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ] - }, - { - "PARABRICKS_FQ2BAM": { - "pbrun": "4.4.0-1" - } - } - ], - "meta": { - "nf-test": "0.9.2", - "nextflow": "24.10.4" - }, - "timestamp": "2025-02-20T10:24:03.653598546" - }, "SRR389222 - fastq - se": { "content": [ "3d5b94990c7fdf90a682edb5ee0f59de", From 9a0081e3a577256f15f119615a00e047f662ee27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 15:02:35 +0100 Subject: [PATCH 25/55] add back cram access --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 0fd10499ff6..4eab7ba9c5e 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -249,7 +249,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot( - //cram(process.out.cram[0][1]).getReadsMD5(), + cram(process.out.cram[0][1]).getReadsMD5(), file(process.out.crai[0][1]).name, process.out.versions, path(process.out.versions[0]).yaml From 9f98ad5250a7ee5b51659df29e0709c56c5777c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 15:11:37 +0100 Subject: [PATCH 26/55] try different assertion --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 4eab7ba9c5e..cca4002b5dc 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -249,7 +249,7 @@ nextflow_process { assertAll( { assert process.success }, { assert snapshot( - cram(process.out.cram[0][1]).getReadsMD5(), + process.out.cram.collect { cram(it[1], it[2]).getSamLines() }, file(process.out.crai[0][1]).name, process.out.versions, path(process.out.versions[0]).yaml From 017bcc5b5883898550a369ce72eed010843c6ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 15:13:07 +0100 Subject: [PATCH 27/55] add fasta to cram eval --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index cca4002b5dc..71aecf14f45 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -246,10 +246,14 @@ nextflow_process { } then { + def fasta = params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta' assertAll( { assert process.success }, { assert snapshot( - process.out.cram.collect { cram(it[1], it[2]).getSamLines() }, + cram( + process.out.cram[0][1], + fasta, + ).getReadsMD5(), file(process.out.crai[0][1]).name, process.out.versions, path(process.out.versions[0]).yaml From bbeab301f45ac5195eefc03abc507f252448ae35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 15:18:12 +0100 Subject: [PATCH 28/55] add fasta to cram eval --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 71aecf14f45..f59a15c47fb 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -246,7 +246,7 @@ nextflow_process { } then { - def fasta = params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta' + def fasta = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta') assertAll( { assert process.success }, { assert snapshot( From af28757753666e78bf5a87ff740c9dcb0016d8de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 15:24:29 +0100 Subject: [PATCH 29/55] add fasta to cram eval --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index f59a15c47fb..dc0eec51f87 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -246,7 +246,7 @@ nextflow_process { } then { - def fasta = file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta') + def fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/modules/data/genomics/sarscov2/genome/genome.fasta" assertAll( { assert process.success }, { assert snapshot( From 7dcdee238e431732b70b03889bb9a4356c916b4f Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 20 Feb 2025 15:32:26 +0100 Subject: [PATCH 30/55] update snaps --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap index 6be3b193244..333386f0e57 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap @@ -212,6 +212,7 @@ }, "sarscov2 - fastq - se - cram": { "content": [ + "30c325e1e032eb1782a280d34c0fb1c7", "test.cram.crai", [ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" @@ -226,7 +227,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-20T14:27:42.828104858" + "timestamp": "2025-02-20T15:28:05.839124732" }, "sarscov2 - fastq - pe - cram - stub": { "content": [ From 9438577b5b9caa9605fead3df7a0ac397482d0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 17:35:10 +0100 Subject: [PATCH 31/55] modify nf-test --- .../nf-core/fastq_align_parabricks/tests/main.nf.test | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test index b663498e9f8..cb6e4e8b12c 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test @@ -53,10 +53,11 @@ nextflow_workflow { } then { + def fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/modules/data/genomics/sarscov2/genome/genome.fasta" assertAll( { assert workflow.success}, { assert snapshot( - workflow.out.cram.collect { meta, cramfile -> cram(cramfile).getReadsMD5() }, + workflow.out.cram.collect { meta, cramfile -> cram(cramfile, fasta).getReadsMD5() }, workflow.out.crai.collect { meta, crai -> file(crai).name }, workflow.out.versions ).match() @@ -92,10 +93,11 @@ nextflow_workflow { } then { + def fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/modules/data/genomics/sarscov2/genome/genome.fasta" assertAll( { assert workflow.success}, { assert snapshot( - workflow.out.cram.collect { meta, cramfile -> cram(cramfile).getReadsMD5() }, + workflow.out.cram.collect { meta, cramfile -> cram(cramfile, fasta).getReadsMD5() }, workflow.out.crai.collect { meta, crai -> file(crai).name }, workflow.out.versions ).match() From 97e3ab6ba1aeb325b7661b9a94853864962e15e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 20 Feb 2025 17:35:32 +0100 Subject: [PATCH 32/55] modify nf-test --- subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test index cb6e4e8b12c..a22abdb8d03 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test @@ -9,8 +9,7 @@ nextflow_workflow { tag "subworkflows_nfcore" tag "subworkflows/fastq_align_parabricks" tag "parabricks" - - tag "parabricks/applybqsr" + tag "parabricks/fq2bam" tag "bwa" tag "bwa/index" tag "gpu" From aa464c55c6bdd29e941fe696d7493221b990ec3e Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 20 Feb 2025 17:45:17 +0100 Subject: [PATCH 33/55] update snaps --- .../fastq_align_parabricks/tests/main.nf.test.snap | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap index 7e648f2e5f9..cff91de13cb 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap @@ -2,10 +2,10 @@ "sarscov2 single-end [fastq_gz]": { "content": [ [ - + "30c325e1e032eb1782a280d34c0fb1c7" ], [ - + "test.cram.crai" ], [ "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6" @@ -15,15 +15,15 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-20T09:48:46.795069344" + "timestamp": "2025-02-20T17:41:02.703286006" }, "sarscov2 paired-end [fastq_gz]": { "content": [ [ - + "2d64e4363d9f3c0e2167fce49d5087cf" ], [ - + "test.cram.crai" ], [ "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6" @@ -33,6 +33,6 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-20T09:49:38.079623864" + "timestamp": "2025-02-20T17:41:54.475172575" } } \ No newline at end of file From d75da76c946c9a89bc8b11eb29388e533b52650a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 24 Feb 2025 10:22:17 +0100 Subject: [PATCH 34/55] add bam output to sbwf --- .../nf-core/fastq_align_parabricks/main.nf | 6 ++ .../fastq_align_parabricks/tests/main.nf.test | 61 +++++++++++++++++++ .../tests/nextflow.config | 4 +- 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_parabricks/main.nf b/subworkflows/nf-core/fastq_align_parabricks/main.nf index 51a7d5e66d9..936de52efa6 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/main.nf +++ b/subworkflows/nf-core/fastq_align_parabricks/main.nf @@ -17,6 +17,8 @@ workflow FASTQ_ALIGN_PARABRICKS { ch_versions = Channel.empty() ch_cram = Channel.empty() ch_crai = Channel.empty() + ch_bam = Channel.empty() + ch_bai = Channel.empty() ch_bqsr_table = Channel.empty() ch_qc_metrics = Channel.empty() ch_duplicate_metrics = Channel.empty() @@ -32,6 +34,8 @@ workflow FASTQ_ALIGN_PARABRICKS { // Collecting FQ2BAM outputs ch_cram = ch_cram.mix(PARABRICKS_FQ2BAM.out.cram) ch_crai = ch_crai.mix(PARABRICKS_FQ2BAM.out.crai) + ch_bam = ch_bam.mix(PARABRICKS_FQ2BAM.out.bam) + ch_bai = ch_bai.mix(PARABRICKS_FQ2BAM.out.bai) ch_qc_metrics = ch_qc_metrics.mix(PARABRICKS_FQ2BAM.out.qc_metrics) ch_bqsr_table = ch_bqsr_table.mix(PARABRICKS_FQ2BAM.out.bqsr_table) ch_duplicate_metrics = ch_duplicate_metrics.mix(PARABRICKS_FQ2BAM.out.duplicate_metrics) @@ -40,6 +44,8 @@ workflow FASTQ_ALIGN_PARABRICKS { emit: cram = ch_cram // channel: [ [meta], cram ] crai = ch_crai // channel: [ [meta], crai ] + bam = ch_bam // channel: [ [meta], bam ] + bai = ch_bai // channel: [ [meta], bai ] qc_metrics = ch_qc_metrics // channel: [ [meta], qc_metrics ] duplicate_metrics = ch_duplicate_metrics // channel: [ [meta], duplicate_metrics ] bqsr_table = ch_bqsr_table // channel: [ [meta], bqsr_table ] diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test index a22abdb8d03..0b3d771f5b4 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test @@ -31,6 +31,13 @@ nextflow_workflow { test("sarscov2 single-end [fastq_gz]") { when { + params { + module_args = '--low-memory' + // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 + // Parabricks’s fq2bam requires 24GB of memory. + // Using --low-memory for testing + module_args2 = '--output-fmt cram' + } workflow { """ input[0] = Channel.of([ @@ -68,6 +75,13 @@ nextflow_workflow { test("sarscov2 paired-end [fastq_gz]") { when { + params { + module_args = '--low-memory' + // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 + // Parabricks’s fq2bam requires 24GB of memory. + // Using --low-memory for testing + module_args2 = '--output-fmt cram' + } workflow { """ input[0] = Channel.of([ @@ -104,4 +118,51 @@ nextflow_workflow { ) } } + + + test("sarscov2 paired-end [fastq_gz] bam") { + + when { + params { + module_args = '--low-memory' + // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 + // Parabricks’s fq2bam requires 24GB of memory. + // Using --low-memory for testing + module_args2 = '--output-fmt bam' + } + workflow { + """ + input[0] = Channel.of([ + [ id:'test', single_end:false ], + [ + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) + ] + ]) + input[1] = Channel.value([ + [id: 'reference'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) + ]) + input[2] = BWA_INDEX.out.index + input[3] = Channel.value([ + [id: 'intervals'], + file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/picard/baits.interval_list', checkIfExists: true) + ]) + input[4] = file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true) + """ + } + } + + then { + assertAll( + { assert workflow.success}, + { assert snapshot( + workflow.out.bam.collect { meta, bamfile -> bam(bamfile, fasta).getReadsMD5() }, + workflow.out.bai.collect { meta, bai -> file(bai).name }, + workflow.out.versions + ).match() + } + ) + } + } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config b/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config index c26523226ac..12e88e3da2d 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config @@ -1,8 +1,8 @@ process { withName: 'PARABRICKS_FQ2BAM' { - ext.args = '--low-memory' - ext.args2 = '--output-fmt cram' + ext.args = params.module_args + ext.args2 = params.module_args2 } // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 From b4b4675c7adc39fcc9a9df1c909c28215d19a463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 24 Feb 2025 11:22:07 +0100 Subject: [PATCH 35/55] correct bam assertion --- subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test index 0b3d771f5b4..e4b8e14d367 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test @@ -157,7 +157,7 @@ nextflow_workflow { assertAll( { assert workflow.success}, { assert snapshot( - workflow.out.bam.collect { meta, bamfile -> bam(bamfile, fasta).getReadsMD5() }, + workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, workflow.out.bai.collect { meta, bai -> file(bai).name }, workflow.out.versions ).match() From 1ad6d9fc97f4a423bdc04e2e33f92628c15ac73d Mon Sep 17 00:00:00 2001 From: famosab Date: Mon, 24 Feb 2025 11:38:09 +0100 Subject: [PATCH 36/55] add bam test --- .../tests/main.nf.test.snap | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap index cff91de13cb..34497810385 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap @@ -34,5 +34,23 @@ "nextflow": "24.10.4" }, "timestamp": "2025-02-20T17:41:54.475172575" + }, + "sarscov2 paired-end [fastq_gz] bam": { + "content": [ + [ + "2d64e4363d9f3c0e2167fce49d5087cf" + ], + [ + "test.bam.bai" + ], + [ + "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.4" + }, + "timestamp": "2025-02-24T11:25:19.099121134" } } \ No newline at end of file From b35dc0cbbe8ef7885095ecb022ce9c3c20b2a52c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 24 Feb 2025 14:16:06 +0100 Subject: [PATCH 37/55] revert changes to nf-test config --- tests/config/nf-test.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index e0974825ee4..8a0d01aca7b 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -23,7 +23,7 @@ profiles { } docker { docker.enabled = true - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64 --gpus all' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } docker_self_hosted { docker.enabled = true @@ -36,7 +36,6 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true - singularity.runOptions = '--nv' } podman { podman.enabled = true From ad7838c5ea73597e94f59e082069720b08a151e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Mon, 24 Feb 2025 14:17:06 +0100 Subject: [PATCH 38/55] revert changes to nf-test config --- tests/config/nf-test.config | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 8a0d01aca7b..d8350fafaea 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -83,4 +83,3 @@ includeConfig 'test_data.config' manifest { nextflowVersion = '!>=24.10.2' } - From 1ee823aa015c6a19d29f712f9c5fe44850b3e08c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Tue, 25 Feb 2025 10:03:03 +0100 Subject: [PATCH 39/55] address review comments --- modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 4 ++-- subworkflows/nf-core/fastq_align_parabricks/main.nf | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index dc0eec51f87..b08780f5105 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -57,7 +57,7 @@ nextflow_process { when { params { - module_args = "--low-memory" + module_args = '--low-memory' // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing @@ -101,7 +101,7 @@ nextflow_process { when { params { - module_args = "--low-memory" + module_args = '' module_args2 = '--output-fmt bam' } process { diff --git a/subworkflows/nf-core/fastq_align_parabricks/main.nf b/subworkflows/nf-core/fastq_align_parabricks/main.nf index 936de52efa6..4d4569f73ce 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/main.nf +++ b/subworkflows/nf-core/fastq_align_parabricks/main.nf @@ -1,8 +1,7 @@ // // Alignment and BQSR with Nvidia CLARA Parabricks // -include { PARABRICKS_FQ2BAM } from '../../../modules/nf-core/parabricks/fq2bam/main' -include { PARABRICKS_APPLYBQSR } from '../../../modules/nf-core/parabricks/applybqsr/main' +include { PARABRICKS_FQ2BAM } from '../../../modules/nf-core/parabricks/fq2bam/main' workflow FASTQ_ALIGN_PARABRICKS { From ca96a18fcece82869b1f1b00f36b1898759283d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 09:55:44 +0100 Subject: [PATCH 40/55] swap bam-cram to val input --- modules/nf-core/parabricks/fq2bam/main.nf | 9 ++++----- modules/nf-core/parabricks/fq2bam/meta.yml | 4 ++++ modules/nf-core/parabricks/fq2bam/tests/main.nf.test | 12 ++++++------ .../nf-core/parabricks/fq2bam/tests/nextflow.config | 1 - 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index 0e1d28bdedd..71b3680ede9 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -12,6 +12,7 @@ process PARABRICKS_FQ2BAM { tuple val(meta3), path(index) tuple val(meta4), path(interval_file) path(known_sites) + val(output_fmt) // either bam or cram output: tuple val(meta), path("*.bam") , emit: bam , optional:true @@ -32,10 +33,9 @@ process PARABRICKS_FQ2BAM { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' - def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def in_fq_command = meta.single_end ? "--in-se-fq $reads" : "--in-fq $reads" - def extension = args2.contains("--output-fmt bam") ? "bam" : "cram" + def extension = "$output_fmt" def known_sites_command = known_sites ? known_sites.collect{"--knownSites $it"}.join(' ') : "" def known_sites_output = known_sites ? "--out-recal-file ${prefix}.table" : "" def interval_file_command = interval_file ? interval_file.collect{"--interval-file $it"}.join(' ') : "" @@ -67,10 +67,9 @@ process PARABRICKS_FQ2BAM { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' - def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def extension = args2.contains("--output-fmt bam") ? "bam" : "cram" - def extension_index = extension == "cram" ? "crai" : "bai" + def extension = "$output_fmt" + def extension_index = "$output_fmt" == "cram" ? "crai" : "bai" def known_sites_output = known_sites ? "touch ${prefix}.table" : "" def qc_metrics_output = args.contains("--out-qc-metrics-dir") ? "mkdir ${prefix}_qc_metrics" : "" def duplicate_metrics_output = args.contains("--out-duplicate-metrics") ? "touch ${prefix}.duplicate-metrics.txt" : "" diff --git a/modules/nf-core/parabricks/fq2bam/meta.yml b/modules/nf-core/parabricks/fq2bam/meta.yml index 5030dea9649..e7ada4bbd46 100644 --- a/modules/nf-core/parabricks/fq2bam/meta.yml +++ b/modules/nf-core/parabricks/fq2bam/meta.yml @@ -54,6 +54,10 @@ input: description: (optional) known sites file(s) for calculating BQSR. markdups must be true to perform BQSR. pattern: "*.vcf.gz" + - - output_fmt: + type: value + description: Output format for the alignment. Options are 'bam' or 'cram' + pattern: "{bam,cram}" output: - bam: - meta: diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index b08780f5105..5400445de5b 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -61,7 +61,6 @@ nextflow_process { // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing - module_args2 = '--output-fmt bam' } process { """ @@ -78,6 +77,7 @@ nextflow_process { input[2] = BWA_INDEX.out.index input[3] = [ [], [] ] input[4] = [] + input[5] = 'bam' """ } } @@ -102,7 +102,6 @@ nextflow_process { when { params { module_args = '' - module_args2 = '--output-fmt bam' } process { """ @@ -119,6 +118,7 @@ nextflow_process { input[2] = BWA_INDEX.out.index input[3] = [ [], [] ] input[4] = [] + input[5] = 'bam' """ } } @@ -142,7 +142,6 @@ nextflow_process { // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing - module_args2 = '--output-fmt bam' } process { """ @@ -160,6 +159,7 @@ nextflow_process { input[2] = BWA_INDEX_PE.out.index input[3] = [ [], [] ] input[4] = [] + input[5] = 'bam' """ } } @@ -185,7 +185,6 @@ nextflow_process { when { params { module_args = '' - module_args2 = '--output-fmt bam' } process { """ @@ -203,6 +202,7 @@ nextflow_process { input[2] = BWA_INDEX_PE.out.index input[3] = [ [], [] ] input[4] = [] + input[5] = 'bam' """ } } @@ -227,7 +227,6 @@ nextflow_process { // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing - module_args2 = '--output-fmt cram' } process { """ @@ -241,6 +240,7 @@ nextflow_process { input[2] = BWA_INDEX_CRAM.out.index input[3] = [ [], [] ] input[4] = [] + input[5] = 'cram' """ } } @@ -270,7 +270,6 @@ nextflow_process { when { params { module_args = '' - module_args2 = '--output-fmt cram' } process { """ @@ -288,6 +287,7 @@ nextflow_process { input[2] = BWA_INDEX_PE.out.index input[3] = [ [], [] ] input[4] = [] + input[5] = 'cram' """ } } diff --git a/modules/nf-core/parabricks/fq2bam/tests/nextflow.config b/modules/nf-core/parabricks/fq2bam/tests/nextflow.config index ebbb03b5004..6f58b220deb 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/nextflow.config +++ b/modules/nf-core/parabricks/fq2bam/tests/nextflow.config @@ -2,7 +2,6 @@ process { withName: 'PARABRICKS_FQ2BAM' { ext.args = params.module_args - ext.args2 = params.module_args2 } } From 9537583746c72890a08757bf70c9aa45bb11c253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 10:17:14 +0100 Subject: [PATCH 41/55] swap bam-cram to val input --- subworkflows/nf-core/fastq_align_parabricks/main.nf | 4 +++- subworkflows/nf-core/fastq_align_parabricks/meta.yml | 5 +++++ .../nf-core/fastq_align_parabricks/tests/main.nf.test | 6 +++--- .../nf-core/fastq_align_parabricks/tests/nextflow.config | 1 - 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/subworkflows/nf-core/fastq_align_parabricks/main.nf b/subworkflows/nf-core/fastq_align_parabricks/main.nf index 4d4569f73ce..7e851b156bf 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/main.nf +++ b/subworkflows/nf-core/fastq_align_parabricks/main.nf @@ -11,6 +11,7 @@ workflow FASTQ_ALIGN_PARABRICKS { ch_index // channel: [mandatory] meta, index ch_interval_file // channel: [optional] meta, intervals_bed_combined ch_known_sites // channel [optional] known_sites_indels + val_output_fmt // either bam or cram main: ch_versions = Channel.empty() @@ -27,7 +28,8 @@ workflow FASTQ_ALIGN_PARABRICKS { ch_fasta, ch_index, ch_interval_file, - ch_known_sites + ch_known_sites, + val_output_fmt ) // Collecting FQ2BAM outputs diff --git a/subworkflows/nf-core/fastq_align_parabricks/meta.yml b/subworkflows/nf-core/fastq_align_parabricks/meta.yml index 27fe1ab34df..aa34c6ec78e 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/meta.yml +++ b/subworkflows/nf-core/fastq_align_parabricks/meta.yml @@ -38,6 +38,11 @@ input: (optional) known sites file(s) for calculating BQSR. markdups must be true to perform BQSR. Structure [ path(vcf) ] + - - output_fmt: + type: value + description: | + Output format for the alignment. Options are 'bam' or 'cram' + pattern: "{bam,cram}" output: - bam: type: file diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test index e4b8e14d367..f8160631b53 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test @@ -36,7 +36,6 @@ nextflow_workflow { // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing - module_args2 = '--output-fmt cram' } workflow { """ @@ -54,6 +53,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/picard/baits.interval_list', checkIfExists: true) ]) input[4] = file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true) + input[5] = 'cram' """ } } @@ -80,7 +80,6 @@ nextflow_workflow { // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing - module_args2 = '--output-fmt cram' } workflow { """ @@ -101,6 +100,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/picard/baits.interval_list', checkIfExists: true) ]) input[4] = file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true) + input[5] = 'cram' """ } } @@ -128,7 +128,6 @@ nextflow_workflow { // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 // Parabricks’s fq2bam requires 24GB of memory. // Using --low-memory for testing - module_args2 = '--output-fmt bam' } workflow { """ @@ -149,6 +148,7 @@ nextflow_workflow { file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/picard/baits.interval_list', checkIfExists: true) ]) input[4] = file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true) + input[5] = 'bam' """ } } diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config b/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config index 12e88e3da2d..5d5fb8b526c 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config +++ b/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config @@ -2,7 +2,6 @@ process { withName: 'PARABRICKS_FQ2BAM' { ext.args = params.module_args - ext.args2 = params.module_args2 } // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 From 4492f6a0eb8d74b5d400bd7b71b33df670afadba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 10:20:03 +0100 Subject: [PATCH 42/55] fix meta for lint --- modules/nf-core/parabricks/fq2bam/meta.yml | 2 +- subworkflows/nf-core/fastq_align_parabricks/meta.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/meta.yml b/modules/nf-core/parabricks/fq2bam/meta.yml index e7ada4bbd46..6f78b015b99 100644 --- a/modules/nf-core/parabricks/fq2bam/meta.yml +++ b/modules/nf-core/parabricks/fq2bam/meta.yml @@ -55,7 +55,7 @@ input: be true to perform BQSR. pattern: "*.vcf.gz" - - output_fmt: - type: value + type: string description: Output format for the alignment. Options are 'bam' or 'cram' pattern: "{bam,cram}" output: diff --git a/subworkflows/nf-core/fastq_align_parabricks/meta.yml b/subworkflows/nf-core/fastq_align_parabricks/meta.yml index aa34c6ec78e..73a0ac6d511 100644 --- a/subworkflows/nf-core/fastq_align_parabricks/meta.yml +++ b/subworkflows/nf-core/fastq_align_parabricks/meta.yml @@ -39,7 +39,7 @@ input: be true to perform BQSR. Structure [ path(vcf) ] - - output_fmt: - type: value + type: string description: | Output format for the alignment. Options are 'bam' or 'cram' pattern: "{bam,cram}" From 67f2c34a9ceb1a1b1d3e1069152a9ed4945e63e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 10:42:39 +0100 Subject: [PATCH 43/55] add meta maps --- modules/nf-core/parabricks/fq2bam/main.nf | 33 ++++++++++--------- modules/nf-core/parabricks/fq2bam/meta.yml | 10 ++++-- .../parabricks/fq2bam/tests/main.nf.test | 12 +++---- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index 71b3680ede9..c42d9829e61 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -7,22 +7,22 @@ process PARABRICKS_FQ2BAM { container "nvcr.io/nvidia/clara/clara-parabricks:4.4.0-1" input: - tuple val(meta), path(reads) + tuple val(meta) , path(reads) tuple val(meta2), path(fasta) tuple val(meta3), path(index) tuple val(meta4), path(interval_file) - path(known_sites) + tuple val(meta5), path(known_sites) val(output_fmt) // either bam or cram output: - tuple val(meta), path("*.bam") , emit: bam , optional:true - tuple val(meta), path("*.bai") , emit: bai , optional:true - tuple val(meta), path("*.cram") , emit: cram , optional:true - tuple val(meta), path("*.crai") , emit: crai , optional:true - tuple val(meta), path("*.table"), emit: bqsr_table , optional:true + tuple val(meta), path("*.bam") , emit: bam , optional:true + tuple val(meta), path("*.bai") , emit: bai , optional:true + tuple val(meta), path("*.cram") , emit: cram , optional:true + tuple val(meta), path("*.crai") , emit: crai , optional:true + tuple val(meta), path("*.table") , emit: bqsr_table , optional:true + tuple val(meta), path("${prefix}_qc_metrics") , emit: qc_metrics , optional:true + tuple val(meta), path("*.duplicate-metrics.txt"), optional: true, emit: duplicate_metrics path("versions.yml") , emit: versions - path("*") , emit: qc_metrics , optional:true - path("*.txt") , emit: duplicate_metrics, optional:true when: task.ext.when == null || task.ext.when @@ -33,12 +33,15 @@ process PARABRICKS_FQ2BAM { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" + prefix = task.ext.prefix ?: "${meta.id}" + def in_fq_command = meta.single_end ? "--in-se-fq $reads" : "--in-fq $reads" def extension = "$output_fmt" - def known_sites_command = known_sites ? known_sites.collect{"--knownSites $it"}.join(' ') : "" - def known_sites_output = known_sites ? "--out-recal-file ${prefix}.table" : "" - def interval_file_command = interval_file ? interval_file.collect{"--interval-file $it"}.join(' ') : "" + + def known_sites_command = known_sites ? (known_sites instanceof List ? known_sites.collect { "--knownSites $it" }.join(' ') : "--knownSites ${known_sites}") : "" + def known_sites_output_cmd = known_sites ? "--out-recal-file ${prefix}.table" : "" + def interval_file_command = interval_file ? (interval_file instanceof List ? interval_file.collect { "--interval-file $it" }.join(' ') : "--interval-file ${interval_file}") : "" + def num_gpus = task.accelerator ? "--num-gpus $task.accelerator.request" : '' """ INDEX=`find -L ./ -name "*.amb" | sed 's/\\.amb\$//'` @@ -50,7 +53,7 @@ process PARABRICKS_FQ2BAM { $in_fq_command \\ --out-bam ${prefix}.${extension} \\ $known_sites_command \\ - $known_sites_output \\ + $known_sites_output_cmd \\ $interval_file_command \\ $num_gpus \\ $args @@ -67,7 +70,7 @@ process PARABRICKS_FQ2BAM { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' - def prefix = task.ext.prefix ?: "${meta.id}" + prefix = task.ext.prefix ?: "${meta.id}" def extension = "$output_fmt" def extension_index = "$output_fmt" == "cram" ? "crai" : "bai" def known_sites_output = known_sites ? "touch ${prefix}.table" : "" diff --git a/modules/nf-core/parabricks/fq2bam/meta.yml b/modules/nf-core/parabricks/fq2bam/meta.yml index 6f78b015b99..79e1c26fafe 100644 --- a/modules/nf-core/parabricks/fq2bam/meta.yml +++ b/modules/nf-core/parabricks/fq2bam/meta.yml @@ -49,7 +49,11 @@ input: description: (optional) file(s) containing genomic intervals for use in base quality score recalibration (BQSR) pattern: "*.{bed,interval_list,picard,list,intervals}" - - - known_sites: + - - meta5: + type: map + description: | + Groovy Map containing known sites information + - known_sites: type: file description: (optional) known sites file(s) for calculating BQSR. markdups must be true to perform BQSR. @@ -119,13 +123,13 @@ output: - "*": type: directory description: (optional) optional directory of qc metrics - pattern: "qc_metrics" + pattern: "*_qc_metrics" - duplicate_metrics: - "*.txt": type: file description: (optional) metrics calculated from marking duplicates in the bam file - pattern: "*-duplicate-metrics.txt" + pattern: "*.duplicate-metrics.txt" authors: - "@bsiranosian" - "@adamrtalbot" diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test index 5400445de5b..afa16c72927 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test @@ -76,7 +76,7 @@ nextflow_process { ]) input[2] = BWA_INDEX.out.index input[3] = [ [], [] ] - input[4] = [] + input[4] = [ [], [] ] input[5] = 'bam' """ } @@ -117,7 +117,7 @@ nextflow_process { ]) input[2] = BWA_INDEX.out.index input[3] = [ [], [] ] - input[4] = [] + input[4] = [ [], [] ] input[5] = 'bam' """ } @@ -158,7 +158,7 @@ nextflow_process { ]) input[2] = BWA_INDEX_PE.out.index input[3] = [ [], [] ] - input[4] = [] + input[4] = [ [], [] ] input[5] = 'bam' """ } @@ -201,7 +201,7 @@ nextflow_process { ]) input[2] = BWA_INDEX_PE.out.index input[3] = [ [], [] ] - input[4] = [] + input[4] = [ [], [] ] input[5] = 'bam' """ } @@ -239,7 +239,7 @@ nextflow_process { input[1] = [[id: 'test'],file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true)] input[2] = BWA_INDEX_CRAM.out.index input[3] = [ [], [] ] - input[4] = [] + input[4] = [ [], [] ] input[5] = 'cram' """ } @@ -286,7 +286,7 @@ nextflow_process { ]) input[2] = BWA_INDEX_PE.out.index input[3] = [ [], [] ] - input[4] = [] + input[4] = [ [], [] ] input[5] = 'cram' """ } From f26fef62293cf27a13ed93d613918c7b442dd612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 10:44:22 +0100 Subject: [PATCH 44/55] align --- modules/nf-core/parabricks/fq2bam/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index c42d9829e61..edac19d50d5 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -21,8 +21,8 @@ process PARABRICKS_FQ2BAM { tuple val(meta), path("*.crai") , emit: crai , optional:true tuple val(meta), path("*.table") , emit: bqsr_table , optional:true tuple val(meta), path("${prefix}_qc_metrics") , emit: qc_metrics , optional:true - tuple val(meta), path("*.duplicate-metrics.txt"), optional: true, emit: duplicate_metrics - path("versions.yml") , emit: versions + tuple val(meta), path("*.duplicate-metrics.txt"), emit: duplicate_metrics, optional: true + path("versions.yml") , emit: versions when: task.ext.when == null || task.ext.when From 65ae2272dcf93850a7237dbd2cdfeb7d12b5bf36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 10:46:28 +0100 Subject: [PATCH 45/55] fix meta --- modules/nf-core/parabricks/fq2bam/meta.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/meta.yml b/modules/nf-core/parabricks/fq2bam/meta.yml index 79e1c26fafe..9307c17398f 100644 --- a/modules/nf-core/parabricks/fq2bam/meta.yml +++ b/modules/nf-core/parabricks/fq2bam/meta.yml @@ -120,12 +120,12 @@ output: description: File containing software versions pattern: "versions.yml" - qc_metrics: - - "*": + - "*_qc_metrics": type: directory description: (optional) optional directory of qc metrics pattern: "*_qc_metrics" - duplicate_metrics: - - "*.txt": + - "*.duplicate-metrics.txt": type: file description: (optional) metrics calculated from marking duplicates in the bam file From 2639ff8e62b6a7a7a0fac41b03a0564f910c6c23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 10:52:12 +0100 Subject: [PATCH 46/55] fix meta --- modules/nf-core/parabricks/fq2bam/meta.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/meta.yml b/modules/nf-core/parabricks/fq2bam/meta.yml index 9307c17398f..abce2aefa69 100644 --- a/modules/nf-core/parabricks/fq2bam/meta.yml +++ b/modules/nf-core/parabricks/fq2bam/meta.yml @@ -114,22 +114,30 @@ output: description: (optional) table from base quality score recalibration calculation, to be used with parabricks/applybqsr pattern: "*.table" - - versions: - - versions.yml: - type: file - description: File containing software versions - pattern: "versions.yml" - qc_metrics: - - "*_qc_metrics": + - meta: + type: map + description: | + Groovy Map containing sample information + - ${prefix}_qc_metrics: type: directory description: (optional) optional directory of qc metrics pattern: "*_qc_metrics" - duplicate_metrics: + - meta: + type: map + description: | + Groovy Map containing sample information - "*.duplicate-metrics.txt": type: file description: (optional) metrics calculated from marking duplicates in the bam file pattern: "*.duplicate-metrics.txt" + - versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" authors: - "@bsiranosian" - "@adamrtalbot" From b94abc1f9c103bf67c174ca511ba3c805d39b12f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 10:56:28 +0100 Subject: [PATCH 47/55] remove single module sbwf --- .../nf-core/fastq_align_parabricks/main.nf | 54 ------ .../nf-core/fastq_align_parabricks/meta.yml | 68 ------- .../fastq_align_parabricks/tests/main.nf.test | 168 ------------------ .../tests/main.nf.test.snap | 56 ------ .../tests/nextflow.config | 15 -- 5 files changed, 361 deletions(-) delete mode 100644 subworkflows/nf-core/fastq_align_parabricks/main.nf delete mode 100644 subworkflows/nf-core/fastq_align_parabricks/meta.yml delete mode 100644 subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test delete mode 100644 subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap delete mode 100644 subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config diff --git a/subworkflows/nf-core/fastq_align_parabricks/main.nf b/subworkflows/nf-core/fastq_align_parabricks/main.nf deleted file mode 100644 index 7e851b156bf..00000000000 --- a/subworkflows/nf-core/fastq_align_parabricks/main.nf +++ /dev/null @@ -1,54 +0,0 @@ -// -// Alignment and BQSR with Nvidia CLARA Parabricks -// -include { PARABRICKS_FQ2BAM } from '../../../modules/nf-core/parabricks/fq2bam/main' - -workflow FASTQ_ALIGN_PARABRICKS { - - take: - ch_reads // channel: [mandatory] meta, reads - ch_fasta // channel: [mandatory] meta, fasta - ch_index // channel: [mandatory] meta, index - ch_interval_file // channel: [optional] meta, intervals_bed_combined - ch_known_sites // channel [optional] known_sites_indels - val_output_fmt // either bam or cram - - main: - ch_versions = Channel.empty() - ch_cram = Channel.empty() - ch_crai = Channel.empty() - ch_bam = Channel.empty() - ch_bai = Channel.empty() - ch_bqsr_table = Channel.empty() - ch_qc_metrics = Channel.empty() - ch_duplicate_metrics = Channel.empty() - - PARABRICKS_FQ2BAM( - ch_reads, - ch_fasta, - ch_index, - ch_interval_file, - ch_known_sites, - val_output_fmt - ) - - // Collecting FQ2BAM outputs - ch_cram = ch_cram.mix(PARABRICKS_FQ2BAM.out.cram) - ch_crai = ch_crai.mix(PARABRICKS_FQ2BAM.out.crai) - ch_bam = ch_bam.mix(PARABRICKS_FQ2BAM.out.bam) - ch_bai = ch_bai.mix(PARABRICKS_FQ2BAM.out.bai) - ch_qc_metrics = ch_qc_metrics.mix(PARABRICKS_FQ2BAM.out.qc_metrics) - ch_bqsr_table = ch_bqsr_table.mix(PARABRICKS_FQ2BAM.out.bqsr_table) - ch_duplicate_metrics = ch_duplicate_metrics.mix(PARABRICKS_FQ2BAM.out.duplicate_metrics) - ch_versions = ch_versions.mix(PARABRICKS_FQ2BAM.out.versions) - - emit: - cram = ch_cram // channel: [ [meta], cram ] - crai = ch_crai // channel: [ [meta], crai ] - bam = ch_bam // channel: [ [meta], bam ] - bai = ch_bai // channel: [ [meta], bai ] - qc_metrics = ch_qc_metrics // channel: [ [meta], qc_metrics ] - duplicate_metrics = ch_duplicate_metrics // channel: [ [meta], duplicate_metrics ] - bqsr_table = ch_bqsr_table // channel: [ [meta], bqsr_table ] - versions = ch_versions // channel: [ versions.yml ] -} diff --git a/subworkflows/nf-core/fastq_align_parabricks/meta.yml b/subworkflows/nf-core/fastq_align_parabricks/meta.yml deleted file mode 100644 index 73a0ac6d511..00000000000 --- a/subworkflows/nf-core/fastq_align_parabricks/meta.yml +++ /dev/null @@ -1,68 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "fastq_align_parabricks" -description: Align a fastq file using GPU-based acceleration -keywords: - - fastq - - align - - parabricks - - gpu - - preprocessing -components: - - parabricks/fq2bam - - parabricks/applybqsr -input: - - ch_reads: - type: file - description: | - Channel containing reads (either one file for se or two files for pe) - Structure: [ val(meta), [ path(fastq1), path(fastq2) ] ] - - ch_fasta: - type: file - description: | - Channel containing reference fasta file - Structure: [ val(meta), path(fasta) ] - - ch_index: - type: file - description: | - Channel containing reference BWA index - Structure: [ val(meta), path(.{amb,ann,bwt,pac,sa}) ] - - ch_interval_file: - type: file - description: | - (optional) file(s) containing genomic intervals for use in base - quality score recalibration (BQSR) - Structure: [ val(meta), path(.{bed,interval_list,picard,list,intervals}) ] - - ch_known_sites: - type: file - description: | - (optional) known sites file(s) for calculating BQSR. markdups must - be true to perform BQSR. - Structure [ path(vcf) ] - - - output_fmt: - type: string - description: | - Output format for the alignment. Options are 'bam' or 'cram' - pattern: "{bam,cram}" -output: - - bam: - type: file - description: | - Channel containing BAM files - Structure: [ val(meta), path(bam) ] - pattern: "*.bam" - - bai: - type: file - description: | - Channel containing indexed BAM (BAI) files - Structure: [ val(meta), path(bai) ] - pattern: "*.bai" - - versions: - type: file - description: | - File containing software versions - Structure: [ path(versions.yml) ] - pattern: "versions.yml" -authors: - - "@famosab" -maintainers: - - "@famosab" diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test deleted file mode 100644 index f8160631b53..00000000000 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test +++ /dev/null @@ -1,168 +0,0 @@ -nextflow_workflow { - - name "Test Subworkflow FASTQ_ALIGN_PARABRICKS" - script "../main.nf" - workflow "FASTQ_ALIGN_PARABRICKS" - config "./nextflow.config" - - tag "subworkflows" - tag "subworkflows_nfcore" - tag "subworkflows/fastq_align_parabricks" - tag "parabricks" - tag "parabricks/fq2bam" - tag "bwa" - tag "bwa/index" - tag "gpu" - - setup { - run("BWA_INDEX") { - script "../../../../modules/nf-core/bwa/index/main.nf" - process { - """ - input[0] = Channel.of([ - [ id:'test' ], // meta map - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ]) - """ - } - } - } - - test("sarscov2 single-end [fastq_gz]") { - - when { - params { - module_args = '--low-memory' - // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 - // Parabricks’s fq2bam requires 24GB of memory. - // Using --low-memory for testing - } - workflow { - """ - input[0] = Channel.of([ - [ id:'test', single_end:true ], - [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true)] - ]) - input[1] = Channel.value([ - [id: 'reference'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ]) - input[2] = BWA_INDEX.out.index - input[3] = Channel.value([ - [id: 'intervals'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/picard/baits.interval_list', checkIfExists: true) - ]) - input[4] = file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true) - input[5] = 'cram' - """ - } - } - - then { - def fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/modules/data/genomics/sarscov2/genome/genome.fasta" - assertAll( - { assert workflow.success}, - { assert snapshot( - workflow.out.cram.collect { meta, cramfile -> cram(cramfile, fasta).getReadsMD5() }, - workflow.out.crai.collect { meta, crai -> file(crai).name }, - workflow.out.versions - ).match() - } - ) - } - } - - test("sarscov2 paired-end [fastq_gz]") { - - when { - params { - module_args = '--low-memory' - // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 - // Parabricks’s fq2bam requires 24GB of memory. - // Using --low-memory for testing - } - workflow { - """ - input[0] = Channel.of([ - [ id:'test', single_end:false ], - [ - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) - ] - ]) - input[1] = Channel.value([ - [id: 'reference'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ]) - input[2] = BWA_INDEX.out.index - input[3] = Channel.value([ - [id: 'intervals'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/picard/baits.interval_list', checkIfExists: true) - ]) - input[4] = file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true) - input[5] = 'cram' - """ - } - } - - then { - def fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/refs/heads/modules/data/genomics/sarscov2/genome/genome.fasta" - assertAll( - { assert workflow.success}, - { assert snapshot( - workflow.out.cram.collect { meta, cramfile -> cram(cramfile, fasta).getReadsMD5() }, - workflow.out.crai.collect { meta, crai -> file(crai).name }, - workflow.out.versions - ).match() - } - ) - } - } - - - test("sarscov2 paired-end [fastq_gz] bam") { - - when { - params { - module_args = '--low-memory' - // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 - // Parabricks’s fq2bam requires 24GB of memory. - // Using --low-memory for testing - } - workflow { - """ - input[0] = Channel.of([ - [ id:'test', single_end:false ], - [ - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), - file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) - ] - ]) - input[1] = Channel.value([ - [id: 'reference'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/genome.fasta', checkIfExists: true) - ]) - input[2] = BWA_INDEX.out.index - input[3] = Channel.value([ - [id: 'intervals'], - file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/picard/baits.interval_list', checkIfExists: true) - ]) - input[4] = file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true) - input[5] = 'bam' - """ - } - } - - then { - assertAll( - { assert workflow.success}, - { assert snapshot( - workflow.out.bam.collect { meta, bamfile -> bam(bamfile).getReadsMD5() }, - workflow.out.bai.collect { meta, bai -> file(bai).name }, - workflow.out.versions - ).match() - } - ) - } - } -} \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap deleted file mode 100644 index 34497810385..00000000000 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/main.nf.test.snap +++ /dev/null @@ -1,56 +0,0 @@ -{ - "sarscov2 single-end [fastq_gz]": { - "content": [ - [ - "30c325e1e032eb1782a280d34c0fb1c7" - ], - [ - "test.cram.crai" - ], - [ - "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6" - ] - ], - "meta": { - "nf-test": "0.9.2", - "nextflow": "24.10.4" - }, - "timestamp": "2025-02-20T17:41:02.703286006" - }, - "sarscov2 paired-end [fastq_gz]": { - "content": [ - [ - "2d64e4363d9f3c0e2167fce49d5087cf" - ], - [ - "test.cram.crai" - ], - [ - "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6" - ] - ], - "meta": { - "nf-test": "0.9.2", - "nextflow": "24.10.4" - }, - "timestamp": "2025-02-20T17:41:54.475172575" - }, - "sarscov2 paired-end [fastq_gz] bam": { - "content": [ - [ - "2d64e4363d9f3c0e2167fce49d5087cf" - ], - [ - "test.bam.bai" - ], - [ - "versions.yml:md5,0d8766379e89038cb5fdcd074f3289f6" - ] - ], - "meta": { - "nf-test": "0.9.2", - "nextflow": "24.10.4" - }, - "timestamp": "2025-02-24T11:25:19.099121134" - } -} \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config b/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config deleted file mode 100644 index 5d5fb8b526c..00000000000 --- a/subworkflows/nf-core/fastq_align_parabricks/tests/nextflow.config +++ /dev/null @@ -1,15 +0,0 @@ -process { - - withName: 'PARABRICKS_FQ2BAM' { - ext.args = params.module_args - } - - // Ref: https://forums.developer.nvidia.com/t/problem-with-gpu/256825/6 - // Parabricks’s fq2bam requires 24GB of memory. - // Using --low-memory for testing - - withName: 'PARABRICKS_APPLYBQSR' { - ext.prefix = { "${meta.id}.recal" } - } - -} From 008a9c0ddade9a74b929b17963d604709d328a3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 11:18:01 +0100 Subject: [PATCH 48/55] swap back to def --- modules/nf-core/parabricks/fq2bam/main.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index edac19d50d5..afa3e59c464 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -70,7 +70,6 @@ process PARABRICKS_FQ2BAM { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' - prefix = task.ext.prefix ?: "${meta.id}" def extension = "$output_fmt" def extension_index = "$output_fmt" == "cram" ? "crai" : "bai" def known_sites_output = known_sites ? "touch ${prefix}.table" : "" From d0d9117b26605acb03ce5c344559c2f8c3ea597d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Wed, 26 Feb 2025 11:18:07 +0100 Subject: [PATCH 49/55] swap back to def --- modules/nf-core/parabricks/fq2bam/main.nf | 5 +++-- modules/nf-core/parabricks/fq2bam/meta.yml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index afa3e59c464..685fa3b38cf 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -20,7 +20,7 @@ process PARABRICKS_FQ2BAM { tuple val(meta), path("*.cram") , emit: cram , optional:true tuple val(meta), path("*.crai") , emit: crai , optional:true tuple val(meta), path("*.table") , emit: bqsr_table , optional:true - tuple val(meta), path("${prefix}_qc_metrics") , emit: qc_metrics , optional:true + tuple val(meta), path("*_qc_metrics") , emit: qc_metrics , optional:true tuple val(meta), path("*.duplicate-metrics.txt"), emit: duplicate_metrics, optional: true path("versions.yml") , emit: versions @@ -33,7 +33,7 @@ process PARABRICKS_FQ2BAM { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' - prefix = task.ext.prefix ?: "${meta.id}" + def prefix = task.ext.prefix ?: "${meta.id}" def in_fq_command = meta.single_end ? "--in-se-fq $reads" : "--in-fq $reads" def extension = "$output_fmt" @@ -70,6 +70,7 @@ process PARABRICKS_FQ2BAM { error "Parabricks module does not support Conda. Please use Docker / Singularity / Podman instead." } def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" def extension = "$output_fmt" def extension_index = "$output_fmt" == "cram" ? "crai" : "bai" def known_sites_output = known_sites ? "touch ${prefix}.table" : "" diff --git a/modules/nf-core/parabricks/fq2bam/meta.yml b/modules/nf-core/parabricks/fq2bam/meta.yml index abce2aefa69..75b508cd5ce 100644 --- a/modules/nf-core/parabricks/fq2bam/meta.yml +++ b/modules/nf-core/parabricks/fq2bam/meta.yml @@ -119,7 +119,7 @@ output: type: map description: | Groovy Map containing sample information - - ${prefix}_qc_metrics: + - "*_qc_metrics": type: directory description: (optional) optional directory of qc metrics pattern: "*_qc_metrics" From 08d39cc0d23515ff190f6cab565f1a368f33e2ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 27 Feb 2025 11:02:58 +0100 Subject: [PATCH 50/55] edit nf-test config --- tests/config/nf-test.config | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index d8350fafaea..45ed73af4e5 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -36,6 +36,7 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true + singularity.runOptions = '--nv' } podman { podman.enabled = true From c5efc916ce01ff7d5b82547ac1fa038a2030cc61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 27 Feb 2025 11:18:29 +0100 Subject: [PATCH 51/55] edit nf-test config --- tests/config/nf-test.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 45ed73af4e5..8d9c06cdc59 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -23,7 +23,7 @@ profiles { } docker { docker.enabled = true - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64 --gpus all' } docker_self_hosted { docker.enabled = true @@ -36,7 +36,6 @@ profiles { singularity { singularity.enabled = true singularity.autoMounts = true - singularity.runOptions = '--nv' } podman { podman.enabled = true From c6e1af7e476426190aa52a17d798b90c17264eda Mon Sep 17 00:00:00 2001 From: famosab Date: Thu, 27 Feb 2025 11:49:58 +0100 Subject: [PATCH 52/55] update snaps --- .../parabricks/fq2bam/tests/main.nf.test.snap | 54 ++++++------------- 1 file changed, 15 insertions(+), 39 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap index 333386f0e57..57d6bb78baf 100644 --- a/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap +++ b/modules/nf-core/parabricks/fq2bam/tests/main.nf.test.snap @@ -30,17 +30,13 @@ ], "5": [ - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], "6": [ - [ - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ] + ], "7": [ - + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ], "bai": [ [ @@ -73,11 +69,7 @@ ], "qc_metrics": [ - [ - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ] + ], "versions": [ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" @@ -93,7 +85,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-20T10:21:54.533223544" + "timestamp": "2025-02-27T11:46:55.631178258" }, "sarscov2 - fastq - pe - stub": { "content": [ @@ -126,17 +118,13 @@ ], "5": [ - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], "6": [ - [ - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ] + ], "7": [ - + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ], "bai": [ [ @@ -169,11 +157,7 @@ ], "qc_metrics": [ - [ - "test.bam:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.bam.bai:md5,d41d8cd98f00b204e9800998ecf8427e", - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ] + ], "versions": [ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" @@ -189,7 +173,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-20T10:09:23.092123799" + "timestamp": "2025-02-27T11:48:09.468174786" }, "sarscov2 - fastq - pe": { "content": [ @@ -260,17 +244,13 @@ ], "5": [ - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" + ], "6": [ - [ - "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e", - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ] + ], "7": [ - + "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" ], "bai": [ @@ -303,11 +283,7 @@ ], "qc_metrics": [ - [ - "test.cram:md5,d41d8cd98f00b204e9800998ecf8427e", - "test.cram.crai:md5,d41d8cd98f00b204e9800998ecf8427e", - "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" - ] + ], "versions": [ "versions.yml:md5,55d1e67ef8fa9d0ea3065363a653ffef" @@ -323,7 +299,7 @@ "nf-test": "0.9.2", "nextflow": "24.10.4" }, - "timestamp": "2025-02-20T10:10:28.426301855" + "timestamp": "2025-02-27T11:49:34.618772489" }, "SRR389222 - fastq - se": { "content": [ From 5565eb0a0c28326c5839f25d0a93fc95e06e80ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Famke=20Ba=CC=88uerle?= Date: Thu, 27 Feb 2025 12:31:19 +0100 Subject: [PATCH 53/55] back --- modules/nf-core/parabricks/fq2bam/main.nf | 4 ++-- tests/config/nf-test.config | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nf-core/parabricks/fq2bam/main.nf b/modules/nf-core/parabricks/fq2bam/main.nf index 685fa3b38cf..c7d95418a6e 100644 --- a/modules/nf-core/parabricks/fq2bam/main.nf +++ b/modules/nf-core/parabricks/fq2bam/main.nf @@ -20,8 +20,8 @@ process PARABRICKS_FQ2BAM { tuple val(meta), path("*.cram") , emit: cram , optional:true tuple val(meta), path("*.crai") , emit: crai , optional:true tuple val(meta), path("*.table") , emit: bqsr_table , optional:true - tuple val(meta), path("*_qc_metrics") , emit: qc_metrics , optional:true - tuple val(meta), path("*.duplicate-metrics.txt"), emit: duplicate_metrics, optional: true + tuple val(meta), path("*_qc_metrics") , emit: qc_metrics , optional:true + tuple val(meta), path("*.duplicate-metrics.txt"), emit: duplicate_metrics, optional:true path("versions.yml") , emit: versions when: diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index 8d9c06cdc59..d8350fafaea 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -23,7 +23,7 @@ profiles { } docker { docker.enabled = true - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64 --gpus all' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } docker_self_hosted { docker.enabled = true From 83a85f8976d96c43fac900b34298ff9d4db25ee3 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 27 Feb 2025 14:31:45 +0100 Subject: [PATCH 54/55] add debug step --- .github/workflows/nf-test-gpu.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index b5b0a8244f3..799ebff6aa0 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -104,6 +104,17 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 with: fetch-depth: 0 + + - name: Debug GPU access + run: | + whoami + groups + nvidia-smi + docker info | grep -i runtime + env | grep -i cuda + env | grep -i nvidia + ls -la /dev/nvidia* + - name: Run nf-test Action uses: ./.github/actions/nf-test-action env: From 9cd3dcc015edb6989e3fe5991e7b877360e1c410 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 27 Feb 2025 15:04:16 +0100 Subject: [PATCH 55/55] more debugging --- .github/workflows/nf-test-gpu.yml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nf-test-gpu.yml b/.github/workflows/nf-test-gpu.yml index 799ebff6aa0..d9464913c90 100644 --- a/.github/workflows/nf-test-gpu.yml +++ b/.github/workflows/nf-test-gpu.yml @@ -105,15 +105,44 @@ jobs: with: fetch-depth: 0 - - name: Debug GPU access + - name: Basic GPU Check run: | whoami groups nvidia-smi docker info | grep -i runtime + # This should succeed based on your output + + - name: Run Application (Will Fail) + run: | + # Run your actual application command here + # For example: python my_gpu_script.py + # Or: docker run your_image your_command + # ADD "set -x" TO SEE EXACTLY WHAT'S HAPPENING + set -x + your_actual_command_that_fails + continue-on-error: true # This allows the workflow to continue even if this step fails + + - name: Debug Environment + run: | + echo "=== Debug After Failure ===" env | grep -i cuda env | grep -i nvidia ls -la /dev/nvidia* + ldconfig -p | grep cuda + continue-on-error: true # This allows the workflow to continue even if this step fails + + - name: Test Different CUDA Versions + run: | + # Test with CUDA 12.0 + docker run --rm --gpus all nvidia/cuda:12.0.1-base-ubuntu22.04 nvidia-smi + + # Test with CUDA 11.8 + docker run --rm --gpus all nvidia/cuda:11.8.0-base-ubuntu22.04 nvidia-smi + + # Try running your application inside a CUDA container + docker run --rm --gpus all -v $(pwd):/app -w /app nvidia/cuda:11.8.0-runtime-ubuntu22.04 your_application_command + continue-on-error: true # This allows the workflow to continue even if this step fails - name: Run nf-test Action uses: ./.github/actions/nf-test-action