Update CombineBatches workflow (#732)

broadinstitute · Jan 14, 2025 · 88dbd05 · 88dbd05
1 parent 174ad3e
commit 88dbd05
Show file tree

Hide file tree

Showing 44 changed files with 932 additions and 1,639 deletions.
diff --git a/...s/templates/terra_workspaces/cohort_mode/workflow_configurations/CombineBatches.json.tmpl b/...s/templates/terra_workspaces/cohort_mode/workflow_configurations/CombineBatches.json.tmpl
@@ -1,10 +1,15 @@
 {
   "CombineBatches.contig_list": "${workspace.primary_contigs_fai}",
-  "CombineBatches.pe_exclude_list": "${workspace.pesr_exclude_list}",
-  "CombineBatches.depth_exclude_list": "${workspace.depth_exclude_list}",
-  "CombineBatches.empty_file" : "${workspace.empty_file}",
+
+  "CombineBatches.clustering_config_part1" : "${workspace.clustering_config_part1}",
+  "CombineBatches.stratification_config_part1" : "${workspace.stratification_config_part1}",
+  "CombineBatches.clustering_config_part2" : "${workspace.clustering_config_part2}",
+  "CombineBatches.stratification_config_part2" : "${workspace.stratification_config_part2}",
+  "CombineBatches.track_bed_files": {{ reference_resources.clustering_tracks | tojson }},
+  "CombineBatches.track_names": {{ reference_resources.clustering_track_names | tojson }},
 
   "CombineBatches.min_sr_background_fail_batches": 0.5,
+  "CombineBatches.gatk_docker": "${workspace.gatk_docker}",
   "CombineBatches.sv_pipeline_docker": "${workspace.sv_pipeline_docker}",
   "CombineBatches.sv_base_mini_docker": "${workspace.sv_base_mini_docker}",
 
@@ -13,6 +18,10 @@
   "CombineBatches.pesr_vcfs": "${this.sample_sets.genotyped_pesr_vcf}",
   "CombineBatches.depth_vcfs": "${this.regenotyped_depth_vcfs}",
   "CombineBatches.raw_sr_bothside_pass_files": "${this.sample_sets.sr_bothside_pass}",
-  "CombineBatches.raw_sr_background_fail_files": "${this.sample_sets.sr_background_fail}"
+  "CombineBatches.raw_sr_background_fail_files": "${this.sample_sets.sr_background_fail}",
+  "CombineBatches.ped_file": "${workspace.cohort_ped_file}",
 
+  "CombineBatches.reference_dict" : "${workspace.reference_dict}",
+  "CombineBatches.reference_fasta" : "${workspace.reference_fasta}",
+  "CombineBatches.reference_fasta_fai" : "${workspace.reference_index}"
 }
diff --git a/...ts/templates/terra_workspaces/cohort_mode/workflow_configurations/MakeCohortVcf.json.tmpl b/...ts/templates/terra_workspaces/cohort_mode/workflow_configurations/MakeCohortVcf.json.tmpl
diff --git a/inputs/templates/terra_workspaces/cohort_mode/workspace.tsv.tmpl b/inputs/templates/terra_workspaces/cohort_mode/workspace.tsv.tmpl
@@ -1,4 +1,6 @@
 workspace:cloud_sdk_docker	{{ dockers.cloud_sdk_docker }}
+clustering_config_part1	{{ reference_resources.clustering_config_part1 }}
+clustering_config_part2	{{ reference_resources.clustering_config_part2 }}
 cnmops_docker	{{ dockers.cnmops_docker }}
 condense_counts_docker	{{ dockers.condense_counts_docker }}
 gatk_docker	{{ dockers.gatk_docker }}
@@ -57,6 +59,8 @@ reference_version	{{ reference_resources.reference_version }}
 rmsk	{{ reference_resources.rmsk }}
 segdups	{{ reference_resources.segdups }}
 seed_cutoffs	{{ reference_resources.seed_cutoffs }}
+stratification_config_part1	{{ reference_resources.stratification_config_part1 }}
+stratification_config_part2	{{ reference_resources.stratification_config_part2 }}
 wgd_scoring_mask	{{ reference_resources.wgd_scoring_mask }}
 wham_include_list_bed_file	{{ reference_resources.wham_include_list_bed_file }}
 chr_x	{{ reference_resources.chr_x }}

diff --git a/inputs/templates/terra_workspaces/single_sample/GATKSVPipelineSingleSample.json.tmpl b/inputs/templates/terra_workspaces/single_sample/GATKSVPipelineSingleSample.json.tmpl
@@ -30,6 +30,7 @@
 
   "GATKSVPipelineSingleSample.linux_docker" : "${workspace.linux_docker}",
   "GATKSVPipelineSingleSample.manta_docker": "${workspace.manta_docker}",
+  "GATKSVPipelineSingleSample.scramble_docker": "${workspace.scramble_docker}",
   "GATKSVPipelineSingleSample.sv_base_docker": "${workspace.sv_base_docker}",
   "GATKSVPipelineSingleSample.sv_base_mini_docker": "${workspace.sv_base_mini_docker}",
   "GATKSVPipelineSingleSample.sv_pipeline_docker": "${workspace.sv_pipeline_docker}",
@@ -42,7 +43,6 @@
   "GATKSVPipelineSingleSample.autosome_file" : "${workspace.reference_autosome_file}",
   "GATKSVPipelineSingleSample.primary_contigs_list" : "${workspace.reference_primary_contigs_list}",
   "GATKSVPipelineSingleSample.primary_contigs_fai" : "${workspace.reference_primary_contigs_fai}",
-  "GATKSVPipelineSingleSample.empty_file" : "${workspace.reference_empty_file}",
   "GATKSVPipelineSingleSample.genome_file" : "${workspace.reference_genome_file}",
   "GATKSVPipelineSingleSample.max_ref_panel_carrier_freq": 0.03,
   "GATKSVPipelineSingleSample.manta_region_bed" : "${workspace.reference_manta_region_bed}",
@@ -86,6 +86,14 @@
   "GATKSVPipelineSingleSample.depth_interval_overlap": "0.8",
   "GATKSVPipelineSingleSample.depth_clustering_algorithm": "SINGLE_LINKAGE",
 
+  "GATKSVPipelineSingleSample.clustering_config_part1" : "${workspace.clustering_config_part1}",
+  "GATKSVPipelineSingleSample.stratification_config_part1" : "${workspace.clustering_config_part1}",
+  "GATKSVPipelineSingleSample.clustering_config_part2" : "${workspace.clustering_config_part2}",
+  "GATKSVPipelineSingleSample.stratification_config_part2" : "${workspace.stratification_config_part2}",
+
+  "GATKSVPipelineSingleSample.clustering_track_names" : {{ reference_resources.clustering_track_names | tojson }},
+  "GATKSVPipelineSingleSample.clustering_track_bed_files" : {{ reference_resources.clustering_tracks | tojson }},
+
   "GATKSVPipelineSingleSample.ref_copy_number_autosomal_contigs" : 2,
   "GATKSVPipelineSingleSample.clean_vcf_min_sr_background_fail_batches": 0.5,
   "GATKSVPipelineSingleSample.max_shard_size_resolve" : 500,

diff --git a/inputs/templates/terra_workspaces/single_sample/workspace.tsv.tmpl b/inputs/templates/terra_workspaces/single_sample/workspace.tsv.tmpl
@@ -1,4 +1,6 @@
 workspace:cloud_sdk_docker	{{ dockers.cloud_sdk_docker }}
+clustering_config_part1	{{ reference_resources.clustering_config_part1 }}
+clustering_config_part2	{{ reference_resources.clustering_config_part2 }}
 cnmops_docker	{{ dockers.cnmops_docker }}
 condense_counts_docker	{{ dockers.condense_counts_docker }}
 gatk_docker	{{ dockers.gatk_docker }}
@@ -12,6 +14,7 @@ sv_base_mini_docker	{{ dockers.sv_base_mini_docker }}
 sv_pipeline_docker	{{ dockers.sv_pipeline_docker }}
 sv_pipeline_qc_docker	{{ dockers.sv_pipeline_qc_docker }}
 wham_docker	{{ dockers.wham_docker }}
+scramble_docker	{{ dockers.scramble_docker }}
 ref_panel_name	{{ ref_panel.name }}
 ref_panel_bincov_matrix	{{ ref_panel.merged_coverage_file }}
 ref_panel_contig_ploidy_model_tar	{{ ref_panel.contig_ploidy_model_tar }}
@@ -77,3 +80,5 @@ reference_segdups	{{ reference_resources.segdups }}
 reference_seed_cutoffs	{{ reference_resources.seed_cutoffs }}
 reference_wgd_scoring_mask	{{ reference_resources.wgd_scoring_mask }}
 reference_wham_include_list_bed_file	{{ reference_resources.wham_include_list_bed_file }}
+stratification_config_part1	{{ reference_resources.stratification_config_part1 }}
+stratification_config_part2	{{ reference_resources.stratification_config_part2 }}
diff --git a/inputs/templates/test/GATKSVPipelineBatch/GATKSVPipelineBatch.FromSampleEvidence.json.tmpl b/inputs/templates/test/GATKSVPipelineBatch/GATKSVPipelineBatch.FromSampleEvidence.json.tmpl
@@ -123,7 +123,6 @@
   "GATKSVPipelineBatch.RegenotypeCNVs.n_per_split": "5000",
 
   "GATKSVPipelineBatch.MakeCohortVcf.bin_exclude": {{ reference_resources.bin_exclude | tojson }},
-  "GATKSVPipelineBatch.MakeCohortVcf.empty_file" : {{ reference_resources.empty_file | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.site_level_comparison_datasets": [
     {{ reference_resources.ccdg_abel_site_level_benchmarking_dataset | tojson }},
     {{ reference_resources.gnomad_v2_collins_site_level_benchmarking_dataset | tojson }},
@@ -133,7 +132,6 @@
   "GATKSVPipelineBatch.MakeCohortVcf.cytobands": {{ reference_resources.cytobands | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.mei_bed": {{ reference_resources.mei_bed | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.pe_exclude_list": {{ reference_resources.pesr_exclude_list | tojson }},
-  "GATKSVPipelineBatch.MakeCohortVcf.depth_exclude_list": {{ reference_resources.depth_exclude_list | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.HERVK_reference": {{ reference_resources.hervk_reference | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.LINE1_reference": {{ reference_resources.line1_reference | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.min_sr_background_fail_batches": 0.5,
@@ -143,5 +141,11 @@
   "GATKSVPipelineBatch.MakeCohortVcf.samples_per_clean_vcf_step2_shard": 100,
   "GATKSVPipelineBatch.MakeCohortVcf.clean_vcf5_records_per_shard": 5000,
   "GATKSVPipelineBatch.MakeCohortVcf.random_seed": 0,
-  "GATKSVPipelineBatch.MakeCohortVcf.max_shard_size_resolve": 500
+  "GATKSVPipelineBatch.MakeCohortVcf.max_shard_size_resolve": 500,
+  "GATKSVPipelineBatch.MakeCohortVcf.clustering_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.stratification_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.clustering_config_part2" : {{ reference_resources.clustering_config_part2 | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.stratification_config_part2" : {{ reference_resources.stratification_config_part2 | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.track_bed_files": {{ reference_resources.clustering_tracks | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.track_names": {{ reference_resources.clustering_track_names | tojson }}
 }
diff --git a/inputs/templates/test/GATKSVPipelineBatch/GATKSVPipelineBatch.json.tmpl b/inputs/templates/test/GATKSVPipelineBatch/GATKSVPipelineBatch.json.tmpl
@@ -116,7 +116,6 @@
   "GATKSVPipelineBatch.RegenotypeCNVs.n_per_split": "5000",
 
   "GATKSVPipelineBatch.MakeCohortVcf.bin_exclude": {{ reference_resources.bin_exclude | tojson }},
-  "GATKSVPipelineBatch.MakeCohortVcf.empty_file" : {{ reference_resources.empty_file | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.site_level_comparison_datasets": [
     {{ reference_resources.ccdg_abel_site_level_benchmarking_dataset | tojson }},
     {{ reference_resources.gnomad_v2_collins_site_level_benchmarking_dataset | tojson }},
@@ -126,7 +125,6 @@
   "GATKSVPipelineBatch.MakeCohortVcf.cytobands": {{ reference_resources.cytobands | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.mei_bed": {{ reference_resources.mei_bed | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.pe_exclude_list": {{ reference_resources.pesr_exclude_list | tojson }},
-  "GATKSVPipelineBatch.MakeCohortVcf.depth_exclude_list": {{ reference_resources.depth_exclude_list | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.HERVK_reference": {{ reference_resources.hervk_reference | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.LINE1_reference": {{ reference_resources.line1_reference | tojson }},
   "GATKSVPipelineBatch.MakeCohortVcf.min_sr_background_fail_batches": 0.5,
@@ -137,6 +135,12 @@
   "GATKSVPipelineBatch.MakeCohortVcf.clean_vcf5_records_per_shard": 5000,
   "GATKSVPipelineBatch.MakeCohortVcf.random_seed": 0,
   "GATKSVPipelineBatch.MakeCohortVcf.max_shard_size_resolve": 500,
+  "GATKSVPipelineBatch.MakeCohortVcf.clustering_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.stratification_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.clustering_config_part2" : {{ reference_resources.clustering_config_part2 | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.stratification_config_part2" : {{ reference_resources.stratification_config_part2 | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.track_bed_files": {{ reference_resources.clustering_tracks | tojson }},
+  "GATKSVPipelineBatch.MakeCohortVcf.track_names": {{ reference_resources.clustering_track_names | tojson }},
 
   "GATKSVPipelineBatch.GATKSVPipelinePhase1.runtime_attr_postprocess": {
     "cpu_cores": 1,

diff --git a/inputs/templates/test/GATKSVPipelineSingleSample/GATKSVPipelineSingleSample.json.tmpl b/inputs/templates/test/GATKSVPipelineSingleSample/GATKSVPipelineSingleSample.json.tmpl
@@ -31,7 +31,6 @@
 
   "GATKSVPipelineSingleSample.linux_docker" : {{ dockers.linux_docker | tojson }},
   "GATKSVPipelineSingleSample.manta_docker": {{ dockers.manta_docker | tojson }},
-  "GATKSVPipelineSingleSample.melt_docker": {{ dockers.melt_docker | tojson }},
   "GATKSVPipelineSingleSample.scramble_docker": {{ dockers.scramble_docker | tojson }},
   "GATKSVPipelineSingleSample.sv_base_docker": {{ dockers.sv_base_docker | tojson }},
   "GATKSVPipelineSingleSample.sv_base_mini_docker": {{ dockers.sv_base_mini_docker | tojson }},
@@ -46,7 +45,6 @@
   "GATKSVPipelineSingleSample.autosome_file" : {{ reference_resources.autosome_file | tojson }},
   "GATKSVPipelineSingleSample.primary_contigs_list" : {{ reference_resources.primary_contigs_list | tojson }},
   "GATKSVPipelineSingleSample.primary_contigs_fai" : {{ reference_resources.primary_contigs_fai | tojson }},
-  "GATKSVPipelineSingleSample.empty_file" : {{ reference_resources.empty_file | tojson }},
   "GATKSVPipelineSingleSample.genome_file" : {{ reference_resources.genome_file | tojson }},
   "GATKSVPipelineSingleSample.max_ref_panel_carrier_freq": 0.03,
   "GATKSVPipelineSingleSample.manta_region_bed" : {{ reference_resources.manta_region_bed | tojson }},
@@ -90,6 +88,13 @@
   "GATKSVPipelineSingleSample.depth_interval_overlap": "0.8",
   "GATKSVPipelineSingleSample.depth_clustering_algorithm": "SINGLE_LINKAGE",
 
+  "GATKSVPipelineSingleSample.clustering_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "GATKSVPipelineSingleSample.stratification_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "GATKSVPipelineSingleSample.clustering_config_part2" : {{ reference_resources.clustering_config_part2 | tojson }},
+  "GATKSVPipelineSingleSample.stratification_config_part2" : {{ reference_resources.stratification_config_part2 | tojson }},
+  "GATKSVPipelineSingleSample.clustering_track_bed_files": {{ reference_resources.clustering_tracks | tojson }},
+  "GATKSVPipelineSingleSample.clustering_track_names": {{ reference_resources.clustering_track_names | tojson }},
+
   "GATKSVPipelineSingleSample.ref_copy_number_autosomal_contigs" : {{ reference_resources.copy_number_autosomal_contigs | tojson }},
   "GATKSVPipelineSingleSample.clean_vcf_min_sr_background_fail_batches": 0.5,
   "GATKSVPipelineSingleSample.max_shard_size_resolve" : 500,

diff --git a/inputs/templates/test/GATKSVPipelineSingleSample/GATKSVPipelineSingleSample.melt.json.tmpl b/inputs/templates/test/GATKSVPipelineSingleSample/GATKSVPipelineSingleSample.melt.json.tmpl
@@ -45,7 +45,6 @@
   "GATKSVPipelineSingleSample.autosome_file" : {{ reference_resources.autosome_file | tojson }},
   "GATKSVPipelineSingleSample.primary_contigs_list" : {{ reference_resources.primary_contigs_list | tojson }},
   "GATKSVPipelineSingleSample.primary_contigs_fai" : {{ reference_resources.primary_contigs_fai | tojson }},
-  "GATKSVPipelineSingleSample.empty_file" : {{ reference_resources.empty_file | tojson }},
   "GATKSVPipelineSingleSample.genome_file" : {{ reference_resources.genome_file | tojson }},
   "GATKSVPipelineSingleSample.max_ref_panel_carrier_freq": 0.03,
   "GATKSVPipelineSingleSample.manta_region_bed" : {{ reference_resources.manta_region_bed | tojson }},
@@ -89,6 +88,13 @@
   "GATKSVPipelineSingleSample.depth_interval_overlap": "0.8",
   "GATKSVPipelineSingleSample.depth_clustering_algorithm": "SINGLE_LINKAGE",
 
+  "GATKSVPipelineSingleSample.clustering_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "GATKSVPipelineSingleSample.stratification_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "GATKSVPipelineSingleSample.clustering_config_part2" : {{ reference_resources.clustering_config_part2 | tojson }},
+  "GATKSVPipelineSingleSample.stratification_config_part2" : {{ reference_resources.stratification_config_part2 | tojson }},
+  "GATKSVPipelineSingleSample.clustering_track_bed_files": {{ reference_resources.clustering_tracks | tojson }},
+  "GATKSVPipelineSingleSample.clustering_track_names": {{ reference_resources.clustering_track_names | tojson }},
+
   "GATKSVPipelineSingleSample.ref_copy_number_autosomal_contigs" : 2,
 
   "GATKSVPipelineSingleSample.MakeCohortVcf.HERVK_reference": {{ reference_resources.hervk_reference | tojson }},

diff --git a/inputs/templates/test/MakeCohortVcf/CombineBatches.json.tmpl b/inputs/templates/test/MakeCohortVcf/CombineBatches.json.tmpl
@@ -1,14 +1,24 @@
 {
   "CombineBatches.contig_list": {{ reference_resources.primary_contigs_fai | tojson }},
-  "CombineBatches.pe_exclude_list": {{ reference_resources.pesr_exclude_list | tojson }},
-  "CombineBatches.depth_exclude_list": {{ reference_resources.depth_exclude_list | tojson }},
-  "CombineBatches.empty_file" : {{ reference_resources.empty_file | tojson }},
+
+  "CombineBatches.clustering_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "CombineBatches.stratification_config_part1" : {{ reference_resources.clustering_config_part1 | tojson }},
+  "CombineBatches.clustering_config_part2" : {{ reference_resources.clustering_config_part2 | tojson }},
+  "CombineBatches.stratification_config_part2" : {{ reference_resources.stratification_config_part2 | tojson }},
+  "CombineBatches.track_bed_files": {{ reference_resources.clustering_tracks | tojson }},
+  "CombineBatches.track_names": {{ reference_resources.clustering_track_names | tojson }},
+
+  "CombineBatches.reference_fasta": {{ reference_resources.reference_fasta | tojson }},
+  "CombineBatches.reference_dict": {{ reference_resources.reference_dict | tojson }},
+  "CombineBatches.reference_fasta_fai": {{ reference_resources.reference_index | tojson }},
 
   "CombineBatches.min_sr_background_fail_batches": 0.5,
+  "CombineBatches.gatk_docker": {{ dockers.gatk_docker | tojson }},
   "CombineBatches.sv_pipeline_docker": {{ dockers.sv_pipeline_docker | tojson }},
   "CombineBatches.sv_base_mini_docker":{{ dockers.sv_base_mini_docker | tojson }},
 
   "CombineBatches.cohort_name": {{ test_batch.name | tojson }},
+  "CombineBatches.ped_file": {{ test_batch.ped_file | tojson }},
   "CombineBatches.batches": [
     {{ test_batch.name | tojson }}
   ],