Update docker images list, triggered by 6077c31

Add json template

Update top-level workflows; concat variant lists

Fix contig list read

Update ref panel outputs; fix CleanVcf template

Fix Terra CleanVcf templates

inputs/templates/terra_workspaces/cohort_mode/workflow_configurations/CleanVcf.json.tmpl

@@ -19,7 +19,7 @@
   "CleanVcf.cohort_name": "${this.sample_set_set_id}",
   "CleanVcf.ped_file": "${workspace.cohort_ped_file}",
   "CleanVcf.complex_genotype_vcfs": "${this.complex_genotype_vcfs}",
-  "CleanVcf.complex_resolve_bothside_pass_lists": "${this.complex_resolve_bothside_pass_lists}",
-  "CleanVcf.complex_resolve_background_fail_lists": "${this.complex_resolve_background_fail_lists}"
+  "CleanVcf.complex_resolve_bothside_pass_list": "${this.complex_resolve_bothside_pass_list}",
+  "CleanVcf.complex_resolve_background_fail_list": "${this.complex_resolve_background_fail_list}"
 
 }

inputs/templates/test/MakeCohortVcf/CleanVcf.json.tmpl

@@ -19,6 +19,6 @@
   "CleanVcf.cohort_name": {{ test_batch.name | tojson }},
   "CleanVcf.ped_file": {{ test_batch.ped_file | tojson }},
   "CleanVcf.complex_genotype_vcfs": {{ test_batch.complex_genotype_vcfs | tojson }},
-  "CleanVcf.complex_resolve_bothside_pass_lists": {{ test_batch.complex_resolve_bothside_pass_lists | tojson }},
-  "CleanVcf.complex_resolve_background_fail_lists": {{ test_batch.complex_resolve_background_fail_lists | tojson }}
+  "CleanVcf.complex_resolve_bothside_pass_list": {{ test_batch.complex_resolve_bothside_pass_list | tojson }},
+  "CleanVcf.complex_resolve_background_fail_list": {{ test_batch.complex_resolve_background_fail_list | tojson }}
 }

wdl/CleanVcf.wdl

@@ -11,8 +11,8 @@ workflow CleanVcf {
     String cohort_name
 
     Array[File] complex_genotype_vcfs
-    Array[File] complex_resolve_bothside_pass_lists
-    Array[File] complex_resolve_background_fail_lists
+    File complex_resolve_bothside_pass_list
+    File complex_resolve_background_fail_list
     File ped_file
 
     File contig_list
@@ -118,9 +118,9 @@ workflow CleanVcf {
       input:
         vcf=complex_genotype_vcfs[i],
         contig=contig,
-        background_list=complex_resolve_background_fail_lists[i],
+        background_list=complex_resolve_background_fail_list,
         ped_file=ped_file,
-        bothsides_pass_list=complex_resolve_bothside_pass_lists[i],
+        bothsides_pass_list=complex_resolve_bothside_pass_list,
         allosome_fai=allosome_fai,
         prefix="~{cohort_name}.~{contig}",
         max_shards_per_chrom_step1=max_shards_per_chrom_step1,

wdl/GATKSVPipelineBatch.wdl

@@ -452,8 +452,8 @@ workflow GATKSVPipelineBatch {
     # ResolveComplexVariants
     Array[File] complex_resolve_vcfs = MakeCohortVcf.complex_resolve_vcfs
     Array[File] complex_resolve_vcf_indexes = MakeCohortVcf.complex_resolve_vcf_indexes
-    Array[File] complex_resolve_bothside_pass_lists = MakeCohortVcf.complex_resolve_bothside_pass_lists
-    Array[File] complex_resolve_background_fail_lists = MakeCohortVcf.complex_resolve_background_fail_lists
+    File complex_resolve_bothside_pass_list = MakeCohortVcf.complex_resolve_bothside_pass_list
+    File complex_resolve_background_fail_list = MakeCohortVcf.complex_resolve_background_fail_list
     Array[File] breakpoint_overlap_dropped_record_vcfs = MakeCohortVcf.breakpoint_overlap_dropped_record_vcfs
     Array[File] breakpoint_overlap_dropped_record_vcf_indexes = MakeCohortVcf.breakpoint_overlap_dropped_record_vcf_indexes
 

wdl/GATKSVPipelineSingleSample.wdl

@@ -440,6 +440,8 @@ workflow GATKSVPipelineSingleSample {
     RuntimeAttr? runtime_override_update_sr_list_fail
     RuntimeAttr? runtime_override_breakpoint_overlap_filter
     RuntimeAttr? runtime_override_concat_resolve
+    RuntimeAttr? runtime_override_concat_bothside_pass
+    RuntimeAttr? runtime_override_concat_background_fail
 
     RuntimeAttr? runtime_override_get_se_cutoff
     RuntimeAttr? runtime_override_shard_vcf_cpx
@@ -1192,6 +1194,8 @@ workflow GATKSVPipelineSingleSample {
       runtime_override_update_sr_list_fail=runtime_override_update_sr_list_fail,
       runtime_override_subset_inversions=runtime_override_subset_inversions,
       runtime_override_concat_resolve=runtime_override_concat_resolve,
+      runtime_override_concat_bothside_pass=runtime_override_concat_bothside_pass,
+      runtime_override_concat_background_fail=runtime_override_concat_background_fail,
       runtime_override_get_se_cutoff=runtime_override_get_se_cutoff,
       runtime_override_shard_vcf_cpx=runtime_override_shard_vcf_cpx,
       runtime_override_shard_vids_resolve=runtime_override_shard_vids_resolve,

wdl/MakeCohortVcf.wdl

@@ -122,6 +122,8 @@ workflow MakeCohortVcf {
     RuntimeAttr? runtime_override_update_sr_list_pass
     RuntimeAttr? runtime_override_update_sr_list_fail
     RuntimeAttr? runtime_override_concat_resolve
+    RuntimeAttr? runtime_override_concat_bothside_pass
+    RuntimeAttr? runtime_override_concat_background_fail
 
     RuntimeAttr? runtime_override_get_se_cutoff
     RuntimeAttr? runtime_override_shard_vcf_cpx
@@ -146,6 +148,9 @@ workflow MakeCohortVcf {
     RuntimeAttr? runtime_override_hail_merge_resolve_inv
     RuntimeAttr? runtime_override_fix_header_resolve_inv
 
+    # overrides for ReshardVcf
+    RuntimeAttr? runtime_override_reshard
+
     # overrides for GenotypeComplexContig
     RuntimeAttr? runtime_override_ids_from_median
     RuntimeAttr? runtime_override_split_vcf_to_genotype
@@ -300,6 +305,8 @@ workflow MakeCohortVcf {
       runtime_override_breakpoint_overlap_filter=runtime_override_breakpoint_overlap_filter,
       runtime_override_subset_inversions=runtime_override_subset_inversions,
       runtime_override_concat=runtime_override_concat_resolve,
+      runtime_override_concat_bothside_pass=runtime_override_concat_bothside_pass,
+      runtime_override_concat_background_fail=runtime_override_concat_background_fail,
 
       runtime_override_get_se_cutoff=runtime_override_get_se_cutoff,
       runtime_override_shard_vcf_cpx=runtime_override_shard_vcf_cpx,
@@ -364,8 +371,8 @@ workflow MakeCohortVcf {
     input:
       cohort_name=cohort_name,
       complex_genotype_vcfs=GenotypeComplexVariants.complex_genotype_vcfs,
-      complex_resolve_bothside_pass_lists=ResolveComplexVariants.complex_resolve_bothside_pass_lists,
-      complex_resolve_background_fail_lists=ResolveComplexVariants.complex_resolve_background_fail_lists,
+      complex_resolve_bothside_pass_list=ResolveComplexVariants.complex_resolve_bothside_pass_list,
+      complex_resolve_background_fail_list=ResolveComplexVariants.complex_resolve_background_fail_list,
       ped_file=ped_file,
       contig_list=contig_list,
       allosome_fai=allosome_fai,
@@ -495,8 +502,8 @@ workflow MakeCohortVcf {
     # ResolveComplexVariants
     Array[File] complex_resolve_vcfs = ResolveComplexVariants.complex_resolve_vcfs
     Array[File] complex_resolve_vcf_indexes = ResolveComplexVariants.complex_resolve_vcf_indexes
-    Array[File] complex_resolve_bothside_pass_lists = ResolveComplexVariants.complex_resolve_bothside_pass_lists
-    Array[File] complex_resolve_background_fail_lists = ResolveComplexVariants.complex_resolve_background_fail_lists
+    File complex_resolve_bothside_pass_list = ResolveComplexVariants.complex_resolve_bothside_pass_list
+    File complex_resolve_background_fail_list = ResolveComplexVariants.complex_resolve_background_fail_list
     Array[File] breakpoint_overlap_dropped_record_vcfs = ResolveComplexVariants.breakpoint_overlap_dropped_record_vcfs
     Array[File] breakpoint_overlap_dropped_record_vcf_indexes = ResolveComplexVariants.breakpoint_overlap_dropped_record_vcf_indexes
 

wdl/ReshardVcf.wdl

@@ -3,20 +3,19 @@ version 1.0
 import "Structs.wdl"
 import "TasksMakeCohortVcf.wdl" as MiniTasks
 
-# Consumes a contig-sharded vcf where some shards may contain a small proportion of records from other
-#  contigs and moves them to their correct shards.
+# Consumes a set of vcfs and distributes them into contig shards
 
 workflow ReshardVcf {
   input {
-    Array[File] vcfs  # Must be sorted and indexed
+    Array[File] vcfs  # Order does not matter but must be sorted and indexed
     File contig_list
     String prefix
     Boolean? use_ssd
     String sv_base_mini_docker
     RuntimeAttr? runtime_override_reshard
   }
 
-  Array[String] contigs = read_lines(contig_list)
+  Array[String] contigs = transpose(read_tsv(contig_list))[0]
 
   scatter (i in range(length(vcfs))) {
     File vcf_indexes = vcfs[i] + ".tbi"

wdl/ResolveComplexVariants.wdl

@@ -1,5 +1,6 @@
 version 1.0
 
+import "ReshardVcf.wdl" as Reshard
 import "ResolveCpxSv.wdl" as ResolveComplexContig
 import "TasksMakeCohortVcf.wdl" as MiniTasks
 
@@ -37,6 +38,8 @@ workflow ResolveComplexVariants {
     RuntimeAttr? runtime_override_breakpoint_overlap_filter
     RuntimeAttr? runtime_override_subset_inversions
     RuntimeAttr? runtime_override_concat
+    RuntimeAttr? runtime_override_concat_bothside_pass
+    RuntimeAttr? runtime_override_concat_background_fail
 
     # overrides for ResolveComplexContig
     RuntimeAttr? runtime_override_get_se_cutoff
@@ -62,6 +65,9 @@ workflow ResolveComplexVariants {
     RuntimeAttr? runtime_override_preconcat_inv
     RuntimeAttr? runtime_override_hail_merge_inv
     RuntimeAttr? runtime_override_fix_header_inv
+
+    # overrides for ReshardVcf
+    RuntimeAttr? runtime_override_reshard
   }
 
   #Scatter per chromosome
@@ -203,6 +209,33 @@ workflow ResolveComplexVariants {
     }
   }
 
+  call Reshard.ReshardVcf {
+    input:
+      vcfs=RenameVariants.renamed_vcf,
+      contig_list=contig_list,
+      prefix="~{cohort_name}.reshard_vcf",
+      sv_base_mini_docker=sv_base_mini_docker,
+      runtime_override_reshard=runtime_override_reshard
+  }
+
+  # Due to vcf resharding, these contig-sharded lists are no longer valid
+  # Here we combine variant list arrays into single genome-wide lists
+  # This is faster than reshuffling and shouldn't have much performance impact in CleanVcf1a
+  call MiniTasks.CatUncompressedFiles as ConcatBothsidePass {
+    input:
+      shards=UpdateBothsidePass.updated_list,
+      outfile_name="~{cohort_name}.all.sr_bothside_pass.updated3.txt",
+      sv_base_mini_docker=sv_base_mini_docker,
+      runtime_attr_override=runtime_override_concat_bothside_pass
+  }
+  call MiniTasks.CatUncompressedFiles as ConcatBackgroundFail {
+    input:
+      shards=UpdateBackgroundFail.updated_list,
+      outfile_name="~{cohort_name}.all.sr_background_fail.updated3.txt",
+      sv_base_mini_docker=sv_base_mini_docker,
+      runtime_attr_override=runtime_override_concat_background_fail
+  }
+
   #Merge resolved vcfs for QC
   if (merge_vcfs) {
     call MiniTasks.ConcatVcfs {
@@ -218,10 +251,10 @@ workflow ResolveComplexVariants {
 
   #Final outputs
   output {
-    Array[File] complex_resolve_vcfs = RenameVariants.renamed_vcf
-    Array[File] complex_resolve_vcf_indexes = RenameVariants.renamed_vcf_index
-    Array[File] complex_resolve_bothside_pass_lists = UpdateBothsidePass.updated_list
-    Array[File] complex_resolve_background_fail_lists = UpdateBackgroundFail.updated_list
+    Array[File] complex_resolve_vcfs = ReshardVcf.resharded_vcfs
+    Array[File] complex_resolve_vcf_indexes = ReshardVcf.resharded_vcf_indexes
+    File complex_resolve_bothside_pass_list = ConcatBothsidePass.outfile
+    File complex_resolve_background_fail_list = ConcatBackgroundFail.outfile
     Array[File] breakpoint_overlap_dropped_record_vcfs = BreakpointOverlap.dropped_record_vcf
     Array[File] breakpoint_overlap_dropped_record_vcf_indexes = BreakpointOverlap.dropped_record_vcf_index
     File? complex_resolve_merged_vcf = ConcatVcfs.concat_vcf