Merge branch 'develop' into lk-PD2432-gex-barcodes-to-fragment-file

pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.changelog.md

@@ -1,3 +1,8 @@
+# 2.1.7
+2023-11-21 (Date of Last Commit)
+
+* Fixes bug so now ReblockGVCFs can take in GVCFs that are not in the same location as their index file
+
 # 2.1.6
 2023-09-18 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl

@@ -5,7 +5,7 @@ import "../../../../../../tasks/broad/Qc.wdl" as QC
 
 workflow ReblockGVCF {
 
-  String pipeline_version = "2.1.6"
+  String pipeline_version = "2.1.7"
 
 
   input {

pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/test_inputs/Plumbing/G96830.NA12878.index.json

@@ -0,0 +1,9 @@
+{
+    "ReblockGVCF.gvcf": "gs://broad-gotc-test-storage/reblock_gvcf/wgs/plumbing/input/G96830.NA12878/NA12878_PLUMBING.g.vcf.gz",
+    "ReblockGVCF.gvcf_index": "gs://broad-gotc-test-storage/reblock_gvcf/wgs/plumbing/input/G96830.NA12878/index_in_different_location/NA12878_PLUMBING.g.vcf.gz.tbi",
+    "ReblockGVCF.calling_interval_list": "gs://gcp-public-data--broad-references/hg38/v0/wgs_calling_regions.hg38.interval_list",
+    "ReblockGVCF.ref_fasta": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta",
+    "ReblockGVCF.ref_fasta_index": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta.fai",
+    "ReblockGVCF.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict"
+  }
+

pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md

@@ -1,3 +1,8 @@
+# 3.1.14
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 3.1.13
 2023-10-10 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl

@@ -44,7 +44,7 @@ import "../../../../../../structs/dna_seq/DNASeqStructs.wdl"
 # WORKFLOW DEFINITION
 workflow ExomeGermlineSingleSample {
 
-  String pipeline_version = "3.1.13"
+  String pipeline_version = "3.1.14"
 
 
   input {

pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.changelog.md

@@ -1,3 +1,8 @@
+# 1.0.11
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 1.0.10
 2023-09-18 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.wdl

@@ -50,7 +50,7 @@ workflow UltimaGenomicsWholeGenomeGermline {
     filtering_model_no_gt_name: "String describing the optional filtering model; default set to rf_model_ignore_gt_incl_hpol_runs"
   }
 
-  String pipeline_version = "1.0.10"
+  String pipeline_version = "1.0.11"
 
 
   References references = alignment_references.references

pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md

@@ -1,3 +1,8 @@
+# 3.1.15
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 3.1.14 
 2023-10-10 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl

@@ -40,7 +40,7 @@ import "../../../../../../structs/dna_seq/DNASeqStructs.wdl"
 workflow WholeGenomeGermlineSingleSample {
 
 
-  String pipeline_version = "3.1.14"
+  String pipeline_version = "3.1.15"
 
 
   input {

pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.changelog.md

@@ -1,3 +1,8 @@
+# 2.1.13
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 2.1.12
 2023-09-18 (Date of Last Commit)
 

pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl

@@ -9,7 +9,7 @@ import "../../../../../tasks/broad/DragenTasks.wdl" as DragenTasks
 workflow VariantCalling {
 
 
-  String pipeline_version = "2.1.12"
+  String pipeline_version = "2.1.13"
 
 
   input {

pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.changelog.md

@@ -1,3 +1,8 @@
+# 1.0.11
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 1.0.10
 2023-09-18 (Date of Last Commit)
 

pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl

@@ -43,7 +43,7 @@ workflow UltimaGenomicsWholeGenomeCramOnly {
     save_bam_file: "If true, then save intermeidate ouputs used by germline pipeline (such as the output BAM) otherwise they won't be kept as outputs."
   }
 
-  String pipeline_version = "1.0.10"
+  String pipeline_version = "1.0.11"
 
   References references = alignment_references.references
 

pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.changelog.md

@@ -1,3 +1,8 @@
+# 1.0.12
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 1.0.11
 2023-09-18 (Date of Last Commit)
 

pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.wdl

@@ -6,7 +6,7 @@ import "../../../../../../../pipelines/broad/qc/CheckFingerprint.wdl" as FP
 
 workflow BroadInternalUltimaGenomics {
 
-  String pipeline_version = "1.0.11"
+  String pipeline_version = "1.0.12"
 
   input {
 

pipelines/broad/reprocessing/exome/ExomeReprocessing.changelog.md

@@ -1,3 +1,8 @@
+# 3.1.14
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 3.1.13
 2023-10-10 (Date of Last Commit)
 

pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl

@@ -7,7 +7,7 @@ import "../../../../structs/dna_seq/DNASeqStructs.wdl"
 workflow ExomeReprocessing {
 
 
-  String pipeline_version = "3.1.13"
+  String pipeline_version = "3.1.14"
 
   input {
     File? input_cram

pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.changelog.md

@@ -1,3 +1,8 @@
+# 3.1.16
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 3.1.15
 2023-10-10 (Date of Last Commit)
 

pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.wdl

@@ -5,7 +5,7 @@ import "../../../../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow ExternalExomeReprocessing {
 
-  String pipeline_version = "3.1.15"
+  String pipeline_version = "3.1.16"
 
 
   input {

pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.changelog.md

@@ -1,3 +1,8 @@
+# 2.1.16
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 2.1.15
 2023-10-10 (Date of Last Commit)
 

pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.wdl

@@ -6,7 +6,7 @@ import "../../../../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
 workflow ExternalWholeGenomeReprocessing {
 
 
-  String pipeline_version = "2.1.15"
+  String pipeline_version = "2.1.16"
 
   input {
     File? input_cram

pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.changelog.md

@@ -1,3 +1,8 @@
+# 3.1.15
+2023-11-21 (Date of Last Commit)
+
+* Fixed bug in ReblockGVCFs; this does not affect this pipeline.
+
 # 3.1.14
 2023-10-10 (Date of Last Commit)
 

pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl

@@ -6,7 +6,7 @@ import "../../../../structs/dna_seq/DNASeqStructs.wdl"
 
 workflow WholeGenomeReprocessing {
 
-  String pipeline_version = "3.1.14"
+  String pipeline_version = "3.1.15"
 
   input {
     File? input_cram

pipelines/skylab/multiome/Multiome.changelog.md

@@ -5,6 +5,11 @@
 * Updated the JoinBarcodes task to bgzip and tabix the final ATAC fragment file
 * Added the tabix index file as an output to Multiome
 
+# 2.3.3
+2023-11-21 (Date of Last Commit)
+
+* Added the latest warp-tools docker to tasks in the Metrics, FastqProcessing and H5adUtils wdls; this incorporates new input parameter for number of output fastq files to fastqprocess
+
 # 2.3.2
 2023-11-20 (Date of Last Commit)
 * Added an optional task to the Multiome.wdl that will run CellBender on the Optimus output h5ad file

pipelines/skylab/multiome/atac.changelog.md

@@ -1,3 +1,8 @@
+# 1.1.2
+2023-11-21 (Date of Last Commit)
+
+Added the latest warp-tools docker to tasks in the Metrics, FastqProcessing and H5adUtils wdls; this incorporates new input parameter for number of output fastq files to fastqprocess
+
 # 1.1.1
 2023-10-20 (Date of Last Commit)
 * Removed the dropna from the JoinBarcodes subtask of the H5adUtils WDL; this change does not impact ATAC outputs

pipelines/skylab/multiome/atac.wdl

@@ -34,7 +34,7 @@ workflow ATAC {
     String adapter_seq_read3 = "TCGTCGGCAGCGTCAGATGTGTATAAGAGACAG"
   }
 
-  String pipeline_version = "1.1.1"
+  String pipeline_version = "1.1.2"
 
   parameter_meta {
     read1_fastq_gzipped: "read 1 FASTQ file as input for the pipeline, contains read 1 of paired reads"

pipelines/skylab/optimus/Optimus.changelog.md

@@ -1,3 +1,7 @@
+# 6.2.2
+2023-11-21 (Date of Last Commit)
+
+* Added the latest warp-tools docker to tasks in the Metrics, FastqProcessing and H5adUtils wdls; this incorporates new input parameter for number of output fastq files to fastqprocess
 
 # 6.2.1
 2023-11-20 (Date of Last Commit)

pipelines/skylab/optimus/Optimus.wdl

@@ -64,7 +64,7 @@ workflow Optimus {
 
   # version of this pipeline
 
-  String pipeline_version = "6.2.1"
+  String pipeline_version = "6.2.2"
 
   # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays
   Array[Int] indices = range(length(r1_fastq))

pipelines/skylab/slideseq/SlideSeq.changelog.md

@@ -1,3 +1,8 @@
+# 2.1.2
+2023-11-21 (Date of Last Commit)
+
+* Added the latest warp-tools docker to tasks in the Metrics, FastqProcessing and H5adUtils wdls; this incorporates new input parameter for number of output fastq files to fastqprocess
+
 # 2.1.1
 2023-11-20 (Date of Last Commit)
 

pipelines/skylab/slideseq/SlideSeq.wdl

@@ -23,7 +23,7 @@ import "../../../tasks/skylab/MergeSortBam.wdl" as Merge
 
 workflow SlideSeq {
 
-    String pipeline_version = "2.1.1"
+    String pipeline_version = "2.1.2"
 
     input {
         Array[File] r1_fastq

tasks/broad/GermlineVariantDiscovery.wdl

@@ -223,7 +223,7 @@ task Reblock {
     gatk --java-options "-Xms3000m -Xmx3000m" \
       ReblockGVCF \
       -R ~{ref_fasta} \
-      -V ~{gvcf} \
+      -V ~{gvcf_basename} \
       -do-qual-approx \
       --floor-blocks -GQB 20 -GQB 30 -GQB 40 \
       ~{annotations_to_keep_command} \

tasks/skylab/FastqProcessing.wdl

@@ -11,7 +11,7 @@ task FastqProcessing {
     String read_struct
 
     #using the latest build of warp-tools in GCR
-    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671"
+    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
     #runtime values
     Int machine_mem_mb = 40000
     Int cpu = 16   
@@ -137,7 +137,7 @@ task FastqProcessingSlidSeq {
 
 
     # Runtime attributes
-    String docker =  "us.gcr.io/broad-gotc-prod/warp-tools:1.0.1-1686932671"
+    String docker =  "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
     Int cpu = 16
     Int machine_mb = 40000
     Int disk = ceil(size(r1_fastq, "GiB")*3 + size(r2_fastq, "GiB")*3) + 50
@@ -246,7 +246,7 @@ task FastqProcessATAC {
 
         # [?] copied from corresponding optimus wdl for fastqprocessing
         # using the latest build of warp-tools in GCR
-        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.7-1695393479"
+        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
 
         # Runtime attributes [?]
         Int mem_size = 5
@@ -255,6 +255,10 @@ task FastqProcessATAC {
         # estimate that bam is approximately equal in size to fastq, add 20% buffer
         Int disk_size = ceil(2 * ( size(read1_fastq, "GiB") + size(read3_fastq, "GiB") + size(barcodes_fastq, "GiB") )) + 400
         Int preemptible = 3
+
+        # Additional parameters for fastqprocess
+        Int num_output_files = 1
+        Int bam_size = 1
     }
 
     meta {
@@ -273,6 +277,8 @@ task FastqProcessATAC {
         mem_size: "(optional) the amount of memory (MiB) to provision for this task"
         cpu: "(optional) the number of cpus to provision for this task"
         disk_size: "(optional) the amount of disk space (GiB) to provision for this task"
+        num_output_files: "(optional) the number of output fastq file shards to produce. if this is set to > 0, bam_size is ignored."
+        bam_size: "(optional) the size of each fastq file produced. this is taken into account if num_output_files == 0."
         preemptible: "(optional) if non-zero, request a pre-emptible instance and allow for this number of preemptions before running the task on a non preemptible machine"
     }
 
@@ -341,7 +347,8 @@ task FastqProcessATAC {
         cd /cromwell_root/output_fastq
 
         fastqprocess \
-        --bam-size 30.0 \
+        --bam-size ~{bam_size} \
+        --num-output-files ~{num_output_files} \
         --sample-id "~{output_base_name}" \
         $R1_FILES_CONCAT \
         $R2_FILES_CONCAT \

tasks/skylab/H5adUtils.wdl

@@ -6,7 +6,8 @@ task OptimusH5adGeneration {
 
   input {
     #runtime values
-    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.6-1692962087"
+    #String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.6-1692962087"
+    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
     # name of the sample
     String input_id
     # user provided id
@@ -105,7 +106,8 @@ task SingleNucleusOptimusH5adOutput {
 
     input {
         #runtime values
-        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.6-1692962087"
+        #String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.6-1692962087"
+        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
         # name of the sample
         String input_id
         # user provided id
@@ -289,4 +291,4 @@ task JoinMultiomeBarcodes {
     File atac_fragment_tsv = "~{atac_fragment_base}.sorted.tsv.gz"
     File atac_fragment_tsv_tbi = "~{atac_fragment_base}.sorted.tsv.gz.tbi"
   }
-}
+}