From a4363d482e1bc257e3f65cf0835fc95339388535 Mon Sep 17 00:00:00 2001 From: Megan Shand Date: Thu, 7 Dec 2023 11:24:01 -0500 Subject: [PATCH 1/6] making memory an arg on ValidateVariants --- tasks/broad/Qc.wdl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tasks/broad/Qc.wdl b/tasks/broad/Qc.wdl index 43da63fbb7..e0d778020a 100644 --- a/tasks/broad/Qc.wdl +++ b/tasks/broad/Qc.wdl @@ -622,12 +622,14 @@ task ValidateVCF { Boolean is_gvcf = true String? extra_args String gatk_docker = "us.gcr.io/broad-gatk/gatk:4.3.0.0" + Int memory_in_mb = 7000 } Boolean calling_intervals_is_vcf = defined(calling_interval_list_index) String calling_interval_list_basename = basename(calling_interval_list) String calling_interval_list_index_basename = if calling_intervals_is_vcf then basename(select_first([calling_interval_list_index])) else "" + Int command_mem_in_mb = memory_in_mb - 1000 Float ref_size = size(ref_fasta, "GiB") + size(ref_fasta_index, "GiB") + size(ref_dict, "GiB") Int disk_size = ceil(size(input_vcf, "GiB") + size(dbsnp_vcf, "GiB") + ref_size) + 20 @@ -641,7 +643,7 @@ task ValidateVCF { fi # Note that WGS needs a lot of memory to do the -L *.vcf if an interval file is not supplied - gatk --java-options "-Xms6000m -Xmx6500m" \ + gatk --java-options "-Xms~{command_mem_in_mb}m -Xmx~{command_mem_in_mb}m" \ ValidateVariants \ -V ~{input_vcf} \ -R ~{ref_fasta} \ @@ -654,7 +656,7 @@ task ValidateVCF { runtime { docker: gatk_docker preemptible: preemptible_tries - memory: "7000 MiB" + memory: memory_in_mb + " MiB" bootDiskSizeGb: 15 disks: "local-disk " + disk_size + " HDD" } From 3efb911859d284849c9976f4f35c95811d4478dc Mon Sep 17 00:00:00 2001 From: Megan Shand Date: Fri, 8 Dec 2023 13:57:09 -0500 Subject: [PATCH 2/6] clean up --- tasks/broad/Qc.wdl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/broad/Qc.wdl b/tasks/broad/Qc.wdl index e0d778020a..3dadae1b72 100644 --- a/tasks/broad/Qc.wdl +++ b/tasks/broad/Qc.wdl @@ -622,14 +622,14 @@ task ValidateVCF { Boolean is_gvcf = true String? extra_args String gatk_docker = "us.gcr.io/broad-gatk/gatk:4.3.0.0" - Int memory_in_mb = 7000 + Int machine_mem_mb = 7000 } Boolean calling_intervals_is_vcf = defined(calling_interval_list_index) String calling_interval_list_basename = basename(calling_interval_list) String calling_interval_list_index_basename = if calling_intervals_is_vcf then basename(select_first([calling_interval_list_index])) else "" - Int command_mem_in_mb = memory_in_mb - 1000 + Int command_mem_mb = machine_mem_mb - 1000 Float ref_size = size(ref_fasta, "GiB") + size(ref_fasta_index, "GiB") + size(ref_dict, "GiB") Int disk_size = ceil(size(input_vcf, "GiB") + size(dbsnp_vcf, "GiB") + ref_size) + 20 @@ -643,7 +643,7 @@ task ValidateVCF { fi # Note that WGS needs a lot of memory to do the -L *.vcf if an interval file is not supplied - gatk --java-options "-Xms~{command_mem_in_mb}m -Xmx~{command_mem_in_mb}m" \ + gatk --java-options "-Xms~{command_mem_mb}m -Xmx~{command_mem_mb}m" \ ValidateVariants \ -V ~{input_vcf} \ -R ~{ref_fasta} \ @@ -656,7 +656,7 @@ task ValidateVCF { runtime { docker: gatk_docker preemptible: preemptible_tries - memory: memory_in_mb + " MiB" + memory: machine_mem_mb + " MiB" bootDiskSizeGb: 15 disks: "local-disk " + disk_size + " HDD" } From 3c180f974bc225534714a2155fd19384a7a01f8b Mon Sep 17 00:00:00 2001 From: Megan Shand Date: Fri, 8 Dec 2023 14:18:14 -0500 Subject: [PATCH 3/6] changelogs --- pipelines/broad/arrays/single_sample/Arrays.changelog.md | 5 +++++ pipelines/broad/arrays/single_sample/Arrays.wdl | 2 +- .../joint_genotyping/reblocking/ReblockGVCF.changelog.md | 5 +++++ .../germline/joint_genotyping/reblocking/ReblockGVCF.wdl | 2 +- .../exome/ExomeGermlineSingleSample.changelog.md | 5 +++++ .../single_sample/exome/ExomeGermlineSingleSample.wdl | 2 +- .../ugwgs/UltimaGenomicsWholeGenomeGermline.changelog.md | 5 +++++ .../ugwgs/UltimaGenomicsWholeGenomeGermline.wdl | 2 +- .../wgs/WholeGenomeGermlineSingleSample.changelog.md | 5 +++++ .../single_sample/wgs/WholeGenomeGermlineSingleSample.wdl | 2 +- .../germline/variant_calling/VariantCalling.changelog.md | 5 +++++ .../dna_seq/germline/variant_calling/VariantCalling.wdl | 2 +- .../ugwgs/UltimaGenomicsWholeGenomeCramOnly.changelog.md | 5 +++++ .../ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl | 2 +- .../genotyping/illumina/IlluminaGenotypingArray.changelog.md | 5 +++++ .../broad/genotyping/illumina/IlluminaGenotypingArray.wdl | 2 +- .../arrays/single_sample/BroadInternalArrays.changelog.md | 5 +++++ .../internal/arrays/single_sample/BroadInternalArrays.wdl | 2 +- .../UltimaGenomics/BroadInternalUltimaGenomics.changelog.md | 5 +++++ .../UltimaGenomics/BroadInternalUltimaGenomics.wdl | 2 +- .../internal/rna_seq/BroadInternalRNAWithUMIs.changelog.md | 5 +++++ .../broad/internal/rna_seq/BroadInternalRNAWithUMIs.wdl | 2 +- pipelines/broad/qc/CheckFingerprint.changelog.md | 5 +++++ pipelines/broad/qc/CheckFingerprint.wdl | 2 +- .../broad/reprocessing/exome/ExomeReprocessing.changelog.md | 5 +++++ pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl | 2 +- .../external/exome/ExternalExomeReprocessing.changelog.md | 5 +++++ .../external/exome/ExternalExomeReprocessing.wdl | 2 +- .../wgs/ExternalWholeGenomeReprocessing.changelog.md | 5 +++++ .../external/wgs/ExternalWholeGenomeReprocessing.wdl | 2 +- .../reprocessing/wgs/WholeGenomeReprocessing.changelog.md | 5 +++++ pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl | 2 +- 32 files changed, 96 insertions(+), 16 deletions(-) diff --git a/pipelines/broad/arrays/single_sample/Arrays.changelog.md b/pipelines/broad/arrays/single_sample/Arrays.changelog.md index 4e3f85640b..dc8494544b 100644 --- a/pipelines/broad/arrays/single_sample/Arrays.changelog.md +++ b/pipelines/broad/arrays/single_sample/Arrays.changelog.md @@ -1,3 +1,8 @@ +# 2.6.21 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 2.6.20 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/arrays/single_sample/Arrays.wdl b/pipelines/broad/arrays/single_sample/Arrays.wdl index 3fb50c9798..e7693b152d 100644 --- a/pipelines/broad/arrays/single_sample/Arrays.wdl +++ b/pipelines/broad/arrays/single_sample/Arrays.wdl @@ -23,7 +23,7 @@ import "../../../../tasks/broad/Utilities.wdl" as utils workflow Arrays { - String pipeline_version = "2.6.20" + String pipeline_version = "2.6.21" input { String chip_well_barcode diff --git a/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.changelog.md b/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.changelog.md index 6e6d1193b9..b67a277f04 100644 --- a/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.changelog.md +++ b/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.changelog.md @@ -1,3 +1,8 @@ +# 2.1.9 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter + # 2.1.8 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl b/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl index dcc0628f76..2e9585b825 100644 --- a/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl +++ b/pipelines/broad/dna_seq/germline/joint_genotyping/reblocking/ReblockGVCF.wdl @@ -5,7 +5,7 @@ import "../../../../../../tasks/broad/Qc.wdl" as QC workflow ReblockGVCF { - String pipeline_version = "2.1.8" + String pipeline_version = "2.1.9" input { diff --git a/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md b/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md index 49c8781aef..4cab7e0b06 100644 --- a/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md +++ b/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.changelog.md @@ -1,3 +1,8 @@ +# 3.1.16 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 3.1.15 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl b/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl index 4517778728..f87f7f8a4f 100644 --- a/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl +++ b/pipelines/broad/dna_seq/germline/single_sample/exome/ExomeGermlineSingleSample.wdl @@ -44,7 +44,7 @@ import "../../../../../../structs/dna_seq/DNASeqStructs.wdl" # WORKFLOW DEFINITION workflow ExomeGermlineSingleSample { - String pipeline_version = "3.1.15" + String pipeline_version = "3.1.16" input { diff --git a/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.changelog.md b/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.changelog.md index 9b1d7d7574..b1d38a4c45 100644 --- a/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.changelog.md +++ b/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.changelog.md @@ -1,3 +1,8 @@ +# 1.0.13 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 1.0.12 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.wdl b/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.wdl index 4202abe21f..e09bcf3a05 100644 --- a/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.wdl +++ b/pipelines/broad/dna_seq/germline/single_sample/ugwgs/UltimaGenomicsWholeGenomeGermline.wdl @@ -50,7 +50,7 @@ workflow UltimaGenomicsWholeGenomeGermline { filtering_model_no_gt_name: "String describing the optional filtering model; default set to rf_model_ignore_gt_incl_hpol_runs" } - String pipeline_version = "1.0.12" + String pipeline_version = "1.0.13" References references = alignment_references.references diff --git a/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md b/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md index e5e4b4f959..81cd464c07 100644 --- a/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md +++ b/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.changelog.md @@ -1,3 +1,8 @@ +# 3.1.17 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 3.1.16 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl b/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl index 62a868d971..e1a08e1315 100644 --- a/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl +++ b/pipelines/broad/dna_seq/germline/single_sample/wgs/WholeGenomeGermlineSingleSample.wdl @@ -40,7 +40,7 @@ import "../../../../../../structs/dna_seq/DNASeqStructs.wdl" workflow WholeGenomeGermlineSingleSample { - String pipeline_version = "3.1.16" + String pipeline_version = "3.1.17" input { diff --git a/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.changelog.md b/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.changelog.md index 7f3886c671..1216d5ff7d 100644 --- a/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.changelog.md +++ b/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.changelog.md @@ -1,3 +1,8 @@ +# 2.1.15 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 2.1.14 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl b/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl index 28154f9277..8741308fab 100644 --- a/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl +++ b/pipelines/broad/dna_seq/germline/variant_calling/VariantCalling.wdl @@ -9,7 +9,7 @@ import "../../../../../tasks/broad/DragenTasks.wdl" as DragenTasks workflow VariantCalling { - String pipeline_version = "2.1.14" + String pipeline_version = "2.1.15" input { diff --git a/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.changelog.md b/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.changelog.md index 2e8e29187a..1883574518 100644 --- a/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.changelog.md +++ b/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.changelog.md @@ -1,3 +1,8 @@ +# 1.0.13 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 1.0.12 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl b/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl index c574cc55ca..af1c5551ea 100644 --- a/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl +++ b/pipelines/broad/dna_seq/somatic/single_sample/ugwgs/UltimaGenomicsWholeGenomeCramOnly.wdl @@ -43,7 +43,7 @@ workflow UltimaGenomicsWholeGenomeCramOnly { save_bam_file: "If true, then save intermeidate ouputs used by germline pipeline (such as the output BAM) otherwise they won't be kept as outputs." } - String pipeline_version = "1.0.12" + String pipeline_version = "1.0.13" References references = alignment_references.references diff --git a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.changelog.md b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.changelog.md index 5505a01326..ba50f4ee6d 100644 --- a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.changelog.md +++ b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.changelog.md @@ -1,3 +1,8 @@ +# 1.12.15 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 1.12.14 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl index efc7b19185..c7a35494db 100644 --- a/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl +++ b/pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl @@ -21,7 +21,7 @@ import "../../../../tasks/broad/Qc.wdl" as Qc workflow IlluminaGenotypingArray { - String pipeline_version = "1.12.14" + String pipeline_version = "1.12.15" input { String sample_alias diff --git a/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.changelog.md b/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.changelog.md index 0e7cbb3af8..ba8009535f 100644 --- a/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.changelog.md +++ b/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.changelog.md @@ -1,3 +1,8 @@ +# 1.1.5 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 1.1.4 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.wdl b/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.wdl index 5d7eb9f43a..185ab12d95 100644 --- a/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.wdl +++ b/pipelines/broad/internal/arrays/single_sample/BroadInternalArrays.wdl @@ -9,7 +9,7 @@ workflow BroadInternalArrays { description: "Push outputs of Arrays.wdl to TDR dataset table ArraysOutputsTable." } - String pipeline_version = "1.1.4" + String pipeline_version = "1.1.5" input { # inputs to wrapper task diff --git a/pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.changelog.md b/pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.changelog.md index b368628cb7..e6d3ee8655 100644 --- a/pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.changelog.md +++ b/pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.changelog.md @@ -1,3 +1,8 @@ +# 1.0.14 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 1.0.13 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.wdl b/pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.wdl index 368d0f5e97..2656884f84 100644 --- a/pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.wdl +++ b/pipelines/broad/internal/dna_seq/germline/single_sample/UltimaGenomics/BroadInternalUltimaGenomics.wdl @@ -6,7 +6,7 @@ import "../../../../../../../pipelines/broad/qc/CheckFingerprint.wdl" as FP workflow BroadInternalUltimaGenomics { - String pipeline_version = "1.0.13" + String pipeline_version = "1.0.14" input { diff --git a/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.changelog.md b/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.changelog.md index 1912295bd9..782832746f 100644 --- a/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.changelog.md +++ b/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.changelog.md @@ -1,3 +1,8 @@ +# 1.0.27 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 1.0.26 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.wdl b/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.wdl index 884f74c62d..466ea8c070 100644 --- a/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.wdl +++ b/pipelines/broad/internal/rna_seq/BroadInternalRNAWithUMIs.wdl @@ -7,7 +7,7 @@ import "../../../../tasks/broad/Utilities.wdl" as utils workflow BroadInternalRNAWithUMIs { - String pipeline_version = "1.0.26" + String pipeline_version = "1.0.27" input { # input needs to be either "hg19" or "hg38" diff --git a/pipelines/broad/qc/CheckFingerprint.changelog.md b/pipelines/broad/qc/CheckFingerprint.changelog.md index b17d4f1494..cd3db0e0e3 100644 --- a/pipelines/broad/qc/CheckFingerprint.changelog.md +++ b/pipelines/broad/qc/CheckFingerprint.changelog.md @@ -1,3 +1,8 @@ +# 1.0.14 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 1.0.13 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/qc/CheckFingerprint.wdl b/pipelines/broad/qc/CheckFingerprint.wdl index 4e6234c409..24d2c5f3ea 100644 --- a/pipelines/broad/qc/CheckFingerprint.wdl +++ b/pipelines/broad/qc/CheckFingerprint.wdl @@ -24,7 +24,7 @@ import "../../../tasks/broad/Qc.wdl" as Qc workflow CheckFingerprint { - String pipeline_version = "1.0.13" + String pipeline_version = "1.0.14" input { File? input_vcf diff --git a/pipelines/broad/reprocessing/exome/ExomeReprocessing.changelog.md b/pipelines/broad/reprocessing/exome/ExomeReprocessing.changelog.md index 636050a591..2776f63f89 100644 --- a/pipelines/broad/reprocessing/exome/ExomeReprocessing.changelog.md +++ b/pipelines/broad/reprocessing/exome/ExomeReprocessing.changelog.md @@ -1,3 +1,8 @@ +# 3.1.16 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 3.1.15 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl b/pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl index 0a10b0a5df..40880130d7 100644 --- a/pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl +++ b/pipelines/broad/reprocessing/exome/ExomeReprocessing.wdl @@ -7,7 +7,7 @@ import "../../../../structs/dna_seq/DNASeqStructs.wdl" workflow ExomeReprocessing { - String pipeline_version = "3.1.15" + String pipeline_version = "3.1.16" input { File? input_cram diff --git a/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.changelog.md b/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.changelog.md index e0aaa15ccb..de9e3ad232 100644 --- a/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.changelog.md +++ b/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.changelog.md @@ -1,3 +1,8 @@ +# 3.1.18 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 3.1.17 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.wdl b/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.wdl index 066e1e4f96..807f0cc1f2 100644 --- a/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.wdl +++ b/pipelines/broad/reprocessing/external/exome/ExternalExomeReprocessing.wdl @@ -5,7 +5,7 @@ import "../../../../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy workflow ExternalExomeReprocessing { - String pipeline_version = "3.1.17" + String pipeline_version = "3.1.18" input { diff --git a/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.changelog.md b/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.changelog.md index 1e7e345782..12844d6afb 100644 --- a/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.changelog.md +++ b/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.changelog.md @@ -1,3 +1,8 @@ +# 2.1.18 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 2.1.17 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.wdl b/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.wdl index 0ce84b3ba4..5814c8b253 100644 --- a/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.wdl +++ b/pipelines/broad/reprocessing/external/wgs/ExternalWholeGenomeReprocessing.wdl @@ -6,7 +6,7 @@ import "../../../../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy workflow ExternalWholeGenomeReprocessing { - String pipeline_version = "2.1.17" + String pipeline_version = "2.1.18" input { File? input_cram diff --git a/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.changelog.md b/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.changelog.md index 138bf607e4..7627d84fec 100644 --- a/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.changelog.md +++ b/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.changelog.md @@ -1,3 +1,8 @@ +# 3.1.17 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 3.1.16 2023-11-29 (Date of Last Commit) diff --git a/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl b/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl index 847554b60c..78c54cd078 100644 --- a/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl +++ b/pipelines/broad/reprocessing/wgs/WholeGenomeReprocessing.wdl @@ -6,7 +6,7 @@ import "../../../../structs/dna_seq/DNASeqStructs.wdl" workflow WholeGenomeReprocessing { - String pipeline_version = "3.1.16" + String pipeline_version = "3.1.17" input { File? input_cram From 5569a5373439d1c4e79f284b24876e179cd782d5 Mon Sep 17 00:00:00 2001 From: Kaylee Mathews <95316074+kayleemathews@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:48:47 -0500 Subject: [PATCH 4/6] update workflow versions in docs (#1145) --- .../Exome_Germline_Single_Sample_Pipeline/README.md | 2 +- .../README.md | 6 +++--- .../Whole_Genome_Germline_Single_Sample_Pipeline/README.md | 2 +- .../wgs.methods.md | 7 ++++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/website/docs/Pipelines/Exome_Germline_Single_Sample_Pipeline/README.md b/website/docs/Pipelines/Exome_Germline_Single_Sample_Pipeline/README.md index 1123f2e971..2165bd249d 100644 --- a/website/docs/Pipelines/Exome_Germline_Single_Sample_Pipeline/README.md +++ b/website/docs/Pipelines/Exome_Germline_Single_Sample_Pipeline/README.md @@ -7,7 +7,7 @@ slug: /Pipelines/Exome_Germline_Single_Sample_Pipeline/README | Pipeline Version | Date Updated | Documentation Author | Questions or Feedback | | :----: | :---: | :----: | :--------------: | -| [ExomeGermlineSingleSample_v3.1.15](https://github.com/broadinstitute/warp/releases?q=ExomeGermlineSingleSample_v3.0.0&expanded=true) | December, 2023 | [Elizabeth Kiernan](mailto:ekiernan@broadinstitute.org) | Please file GitHub issues in WARP or contact [the WARP team](mailto:warp-pipelines-help@broadinstitute.org) | +| [ExomeGermlineSingleSample_v3.1.16](https://github.com/broadinstitute/warp/releases?q=ExomeGermlineSingleSample_v3.0.0&expanded=true) | December, 2023 | [Elizabeth Kiernan](mailto:ekiernan@broadinstitute.org) | Please file GitHub issues in WARP or contact [the WARP team](mailto:warp-pipelines-help@broadinstitute.org) | The Exome Germline Single Sample pipeline implements data pre-processing and initial variant calling according to the GATK Best Practices for germline SNP and Indel discovery in human exome sequencing data. diff --git a/website/docs/Pipelines/Ultima_Genomics_Whole_Genome_Germline_Pipeline/README.md b/website/docs/Pipelines/Ultima_Genomics_Whole_Genome_Germline_Pipeline/README.md index af24764132..bd97ab96fa 100644 --- a/website/docs/Pipelines/Ultima_Genomics_Whole_Genome_Germline_Pipeline/README.md +++ b/website/docs/Pipelines/Ultima_Genomics_Whole_Genome_Germline_Pipeline/README.md @@ -7,7 +7,7 @@ slug: /Pipelines/Ultima_Genomics_Whole_Genome_Germline_Pipeline/README | Pipeline Version | Date Updated | Documentation Authors | Questions or Feedback | | :----: | :---: | :----: | :--------------: | -| [UltimaGenomicsWholeGenomeGermline_v1.0.0](https://github.com/broadinstitute/warp/releases) | June, 2022 | [Elizabeth Kiernan](mailto:ekiernan@broadinstitute.org) & [Kaylee Mathews](mailto:kmathews@broadinstitute.org)| Please file GitHub issues in warp or contact [the wARP team](mailto:warp-pipelines-help@broadinstitute.org) | +| [UltimaGenomicsWholeGenomeGermline_v1.0.13](https://github.com/broadinstitute/warp/releases) | December, 2023 | [Elizabeth Kiernan](mailto:ekiernan@broadinstitute.org) & [Kaylee Mathews](mailto:kmathews@broadinstitute.org)| Please file GitHub issues in warp or contact [the wARP team](mailto:warp-pipelines-help@broadinstitute.org) | ![UG_diagram](ug_diagram.png) @@ -19,7 +19,7 @@ The [Ultima Genomics Whole Genome Germline (UG_WGS) workflow](https://github.com Ultima Genomics sequencing is a novel technology that produces single-read, flow-based data ([Almogy et al., 2022](https://www.biorxiv.org/content/10.1101/2022.05.29.493900v1)). The sequencing platform works by flowing one nucleotide at a time in order, iteratively. This is in contrast to traditional technologies that do all four nucleotides at once. This iterative approach ensures that only one dNTP is responsible for the signal and it does not require the blocking of dNTPs. -### What does the workflow do? +### What does the workflow do? The workflow requires either an aligned CRAM output of the sequencing platform or an unmapped BAM as input. Overall, it aligns reads to a reference genome, marks duplicate reads, calls variants, post-processes variants in the output VCF in preparation for joint calling, and calculates quality control metrics. The workflow outputs a (re)aligned CRAM, an annotated GVCF with index, and quality metrics. @@ -201,7 +201,7 @@ The workflow implements initial variant calling with a version of HaplotypeCalle | [Utilities.ScatterIntervalList](https://github.com/broadinstitute/warp/blob/develop/tasks/broad/Utilities.wdl) | IntervalListTools | Picard | Splits the calling interval list into sub-intervals in order to perform variant calling on the sub-intervals. | | [Tasks.HaplotypeCaller](https://github.com/broadinstitute/warp/blob/develop/tasks/broad/UltimaGenomicsWholeGenomeGermlineTasks.wdl) | HaplotypeCaller | GATK | Performs initial variant calling on the aligned BAM file and outputs sub-interval GVCFs and a bamout file. | -### 6. Merge VCFs and BAMs and convert GVCF to VCF +### 6. Merge VCFs and BAMs and convert GVCF to VCF The workflow performs multiple post-processing steps to prepare the VCF for downstream joint calling. The HaplotypeCaller GVCF outputs are merged into a single GVCF and then converted to VCF in preparation for this post-processing. | Task name and WDL link | Tool | Software | Description | diff --git a/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/README.md b/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/README.md index 3b77969d2c..27ad3c9355 100644 --- a/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/README.md +++ b/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/README.md @@ -7,7 +7,7 @@ slug: /Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/README | Pipeline Version | Date Updated | Documentation Author | Questions or Feedback | | :----: | :---: | :----: | :--------------: | -| WholeGenomeGermlineSingleSample_v3.1.14 (see [releases page](https://github.com/broadinstitute/warp/releases)) | August, 2022 | [Elizabeth Kiernan](mailto:ekiernan@broadinstitute.org) | Please file GitHub issues in WARP or contact [the WARP team](mailto:warp-pipelines-help@broadinstitute.org) | +| WholeGenomeGermlineSingleSample_v3.1.17 (see [releases page](https://github.com/broadinstitute/warp/releases)) | December, 2023 | [Elizabeth Kiernan](mailto:ekiernan@broadinstitute.org) | Please file GitHub issues in WARP or contact [the WARP team](mailto:warp-pipelines-help@broadinstitute.org) | ## Introduction to the Whole Genome Germline Single Sample Pipeline The Whole Genome Germline Single Sample (WGS) pipeline implements data pre-processing and initial variant calling according to the GATK Best Practices for germline SNP and Indel discovery in human whole-genome sequencing data. It includes the DRAGEN-GATK mode, which makes the pipeline functionally equivalent to DRAGEN’s analysis pipeline (read more in this [DRAGEN-GATK blog](https://gatk.broadinstitute.org/hc/en-us/articles/360039984151)). diff --git a/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/wgs.methods.md b/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/wgs.methods.md index dd19f5bd03..842dd9ec59 100644 --- a/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/wgs.methods.md +++ b/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/wgs.methods.md @@ -2,13 +2,13 @@ sidebar_position: 2 --- -# Whole Genome Germline Single Sample v3.1.6 Methods (Default workflow) +# Whole Genome Germline Single Sample v3.1.17 Methods (Default workflow) The following contains a detailed methods description outlining the pipeline’s process, software, and tools that can be modified for a publication methods section. ## Detailed methods for the default Whole Genome Germline Single Sample workflow -Preprocessing and variant calling was performed using the WholeGenomeGermlineSingleSample v3.1.6 pipeline using Picard v2.26.10, GATK v4.2.6.1, and Samtools v1.11 with default tool parameters unless otherwise specified. All reference files are available in the public [Broad References Google Bucket](https://console.cloud.google.com/storage/browser/gcp-public-data--broad-references/hg38/v0). The pipeline follows GATK Best Practices as previously described ([Van der Auwera & O'Connor, 2020](https://www.oreilly.com/library/view/genomics-in-the/9781491975183/)) as well as the Functional Equivalence specification ([Regier et al., 2018](https://www.nature.com/articles/s41467-018-06159-4)). +Preprocessing and variant calling was performed using the WholeGenomeGermlineSingleSample v3.1.17 pipeline using Picard v2.26.10, GATK v4.3.0.0, and Samtools v1.11 with default tool parameters unless otherwise specified. All reference files are available in the public [Broad References Google Bucket](https://console.cloud.google.com/storage/browser/gcp-public-data--broad-references/hg38/v0). The pipeline follows GATK Best Practices as previously described ([Van der Auwera & O'Connor, 2020](https://www.oreilly.com/library/view/genomics-in-the/9781491975183/)) as well as the Functional Equivalence specification ([Regier et al., 2018](https://www.nature.com/articles/s41467-018-06159-4)). ### Pre-processing and quality control metrics @@ -34,7 +34,7 @@ The pipeline’s final outputs included metrics, validation reports, an aligned ## Detailed methods for the Functional Equivalence mode of the Whole Genome Germline Single Sample workflow -Preprocessing and variant calling was performed using the WholeGenomeGermlineSingleSample v3.1.6 pipeline using v2.26.10, GATK v4.2.6.1, and Samtools v1.11 with default tool parameters unless otherwise specified. All reference files are available in the public [Broad References Google Bucket](https://console.cloud.google.com/storage/browser/gcp-public-data--broad-references/hg38/v0). The pipeline is functionally equivalent (as described in GATK Support: https://gatk.broadinstitute.org/hc/en-us/articles/4410456501915) to DRAGEN v3.4.12. +Preprocessing and variant calling was performed using the WholeGenomeGermlineSingleSample v3.1.17 pipeline using v2.26.10, GATK v4.3.0.0, and Samtools v1.11 with default tool parameters unless otherwise specified. All reference files are available in the public [Broad References Google Bucket](https://console.cloud.google.com/storage/browser/gcp-public-data--broad-references/hg38/v0). The pipeline is functionally equivalent (as described in GATK Support: https://gatk.broadinstitute.org/hc/en-us/articles/4410456501915) to DRAGEN v3.4.12. ### Pre-processing and quality control metrics @@ -57,6 +57,7 @@ Prior to variant calling, the DRAGEN STR model was calibrated using the Calibrat The pipeline’s final outputs included metrics, validation reports, an aligned CRAM with index, and a reblocked GVCF containing variant calls with an accompanying index. ## Previous methods documents +- [WholeGenomeGermlineSingleSample_v3.1.6](https://github.com/broadinstitute/warp/blob/WholeGenomeGermlineSingleSample_v3.1.8/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/wgs.methods.md) - [WholeGenomeGermlineSingleSample_v3.0.0](https://github.com/broadinstitute/warp/blob/WholeGenomeGermlineSingleSample_v3.0.0/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/wgs.methods.md) - [WholeGenomeGermlineSingleSample_v2.5.0](https://github.com/broadinstitute/warp/blob/WholeGenomeGermlineSingleSample_v2.5.0/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/wgs.methods.md) - [WholeGenomeGermlineSingleSample_v2.3.7](https://github.com/broadinstitute/warp/blob/WholeGenomeGermlineSingleSample_v2.3.7/website/docs/Pipelines/Whole_Genome_Germline_Single_Sample_Pipeline/wgs.methods.md) From 66a05975006a99cd4d3a75f6068592e5e5254b73 Mon Sep 17 00:00:00 2001 From: Megan Shand Date: Mon, 11 Dec 2023 15:55:14 -0500 Subject: [PATCH 5/6] another changelog --- pipelines/skylab/multiome/Multiome.changelog.md | 5 +++++ pipelines/skylab/multiome/Multiome.wdl | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md index 392fa73728..f11e92137b 100644 --- a/pipelines/skylab/multiome/Multiome.changelog.md +++ b/pipelines/skylab/multiome/Multiome.changelog.md @@ -1,3 +1,8 @@ +# 3.0.1 +2023-12-08 (Date of Last Commit) + +* ValidateVcfs now has optional memory parameter; this does not affect this pipeline + # 3.0.0 2023-11-22 (Date of Last Commit) diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl index a85ac47041..e7fe42070f 100644 --- a/pipelines/skylab/multiome/Multiome.wdl +++ b/pipelines/skylab/multiome/Multiome.wdl @@ -6,7 +6,7 @@ import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender workflow Multiome { - String pipeline_version = "3.0.0" + String pipeline_version = "3.0.1" input { String input_id From bd5100e89fea3d907799df5052b907bc09eb33ba Mon Sep 17 00:00:00 2001 From: kayleemathews Date: Tue, 12 Dec 2023 15:19:46 -0500 Subject: [PATCH 6/6] update multiome readme --- website/docs/Pipelines/Multiome_Pipeline/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/Pipelines/Multiome_Pipeline/README.md b/website/docs/Pipelines/Multiome_Pipeline/README.md index c2cb4277ac..0efc2ab86b 100644 --- a/website/docs/Pipelines/Multiome_Pipeline/README.md +++ b/website/docs/Pipelines/Multiome_Pipeline/README.md @@ -7,7 +7,7 @@ slug: /Pipelines/Multiome_Pipeline/README | Pipeline Version | Date Updated | Documentation Author | Questions or Feedback | | :----: | :---: | :----: | :--------------: | -| [Multiome v3.0.0](https://github.com/broadinstitute/warp/releases) | December, 2023 | Kaylee Mathews | Please file GitHub issues in warp or contact the [WARP Pipeline Development team](mailto:warp-pipelines-help@broadinstitute.org) | +| [Multiome v3.0.1](https://github.com/broadinstitute/warp/releases) | December, 2023 | Kaylee Mathews | Please file GitHub issues in warp or contact the [WARP Pipeline Development team](mailto:warp-pipelines-help@broadinstitute.org) | ![Multiome_diagram](./multiome_diagram.png)