From 801c59fa010e495cc199cded20c17ab1f4d5b1b0 Mon Sep 17 00:00:00 2001 From: ekiernan Date: Thu, 25 Apr 2024 13:07:20 -0400 Subject: [PATCH 01/18] Made changes for SnapATAC2 v2.6.0 --- pipelines/skylab/multiome/Multiome.wdl | 2 +- pipelines/skylab/multiome/atac.wdl | 11 ++--------- pipelines/skylab/paired_tag/PairedTag.wdl | 4 ++-- tasks/skylab/H5adUtils.wdl | 7 ++++++- tasks/skylab/PairedTagUtils.wdl | 7 ++++++- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl index 968e37723d..63c2dc7b2d 100644 --- a/pipelines/skylab/multiome/Multiome.wdl +++ b/pipelines/skylab/multiome/Multiome.wdl @@ -80,7 +80,6 @@ workflow Multiome { read3_fastq_gzipped = atac_r3_fastq, input_id = input_id + "_atac", tar_bwa_reference = tar_bwa_reference, - annotations_gtf = annotations_gtf, chrom_sizes = chrom_sizes, whitelist = atac_whitelist, adapter_seq_read1 = adapter_seq_read1, @@ -92,6 +91,7 @@ workflow Multiome { gex_h5ad = Optimus.h5ad_output_file, gex_whitelist = gex_whitelist, atac_whitelist = atac_whitelist, + annotations_gtf = annotations_gtf, atac_fragment = Atac.fragment_file } diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl index 988d7e2bd6..3e22d5c2f8 100644 --- a/pipelines/skylab/multiome/atac.wdl +++ b/pipelines/skylab/multiome/atac.wdl @@ -29,8 +29,6 @@ workflow ATAC { Int mem_size_bwa = 512 String cpu_platform_bwa = "Intel Ice Lake" - # GTF for SnapATAC2 to calculate TSS sites of fragment file - File annotations_gtf # Text file containing chrom_sizes for genome build (i.e. hg38) File chrom_sizes # Whitelist @@ -104,7 +102,6 @@ workflow ATAC { input: bam = BBTag.bb_bam, chrom_sizes = chrom_sizes, - annotations_gtf = annotations_gtf, preindex = preindex } } @@ -113,7 +110,6 @@ workflow ATAC { input: bam = BWAPairedEndAlignment.bam_aligned_output, chrom_sizes = chrom_sizes, - annotations_gtf = annotations_gtf, preindex = preindex } @@ -435,7 +431,6 @@ task BWAPairedEndAlignment { task CreateFragmentFile { input { File bam - File annotations_gtf File chrom_sizes Boolean preindex Int disk_size = 500 @@ -448,7 +443,6 @@ task CreateFragmentFile { parameter_meta { bam: "Aligned bam with CB in CB tag. This is the output of the BWAPairedEndAlignment task." - annotations_gtf: "GTF for SnapATAC2 to calculate TSS sites of fragment file." chrom_sizes: "Text file containing chrom_sizes for genome build (i.e. hg38)." disk_size: "Disk size used in create fragment file step." mem_size: "The size of memory used in create fragment file." @@ -460,7 +454,6 @@ task CreateFragmentFile { python3 <>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.4-2.3.1" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.4-2.6.0-1714058747" disks: "local-disk ${disk_size} SSD" memory: "${mem_size} GiB" cpu: nthreads diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl index a4de0f85d7..4de07d6e7f 100644 --- a/pipelines/skylab/paired_tag/PairedTag.wdl +++ b/pipelines/skylab/paired_tag/PairedTag.wdl @@ -84,7 +84,6 @@ workflow PairedTag { read3_fastq_gzipped = demultiplex.fastq3, input_id = input_id + "_atac", tar_bwa_reference = tar_bwa_reference, - annotations_gtf = annotations_gtf, chrom_sizes = chrom_sizes, whitelist = atac_whitelist, adapter_seq_read1 = adapter_seq_read1, @@ -96,7 +95,8 @@ workflow PairedTag { call Demultiplexing.ParseBarcodes as ParseBarcodes { input: atac_h5ad = Atac_preindex.snap_metrics, - atac_fragment = Atac_preindex.fragment_file + atac_fragment = Atac_preindex.fragment_file, + annotations_gtf = annotations_gtf } } diff --git a/tasks/skylab/H5adUtils.wdl b/tasks/skylab/H5adUtils.wdl index 18fed45fc1..10451867db 100644 --- a/tasks/skylab/H5adUtils.wdl +++ b/tasks/skylab/H5adUtils.wdl @@ -190,6 +190,7 @@ task JoinMultiomeBarcodes { File gex_h5ad File gex_whitelist File atac_whitelist + File annotations_gtf Int nthreads = 1 String cpuPlatform = "Intel Cascade Lake" @@ -214,6 +215,7 @@ task JoinMultiomeBarcodes { python3 <>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.4-2.3.1-1700590229" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.4-2.6.0-1714058747" disks: "local-disk ~{disk} HDD" memory: "${machine_mem_mb} MiB" cpu: nthreads diff --git a/tasks/skylab/PairedTagUtils.wdl b/tasks/skylab/PairedTagUtils.wdl index ceb37b273e..0c5731004b 100644 --- a/tasks/skylab/PairedTagUtils.wdl +++ b/tasks/skylab/PairedTagUtils.wdl @@ -185,6 +185,7 @@ task ParseBarcodes { input { File atac_h5ad File atac_fragment + File annotations_gtf Int nthreads = 1 String cpuPlatform = "Intel Cascade Lake" } @@ -206,12 +207,14 @@ task ParseBarcodes { python3 < 1].index), 'duplicates'] = 1 + # calculate tsse metrics + snap.metrics.tsse(atac_data, atac_gtf) # Idenitfy the barcodes in the whitelist that match barcodes in datasets atac_data.write_h5ad("~{atac_base_name}.h5ad") test_fragment.to_csv("~{atac_fragment_base}.tsv", sep='\t', index=False, header = False) @@ -256,7 +261,7 @@ task ParseBarcodes { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.4-2.3.1-1700590229" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.4-2.6.0-1714058747" disks: "local-disk ~{disk} HDD" memory: "${machine_mem_mb} MiB" cpu: nthreads From b69f57b658ffbc1af4b3c1f4c8077ae49335f945 Mon Sep 17 00:00:00 2001 From: ekiernan Date: Thu, 25 Apr 2024 13:32:31 -0400 Subject: [PATCH 02/18] updated min tsse parameter --- pipelines/skylab/multiome/atac.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl index 3e22d5c2f8..0013913e94 100644 --- a/pipelines/skylab/multiome/atac.wdl +++ b/pipelines/skylab/multiome/atac.wdl @@ -479,7 +479,7 @@ task CreateFragmentFile { # calculate quality metrics; note min_num_fragments and min_tsse are set to 0 instead of default # those settings allow us to retain all barcodes - pp.import_data("~{bam_base_name}.fragments.tsv", file="~{bam_base_name}.metrics.h5ad", chrom_sizes=chrom_size_dict, min_num_fragments=0, min_tsse=0) + pp.import_data("~{bam_base_name}.fragments.tsv", file="~{bam_base_name}.metrics.h5ad", chrom_sizes=chrom_size_dict, min_num_fragments=0) CODE >>> From 7f31bf69168c48f997c6f3eba605b5c7737f6c36 Mon Sep 17 00:00:00 2001 From: ekiernan Date: Thu, 25 Apr 2024 15:04:02 -0400 Subject: [PATCH 03/18] updated docker to include one with bgzip --- pipelines/skylab/multiome/atac.wdl | 2 +- tasks/skylab/H5adUtils.wdl | 2 +- tasks/skylab/PairedTagUtils.wdl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl index 0013913e94..35d2a6c5b5 100644 --- a/pipelines/skylab/multiome/atac.wdl +++ b/pipelines/skylab/multiome/atac.wdl @@ -485,7 +485,7 @@ task CreateFragmentFile { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.4-2.6.0-1714058747" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.5-2.6.0-1714070943" disks: "local-disk ${disk_size} SSD" memory: "${mem_size} GiB" cpu: nthreads diff --git a/tasks/skylab/H5adUtils.wdl b/tasks/skylab/H5adUtils.wdl index 10451867db..357ad00348 100644 --- a/tasks/skylab/H5adUtils.wdl +++ b/tasks/skylab/H5adUtils.wdl @@ -282,7 +282,7 @@ task JoinMultiomeBarcodes { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.4-2.6.0-1714058747" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.5-2.6.0-1714070943" disks: "local-disk ~{disk} HDD" memory: "${machine_mem_mb} MiB" cpu: nthreads diff --git a/tasks/skylab/PairedTagUtils.wdl b/tasks/skylab/PairedTagUtils.wdl index 0c5731004b..0bf8552dd1 100644 --- a/tasks/skylab/PairedTagUtils.wdl +++ b/tasks/skylab/PairedTagUtils.wdl @@ -261,7 +261,7 @@ task ParseBarcodes { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.4-2.6.0-1714058747" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.5-2.6.0-1714070943" disks: "local-disk ~{disk} HDD" memory: "${machine_mem_mb} MiB" cpu: nthreads From 2416c9a5e3d62275a3eff89c471de91a73249061 Mon Sep 17 00:00:00 2001 From: ekiernan Date: Mon, 29 Apr 2024 09:57:57 -0400 Subject: [PATCH 04/18] calculate snap metrics earlier --- tasks/skylab/H5adUtils.wdl | 6 ++++-- tasks/skylab/PairedTagUtils.wdl | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tasks/skylab/H5adUtils.wdl b/tasks/skylab/H5adUtils.wdl index 357ad00348..e526fd73de 100644 --- a/tasks/skylab/H5adUtils.wdl +++ b/tasks/skylab/H5adUtils.wdl @@ -238,6 +238,9 @@ task JoinMultiomeBarcodes { whitelist_gex = pd.read_csv("~{gex_whitelist}", header=None, names=["gex_barcodes"]) whitelist_atac = pd.read_csv("~{atac_whitelist}", header=None, names=["atac_barcodes"]) + # calculate tsse metrics + snap.metrics.tsse(atac_data, atac_gtf) + # get dataframes df_atac = atac_data.obs df_gex = gex_data.obs @@ -264,8 +267,7 @@ task JoinMultiomeBarcodes { # set gene_data.obs to new dataframe print("Setting Optimus obs to new dataframe") gex_data.obs = df_gex - # calculate tsse metrics - snap.metrics.tsse(atac_data, atac_gtf) + # write out the files gex_data.write("~{gex_base_name}.h5ad") atac_data.write_h5ad("~{atac_base_name}.h5ad") diff --git a/tasks/skylab/PairedTagUtils.wdl b/tasks/skylab/PairedTagUtils.wdl index 0bf8552dd1..a9cc2a87e8 100644 --- a/tasks/skylab/PairedTagUtils.wdl +++ b/tasks/skylab/PairedTagUtils.wdl @@ -220,6 +220,9 @@ task ParseBarcodes { print("Reading ATAC fragment file:") test_fragment = pd.read_csv("~{atac_fragment}", sep="\t", names=['chr','start', 'stop', 'barcode','n_reads']) + # calculate tsse metrics + snap.metrics.tsse(atac_data, atac_gtf) + # Separate out CB and preindex in the h5ad and identify sample barcodes assigned to more than one cell barcode print("Setting preindex and CB columns in h5ad") df_h5ad = atac_data.obs @@ -243,8 +246,6 @@ task ParseBarcodes { # Update the 'duplicates' column for rows with more than one unique 'preindex' for a 'CB' test_fragment.loc[test_fragment['CB'].isin(preindex_counts[preindex_counts > 1].index), 'duplicates'] = 1 - # calculate tsse metrics - snap.metrics.tsse(atac_data, atac_gtf) # Idenitfy the barcodes in the whitelist that match barcodes in datasets atac_data.write_h5ad("~{atac_base_name}.h5ad") test_fragment.to_csv("~{atac_fragment_base}.tsv", sep='\t', index=False, header = False) From ff8544758842a9dc07cc84b4e33d3dcafd0901a8 Mon Sep 17 00:00:00 2001 From: ekiernan Date: Fri, 3 May 2024 13:51:38 -0400 Subject: [PATCH 05/18] adding tsse metrics back into ATAC workflow --- pipelines/skylab/multiome/Multiome.wdl | 2 +- pipelines/skylab/multiome/atac.wdl | 16 ++++++++++++++-- tasks/skylab/H5adUtils.wdl | 5 ----- tasks/skylab/PairedTagUtils.wdl | 6 +----- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl index 63c2dc7b2d..499bbc610f 100644 --- a/pipelines/skylab/multiome/Multiome.wdl +++ b/pipelines/skylab/multiome/Multiome.wdl @@ -83,6 +83,7 @@ workflow Multiome { chrom_sizes = chrom_sizes, whitelist = atac_whitelist, adapter_seq_read1 = adapter_seq_read1, + annotations_gtf = annotations_gtf, adapter_seq_read3 = adapter_seq_read3 } call H5adUtils.JoinMultiomeBarcodes as JoinBarcodes { @@ -91,7 +92,6 @@ workflow Multiome { gex_h5ad = Optimus.h5ad_output_file, gex_whitelist = gex_whitelist, atac_whitelist = atac_whitelist, - annotations_gtf = annotations_gtf, atac_fragment = Atac.fragment_file } diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl index 35d2a6c5b5..de0c740154 100644 --- a/pipelines/skylab/multiome/atac.wdl +++ b/pipelines/skylab/multiome/atac.wdl @@ -31,6 +31,8 @@ workflow ATAC { # Text file containing chrom_sizes for genome build (i.e. hg38) File chrom_sizes + #File for annotations for calculating ATAC TSSE + File annotations_gtf # Whitelist File whitelist @@ -102,6 +104,7 @@ workflow ATAC { input: bam = BBTag.bb_bam, chrom_sizes = chrom_sizes, + annotations_gtf = annotations_gtf, preindex = preindex } } @@ -110,6 +113,7 @@ workflow ATAC { input: bam = BWAPairedEndAlignment.bam_aligned_output, chrom_sizes = chrom_sizes, + annotations_gtf = annotations_gtf, preindex = preindex } @@ -432,6 +436,7 @@ task CreateFragmentFile { input { File bam File chrom_sizes + File annotations_gtf Boolean preindex Int disk_size = 500 Int mem_size = 16 @@ -444,6 +449,7 @@ task CreateFragmentFile { parameter_meta { bam: "Aligned bam with CB in CB tag. This is the output of the BWAPairedEndAlignment task." chrom_sizes: "Text file containing chrom_sizes for genome build (i.e. hg38)." + annotations_gtf: "GTF for SnapATAC2 to calculate TSS sites of fragment file." disk_size: "Disk size used in create fragment file step." mem_size: "The size of memory used in create fragment file." } @@ -457,6 +463,7 @@ task CreateFragmentFile { bam = "~{bam}" bam_base_name = "~{bam_base_name}" chrom_sizes = "~{chrom_sizes}" + atac_gtf = "~{annotations_gtf}" preindex = "~{preindex}" # calculate chrom size dictionary based on text file @@ -479,13 +486,18 @@ task CreateFragmentFile { # calculate quality metrics; note min_num_fragments and min_tsse are set to 0 instead of default # those settings allow us to retain all barcodes - pp.import_data("~{bam_base_name}.fragments.tsv", file="~{bam_base_name}.metrics.h5ad", chrom_sizes=chrom_size_dict, min_num_fragments=0) + pp.import_data("~{bam_base_name}.fragments.tsv", file="temp_metrics.h5ad", chrom_sizes=chrom_size_dict, min_num_fragments=0) + atac_data = ad.read_h5ad("temp_metrics.h5ad") + # calculate tsse metrics + snap.metrics.tsse(atac_data, atac_gtf) + # Write new atac file + atac_data.write_h5ad("~{bam_base_name}.metrics.h5ad") CODE >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.5-2.6.0-1714070943" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.6-2.6.1-1714757147" disks: "local-disk ${disk_size} SSD" memory: "${mem_size} GiB" cpu: nthreads diff --git a/tasks/skylab/H5adUtils.wdl b/tasks/skylab/H5adUtils.wdl index e526fd73de..06d5731408 100644 --- a/tasks/skylab/H5adUtils.wdl +++ b/tasks/skylab/H5adUtils.wdl @@ -190,7 +190,6 @@ task JoinMultiomeBarcodes { File gex_h5ad File gex_whitelist File atac_whitelist - File annotations_gtf Int nthreads = 1 String cpuPlatform = "Intel Cascade Lake" @@ -215,7 +214,6 @@ task JoinMultiomeBarcodes { python3 < Date: Fri, 3 May 2024 14:05:46 -0400 Subject: [PATCH 06/18] took out annotations gtf from parsebarcodes call --- pipelines/skylab/paired_tag/PairedTag.wdl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl index 4de07d6e7f..76aea95926 100644 --- a/pipelines/skylab/paired_tag/PairedTag.wdl +++ b/pipelines/skylab/paired_tag/PairedTag.wdl @@ -95,8 +95,7 @@ workflow PairedTag { call Demultiplexing.ParseBarcodes as ParseBarcodes { input: atac_h5ad = Atac_preindex.snap_metrics, - atac_fragment = Atac_preindex.fragment_file, - annotations_gtf = annotations_gtf + atac_fragment = Atac_preindex.fragment_file } } From d50aebc815b9e7307ef87fb62d670fe2f44c827a Mon Sep 17 00:00:00 2001 From: ekiernan Date: Mon, 6 May 2024 16:52:34 -0400 Subject: [PATCH 07/18] added anndata to atac --- pipelines/skylab/multiome/atac.wdl | 1 + 1 file changed, 1 insertion(+) diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl index de0c740154..0adf2fbd26 100644 --- a/pipelines/skylab/multiome/atac.wdl +++ b/pipelines/skylab/multiome/atac.wdl @@ -476,6 +476,7 @@ task CreateFragmentFile { # use snap atac2 import snapatac2.preprocessing as pp import snapatac2 as snap + import anndata as ad # extract CB or BB (if preindex is true) tag from bam file to create fragment file if preindex == "true": From df1729aa70e7567ada1ee25e5519474e712953a0 Mon Sep 17 00:00:00 2001 From: ekiernan Date: Thu, 9 May 2024 13:11:46 -0400 Subject: [PATCH 08/18] update snapatac2 dockers to v2.6.2 --- pipelines/skylab/multiome/atac.wdl | 2 +- tasks/skylab/H5adUtils.wdl | 2 +- tasks/skylab/PairedTagUtils.wdl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl index 0adf2fbd26..d1a4687c35 100644 --- a/pipelines/skylab/multiome/atac.wdl +++ b/pipelines/skylab/multiome/atac.wdl @@ -498,7 +498,7 @@ task CreateFragmentFile { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.6-2.6.1-1714757147" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.7-2.6.2-1715265458" disks: "local-disk ${disk_size} SSD" memory: "${mem_size} GiB" cpu: nthreads diff --git a/tasks/skylab/H5adUtils.wdl b/tasks/skylab/H5adUtils.wdl index 06d5731408..512c408a04 100644 --- a/tasks/skylab/H5adUtils.wdl +++ b/tasks/skylab/H5adUtils.wdl @@ -279,7 +279,7 @@ task JoinMultiomeBarcodes { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.5-2.6.0-1714070943" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.7-2.6.2-1715265458" disks: "local-disk ~{disk} HDD" memory: "${machine_mem_mb} MiB" cpu: nthreads diff --git a/tasks/skylab/PairedTagUtils.wdl b/tasks/skylab/PairedTagUtils.wdl index 35c7cfa99f..6b5158dd45 100644 --- a/tasks/skylab/PairedTagUtils.wdl +++ b/tasks/skylab/PairedTagUtils.wdl @@ -258,7 +258,7 @@ task ParseBarcodes { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.5-2.6.0-1714070943" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.7-2.6.2-1715265458" disks: "local-disk ~{disk} HDD" memory: "${machine_mem_mb} MiB" cpu: nthreads From f48ef54ff56f0e2c3f3df77301834f8a50eec331 Mon Sep 17 00:00:00 2001 From: ekiernan Date: Mon, 13 May 2024 09:57:01 -0400 Subject: [PATCH 09/18] added more threads --- pipelines/skylab/multiome/atac.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl index 9d50ade67b..0aa492625a 100644 --- a/pipelines/skylab/multiome/atac.wdl +++ b/pipelines/skylab/multiome/atac.wdl @@ -440,7 +440,7 @@ task CreateFragmentFile { Boolean preindex Int disk_size = 500 Int mem_size = 16 - Int nthreads = 1 + Int nthreads = 4 String cpuPlatform = "Intel Cascade Lake" } From 58c081897bfbfb2e03fa3c5f02da5bb7cf18986a Mon Sep 17 00:00:00 2001 From: ekiernan Date: Thu, 16 May 2024 09:42:37 -0400 Subject: [PATCH 10/18] updated snapatac docker to snapatac v2.6.3 to fix bug --- pipelines/skylab/multiome/atac.wdl | 2 +- tasks/skylab/H5adUtils.wdl | 2 +- tasks/skylab/PairedTagUtils.wdl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl index 0aa492625a..8c9aa0fe46 100644 --- a/pipelines/skylab/multiome/atac.wdl +++ b/pipelines/skylab/multiome/atac.wdl @@ -498,7 +498,7 @@ task CreateFragmentFile { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.7-2.6.2-1715265458" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.9-2.6.3-1715865353" disks: "local-disk ${disk_size} SSD" memory: "${mem_size} GiB" cpu: nthreads diff --git a/tasks/skylab/H5adUtils.wdl b/tasks/skylab/H5adUtils.wdl index 512c408a04..924b19a770 100644 --- a/tasks/skylab/H5adUtils.wdl +++ b/tasks/skylab/H5adUtils.wdl @@ -279,7 +279,7 @@ task JoinMultiomeBarcodes { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.7-2.6.2-1715265458" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.9-2.6.3-1715865353" disks: "local-disk ~{disk} HDD" memory: "${machine_mem_mb} MiB" cpu: nthreads diff --git a/tasks/skylab/PairedTagUtils.wdl b/tasks/skylab/PairedTagUtils.wdl index cf2db37517..87f695b066 100644 --- a/tasks/skylab/PairedTagUtils.wdl +++ b/tasks/skylab/PairedTagUtils.wdl @@ -273,7 +273,7 @@ task ParseBarcodes { >>> runtime { - docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.7-2.6.2-1715265458" + docker: "us.gcr.io/broad-gotc-prod/snapatac2:1.0.9-2.6.3-1715865353" disks: "local-disk ~{disk} HDD" memory: "${machine_mem_mb} MiB" cpu: nthreads From c528e2323e6897a53b9b7020daa6c0fafeaf8e4a Mon Sep 17 00:00:00 2001 From: ekiernan Date: Thu, 16 May 2024 09:56:07 -0400 Subject: [PATCH 11/18] Update PairedTag.changelog.md --- pipelines/skylab/paired_tag/PairedTag.changelog.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/skylab/paired_tag/PairedTag.changelog.md b/pipelines/skylab/paired_tag/PairedTag.changelog.md index ef69a9376a..0a020c9eec 100644 --- a/pipelines/skylab/paired_tag/PairedTag.changelog.md +++ b/pipelines/skylab/paired_tag/PairedTag.changelog.md @@ -1,6 +1,7 @@ # 0.6.0 -2024-05-10 (Date) +2024-05-16 (Date) +* Added testing infrastructure for paired-tag plumbing data * Updated the UPStools docker to version 2.0.0 which reinstates a barcode orientation check script * Updated the demultiplex task so that output FASTQ files are renamed according to the original input file name, avoiding naming collisions in downstream tasks From a61798f401dcd15a56f6e70532c6ea4788b3b551 Mon Sep 17 00:00:00 2001 From: ekiernan Date: Thu, 16 May 2024 09:57:09 -0400 Subject: [PATCH 12/18] Update PairedTag.changelog.md --- pipelines/skylab/paired_tag/PairedTag.changelog.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pipelines/skylab/paired_tag/PairedTag.changelog.md b/pipelines/skylab/paired_tag/PairedTag.changelog.md index 0a020c9eec..263289644d 100644 --- a/pipelines/skylab/paired_tag/PairedTag.changelog.md +++ b/pipelines/skylab/paired_tag/PairedTag.changelog.md @@ -1,7 +1,11 @@ -# 0.6.0 -2024-05-16 (Date) +# 0.7.0 +2024-05-16 * Added testing infrastructure for paired-tag plumbing data + +# 0.6.0 +2024-05-10 (Date) + * Updated the UPStools docker to version 2.0.0 which reinstates a barcode orientation check script * Updated the demultiplex task so that output FASTQ files are renamed according to the original input file name, avoiding naming collisions in downstream tasks From 480ab579b2b84025227aa1ef2de48e1c35dfc18a Mon Sep 17 00:00:00 2001 From: ekiernan Date: Thu, 16 May 2024 09:58:31 -0400 Subject: [PATCH 13/18] Update PairedTag.wdl --- pipelines/skylab/paired_tag/PairedTag.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl index fe9158c55d..c8579b0ab7 100644 --- a/pipelines/skylab/paired_tag/PairedTag.wdl +++ b/pipelines/skylab/paired_tag/PairedTag.wdl @@ -5,7 +5,7 @@ import "../../../pipelines/skylab/optimus/Optimus.wdl" as optimus import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils import "../../../tasks/skylab/PairedTagUtils.wdl" as Demultiplexing workflow PairedTag { - String pipeline_version = "0.6.0" + String pipeline_version = "0.7.0" input { String input_id From 7a84034e98b1518acf5fd2fd1e6c1d4299b1359b Mon Sep 17 00:00:00 2001 From: ekiernan Date: Mon, 20 May 2024 15:51:51 -0400 Subject: [PATCH 14/18] added annotations gtf back into atac preindex --- pipelines/skylab/paired_tag/PairedTag.wdl | 1 + 1 file changed, 1 insertion(+) diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl index 9c417cbeed..5e5a0d667b 100644 --- a/pipelines/skylab/paired_tag/PairedTag.wdl +++ b/pipelines/skylab/paired_tag/PairedTag.wdl @@ -90,6 +90,7 @@ workflow PairedTag { whitelist = atac_whitelist, adapter_seq_read1 = adapter_seq_read1, adapter_seq_read3 = adapter_seq_read3, + annotations_gtf = annotations_gtf, preindex = preindex } From e9c8e00f5156308c9798204646e547df23e22eaa Mon Sep 17 00:00:00 2001 From: ekiernan Date: Mon, 20 May 2024 20:40:55 -0400 Subject: [PATCH 15/18] updated changelogs for snapatac2 --- pipelines/skylab/multiome/Multiome.changelog.md | 5 +++++ pipelines/skylab/multiome/Multiome.wdl | 2 +- pipelines/skylab/multiome/atac.changelog.md | 5 +++++ pipelines/skylab/multiome/atac.wdl | 2 +- pipelines/skylab/optimus/Optimus.changelog.md | 5 +++++ pipelines/skylab/optimus/Optimus.wdl | 2 +- pipelines/skylab/slideseq/SlideSeq.changelog.md | 5 +++++ 7 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md index f0b69bdcd5..d28a4780da 100644 --- a/pipelines/skylab/multiome/Multiome.changelog.md +++ b/pipelines/skylab/multiome/Multiome.changelog.md @@ -1,3 +1,8 @@ +# 4.0.0 +2024-05-20 (Date of Last Commit) + +* Updated SnapATAC2 docker to SnapATAC2 v2.6.3; this impacts the workflow output metrics + # 3.4.5 2024-05-14 (Date of Last Commit) diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl index d92acd5e64..6bac8ea364 100644 --- a/pipelines/skylab/multiome/Multiome.wdl +++ b/pipelines/skylab/multiome/Multiome.wdl @@ -7,7 +7,7 @@ import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/c workflow Multiome { - String pipeline_version = "3.4.5" + String pipeline_version = "4.0.0" input { String input_id diff --git a/pipelines/skylab/multiome/atac.changelog.md b/pipelines/skylab/multiome/atac.changelog.md index 54948b799f..7478a49e0d 100644 --- a/pipelines/skylab/multiome/atac.changelog.md +++ b/pipelines/skylab/multiome/atac.changelog.md @@ -1,3 +1,8 @@ +# 2.0.0 +2024-05-20 (Date of Last Commit) + +* Updated SnapATAC2 docker to SnapATAC2 v2.6.3; this impacts the workflow output metrics + # 1.2.3 2024-05-14 (Date of Last Commit) diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl index 3a556359ce..e3a63d7e4c 100644 --- a/pipelines/skylab/multiome/atac.wdl +++ b/pipelines/skylab/multiome/atac.wdl @@ -41,7 +41,7 @@ workflow ATAC { String adapter_seq_read3 = "TCGTCGGCAGCGTCAGATGTGTATAAGAGACAG" } - String pipeline_version = "1.2.3" + String pipeline_version = "2.0.0" parameter_meta { read1_fastq_gzipped: "read 1 FASTQ file as input for the pipeline, contains read 1 of paired reads" diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md index 1b6844d493..a507767e03 100644 --- a/pipelines/skylab/optimus/Optimus.changelog.md +++ b/pipelines/skylab/optimus/Optimus.changelog.md @@ -1,3 +1,8 @@ +# 6.6.3 +2024-05-20 (Date of Last Commit) + +* Updated SnapATAC2 docker to SnapATAC2 v2.6.3; this does not impact the Optimus workflow + # 6.6.2 2024-04-24 (Date of Last Commit) diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl index 59c2610b97..fe78a02f90 100644 --- a/pipelines/skylab/optimus/Optimus.wdl +++ b/pipelines/skylab/optimus/Optimus.wdl @@ -65,7 +65,7 @@ workflow Optimus { # version of this pipeline - String pipeline_version = "6.6.2" + String pipeline_version = "6.6.3" # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays diff --git a/pipelines/skylab/slideseq/SlideSeq.changelog.md b/pipelines/skylab/slideseq/SlideSeq.changelog.md index 3910f51df0..52672034ec 100644 --- a/pipelines/skylab/slideseq/SlideSeq.changelog.md +++ b/pipelines/skylab/slideseq/SlideSeq.changelog.md @@ -1,3 +1,8 @@ +# 3.1.6 +2024-05-20 (Date of Last Commit) + +* Updated SnapATAC2 docker to SnapATAC2 v2.6.3; this does not impact the SlideSeq workflow + # 3.1.5 2024-04-12 (Date of Last Commit) From b55654d73e2fabcb4aa49269e247380aae5c4e46 Mon Sep 17 00:00:00 2001 From: ekiernan Date: Mon, 20 May 2024 20:43:32 -0400 Subject: [PATCH 16/18] updated slideseq version --- pipelines/skylab/slideseq/SlideSeq.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/skylab/slideseq/SlideSeq.wdl b/pipelines/skylab/slideseq/SlideSeq.wdl index ca41374852..2e9346de98 100644 --- a/pipelines/skylab/slideseq/SlideSeq.wdl +++ b/pipelines/skylab/slideseq/SlideSeq.wdl @@ -23,7 +23,7 @@ import "../../../tasks/skylab/MergeSortBam.wdl" as Merge workflow SlideSeq { - String pipeline_version = "3.1.5" + String pipeline_version = "3.1.6" input { Array[File] r1_fastq From 088f95f0b77df2da0f43e9cfa088ae9ed2650c87 Mon Sep 17 00:00:00 2001 From: ekiernan <55763654+ekiernan@users.noreply.github.com> Date: Tue, 21 May 2024 11:44:22 -0400 Subject: [PATCH 17/18] Update pipelines/skylab/paired_tag/PairedTag.changelog.md Co-authored-by: Nikelle Petrillo <38223776+nikellepetrillo@users.noreply.github.com> --- pipelines/skylab/paired_tag/PairedTag.changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/skylab/paired_tag/PairedTag.changelog.md b/pipelines/skylab/paired_tag/PairedTag.changelog.md index 852e741097..3638009b7d 100644 --- a/pipelines/skylab/paired_tag/PairedTag.changelog.md +++ b/pipelines/skylab/paired_tag/PairedTag.changelog.md @@ -2,7 +2,7 @@ 2024-05-20 * Updated SnapATAC2 docker and tasks to run SnapATAC v2.6.3 -* Added testing infrastructure for paired-tag plumbing data and example data sts +* Added testing infrastructure for paired-tag plumbing data and example data sets # 0.6.1 From 08af91f8681b458c78ea2f0ceed1bdfff861c90c Mon Sep 17 00:00:00 2001 From: ekiernan Date: Wed, 22 May 2024 09:44:15 -0400 Subject: [PATCH 18/18] fixing multiome and optimus versions and changelogs --- pipelines/skylab/multiome/Multiome.changelog.md | 8 ++++---- pipelines/skylab/multiome/Multiome.wdl | 2 +- pipelines/skylab/optimus/Optimus.changelog.md | 4 ++-- pipelines/skylab/optimus/Optimus.wdl | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md index d28a4780da..a2737a1c95 100644 --- a/pipelines/skylab/multiome/Multiome.changelog.md +++ b/pipelines/skylab/multiome/Multiome.changelog.md @@ -1,19 +1,19 @@ -# 4.0.0 +# 5.0.0 2024-05-20 (Date of Last Commit) * Updated SnapATAC2 docker to SnapATAC2 v2.6.3; this impacts the workflow output metrics -# 3.4.5 +# 4.0.2 2024-05-14 (Date of Last Commit) * Updated the Paired-tag Demultiplex task so that some intermediate input names have been renamed; this change does not impact the Multiome workflow -# 3.4.4 +# 4.0.1 2024-05-10 (Date of Last Commit) * Updated the Paired-tag Demultiplex task; this change does not impact the Multiome workflow -# 3.4.3 +# 4.0.0 2024-04-24 (Date of Last Commit) * Updated the input parameters for STARsolo in STARsoloFastq task. These include the parameters: soloCBmatchWLtype, soloUMIdedup and soloUMIfiltering diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl index 6bac8ea364..e6cd1017a0 100644 --- a/pipelines/skylab/multiome/Multiome.wdl +++ b/pipelines/skylab/multiome/Multiome.wdl @@ -7,7 +7,7 @@ import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/c workflow Multiome { - String pipeline_version = "4.0.0" + String pipeline_version = "5.0.0" input { String input_id diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md index a507767e03..dbbf6814ab 100644 --- a/pipelines/skylab/optimus/Optimus.changelog.md +++ b/pipelines/skylab/optimus/Optimus.changelog.md @@ -1,9 +1,9 @@ -# 6.6.3 +# 7.1.0 2024-05-20 (Date of Last Commit) * Updated SnapATAC2 docker to SnapATAC2 v2.6.3; this does not impact the Optimus workflow -# 6.6.2 +# 7.0.0 2024-04-24 (Date of Last Commit) * Updated the input parameters for STARsolo in STARsoloFastq task. These include the parameters: soloCBmatchWLtype, soloUMIdedup and soloUMIfiltering diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl index fe78a02f90..2163740eb9 100644 --- a/pipelines/skylab/optimus/Optimus.wdl +++ b/pipelines/skylab/optimus/Optimus.wdl @@ -65,7 +65,7 @@ workflow Optimus { # version of this pipeline - String pipeline_version = "6.6.3" + String pipeline_version = "7.1.0" # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays