From eb936a52d908af729a30c435b675e17c23056e93 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Tue, 22 Oct 2024 14:16:58 -0400 Subject: [PATCH 1/6] fix: set container for create_circExplorer_merged_found_counts_table resolves #124 --- .../rules/post_findcircrna_processing.smk | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/workflow/rules/post_findcircrna_processing.smk b/workflow/rules/post_findcircrna_processing.smk index 05fe341..e314bc9 100644 --- a/workflow/rules/post_findcircrna_processing.smk +++ b/workflow/rules/post_findcircrna_processing.smk @@ -278,20 +278,21 @@ rule create_circExplorer_merged_found_counts_table: ), outdir=join(WORKDIR, "results", "{sample}", "circExplorer"), tmpdir=f"{TEMPDIR}/{str(uuid.uuid4())}", + container: config['containers']['star_ucsc_cufflinks'] shell: """ -set -exo pipefail -mkdir -p {params.tmpdir} -python3 {params.pythonscript} \\ - -b {input.bsj_found_counts} \\ - -l {input.linear_spliced_counts} \\ - -o {output.found_counts_table} - -python3 {params.pythonscript2} \\ - --annotationcounts {input.annotation_counts} \\ - --allfoundcounts {output.found_counts_table} \\ - --countstable {output.count_counts_table} -""" + set -exo pipefail + mkdir -p {params.tmpdir} + python3 {params.pythonscript} \\ + -b {input.bsj_found_counts} \\ + -l {input.linear_spliced_counts} \\ + -o {output.found_counts_table} + + python3 {params.pythonscript2} \\ + --annotationcounts {input.annotation_counts} \\ + --allfoundcounts {output.found_counts_table} \\ + --countstable {output.count_counts_table} + """ if RUN_MAPSPLICE: From 5f0f3a310f9695921f6fff1a040012ef1524103c Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 23 Oct 2024 11:26:06 -0400 Subject: [PATCH 2/6] style: fix shell indent --- .../rules/post_findcircrna_processing.smk | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/workflow/rules/post_findcircrna_processing.smk b/workflow/rules/post_findcircrna_processing.smk index e314bc9..f44f6b6 100644 --- a/workflow/rules/post_findcircrna_processing.smk +++ b/workflow/rules/post_findcircrna_processing.smk @@ -444,24 +444,25 @@ rule create_hq_bams: additives=ADDITIVES, viruses=VIRUSES, container: config['containers']["base"] - shell:""" -set -exo pipefail -outdir=$(dirname {output.outbam}) -if [ ! -d $outdir ];then mkdir -p $outdir;fi -cd $outdir -python3 {params.script} \\ - -i {input.inbam} \\ - -t {input.countstable} \\ - -o {output.outbam} \\ - --regions {params.regions} \\ - --host "{params.host}" \\ - --additives "{params.additives}" \\ - --viruses "{params.viruses}" \\ - --sample_name {params.samplename} -samtools index {output.outbam} -for bam in $(ls {params.samplename}.*.HQ_only.BSJ.bam);do - if [ ! -f "${{bam}}.bai" ];then - samtools index $bam - fi -done -""" + shell: + """ + set -exo pipefail + outdir=$(dirname {output.outbam}) + if [ ! -d $outdir ];then mkdir -p $outdir;fi + cd $outdir + python3 {params.script} \\ + -i {input.inbam} \\ + -t {input.countstable} \\ + -o {output.outbam} \\ + --regions {params.regions} \\ + --host "{params.host}" \\ + --additives "{params.additives}" \\ + --viruses "{params.viruses}" \\ + --sample_name {params.samplename} + samtools index {output.outbam} + for bam in $(ls {params.samplename}.*.HQ_only.BSJ.bam);do + if [ ! -f "${{bam}}.bai" ];then + samtools index $bam + fi + done + """ From ce2792f0351a45d225ecfd06e64798fc10db1198 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 23 Oct 2024 11:27:44 -0400 Subject: [PATCH 3/6] fix: upgrade base container to v7 --- config/containers.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/containers.yaml b/config/containers.yaml index baca596..2f5f5ed 100644 --- a/config/containers.yaml +++ b/config/containers.yaml @@ -1,5 +1,5 @@ containers: - base: "docker://nciccbr/ccbr_ubuntu_base_20.04:v6" + base: "docker://nciccbr/ccbr_ubuntu_base_20.04:v7" bowtie1: "docker://nciccbr/charlie_bowtie1:v0.1.1" circexplorer: "docker://nciccbr/ccbr_circexplorer:v1.0" circRNA_finder: "docker://nciccbr/charlie_circrna_finder:v1.0.1" From 8e95d349c970beebaf1be48f989e525eda155d3b Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 23 Oct 2024 11:27:56 -0400 Subject: [PATCH 4/6] chore: update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cbc750..5860a06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - CHARLIE was falsely throwing a file permissions error for tempdir values containing bash variables. (#118, @kelly-sovacool) - Singularity bind paths were not being set properly. (#119, @kelly-sovacool) -- Update docker containers to set `$PYTHONPATH`. (#119, @kelly-sovacool) +- Update docker containers to set `$PYTHONPATH`. (#119, #125, @kelly-sovacool) - Otherwise, this environment variable can be carried over and cause package conflicts when singularity is not run with `-C`. - Fix `reconfig` to correctly replace variables in the config file. (#121, @kelly-sovacool) From fb1cc06f7cdb5508263caf5d77070fc6b673be5f Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 24 Oct 2024 13:38:47 -0400 Subject: [PATCH 5/6] fix: do not read entire files in memory --- workflow/rules/init.smk | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/workflow/rules/init.smk b/workflow/rules/init.smk index 380271a..ce1a2ef 100644 --- a/workflow/rules/init.smk +++ b/workflow/rules/init.smk @@ -53,9 +53,8 @@ def append_files_in_list(flist, ofile): with open(ofile, "w") as outfile: for fname in flist: with open(fname) as infile: - l = infile.read() - l = l.strip() - outfile.write("%s\n" % (l)) + for line in infile: + outfile.write(line) return True From fcec511ee4b29f2884217070c41649d7feb949f2 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 24 Oct 2024 13:41:24 -0400 Subject: [PATCH 6/6] chore: update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cbc750..4a9bec4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Update docker containers to set `$PYTHONPATH`. (#119, @kelly-sovacool) - Otherwise, this environment variable can be carried over and cause package conflicts when singularity is not run with `-C`. - Fix `reconfig` to correctly replace variables in the config file. (#121, @kelly-sovacool) +- Prevent using excessive memory when copying reference files. (#126, @kelly-sovacool) ## CHARLIE 0.11.0