reservation update

tdayris · Dec 18, 2023 · 3c3b374 · 3c3b374
1 parent 5c808dc
commit 3c3b374
Show file tree

Hide file tree

Showing 9 changed files with 62 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,17 @@
+# 2.3.1:
+
+## Features:
+
+* Memory/Time/tmp reservation for each rule, with attempt
+* Tests on Flamingo (Gustave Roussy's computing cluster)
+
+## Fix:
+
+* Agat logs overlapping
+* snakemake-wrappers version update to 3.2.0
+* format error on latest python/snakemake
+
+
 # 2.3.0:
 
 ## Feature:

diff --git a/workflow/rules/agat/gff2gtf.smk b/workflow/rules/agat/gff2gtf.smk
@@ -4,6 +4,12 @@ rule agat_gff2gtf:
     output:
         gtf="reference/{species}.{build}.{release}.gtf",
     threads: 1
+    resources:
+        # Reserve 1Gb per attempt (max_vms: ??? on Flamingo)
+        mem_mb=lambda wildcards, attempt: 1024 * attempt,
+        # Reserve 30min per attempts (hg38: ??? on Flamingo)
+        runtime=lambda wildcards, attempt: 30 * attempt,
+        tmpdir="tmp",
     log:
         "logs/agat/gff2gtf/{species}.{build}.{release}.log",
     benchmark:

diff --git a/workflow/rules/bedtools/merge_blacklist.smk b/workflow/rules/bedtools/merge_blacklist.smk
@@ -4,6 +4,12 @@ rule bedtools_merge_blacklist:
     output:
         "reference/blacklist/{species}.{build}.{release}.merged.bed",
     threads: 2
+    resources:
+        # Reserve 500Mb per attempt (max_vms: 373.04 on Flamingo)
+        mem_mb=lambda wildcards, attempt: 500 * attempt,
+        # Reserve 10min per attempts (hg38: 0:4:03 on Flamingo)
+        runtime=lambda wildcards, attempt: 10 * attempt,
+        tmpdir="tmp",
     log:
         "logs/bedtools/merge/blacklist/{species}.{build}.{release}.log",
     benchmark:

diff --git a/workflow/rules/gatk/picard_dict.smk b/workflow/rules/gatk/picard_dict.smk
@@ -4,6 +4,12 @@ rule picard_create_dict:
     output:
         "reference/{species}.{build}.{release}.{datatype}.dict",
     threads: 1
+    resources:
+        # Reserve 9Gb per attempt (max_vms: 8247.39 on Flamingo)
+        mem_mb=lambda wildcards, attempt: (1024 * 9) * attempt,
+        # Reserve 10min per attempts (hg38: 0:01:03 on Flamingo)
+        runtime=lambda wildcards, attempt: 10 * attempt,
+        tmpdir="tmp",
     log:
         "logs/picard/create_dict/{species}.{build}.{release}.{datatype}.log",
     benchmark:

diff --git a/workflow/rules/pyroe/id_to_name.smk b/workflow/rules/pyroe/id_to_name.smk
@@ -4,6 +4,12 @@ rule pyroe_id_to_name:
     output:
         "resources/{species}.{build}.{release}.id_to_gene.tsv",
     threads: 1
+    resources:
+        # Reserve 9Gb per attempt (max_vms: 8247.39 on Flamingo)
+        mem_mb=lambda wildcards, attempt: (1024 * 9) * attempt,
+        # Reserve 10min per attempts (hg38: 0:01:03 on Flamingo)
+        runtime=lambda wildcards, attempt: 10 * attempt,
+        tmpdir="tmp",
     log:
         "logs/pyroe/id2name/{species}.{build}.{release}.log",
     benchmark:

diff --git a/workflow/rules/raw_data/blacklist.smk b/workflow/rules/raw_data/blacklist.smk
@@ -3,6 +3,12 @@ rule blacklist_grch38:
     output:
         "reference/blacklist/homo_sapiens.GRCh38.{release}.bed.gz",
     threads: 1
+    resources:
+        # Reserve 500Mb per attempt (max_vms: 347.80 on Flamingo)
+        mem_mb=lambda wildcards, attempt: 500 * attempt,
+        # Reserve 10min per attempts (hg38: 0:2:07 on Flamingo)
+        runtime=lambda wildcards, attempt: 10 * attempt,
+        tmpdir="tmp",
     log:
         "logs/ftp/blacklist/homo_sapiens.GRCh38.{release}.log",
     benchmark:

diff --git a/workflow/rules/raw_data/genome_annotation.smk b/workflow/rules/raw_data/genome_annotation.smk
@@ -2,6 +2,12 @@ rule get_genome_gtf_annotation:
     output:
         temp("tmp/{species}.{build}.{release}.gtf"),
     threads: 1
+    resources:
+        # Reserve 700Mb per attempt (max_vms: 695.41 on Flamingo)
+        mem_mb=lambda wildcards, attempt: 700 * attempt,
+        # Reserve 10min per attempts (hg38: 0:5:19 on Flamingo)
+        runtime=lambda wildcards, attempt: 10 * attempt,
+        tmpdir="tmp",
     log:
         "logs/get_genome/gtf_annotation/{species}.{build}.{release}.log",
     benchmark:

diff --git a/workflow/rules/raw_data/genome_sequence.smk b/workflow/rules/raw_data/genome_sequence.smk
@@ -2,6 +2,12 @@ rule get_genome_fasta_sequence:
     output:
         "reference/{species}.{build}.{release}.{datatype}.fasta",
     threads: 1
+    resources:
+        # Reserve 700Mb per attempt (max_vms: 391.72 on Flamingo)
+        mem_mb=lambda wildcards, attempt: 500 * attempt,
+        # Reserve 20min per attempts (hg38: 0:05:32 on Flamingo)
+        runtime=lambda wildcards, attempt: 10 * attempt,
+        tmpdir="tmp",
     log:
         "logs/get_genome/fasta_sequence/{species}.{build}.{release}.{datatype}.log",
     benchmark:

diff --git a/workflow/rules/raw_data/genome_variation.smk b/workflow/rules/raw_data/genome_variation.smk
@@ -2,6 +2,12 @@ rule get_genome_vcf_variations:
     output:
         "reference/{species}.{build}.{release}.{datatype}.vcf",
     threads: 1
+    resources:
+        # Reserve 700Mb per attempt (max_vms: 691.63 on Flamingo)
+        mem_mb=lambda wildcards, attempt: 700 * attempt,
+        # Reserve 20min per attempts (hg38: 0:15:44 on Flamingo)
+        runtime=lambda wildcards, attempt: 20 * attempt,
+        tmpdir="tmp",
     log:
         "logs/get_genome/fasta_sequence/{species}.{build}.{release}.{datatype}.log",
     benchmark: