-
Notifications
You must be signed in to change notification settings - Fork 285
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #824 from husensofteng/master
Update kaust profile
- Loading branch information
Showing
7 changed files
with
301 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,192 @@ | ||
| /* | ||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| nf-core/mag Nextflow KAUST config file | ||
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | ||
| A config file, appropriate for general use on KAUST high performance | ||
| compute environment - Ibex when using the KAUST profile. | ||
| ---------------------------------------------------------------------------------------- | ||
| */ | ||
| params { | ||
| // There is an error on Ibex related to the latest version of meta-spades | ||
| skip_spades = true | ||
| } | ||
|
|
||
| process { | ||
| cpus = { 4 * task.attempt } | ||
| memory = { 16.GB * task.attempt } | ||
| time = { 4.h * task.attempt } | ||
|
|
||
| errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } | ||
| maxRetries = 3 | ||
| maxErrors = '-1' | ||
|
|
||
| // Process-specific resource requirements | ||
| // NOTE - Please try and reuse the labels below as much as possible. | ||
| // These labels are used and recognised by default in DSL2 files hosted on nf-core/modules. | ||
| // If possible, it would be nice to keep the same label naming convention when | ||
| // adding in your local modules too. | ||
| // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors | ||
| withLabel: process_single { | ||
| cpus = { 1 } | ||
| memory = { 16.GB * task.attempt } | ||
| time = { 6.h * task.attempt } | ||
| } | ||
| withLabel: process_low { | ||
| cpus = { 4 * task.attempt } | ||
| memory = { 16.GB * task.attempt } | ||
| time = { 6.h * task.attempt } | ||
| } | ||
| withLabel: process_medium { | ||
| cpus = { 12 * task.attempt } | ||
| memory = { 97.GB * task.attempt } | ||
| time = { 12.h * task.attempt } | ||
| } | ||
| withLabel: process_high { | ||
| cpus = { 32 } | ||
| memory = { 240.GB * task.attempt } | ||
| time = { 20.h * task.attempt } | ||
| } | ||
| withLabel:process_long { | ||
| cpus = { 8 * task.attempt } | ||
| memory = { 32.GB * task.attempt } | ||
| time = { 96.h * task.attempt } | ||
| } | ||
| withLabel: process_high_memory { | ||
| memory = { 360.GB * task.attempt } | ||
| } | ||
| withLabel: error_ignore { | ||
| errorStrategy = 'ignore' | ||
| } | ||
| withLabel: error_retry { | ||
| errorStrategy = 'retry' | ||
| maxRetries = 3 | ||
| } | ||
| withName: BOWTIE2_HOST_REMOVAL_BUILD { | ||
| cpus = { 10 * task.attempt } | ||
| memory = { 40.GB * task.attempt } | ||
| time = { 4.h * task.attempt } | ||
| } | ||
| // Process-specific resource requirements | ||
| withName: BOWTIE2_HOST_REMOVAL_BUILD { | ||
| cpus = { 12 * task.attempt } | ||
| memory = { 48.GB * task.attempt } | ||
| time = { 8.h * task.attempt } | ||
| } | ||
| withName: BOWTIE2_HOST_REMOVAL_ALIGN { | ||
| cpus = { 6 * task.attempt } | ||
| memory = { 24.GB * task.attempt } | ||
| time = { 6.h * task.attempt } | ||
| } | ||
| withName: BOWTIE2_PHIX_REMOVAL_ALIGN { | ||
| cpus = { 6 * task.attempt } | ||
| memory = { 24.GB * task.attempt } | ||
| time = { 6.h * task.attempt } | ||
| } | ||
| withName: PORECHOP_PORECHOP { | ||
| cpus = { 12 * task.attempt } | ||
| memory = { 48.GB * task.attempt } | ||
| time = { 4.h * task.attempt } | ||
| } | ||
| withName: NANOLYSE { | ||
| cpus = { 6 * task.attempt } | ||
| memory = { 24.GB * task.attempt } | ||
| time = { 3.h * task.attempt } | ||
| } | ||
| //filtlong: exponential increase of memory and time with attempts | ||
| withName: FILTLONG { | ||
| cpus = { 16 * task.attempt } | ||
| memory = { 97.GB * (2 ** (task.attempt - 1)) } | ||
| time = { 24.h * (2 ** (task.attempt - 1)) } | ||
| } | ||
| withName: CENTRIFUGE_CENTRIFUGE { | ||
| cpus = { 12 * task.attempt } | ||
| memory = { 48.GB * task.attempt } | ||
| time = { 12.h * task.attempt } | ||
| } | ||
| withName: KRAKEN2 { | ||
| cpus = { 16 * task.attempt } | ||
| memory = { 240.GB * task.attempt } | ||
| time = { 20.h * task.attempt } | ||
| } | ||
| withName: KRONA_KTIMPORTTAXONOMY { | ||
| cpus = { 16 * task.attempt } | ||
| memory = { 240.GB * task.attempt } | ||
| time = { 20.h * task.attempt } | ||
| } | ||
| withName: CAT_DB_GENERATE { | ||
| memory = { 240.GB * task.attempt } | ||
| time = { 16.h * task.attempt } | ||
| } | ||
| withName: CAT { | ||
| cpus = { 16 * task.attempt } | ||
| memory = { 240.GB * task.attempt } | ||
| time = { 20.h * task.attempt } | ||
| } | ||
| withName: GTDBTK_CLASSIFYWF { | ||
| cpus = { 16 * task.attempt } | ||
| memory = { 240.GB * task.attempt } | ||
| time = { 20.h * task.attempt } | ||
| } | ||
| //MEGAHIT returns exit code 250 when running out of memory | ||
| withName: MEGAHIT { | ||
| cpus = { params.megahit_fix_cpu_1 ? 1 : (16 * task.attempt) } | ||
| memory = { 240.GB * task.attempt } | ||
| time = { 20.h * task.attempt } | ||
| errorStrategy = { task.exitStatus in ((130..145) + 104 + 247 + 250) ? 'retry' : 'finish' } | ||
| } | ||
| //SPAdes returns error(1) if it runs out of memory (and for other reasons as well...)! | ||
| //exponential increase of memory and time with attempts, keep number of threads to enable reproducibility | ||
| withName: SPADES { | ||
| cpus = { params.spades_fix_cpus != -1 ? params.spades_fix_cpus : (12 * task.attempt) } | ||
| memory = { 240.GB * (2 ** (task.attempt - 1)) } | ||
| time = { 24.h * (2 ** (task.attempt - 1)) } | ||
| errorStrategy = { task.exitStatus in ((130..145) + 104 + 21 + 12 + 1 + 247) ? 'retry' : 'finish' } | ||
| maxRetries = 5 | ||
| } | ||
| withName: SPADESHYBRID { | ||
| cpus = { params.spadeshybrid_fix_cpus != -1 ? params.spadeshybrid_fix_cpus : (12 * task.attempt) } | ||
| memory = { 240.GB * (2 ** (task.attempt - 1)) } | ||
| time = { 24.h * (2 ** (task.attempt - 1)) } | ||
| errorStrategy = { task.exitStatus in ((130..145) + 104 + 21 + 12 + 1 + 247) ? 'retry' : 'finish' } | ||
| maxRetries = 5 | ||
| } | ||
| //returns exit code 247 when running out of memory | ||
| withName: BOWTIE2_ASSEMBLY_ALIGN { | ||
| cpus = { 12 * task.attempt } | ||
| memory = { 97.GB * task.attempt } | ||
| time = { 16.h * task.attempt } | ||
| errorStrategy = { task.exitStatus in ((130..145) + 104 + 247) ? 'retry' : 'finish' } | ||
| } | ||
| withName: METABAT2_METABAT2 { | ||
| cpus = { 20 * task.attempt } | ||
| memory = { 180.GB * task.attempt } | ||
| time = { 8.h * task.attempt } | ||
| } | ||
| withName: MAG_DEPTHS { | ||
| memory = { 97.GB * task.attempt } | ||
| } | ||
| withName: BUSCO { | ||
| cpus = { 20 * task.attempt } | ||
| memory = { 180.GB * task.attempt } | ||
| } | ||
| withName: MAXBIN2 { | ||
| errorStrategy = { task.exitStatus in [1, 255] ? 'ignore' : 'retry' } | ||
| cpus = { 20 * task.attempt } | ||
| memory = { 180.GB * task.attempt } | ||
| } | ||
| withName: DASTOOL_DASTOOL { | ||
| errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : task.exitStatus == 1 ? 'ignore' : 'finish' } | ||
| } | ||
| withName: BOWTIE2_ASSEMBLY_BUILD { | ||
| cpus = { 20 * task.attempt } | ||
| memory = { 180.GB * task.attempt } | ||
| } | ||
| withName: BOWTIE2_ASSEMBLY_ALIGN { | ||
| cpus = { 20 * task.attempt } | ||
| memory = { 180.GB * task.attempt } | ||
| } | ||
| withName: PRODIGAL { | ||
| cpus = { 20 * task.attempt } | ||
| memory = { 180.GB * task.attempt } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| process { | ||
|
|
||
| cpus = { 4 * task.attempt } | ||
| memory = { 16.GB * task.attempt } | ||
| time = { 4.h * task.attempt } | ||
|
|
||
| errorStrategy = { task.exitStatus in [143,137,104,134,139,151,140,247,12] ? 'retry' : 'finish' } | ||
| maxRetries = 3 | ||
| maxErrors = '-1' | ||
|
|
||
| withLabel:process_single { | ||
| cpus = { 4 } | ||
| memory = { 32.GB * task.attempt } | ||
| time = { 4.h * task.attempt } | ||
| } | ||
| withLabel:process_low { | ||
| cpus = { 4 * task.attempt } | ||
| memory = { 16.GB * task.attempt } | ||
| time = { 6.h * task.attempt } | ||
| } | ||
| withLabel:process_medium { | ||
| cpus = { 12 * task.attempt } | ||
| memory = { 97.GB * task.attempt } | ||
| time = { 12.h * task.attempt } | ||
| } | ||
| withLabel:process_high { | ||
| cpus = 32 | ||
| memory = { 240.GB * task.attempt } | ||
| time = { 20.h * task.attempt } | ||
| } | ||
| withLabel:process_long { | ||
| cpus = { 12 * task.attempt } | ||
| memory = { 48.GB * task.attempt } | ||
| time = { 96.h * task.attempt } | ||
| } | ||
| withLabel:process_high_memory { | ||
| memory = { 240.GB * task.attempt } | ||
| } | ||
| withLabel:error_ignore { | ||
| errorStrategy = 'ignore' | ||
| } | ||
| withLabel:error_retry { | ||
| errorStrategy = 'retry' | ||
| maxRetries = 3 | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| # nf-core/configs: kaust mag specific configuration | ||
|
|
||
| Extra specific configuration for [nf-co.re/mag](https://nf-co.re/mag) pipeline. | ||
|
|
||
| ## Usage | ||
|
|
||
| To use, run the pipeline with `-profile kaust`. | ||
|
|
||
| This will download and launch the mag specific [`kaust.config`](../../../conf/pipeline/mag/kaust.config) which has been pre-configured with a setup suitable for the Ibex cluster. | ||
|
|
||
| Example: `nextflow run nf-core/mag -profile kaust` | ||
|
|
||
| ## mag specific configurations for kaust | ||
|
|
||
| Specific configurations for kaust has been made for mag, primarily increasing the default resource allocations especeially for high demand processes. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| # nf-core/configs: kaust rnaseq specific configuration | ||
|
|
||
| Specific configuration for [nf-co.re/rnaseq](https://nf-co.re/rnaseq) pipeline. | ||
|
|
||
| ## Usage | ||
|
|
||
| To use, run the pipeline with `-profile kaust`. | ||
|
|
||
| This will download and launch the rnaseq specific [`kaust.config`](../../../conf/pipeline/rnaseq/kaust.config) which has been pre-configured with a setup suitable for the Ibex cluster. | ||
|
|
||
| Example: `nextflow run nf-core/rnaseq -profile kaust` | ||
|
|
||
| ## rnaseq specific configurations for kaust | ||
|
|
||
| Specific configurations for kaust has been made for rnaseq, primarily increasing the default resource allocations especeially for high demand processes. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters