From f8d596515c740b9f7f3f45805e6c5536988cad5f Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Thu, 9 Nov 2023 09:20:28 +0000 Subject: [PATCH 01/18] Don't need a complete process for that --- subworkflows/local/genome_statistics.nf | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/subworkflows/local/genome_statistics.nf b/subworkflows/local/genome_statistics.nf index a73cd364..6ed69afb 100644 --- a/subworkflows/local/genome_statistics.nf +++ b/subworkflows/local/genome_statistics.nf @@ -73,7 +73,9 @@ workflow GENOME_STATISTICS { | join ( FASTK_FASTK.out.ktab ) | set { ch_combo } - ch_grab = GrabFiles ( ch_pacbio.dir ) + ch_pacbio.dir + | map { meta, dir -> [ meta, [dir.listFiles().findAll { it.toString().endsWth(".hist") }], [dir.listFiles().findAll { it.toString().contains(".ktab") }] ] } + | set { ch_grab } ch_combo | mix ( ch_grab ) @@ -119,16 +121,3 @@ workflow GENOME_STATISTICS { } - -process GrabFiles { - tag "${meta.id}" - executor 'local' - - input: - tuple val(meta), path("in") - - output: - tuple val(meta), path("in/*.hist"), path("in/*.ktab*", hidden:true) - - "true" -} From 57df724249e923bc36dea570c283d8745ff98341 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Thu, 9 Nov 2023 09:23:01 +0000 Subject: [PATCH 02/18] Take 100M out for the various overheads --- conf/modules.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 7528da03..49e37a0f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -24,12 +24,12 @@ process { } withName: BED_SORT { - ext.args = { "-k4 --parallel=${task.cpus} -S${task.memory.giga}G" } + ext.args = { "-k4 --parallel=${task.cpus} -S${task.memory.mega - 100}M" } ext.prefix = { "${meta.id}_sorted" } } withName: FILTER_SORT { - ext.args = { "-k3,3d -k7,7d --parallel=${task.cpus} -S${task.memory.giga}G" } + ext.args = { "-k3,3d -k7,7d --parallel=${task.cpus} -S${task.memory.mega - 100}M" } ext.prefix = { "${meta.id}_sorted" } } From 0ca1e3e4ba08a8ddcea59c4493ddc694b5485dcd Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 10 Nov 2023 10:28:48 +0000 Subject: [PATCH 03/18] Make sure the temporary files are created locally rather than in /tmp --- conf/modules.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 49e37a0f..8519f362 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -24,12 +24,12 @@ process { } withName: BED_SORT { - ext.args = { "-k4 --parallel=${task.cpus} -S${task.memory.mega - 100}M" } + ext.args = { "-k4 --parallel=${task.cpus} -S${task.memory.mega - 100}M -T ." } ext.prefix = { "${meta.id}_sorted" } } withName: FILTER_SORT { - ext.args = { "-k3,3d -k7,7d --parallel=${task.cpus} -S${task.memory.mega - 100}M" } + ext.args = { "-k3,3d -k7,7d --parallel=${task.cpus} -S${task.memory.mega - 100}M -T ." } ext.prefix = { "${meta.id}_sorted" } } From f85341f2354d551aca5e2d48c7beaed4aaad63df Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 10 Nov 2023 10:29:58 +0000 Subject: [PATCH 04/18] First stab at defining optimised resource requirements for the pipeline [not tested !] --- conf/base.config | 127 ++++++++++++++++-------- subworkflows/local/genome_statistics.nf | 2 +- workflows/genomenote.nf | 10 +- 3 files changed, 92 insertions(+), 47 deletions(-) diff --git a/conf/base.config b/conf/base.config index bd8c3d88..3e5811cd 100644 --- a/conf/base.config +++ b/conf/base.config @@ -2,66 +2,107 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ sanger-tol/genomenote Nextflow base config file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - A 'blank slate' config file, appropriate for general use on most high performance - compute environments. Assumes that all software is installed and available on - the PATH. Runs in `local` mode - all jobs will be run on the logged in environment. ----------------------------------------------------------------------------------------- */ -process { +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Increasing the number of CPUs often gives diminishing returns, so we increase it + following a logarithm curve. Example: + - 0 < value <= 1: start + step + - 1 < value <= 2: start + 2*step + - 2 < value <= 4: start + 3*step + - 4 < value <= 8: start + 4*step + In order to support re-runs, the step increase may be multiplied by the attempt + number prior to calling this function. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ - cpus = { check_max( 1 * task.attempt, 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } +// Modified logarithm function that doesn't return negative numbers +def positive_log(value, base) { + if (value <= 1) { + return 0 + } else { + return Math.log(value)/Math.log(base) + } +} + +def log_increase_cpus(start, step, value, base) { + return check_max(start + step * (1 + Math.ceil(positive_log(value, base))), 'cpus') +} + + +process { errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } - maxRetries = 5 + maxRetries = 3 maxErrors = '-1' - // Process-specific resource requirements - // NOTE - Please try and re-use the labels below as much as possible. - // These labels are used and recognised by default in DSL2 files hosted on nf-core/modules. - // If possible, it would be nice to keep the same label naming convention when - // adding in your local modules too. - // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors - withLabel:process_single { - cpus = { check_max( 1 , 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + // In this configuration file, we give little resources by default and + // explicitly bump them up for some processes. + // All rules should still increase resources every attempt to allow the + // pipeline to self-heal from MEMLIMIT/RUNLIMIT. + + // Default + cpus = 1 + memory = { check_max( 50.MB * task.attempt, 'memory' ) } + time = { check_max( 30.min * task.attempt, 'time' ) } + + // These processes can take more than 30 min, and sometimes several hours. + // Let's give them 8 hours, which should be plenty of time. + withName: 'BUSCO|COOLER_ZOOMIFY|BEDTOOLS_BAMTOBED|COOLER_CLOAD|FILTER_BED|FILTER_SORT|BED_SORT' { + time = { check_max( 8.hour * task.attempt, 'time' ) } } - withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + + withName: SAMTOOLS_VIEW { + memory = { check_max( 1.GB * task.attempt, 'memory' ) } } - withLabel:process_medium { + + withName: FASTK_FASTK { + memory = { check_max( 12.GB * task.attempt, 'memory' ) } cpus = { check_max( 6 * task.attempt, 'cpus' ) } - memory = { check_max( 36.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } } - withLabel:process_high { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 72.GB * task.attempt, 'memory' ) } - time = { check_max( 16.h * task.attempt, 'time' ) } + + withName: MERQURYFK_MERQURYFK { + memory = { check_max( meta.genome_size < 840000000 ? (Math.ceil(meta.genome_size/60000000) * 1.G * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.G * task.attempt + 12.G), 'cpus' ) } + cpus = { check_max( 6 * task.attempt, 'cpus' ) } } - withLabel:process_long { - time = { check_max( 20.h * task.attempt, 'time' ) } + + withName: BUSCO { + //memory = { check_max(4.GB + Math.ceil(meta.genome_size/60000000) * 8.G * task.attempt, 'memory' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } + cpus = { check_max( 6 * task.attempt, 'cpus' ) } + // duration = 71.45*x+105.28 + // memory = y=6483.28*x+3837.4 + // above with 6 CPUs } - withLabel:process_high_memory { - memory = { check_max( 200.GB * task.attempt, 'memory' ) } + + withName: BEDTOOLS_BAMTOBED { + memory = { check_max( 12.GB * task.attempt, 'memory' ) } + } + + withName: 'BED_SORT|FILTER_SORT' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + //time = { check_max( 4.h * task.attempt, 'time' ) } } - withName:"COOLER_.*" { - cpus = { check_max( 6 * task.attempt, 'cpus' ) } - memory = { check_max( 36.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + + withName: COOLER_CLOAD { + memory = { check_max( 6.GB * task.attempt, 'memory' ) } } - withLabel:error_ignore { - errorStrategy = 'ignore' + + withName: COOLER_DUMP { + memory = { check_max( 100.MB * task.attempt, 'memory' ) } } - withLabel:error_retry { - errorStrategy = 'retry' - maxRetries = 2 + + withName: COOLER_ZOOMIFY { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( (meta.genome_size < 1000000000 ? 16.GB : 24.GB) * task.attempt, 'cpus' ) } +//(Math.ceil(meta.genome_size/60000000) * 1.G * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.G * task.attempt + 12.G), 'cpus' ) } + // RAM 16 GB if if < 1 GB else 24 GB + // 4 CPUs (or 3 ?) + // duration = 30 min if < 1 GB else a few hours } + withName:CUSTOM_DUMPSOFTWAREVERSIONS { cache = false } diff --git a/subworkflows/local/genome_statistics.nf b/subworkflows/local/genome_statistics.nf index 6ed69afb..520ca12a 100644 --- a/subworkflows/local/genome_statistics.nf +++ b/subworkflows/local/genome_statistics.nf @@ -80,7 +80,7 @@ workflow GENOME_STATISTICS { ch_combo | mix ( ch_grab ) | combine ( genome ) - | map { meta, hist, ktab, meta2, fasta -> [ meta, hist, ktab, fasta, [] ] } + | map { meta, hist, ktab, meta2, fasta -> [ meta + [genome_size: meta2.genome_size], hist, ktab, fasta, [] ] } | set { ch_merq } diff --git a/workflows/genomenote.nf b/workflows/genomenote.nf index 6e9d2804..94bca532 100644 --- a/workflows/genomenote.nf +++ b/workflows/genomenote.nf @@ -97,16 +97,20 @@ workflow GENOMENOTE { // MODULE: Uncompress fasta file if needed // ch_fasta - | map { file -> [ [ 'id': file.baseName.tokenize('.')[0..1].join('.') ], file ] } + | map { file -> [ [ 'id': file.baseName ], file ] } | set { ch_genome } if ( params.fasta.endsWith('.gz') ) { - ch_fasta = GUNZIP ( ch_genome ).gunzip + ch_unzipped = GUNZIP ( ch_genome ).gunzip ch_versions = ch_versions.mix ( GUNZIP.out.versions.first() ) } else { - ch_fasta = ch_genome + ch_unzipped = ch_genome } + ch_unzipped + | map { meta, fa -> [ meta + [id: fa.baseName, genome_size: fa.size()], fa] } + | set { ch_fasta } + // // SUBWORKFLOW: Create contact map matrices from HiC alignment files From 6dc2da7e7662be13dad6b7d3f034eb9d1be84045 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 10 Nov 2023 10:44:48 +0000 Subject: [PATCH 05/18] Indentation should be a multiple of 4 --- conf/base.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/base.config b/conf/base.config index 3e5811cd..79564740 100644 --- a/conf/base.config +++ b/conf/base.config @@ -8,10 +8,10 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Increasing the number of CPUs often gives diminishing returns, so we increase it following a logarithm curve. Example: - - 0 < value <= 1: start + step - - 1 < value <= 2: start + 2*step - - 2 < value <= 4: start + 3*step - - 4 < value <= 8: start + 4*step + - 0 < value <= 1: start + step + - 1 < value <= 2: start + 2*step + - 2 < value <= 4: start + 3*step + - 4 < value <= 8: start + 4*step In order to support re-runs, the step increase may be multiplied by the attempt number prior to calling this function. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From ec978ee9aad3163a0c8d40fe14e2a6743023be83 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 10 Nov 2023 10:49:35 +0000 Subject: [PATCH 06/18] typo --- conf/base.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/base.config b/conf/base.config index 79564740..fc668fa9 100644 --- a/conf/base.config +++ b/conf/base.config @@ -63,12 +63,12 @@ process { } withName: MERQURYFK_MERQURYFK { - memory = { check_max( meta.genome_size < 840000000 ? (Math.ceil(meta.genome_size/60000000) * 1.G * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.G * task.attempt + 12.G), 'cpus' ) } + memory = { check_max( meta.genome_size < 840000000 ? (Math.ceil(meta.genome_size/60000000) * 1.GB * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.GB * task.attempt + 12.GB), 'cpus' ) } cpus = { check_max( 6 * task.attempt, 'cpus' ) } } withName: BUSCO { - //memory = { check_max(4.GB + Math.ceil(meta.genome_size/60000000) * 8.G * task.attempt, 'memory' ) } + //memory = { check_max(4.GB + Math.ceil(meta.genome_size/60000000) * 8.GB * task.attempt, 'memory' ) } memory = { check_max( 32.GB * task.attempt, 'memory' ) } cpus = { check_max( 6 * task.attempt, 'cpus' ) } // duration = 71.45*x+105.28 @@ -97,7 +97,7 @@ process { withName: COOLER_ZOOMIFY { cpus = { check_max( 4 * task.attempt, 'cpus' ) } memory = { check_max( (meta.genome_size < 1000000000 ? 16.GB : 24.GB) * task.attempt, 'cpus' ) } -//(Math.ceil(meta.genome_size/60000000) * 1.G * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.G * task.attempt + 12.G), 'cpus' ) } +//(Math.ceil(meta.genome_size/60000000) * 1.GB * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.GB * task.attempt + 12.GB), 'cpus' ) } // RAM 16 GB if if < 1 GB else 24 GB // 4 CPUs (or 3 ?) // duration = 30 min if < 1 GB else a few hours From 3e40be17e0d1f21cb4b8f06e9f7fb05eb0897548 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 10 Nov 2023 10:54:13 +0000 Subject: [PATCH 07/18] Another typo --- conf/base.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/base.config b/conf/base.config index fc668fa9..35f168f0 100644 --- a/conf/base.config +++ b/conf/base.config @@ -63,7 +63,7 @@ process { } withName: MERQURYFK_MERQURYFK { - memory = { check_max( meta.genome_size < 840000000 ? (Math.ceil(meta.genome_size/60000000) * 1.GB * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.GB * task.attempt + 12.GB), 'cpus' ) } + memory = { check_max( meta.genome_size < 840000000 ? (Math.ceil(meta.genome_size/60000000) * 1.GB * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.GB * task.attempt + 12.GB), 'memory' ) } cpus = { check_max( 6 * task.attempt, 'cpus' ) } } @@ -96,7 +96,7 @@ process { withName: COOLER_ZOOMIFY { cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( (meta.genome_size < 1000000000 ? 16.GB : 24.GB) * task.attempt, 'cpus' ) } + memory = { check_max( (meta.genome_size < 1000000000 ? 16.GB : 24.GB) * task.attempt, 'memory' ) } //(Math.ceil(meta.genome_size/60000000) * 1.GB * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.GB * task.attempt + 12.GB), 'cpus' ) } // RAM 16 GB if if < 1 GB else 24 GB // 4 CPUs (or 3 ?) From 7372b6e471715fe8416366e56d51771c28c573a7 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 10 Nov 2023 11:27:35 +0000 Subject: [PATCH 08/18] Removed trailing whitespace --- conf/base.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/base.config b/conf/base.config index 35f168f0..05c32eb0 100644 --- a/conf/base.config +++ b/conf/base.config @@ -78,7 +78,7 @@ process { withName: BEDTOOLS_BAMTOBED { memory = { check_max( 12.GB * task.attempt, 'memory' ) } - } + } withName: 'BED_SORT|FILTER_SORT' { cpus = { check_max( 4 * task.attempt, 'cpus' ) } From 381536e32ed03ce7d26c27648bd9c919d849a9eb Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 13 Nov 2023 09:18:30 +0000 Subject: [PATCH 09/18] Another round of resource updates --- conf/base.config | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/conf/base.config b/conf/base.config index 05c32eb0..9e1535e4 100644 --- a/conf/base.config +++ b/conf/base.config @@ -49,7 +49,7 @@ process { // These processes can take more than 30 min, and sometimes several hours. // Let's give them 8 hours, which should be plenty of time. - withName: 'BUSCO|COOLER_ZOOMIFY|BEDTOOLS_BAMTOBED|COOLER_CLOAD|FILTER_BED|FILTER_SORT|BED_SORT' { + withName: 'BED_SORT|BEDTOOLS_BAMTOBED|BUSCO|COOLER_CLOAD|COOLER_ZOOMIFY|FILTER_BED|FILTER_SORT|SAMTOOLS_VIEW' { time = { check_max( 8.hour * task.attempt, 'time' ) } } @@ -63,27 +63,22 @@ process { } withName: MERQURYFK_MERQURYFK { - memory = { check_max( meta.genome_size < 840000000 ? (Math.ceil(meta.genome_size/60000000) * 1.GB * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.GB * task.attempt + 12.GB), 'memory' ) } + // Memory usage seems to be following two different linear rules: + // - 1 GB for every 60 Mbp for genomes < 840 Mbp + // - 2 GB for every 1 Gbp for genomes > 840 Mbp, with a 12 GB offset to match the previous rule + memory = { check_max( 1.GB + ((meta.genome_size < 840000000) ? (Math.ceil(meta.genome_size/60000000) * 1.GB * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.GB * task.attempt + 12.GB)), 'memory' ) } cpus = { check_max( 6 * task.attempt, 'cpus' ) } } withName: BUSCO { - //memory = { check_max(4.GB + Math.ceil(meta.genome_size/60000000) * 8.GB * task.attempt, 'memory' ) } - memory = { check_max( 32.GB * task.attempt, 'memory' ) } + memory = { check_max(1.GB * Math.max(Math.pow(2, positive_log(meta.genome_size/1000000, 10)+task.attempt), Math.floor(meta.genome_size/1000000000) * 16 * task.attempt), 'cpus' ) } cpus = { check_max( 6 * task.attempt, 'cpus' ) } - // duration = 71.45*x+105.28 - // memory = y=6483.28*x+3837.4 - // above with 6 CPUs - } - - withName: BEDTOOLS_BAMTOBED { - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + time = { check_max( 2.h * Math.ceil(meta.genome_size/1000000000), 'time') } } withName: 'BED_SORT|FILTER_SORT' { cpus = { check_max( 4 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - //time = { check_max( 4.h * task.attempt, 'time' ) } } withName: COOLER_CLOAD { @@ -96,11 +91,7 @@ process { withName: COOLER_ZOOMIFY { cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( (meta.genome_size < 1000000000 ? 16.GB : 24.GB) * task.attempt, 'memory' ) } -//(Math.ceil(meta.genome_size/60000000) * 1.GB * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.GB * task.attempt + 12.GB), 'cpus' ) } - // RAM 16 GB if if < 1 GB else 24 GB - // 4 CPUs (or 3 ?) - // duration = 30 min if < 1 GB else a few hours + memory = { check_max( (meta.genome_size < 1000000000 ? 12.GB : 16.GB) * task.attempt, 'memory' ) } } withName:CUSTOM_DUMPSOFTWAREVERSIONS { From ca96b15aaf1fc932528b17ab46d72eff674d38f9 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Tue, 14 Nov 2023 13:31:51 +0000 Subject: [PATCH 10/18] Everything should be increasing the number of CPUs with a log function The default multiplication rule is too greedy --- conf/base.config | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/base.config b/conf/base.config index 9e1535e4..22277171 100644 --- a/conf/base.config +++ b/conf/base.config @@ -59,7 +59,7 @@ process { withName: FASTK_FASTK { memory = { check_max( 12.GB * task.attempt, 'memory' ) } - cpus = { check_max( 6 * task.attempt, 'cpus' ) } + cpus = { log_increase_cpus(4, 2*task.attempt, 1, 2) } } withName: MERQURYFK_MERQURYFK { @@ -67,17 +67,17 @@ process { // - 1 GB for every 60 Mbp for genomes < 840 Mbp // - 2 GB for every 1 Gbp for genomes > 840 Mbp, with a 12 GB offset to match the previous rule memory = { check_max( 1.GB + ((meta.genome_size < 840000000) ? (Math.ceil(meta.genome_size/60000000) * 1.GB * task.attempt) : (Math.ceil(meta.genome_size/1000000000) * 2.GB * task.attempt + 12.GB)), 'memory' ) } - cpus = { check_max( 6 * task.attempt, 'cpus' ) } + cpus = { log_increase_cpus(4, 2*task.attempt, 1, 2) } } withName: BUSCO { memory = { check_max(1.GB * Math.max(Math.pow(2, positive_log(meta.genome_size/1000000, 10)+task.attempt), Math.floor(meta.genome_size/1000000000) * 16 * task.attempt), 'cpus' ) } - cpus = { check_max( 6 * task.attempt, 'cpus' ) } + cpus = { log_increase_cpus(4, 2*task.attempt, Math.ceil(meta.genome_size/1000000000), 2) } time = { check_max( 2.h * Math.ceil(meta.genome_size/1000000000), 'time') } } withName: 'BED_SORT|FILTER_SORT' { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } + cpus = { log_increase_cpus(4, 2*task.attempt, 1, 2) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } } @@ -90,7 +90,7 @@ process { } withName: COOLER_ZOOMIFY { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } + cpus = { log_increase_cpus(4, 2*task.attempt, 1, 2) } memory = { check_max( (meta.genome_size < 1000000000 ? 12.GB : 16.GB) * task.attempt, 'memory' ) } } From e8e093f1a232186dac3f9152b3f9a2cda8a05c80 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Wed, 15 Nov 2023 15:09:04 +0000 Subject: [PATCH 11/18] Wrongly placed ifEmpty --- subworkflows/local/genome_statistics.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/genome_statistics.nf b/subworkflows/local/genome_statistics.nf index 520ca12a..4fed91d9 100644 --- a/subworkflows/local/genome_statistics.nf +++ b/subworkflows/local/genome_statistics.nf @@ -100,9 +100,9 @@ workflow GENOME_STATISTICS { MERQURYFK_MERQURYFK.out.qv | join ( MERQURYFK_MERQURYFK.out.stats ) - | ifEmpty ( [ [], [], [] ] ) | map { meta, qv, comp -> [ meta + [ id: "merq" ], qv, comp ] } | groupTuple () + | ifEmpty ( [ [], [], [] ] ) | set { ch_merqury } CREATETABLE ( ch_summary, ch_busco, ch_merqury, flagstat ) From 08784560650630e0f25de3dd1db3f7e251b12a35 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Thu, 16 Nov 2023 09:18:42 +0000 Subject: [PATCH 12/18] Comment --- conf/base.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/base.config b/conf/base.config index 22277171..a56952a6 100644 --- a/conf/base.config +++ b/conf/base.config @@ -71,6 +71,7 @@ process { } withName: BUSCO { + // Weird memory growth. The formula below is to fit the actual usage and avoid BUSCO being killed. memory = { check_max(1.GB * Math.max(Math.pow(2, positive_log(meta.genome_size/1000000, 10)+task.attempt), Math.floor(meta.genome_size/1000000000) * 16 * task.attempt), 'cpus' ) } cpus = { log_increase_cpus(4, 2*task.attempt, Math.ceil(meta.genome_size/1000000000), 2) } time = { check_max( 2.h * Math.ceil(meta.genome_size/1000000000), 'time') } From 00168f54f0a2e7ba9cd5bd843426c90af3d09a91 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Thu, 16 Nov 2023 21:32:43 +0000 Subject: [PATCH 13/18] Both FastK and MerquryFK need `-P.` to avoid polluting /tmp --- conf/modules.config | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 8519f362..422d49dc 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -62,7 +62,11 @@ process { withName: FASTK_FASTK { scratch = false // Disable scratch to get around bug https://github.com/nextflow-io/nextflow/issues/2983 - from genomeassembler - ext.args = { "-t1 -k${params.kmer_size}" } + ext.args = { "-t1 -k${params.kmer_size} -P." } + } + + withName: MERQURYFK_MERQURYFK { + ext.args = '-P.' } withName: CREATETABLE { From 0c13d223f2ed6150250b050a377dd76a3aca38d0 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Thu, 16 Nov 2023 21:47:19 +0000 Subject: [PATCH 14/18] typo --- subworkflows/local/genome_statistics.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/genome_statistics.nf b/subworkflows/local/genome_statistics.nf index 4fed91d9..ad8c9435 100644 --- a/subworkflows/local/genome_statistics.nf +++ b/subworkflows/local/genome_statistics.nf @@ -74,7 +74,7 @@ workflow GENOME_STATISTICS { | set { ch_combo } ch_pacbio.dir - | map { meta, dir -> [ meta, [dir.listFiles().findAll { it.toString().endsWth(".hist") }], [dir.listFiles().findAll { it.toString().contains(".ktab") }] ] } + | map { meta, dir -> [ meta, [dir.listFiles().findAll { it.toString().endsWith(".hist") }], [dir.listFiles().findAll { it.toString().contains(".ktab") }] ] } | set { ch_grab } ch_combo From 7063fb44011af64e74c73eea999ce49a098090e2 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 17 Nov 2023 16:07:51 +0000 Subject: [PATCH 15/18] Increased a few more memory requirements that were too low --- conf/base.config | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conf/base.config b/conf/base.config index a56952a6..1f10ac69 100644 --- a/conf/base.config +++ b/conf/base.config @@ -92,7 +92,11 @@ process { withName: COOLER_ZOOMIFY { cpus = { log_increase_cpus(4, 2*task.attempt, 1, 2) } - memory = { check_max( (meta.genome_size < 1000000000 ? 12.GB : 16.GB) * task.attempt, 'memory' ) } + memory = { check_max( (meta.genome_size < 1000000000 ? 16.GB : 24.GB) * task.attempt, 'memory' ) } + } + + withName: MULTIQC { + memory = { check_max( 150.MB * task.attempt, 'memory' ) } } withName:CUSTOM_DUMPSOFTWAREVERSIONS { From 1c561ebf498256a332c9be543c6b8ad6e1024eed Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 17 Nov 2023 16:10:57 +0000 Subject: [PATCH 16/18] Reduced a few CPU requirements that were unnecessary high --- conf/base.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/base.config b/conf/base.config index 1f10ac69..65082d52 100644 --- a/conf/base.config +++ b/conf/base.config @@ -78,7 +78,7 @@ process { } withName: 'BED_SORT|FILTER_SORT' { - cpus = { log_increase_cpus(4, 2*task.attempt, 1, 2) } + cpus = { log_increase_cpus(2, 2*task.attempt, 1, 2) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } } @@ -91,7 +91,7 @@ process { } withName: COOLER_ZOOMIFY { - cpus = { log_increase_cpus(4, 2*task.attempt, 1, 2) } + cpus = { log_increase_cpus(2, 2*task.attempt, 1, 2) } memory = { check_max( (meta.genome_size < 1000000000 ? 16.GB : 24.GB) * task.attempt, 'memory' ) } } From 84ae909173978b8553a88f127411576a02fbd35b Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Fri, 17 Nov 2023 16:41:25 +0000 Subject: [PATCH 17/18] Decreased some time requirements --- conf/base.config | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/conf/base.config b/conf/base.config index 65082d52..ee8ba557 100644 --- a/conf/base.config +++ b/conf/base.config @@ -47,9 +47,13 @@ process { memory = { check_max( 50.MB * task.attempt, 'memory' ) } time = { check_max( 30.min * task.attempt, 'time' ) } - // These processes can take more than 30 min, and sometimes several hours. - // Let's give them 8 hours, which should be plenty of time. - withName: 'BED_SORT|BEDTOOLS_BAMTOBED|BUSCO|COOLER_CLOAD|COOLER_ZOOMIFY|FILTER_BED|FILTER_SORT|SAMTOOLS_VIEW' { + // These processes typically complete within 30 min to 1.5 hours. + withName: 'BED_SORT|BEDTOOLS_BAMTOBED|COOLER_CLOAD|COOLER_ZOOMIFY|FILTER_BED' { + time = { check_max( 4.hour * task.attempt, 'time' ) } + } + + // These processes may take a few hours. + withName: 'FILTER_SORT|SAMTOOLS_VIEW' { time = { check_max( 8.hour * task.attempt, 'time' ) } } From b5b11f0a966e63450fe9269333e2b85d92bf98b5 Mon Sep 17 00:00:00 2001 From: Matthieu Muffato Date: Mon, 20 Nov 2023 15:27:57 +0000 Subject: [PATCH 18/18] More trace fields to be able to optimise the parameters later --- nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nextflow.config b/nextflow.config index 775babf2..a3d96dd9 100644 --- a/nextflow.config +++ b/nextflow.config @@ -204,6 +204,7 @@ report { trace { enabled = true file = "${params.tracedir}/execution_trace_${trace_timestamp}.txt" + fields = 'task_id,hash,native_id,process,tag,status,exit,cpus,memory,time,attempt,submit,start,complete,duration,%cpu,%mem,peak_rss,rchar,wchar' } dag { enabled = true