Merge pull request #103 from sanger-tol/more_bugfixes

More bugfixes
sanger-tol · Jul 18, 2024 · 5420de2 · 5420de2
2 parents 7547ead + b83f05c
commit 5420de2
Show file tree

Hide file tree

Showing 13 changed files with 106 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,6 +12,7 @@ General tidy up of the configuration and the pipeline
 - Increased the resources for blastn
 - Removed some options that were not used or not needed
 - All relevant outputs are now copied to the output directory
+- Fixed some blast parameters to match the behaviour of the Snakemake pipeline
 
 ### Parameters
 

diff --git a/conf/base.config b/conf/base.config
@@ -105,9 +105,15 @@ process {
     }
 
     withName: "BLAST_BLASTN" {
-        cpus   = { check_max( 24    * task.attempt, 'cpus'    ) }
-        memory = { check_max( 100.MB * task.attempt, 'memory' ) }
-        time   = { check_max( 12.h  * task.attempt, 'time'    ) }
+
+        // There are blast failures we don't know how to fix. Just ignore for now
+        errorStrategy = { task.exitStatus in ((130..145) + 104) ? (task.attempt == process.maxRetries ? 'ignore' : 'retry') : 'finish' }
+
+        // Most jobs complete quickly but some need a lot longer. For those outliers,
+        // the CPU usage remains usually low, often nearing a single CPU
+        cpus   = { check_max( 6    -            (task.attempt-1), 'cpus'   ) }
+        memory = { check_max( 1.GB * Math.pow(4, task.attempt-1), 'memory' ) }
+        time   = { check_max( 10.h * Math.pow(4, task.attempt-1), 'time'   ) }
     }
 
     withName:CUSTOM_DUMPSOFTWAREVERSIONS {

diff --git a/conf/modules.config b/conf/modules.config
@@ -138,7 +138,7 @@ process {
     }
 
     withName: "BLAST_BLASTN" {
-        ext.args = "-outfmt '6 qseqid staxids bitscore std' -max_target_seqs 10 -max_hsps 1 -evalue 1.0e-10 -lcase_masking -dust '20 64 1'"
+        ext.args = "-task megablast -outfmt '6 qseqid staxids bitscore std' -max_target_seqs 10 -max_hsps 1 -evalue 1.0e-10 -lcase_masking -dust '20 64 1'"
     }
 
     withName: "CUSTOM_DUMPSOFTWAREVERSIONS" {

diff --git a/modules.json b/modules.json
@@ -30,12 +30,14 @@
                     "diamond/blastp": {
                         "branch": "master",
                         "git_sha": "b29f6beb86d1d24d680277fb1a3f4de7b8b8a92c",
-                        "installed_by": ["modules"]
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/diamond/blastp/diamond-blastp.diff"
                     },
                     "diamond/blastx": {
                         "branch": "master",
                         "git_sha": "b29f6beb86d1d24d680277fb1a3f4de7b8b8a92c",
-                        "installed_by": ["modules"]
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/diamond/blastx/diamond-blastx.diff"
                     },
                     "fastawindows": {
                         "branch": "master",

diff --git a/modules/nf-core/blast/blastn/blast-blastn.diff b/modules/nf-core/blast/blastn/blast-blastn.diff
diff --git a/modules/nf-core/blast/blastn/main.nf b/modules/nf-core/blast/blastn/main.nf
diff --git a/modules/nf-core/diamond/blastp/diamond-blastp.diff b/modules/nf-core/diamond/blastp/diamond-blastp.diff
diff --git a/modules/nf-core/diamond/blastp/main.nf b/modules/nf-core/diamond/blastp/main.nf
diff --git a/modules/nf-core/diamond/blastx/diamond-blastx.diff b/modules/nf-core/diamond/blastx/diamond-blastx.diff
diff --git a/modules/nf-core/diamond/blastx/main.nf b/modules/nf-core/diamond/blastx/main.nf
diff --git a/subworkflows/local/busco_diamond_blastp.nf b/subworkflows/local/busco_diamond_blastp.nf
@@ -39,6 +39,7 @@ workflow BUSCO_DIAMOND {
     | transpose()
     | filter { rank,id -> rank =~ /species/ }
     | map { rank, id -> id}
+    | first
     | set { ch_taxid }
 
 
@@ -116,7 +117,7 @@ workflow BUSCO_DIAMOND {
     // Hardcoded to match the format expected by blobtools
     def outext = 'txt'
     def cols   = 'qseqid staxids bitscore qseqid sseqid pident length mismatch gapopen qstart qend sstart send evalue bitscore'
-    DIAMOND_BLASTP ( ch_busco_genes, blastp, outext, cols )
+    DIAMOND_BLASTP ( ch_busco_genes, blastp, outext, cols, ch_taxid )
     ch_versions = ch_versions.mix ( DIAMOND_BLASTP.out.versions.first() )
 
 

diff --git a/subworkflows/local/run_blastx.nf b/subworkflows/local/run_blastx.nf
@@ -11,6 +11,7 @@ workflow RUN_BLASTX {
     fasta      // channel: [ val(meta), path(fasta) ]
     table      // channel: [ val(meta), path(busco_table) ]
     blastx     // channel: [ val(meta), path(blastx_db) ]
+    taxon_id   // channel: val(taxon_id)
 
 
     main:
@@ -30,7 +31,7 @@ workflow RUN_BLASTX {
     // Hardocded to match the format expected by blobtools
     def outext = 'txt'
     def cols   = 'qseqid staxids bitscore qseqid sseqid pident length mismatch gapopen qstart qend sstart send evalue bitscore'
-    DIAMOND_BLASTX ( BLOBTOOLKIT_CHUNK.out.chunks, blastx, outext, cols)
+    DIAMOND_BLASTX ( BLOBTOOLKIT_CHUNK.out.chunks, blastx, outext, cols, taxon_id )
     ch_versions = ch_versions.mix ( DIAMOND_BLASTX.out.versions.first() )
 
 

diff --git a/workflows/blobtoolkit.nf b/workflows/blobtoolkit.nf
@@ -140,6 +140,7 @@ workflow BLOBTOOLKIT {
         PREPARE_GENOME.out.genome,
         BUSCO_DIAMOND.out.first_table,
         ch_blastx,
+        BUSCO_DIAMOND.out.taxon_id,
     )
     ch_versions = ch_versions.mix ( RUN_BLASTX.out.versions )
 
@@ -151,7 +152,7 @@ workflow BLOBTOOLKIT {
         RUN_BLASTX.out.blastx_out,
         PREPARE_GENOME.out.genome,
         ch_blastn,
-        BUSCO_DIAMOND.out.taxon_id
+        BUSCO_DIAMOND.out.taxon_id,
     )
 
     //