sanger-tol · DLBPointon · Dec 10, 2024 · Dec 5, 2024 · Dec 5, 2024 · Dec 5, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -39,7 +39,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install Nextflow
-        uses: nf-core/setup-nextflow@v1
+        uses: nf-core/setup-nextflow@v2
         with:
           version: "${{ matrix.NXF_VER }}"
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,67 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [[1.1.0](https://github.com/sanger-tol/curationpretext/releases/tag/1.1.0)] - UNSC Delphi - [2024-12-09]
+
+### Added
+
+- Added map_order so that the output maps are defaulted to unsorted and can be selected as sorted.
+- Updating all modules.
+- Removing Anaconda 'defaults' channel.
+- Updating local module containers.
+- Update to LISCENSES and CITATIONS files.
+- Update algorithms at play for memory allocation, particulary minimap2.
+- Parity update to TreeVal as the mapping subworkflow is based on the treeval implementation.
+- Fixed some version output being generated incorrectly.
+
+### Paramters
+
+| Old Version | New Versions |
+| - | --map_order |
+
+### Software Dependencies
+
+Note, since the pipeline is using Nextflow DSL2, each process will be run with its own Biocontainer. This means that on occasion it is entirely possible for the pipeline to be using different versions of the same tool. However, the overall software dependency changes compared to the last release have been listed below for reference.
+
+| Module                                       | Old Version   | New Versions               |
+| -------------------------------------------- | ------------- | -------------------------- |
+| get_avcov                                    | -             | 1.0.0                      |
+| bamtobed_sort ( bedtools + samtools )        | 2.31.0 + 1.17 | 2.31.1 + 1.17              |
+| bedtools ( all modules)                      | 2.31.1        | -                          |
+| bwamem2 index                                | -             | 2.2.1                      |
+| cram_filter_align_bwamem2_fixmate_sort       | -             |                            |
+| ^ ( samtools + bwamem2 ) ^                   | 1.17 + 2.2.1  | -                          |
+| cram_filter_minimap2_filter5end_fixmate_sort | -             |                            |
+| custom_dumpsoftwareversions                  | -             | Python 3.11.7 + yaml 5.4.1 |
+| ^ ( samtools + minimap2 ) ^                  | 1.17 + 2.24   | -                          |
+| extract_cov_id ( coreutils )                 | 9.1           | 9.3                        |
+| extract_repeat ( perl )                      | 5.26.2        | -                          |
+| extract_telo ( coreutils )                   | -             | 9.1                        |
+| find_telomere_regions ( gcc )                | 7.1.0         | 7.1.0 + 1.0                |
+| find_telomere_windows ( java-jdk )           | 8.0.112       | 8.0.112 + 1.0              |
+| findhalfcoverage ( python )                  | -             | Python 3.9.1 + 1.0         |
+| gap_length ( coreutils )                     | 9.1           | -                          |
+| generate_cram_csv ( samtools )               | 1.17          | -                          |
+| get_largest_scaff ( coreutils )              | 9.1           | -                          |
+| getminmaxpunches ( coreutils )               | 9.1           | -                          |
+| graphoverallcoverage ( perl )                | -             | 5.26.2 + 1.0               |
+| gnu-sort                                     | 8.25          | 9.3                        |
+| longreadcoveragescalelog                     | -             | Python 3.9.1 + 1.0         |
+| minimap2 + samtools (align, map)             |               | 2.28-r1209 + 1.20          |
+| pretextmap + samtools                        | 0.1.9 + 1.18  | 0.1.9\* + 1.20             |
+| pretextgraph                                 | 0.0.4         | 0.0.6                      |
+| pretextsnapshot + UCSC                       | 0.0.6b + 447  | 0.0.4 (official version)   |
+| rename_ids ( coreutils )                     | -             | 9.1                        |
+| replace_dots ( coreutils )                   | -             | 9.1                        |
+| seqtk                                        | 1.4           | 1.4-r122                   |
+| samtools (faidx,merge,sort,view)             | 1.18          | 1.21                       |
+| ucsc                                         | 445           | 469                        |
+| windowmasker (blast)                         | -             | 2.14.0 + 1.0.0             |
+
+Even modules which have not had a version bump have indeed been updated through NF-core to remove defaults.
+
+Some modules now have two verions, the new addition is the script version rather than just the dependency version.
+
 ## [[1.0.0](https://github.com/sanger-tol/curationpretext/releases/tag/1.0.0)] - UNSC Cradle - [2024-02-22]
 
 ### Added

diff --git a/CITATIONS.md b/CITATIONS.md
@@ -72,9 +72,9 @@
 
 ## Software packaging/containerisation tools
 
-- [Anaconda](https://anaconda.com)
+- [Conda](https://conda.org/)
 
-  > Anaconda Software Distribution. Computer software. Vers. 2-2.4.0. Anaconda, Nov. 2016. Web.
+  > conda contributors. conda: A system-level, binary package and environment manager running on all major operating systems and platforms. Computer software. https://github.com/conda/conda
 
 - [Bioconda](https://pubmed.ncbi.nlm.nih.gov/29967506/)
 

diff --git a/bin/get_avgcov.sh b/bin/get_avgcov.sh
@@ -6,9 +6,10 @@
 # into bed format for use
 # -------------------
 # Author = yy5
+# Modified = dp24
 # -------------------
 version='1.0.0'
-if [ $1 == '-v'];
+if [ $1 == '-v' ];
 then
     echo "$version"
 else

diff --git a/conf/local_testing/test.config b/conf/local_testing/test.config
@@ -0,0 +1,28 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run sanger-tol/curationpretext -profile test,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    config_profile_name        = 'GitHub Test profile'
+    config_profile_description = 'Minimal test dataset to check pipeline function'
+
+    // Limit resources so that this can run on GitHub Actions
+    max_cpus    = 20
+    max_memory  = '30.GB'
+    max_time    = '6.h'
+
+    input       = "/nfs/treeoflife-01/teams/tola/users/dp24/curationpretext/TreeValTinyData/assembly/draft/grTriPseu1.fa"
+    outdir      = "./results"
+    longread    = "/nfs/treeoflife-01/teams/tola/users/dp24/curationpretext/TreeValTinyData/genomic_data/pacbio/"
+    cram        = "/nfs/treeoflife-01/teams/tola/users/dp24/curationpretext/TreeValTinyData/genomic_data/hic-arima/"
+    sample      = "CurationPretextTest"
+    teloseq     = "TTAGGG"
+}
diff --git a/conf/modules.config b/conf/modules.config
@@ -133,7 +133,7 @@ process {
     }
 
     withName: PRETEXTMAP_STANDRD {
-        ext.args        = { "${meta.map_order.equals("length") ? "--sortby length": "--sortby nosort" } --mapq 0 --memory ${task.memory.giga}" }
+        ext.args        = { "${meta.map_order.equals("length") ? "--sortby length": "--sortby nosort" } --mapq 0" }
         ext.prefix      = { "${meta.id}_normal_pi" }
     }
 

diff --git a/conf/test_full.config b/conf/test_full.config
@@ -12,15 +12,16 @@
 
 cleanup = true
 
+executor.cpus = 50
+executor.memory = 50.GB
+executor.time = 1.h
+
 params {
     config_profile_name        = 'Full test profile'
     config_profile_description = 'Full test dataset to check pipeline function'
 
     // Input data for full size test
     // Limit resources so that this can run on GitHub Actions
-    max_cpus    = 4
-    max_memory  = '10.GB'
-    max_time    = '6.h'
 
     sample      = "testing"
     input       = "/lustre/scratch123/tol/resources/treeval/treeval-testdata/TreeValTinyData/assembly/draft/grTriPseu1.fa"

diff --git a/lib/Utils.groovy b/lib/Utils.groovy
@@ -22,7 +22,7 @@ class Utils {
 
         // Check that all channels are present
         // This channel list is ordered by required channel priority.
-        def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults']
+        def required_channels_in_order = ['conda-forge', 'bioconda']
         def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean
 
         // Check that they are in the right order

diff --git a/modules.json b/modules.json
@@ -7,114 +7,114 @@
                 "nf-core": {
                     "bedtools/bamtobed": {
                         "branch": "master",
-                        "git_sha": "1d1cb7bfef6cf67fbc7faafa6992ad8bdc3045b3",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "bedtools/genomecov": {
                         "branch": "master",
-                        "git_sha": "575e1bc54b083fb15e7dd8b5fcc40bea60e8ce83",
+                        "git_sha": "bfa8975eefb8df3e480a44ac9e594f23f52b2963",
                         "installed_by": ["modules"]
                     },
                     "bedtools/intersect": {
                         "branch": "master",
-                        "git_sha": "575e1bc54b083fb15e7dd8b5fcc40bea60e8ce83",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "bedtools/makewindows": {
                         "branch": "master",
-                        "git_sha": "3b248b84694d1939ac4bb33df84bf6233a34d668",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "bedtools/map": {
                         "branch": "master",
-                        "git_sha": "3b248b84694d1939ac4bb33df84bf6233a34d668",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "bedtools/merge": {
                         "branch": "master",
-                        "git_sha": "575e1bc54b083fb15e7dd8b5fcc40bea60e8ce83",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "bwamem2/index": {
                         "branch": "master",
-                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
-                        "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "gnu/sort": {
                         "branch": "master",
-                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "minimap2/align": {
                         "branch": "master",
-                        "git_sha": "efbf86bb487f288ac30660282709d9620dd6048e",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "minimap2/index": {
                         "branch": "master",
-                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "pretextmap": {
                         "branch": "master",
-                        "git_sha": "6f4299292ef2c5b66e6829527b2647c301b77cc9",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/pretextmap/pretextmap.diff"
                     },
                     "pretextsnapshot": {
                         "branch": "master",
-                        "git_sha": "b119603b5ebefa3075cfa513d1d91b1dc875a679",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/pretextsnapshot/pretextsnapshot.diff"
                     },
                     "samtools/faidx": {
                         "branch": "master",
-                        "git_sha": "ce0b1aed7d504883061e748f492a31bf44c5777c",
+                        "git_sha": "b13f07be4c508d6ff6312d354d09f2493243e208",
                         "installed_by": ["modules"]
                     },
                     "samtools/merge": {
                         "branch": "master",
-                        "git_sha": "ce0b1aed7d504883061e748f492a31bf44c5777c",
+                        "git_sha": "b13f07be4c508d6ff6312d354d09f2493243e208",
                         "installed_by": ["modules"]
                     },
                     "samtools/sort": {
                         "branch": "master",
-                        "git_sha": "ce0b1aed7d504883061e748f492a31bf44c5777c",
+                        "git_sha": "b7800db9b069ed505db3f9d91b8c72faea9be17b",
                         "installed_by": ["modules"]
                     },
                     "samtools/view": {
                         "branch": "master",
-                        "git_sha": "ce0b1aed7d504883061e748f492a31bf44c5777c",
+                        "git_sha": "2d20463181b1c38981a02e90d3084b5f9fa8d540",
                         "installed_by": ["modules"]
                     },
                     "seqtk/cutn": {
                         "branch": "master",
-                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "tabix/bgziptabix": {
                         "branch": "master",
-                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "git_sha": "f448e846bdadd80fc8be31fbbc78d9f5b5131a45",
                         "installed_by": ["modules"]
                     },
                     "ucsc/bedgraphtobigwig": {
                         "branch": "master",
-                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "git_sha": "49f4e50534fe4b64101e62ea41d5dc43b1324358",
                         "installed_by": ["modules"]
                     },
                     "windowmasker/mkcounts": {
                         "branch": "master",
-                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "windowmasker/ustat": {
                         "branch": "master",
-                        "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     }
                 }

diff --git a/modules/local/extract_repeat.nf b/modules/local/extract_repeat.nf
@@ -25,7 +25,7 @@ process EXTRACT_REPEAT {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        perl: \$(echo \$(perl --version 2>&1) | sed 's/^.*perl //; s/Using.*\$//')
+        perl: \$(echo \$(perl --version 2>&1) | awk '/This/ {print \$9}'))
         extract_repeat.pl: $VERSION
     END_VERSIONS
     """
@@ -38,7 +38,7 @@ process EXTRACT_REPEAT {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        perl: \$(echo \$(perl --version 2>&1) | sed 's/^.*perl //; s/Using.*\$//')
+        perl: \$(echo \$(perl --version 2>&1) | awk '/This/ {print \$9}'))
         extract_repeat.pl: $VERSION
     END_VERSIONS
     """

diff --git a/modules/local/find_telomere_regions.nf b/modules/local/find_telomere_regions.nf
@@ -29,7 +29,7 @@ process FIND_TELOMERE_REGIONS {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        find_telomere: \$(find_telomere -v)
+        find_telomere: \$(VERSION)
     END_VERSIONS
     """
 
@@ -42,7 +42,7 @@ process FIND_TELOMERE_REGIONS {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        find_telomere: \$(find_telomere -v)
+        find_telomere: \$(VERSION)
     END_VERSIONS
     """
 

diff --git a/modules/local/generate_cram_csv.nf b/modules/local/generate_cram_csv.nf
@@ -2,7 +2,9 @@ process GENERATE_CRAM_CSV {
     tag "${meta.id}"
     label 'process_single'
 
-    container 'quay.io/sanger-tol/cramfilter_bwamem2_minimap2_samtools_perl:0.001-c1'
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' :
+        'biocontainers/mulled-v2-1a6fe65bd6674daba65066aa796ed8f5e8b4687b:688e175eb0db54de17822ba7810cc9e20fa06dd5-0' }"
 
     // Exit if running this module with -profile conda / -profile mamba
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {

diff --git a/modules/local/graphoverallcoverage.nf b/modules/local/graphoverallcoverage.nf
@@ -25,7 +25,7 @@ process GRAPHOVERALLCOVERAGE {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        perl: \$(echo \$(perl --version 2>&1) | sed 's/^.*perl //; s/Using.*\$//')
+        perl: \$(echo \$(perl --version 2>&1) | awk '/This/ {print \$9}'))
         graph_overall_coverage.pl: \$(graph_overall_coverage.pl --version)
     END_VERSIONS
     """
@@ -37,8 +37,7 @@ process GRAPHOVERALLCOVERAGE {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        perl: \$(echo \$(perl --version 2>&1) | sed 's/^.*perl //; s/Using.*\$//')
-        graph_overall_coverage.pl: \$(graph_overall_coverage.pl --version)
+        perl: \$(echo \$(perl --version 2>&1) | awk '/This/ {print \$9}'))        graph_overall_coverage.pl: \$(graph_overall_coverage.pl --version)
     END_VERSIONS
     """
 }
diff --git a/modules/local/pretext_graph.nf b/modules/local/pretext_graph.nf
@@ -2,7 +2,7 @@ process PRETEXT_GRAPH {
     tag "$meta.id"
     label 'process_single'
 
-    container "quay.io/sanger-tol/pretext:0.0.2-yy5-c3"
+    container "quay.io/sanger-tol/pretext:0.0.3-yy5-c1"
 
     // Exit if running this module with -profile conda / -profile mamba
     if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) {

diff --git a/modules/nf-core/bedtools/bamtobed/environment.yml b/modules/nf-core/bedtools/bamtobed/environment.yml