Implement chunking to speed up alignment

conf/base.config

@@ -100,4 +100,16 @@ process {
     withName:CUSTOM_DUMPSOFTWAREVERSIONS {
         cache = false
     }
+
+    withName: SEQKIT_SPLIT2 {
+        cpus   = { log_increase_cpus(2, 1*task.attempt, meta.read_count/1000000, 2) }
+        memory = { check_max( 2.GB + 500.MB * log_increase_cpus(2, 1*task.attempt, meta.read_count/1000000, 2), 'memory' ) }
+        time   = { check_max( 1.h * Math.ceil( meta.read_count / 1000000 ) * task.attempt, 'time' ) }
+    }
+
+    withName: SAMTOOLS_SPLIT {
+        cpus   = { log_increase_cpus(2, 2*task.attempt, 1, 2) }
+        memory = { check_max( 1.GB * task.attempt, 'memory' ) }
+        time   = { check_max( 2.hour * task.attempt, 'time' ) }
+    }
 }

conf/modules.config

@@ -28,6 +28,11 @@ process {
         ext.prefix = { "${meta.id}.merge" }
     }
 
+    withName: SAMTOOLS_MERGE_CHUNKS {
+        ext.args = { "-c -p" }
+        ext.prefix = { "${meta.id}.merge" }
+    }
+
     // If custom header provided, this is inserted in place of existing
     // @HD and @SQ lines, while preserving any other header entries
     withName: SAMTOOLS_REHEADER {
@@ -100,5 +105,13 @@ process {
             pattern: '*_versions.yml'
         ]
     }
+
+    withName: SEQKIT_SPLIT2 {
+        ext.args = "--by-part 10"
+    }
 
+    withName: SAMTOOLS_SPLIT {
+        ext.args = "-h"
+        ext.chunk_size = 10000
+    }
 }

modules.json

@@ -84,7 +84,17 @@
                     "samtools/view": {
                         "branch": "master",
                         "git_sha": "6c2309aaec566c0d44a6cf14d4b2d0c51afe2e91",
-                        "installed_by": ["modules"]
+                        "installed_by": [
+                            "modules"
+                        ]
+                    },
+                    "seqkit/split2": {
+                        "branch": "master",
+                        "git_sha": "2be41ca2cc780eca4293d1b0dd3850b0b7ac40a3",
+                        "installed_by": [
+                            "modules"
+                        ],
+                        "patch": "modules/nf-core/seqkit/split2/seqkit-split2.diff"
                     },
                     "untar": {
                         "branch": "master",

modules/local/samtools_split.nf

@@ -0,0 +1,62 @@
+process SAMTOOLS_SPLIT {
+    tag "$meta.id"
+    label "process_high"
+
+    conda "${moduleDir}/environment.yml"
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/samtools:1.20--h50ea8bc_0' :
+        'biocontainers/samtools:1.20--h50ea8bc_0' }"
+
+    input:
+    tuple val(meta), path(reads)
+    tuple val(meta2), path(fasta)
+
+    output:
+    tuple val(meta), path("*.bam"), optional: true, emit: chunked_bam
+    tuple val(meta), path("*.cram"), optional: true, emit: chunked_cram
+
+    path "versions.yml"           , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    def chunk_size = task.ext.chunk_size ?: 10000
+    def reference = fasta ? "--reference ${fasta}" : ""
+    def file_extension = reads.name.split('\\.')[-1]
+    def cpus = task.ext.cpus ?: 1
+
+    """
+    # Convert BAM or CRAM to SAM
+    if [ "${file_extension}" == "bam" ]; then
+        samtools view -@ ${cpus} ${args} ${reads} > ${prefix}.sam
+
+        # Split SAM file into chunks
+        split -l ${chunk_size} ${prefix}.sam ${prefix}_part_
+
+        # Convert each chunk back to BAM
+        for chunk in ${prefix}_part_*; do
+            samtools view -@ ${cpus} -b -h \$chunk > \${chunk}.bam
+        done
+    elif [ "${file_extension}" == "cram" ]; then
+        samtools view -@ ${cpus} ${args} ${reference} ${reads} > ${prefix}.sam
+        # Split SAM file into chunks
+        split -l ${chunk_size} ${prefix}.sam ${prefix}_part_
+
+        # Convert each chunk back to BAM
+        for chunk in ${prefix}_part_*; do
+            samtools view -@ ${cpus} -b -h \$chunk > \${chunk}.cram
+        done
+    else
+        echo "Unsupported file type: ${file_extension}"
+        exit 1
+    fi
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' )
+    END_VERSIONS
+    """
+}