diff --git a/.editorconfig b/.editorconfig index b6b3190..9b99008 100644 --- a/.editorconfig +++ b/.editorconfig @@ -22,3 +22,11 @@ indent_size = unset [/assets/email*] indent_size = unset + +# ignore Readme +[README.md] +indent_style = unset + +# ignore python +[*.{py}] +indent_style = unset diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 41734bd..98cc2fc 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -27,6 +27,9 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests +You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to +receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir `. + When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5f925ad..754539c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -19,6 +19,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/epit - [ ] If necessary, also make a PR on the nf-core/epitopeprediction _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. - [ ] `CHANGELOG.md` is updated. diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 469824a..3f8e42f 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -28,7 +28,7 @@ jobs: } profiles: test_full - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: Tower debug log file path: | diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index eee9395..447968e 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -25,7 +25,7 @@ jobs: } profiles: test - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: Tower debug log file path: | diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 1abbb9b..c86cbc3 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -19,7 +19,7 @@ jobs: # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - name: Post PR comment if: failure() - uses: mshick/add-pr-comment@v1 + uses: mshick/add-pr-comment@v2 with: message: | ## This PR is against the `master` branch :x: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15ea7e4..f2cacf6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml index 694e90e..e37cfda 100644 --- a/.github/workflows/clean-up.yml +++ b/.github/workflows/clean-up.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v7 + - uses: actions/stale@v9 with: stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml new file mode 100644 index 0000000..8a33004 --- /dev/null +++ b/.github/workflows/download_pipeline.yml @@ -0,0 +1,67 @@ +name: Test successful pipeline download with 'nf-core download' + +# Run the workflow when: +# - dispatched manually +# - when a PR is opened or reopened to master branch +# - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. +on: + workflow_dispatch: + pull_request: + types: + - opened + branches: + - master + pull_request_target: + branches: + - master + +env: + NXF_ANSI_LOG: false + +jobs: + download: + runs-on: ubuntu-latest + steps: + - name: Install Nextflow + uses: nf-core/setup-nextflow@v1 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + architecture: "x64" + - uses: eWaterCycle/setup-singularity@v7 + with: + singularity-version: 3.8.3 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install git+https://github.com/nf-core/tools.git@dev + + - name: Get the repository name and current branch set as environment variable + run: | + echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} + echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} + echo "REPO_BRANCH=${GITHUB_REF#refs/heads/}" >> ${GITHUB_ENV} + + - name: Download the pipeline + env: + NXF_SINGULARITY_CACHEDIR: ./ + run: | + nf-core download ${{ env.REPO_LOWERCASE }} \ + --revision ${{ env.REPO_BRANCH }} \ + --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ + --compress "none" \ + --container-system 'singularity' \ + --container-library "quay.io" -l "docker.io" -l "ghcr.io" \ + --container-cache-utilisation 'amend' \ + --download-configuration + + - name: Inspect download + run: tree ./${{ env.REPOTITLE_LOWERCASE }} + + - name: Run the downloaded pipeline + env: + NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_HOME_MOUNT: true + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 9ebf496..5b20cfe 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -4,7 +4,7 @@ on: types: [created] jobs: - deploy: + fix-linting: # Only run if comment is on a PR with the main repo, and if it contains the magic keywords if: > contains(github.event.comment.html_url, '/pull/') && @@ -13,10 +13,17 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} + # indication that the linting is being fixed + - name: React on comment + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: eyes + # Action runs on the issue comment, so we don't get the PR by default # Use the gh cli to check out the PR - name: Checkout Pull Request @@ -24,32 +31,59 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v3 + # Install and run pre-commit + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + with: + python-version: 3.11 - - name: Install Prettier - run: npm install -g prettier @prettier/plugin-php + - name: Install pre-commit + run: pip install pre-commit - # Check that we actually need to fix something - - name: Run 'prettier --check' - id: prettier_status - run: | - if prettier --check ${GITHUB_WORKSPACE}; then - echo "result=pass" >> $GITHUB_OUTPUT - else - echo "result=fail" >> $GITHUB_OUTPUT - fi + - name: Run pre-commit + id: pre-commit + run: pre-commit run --all-files + continue-on-error: true - - name: Run 'prettier --write' - if: steps.prettier_status.outputs.result == 'fail' - run: prettier --write ${GITHUB_WORKSPACE} + # indication that the linting has finished + - name: react if linting finished succesfully + if: steps.pre-commit.outcome == 'success' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: "+1" - name: Commit & push changes - if: steps.prettier_status.outputs.result == 'fail' + id: commit-and-push + if: steps.pre-commit.outcome == 'failure' run: | git config user.email "core@nf-co.re" git config user.name "nf-core-bot" git config push.default upstream git add . git status - git commit -m "[automated] Fix linting with Prettier" + git commit -m "[automated] Fix code linting" git push + + - name: react if linting errors were fixed + id: react-if-fixed + if: steps.commit-and-push.outcome == 'success' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: hooray + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: confused + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + issue-number: ${{ github.event.issue.number }} + body: | + @${{ github.actor }} I tried to fix the linting errors, but it didn't work. Please fix them manually. + See [CI log](https://github.com/nf-core/epitopeprediction/actions/runs/${{ github.run_id }}) for more details. diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index b8bdd21..81cd098 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -11,72 +11,33 @@ on: types: [published] jobs: - EditorConfig: + pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') - - Prettier: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-node@v3 - - - name: Install Prettier - run: npm install -g prettier - - - name: Run Prettier --check - run: prettier --check ${GITHUB_WORKSPACE} - - PythonBlack: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Check code lints with Black - uses: psf/black@stable - - # If the above check failed, post a comment on the PR explaining the failure - - name: Post PR comment - if: failure() - uses: mshick/add-pr-comment@v1 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 with: - message: | - ## Python linting (`black`) is failing - - To keep the code consistent with lots of contributors, we run automated code consistency checks. - To fix this CI test, please run: - - * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` - * Fix formatting errors in your pipeline: `black .` - - Once you push these changes the test should pass, and you can hide this comment :+1: + python-version: 3.11 + cache: "pip" - We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + - name: Install pre-commit + run: pip install pre-commit - Thanks again for your contribution! - repo-token: ${{ secrets.GITHUB_TOKEN }} - allow-repeats: false + - name: Run pre-commit + run: pre-commit run --all-files nf-core: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" architecture: "x64" @@ -99,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: linting-logs path: | diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 0bbcd30..147bcd1 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v3 with: workflow: linting.yml workflow_conclusion: completed diff --git a/.github/workflows/release-announcments.yml b/.github/workflows/release-announcements.yml similarity index 96% rename from .github/workflows/release-announcments.yml rename to .github/workflows/release-announcements.yml index 6ad3392..21ac3f0 100644 --- a/.github/workflows/release-announcments.yml +++ b/.github/workflows/release-announcements.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install dependencies @@ -56,7 +56,7 @@ jobs: bsky-post: runs-on: ubuntu-latest steps: - - uses: zentered/bluesky-post-action@v0.0.2 + - uses: zentered/bluesky-post-action@v0.1.0 with: post: | Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! diff --git a/.gitpod.yml b/.gitpod.yml index 25488dc..363d5b1 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -4,6 +4,9 @@ tasks: command: | pre-commit install --install-hooks nextflow self-update + - name: unset JAVA_TOOL_OPTIONS + command: | + unset JAVA_TOOL_OPTIONS vscode: extensions: # based on nf-core.nf-core-extensionpack diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c31cdb..af57081 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,10 @@ repos: - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.7.1" + rev: "v3.1.0" hooks: - id: prettier + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: "2.7.3" + hooks: + - id: editorconfig-checker + alias: ec diff --git a/CHANGELOG.md b/CHANGELOG.md index c4f3525..01c58a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,16 +3,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v2.3dev - 2023-08-29 +## v2.3dev - 2024-02-05 ### `Changed` +- [#228](https://github.com/nf-core/epitopeprediction/pull/228) - Update to nf-core template `2.12` +- [#228](https://github.com/nf-core/epitopeprediction/pull/228) - Update to nf-core template `2.11` - [#220](https://github.com/nf-core/epitopeprediction/pull/220) - Switch to nf-validation to parse samplesheet -- [#213](https://github.com/nf-core/epitopeprediction/pull/203) - Rename param `genome_version` to `genome_reference`, add functionality to handle BioMart archive urls - [#213](https://github.com/nf-core/epitopeprediction/pull/203) - Update to nf-core template `2.10` +- [#206](https://github.com/nf-core/epitopeprediction/issues/206) - Update the row checker class. +- [#203](https://github.com/nf-core/epitopeprediction/pull/203) - Rename param `genome_version` to `genome_reference`, add functionality to handle BioMart archive urls - [#203](https://github.com/nf-core/epitopeprediction/pull/203) - Update to nf-core template `2.9` - [#203](https://github.com/nf-core/epitopeprediction/pull/203) - Update to nf-core template `2.8` -- [#206](https://github.com/nf-core/epitopeprediction/issues/206) - Update the row checker class. ### `Fixed` diff --git a/README.md b/README.md index e3b0729..dd15ced 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ -# ![nf-core/epitopeprediction](docs/images/nf-core-epitopeprediction_logo_light.png#gh-light-mode-only) ![nf-core/epitopeprediction](docs/images/nf-core-epitopeprediction_logo_dark.png#gh-dark-mode-only) - +

+ + + nf-core/epitopeprediction + +

[![GitHub Actions CI Status](https://github.com/nf-core/epitopeprediction/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/epitopeprediction/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/nf-core/epitopeprediction/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/epitopeprediction/actions?query=workflow%3A%22nf-core+linting%22)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/epitopeprediction/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.3564666-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.3564666) @@ -26,7 +30,8 @@ On release, automated continuous integration tests run the pipeline on a full-si 2. Generate peptides from variants or proteins or use peptides directly 3. Predict HLA-binding peptides for the given set of HLA alleles -## Usage +> [!NOTE] +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data. > [!NOTE] > If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how @@ -54,8 +59,7 @@ nextflow run nf-core/epitopeprediction \ ```` > [!WARNING] -> Please provide pipeline parameters via the CLI or Nextflow > > `-params-file` option. Custom config files including those -> provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; > see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/epitopeprediction/usage) and the [parameter documentation](https://nf-co.re/epitopeprediction/parameters). diff --git a/assets/email_template.html b/assets/email_template.html index 08d9a41..251ba8c 100644 --- a/assets/email_template.html +++ b/assets/email_template.html @@ -12,7 +12,7 @@ -

nf-core/epitopeprediction v${version}

+

nf-core/epitopeprediction ${version}

Run Name: $runName

<% if (!success){ diff --git a/assets/email_template.txt b/assets/email_template.txt index 8b39053..9d39536 100644 --- a/assets/email_template.txt +++ b/assets/email_template.txt @@ -4,7 +4,7 @@ |\\ | |__ __ / ` / \\ |__) |__ } { | \\| | \\__, \\__/ | \\ |___ \\`-._,-`-, `._,._,' - nf-core/epitopeprediction v${version} + nf-core/epitopeprediction ${version} ---------------------------------------------------- Run Name: $runName diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index ebfcc68..23973f9 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,5 +1,5 @@ report_comment: > - This report has been generated by the nf-core/epitopeprediction + This report has been generated by the nf-core/epitopeprediction analysis pipeline. For information about how to interpret these results, please see the documentation. report_section_order: diff --git a/assets/nf-core-epitopeprediction_logo_light.png b/assets/nf-core-epitopeprediction_logo_light.png index 40ba75d..9850b6b 100644 Binary files a/assets/nf-core-epitopeprediction_logo_light.png and b/assets/nf-core-epitopeprediction_logo_light.png differ diff --git a/assets/slackreport.json b/assets/slackreport.json index 80b50b7..f27f64a 100644 --- a/assets/slackreport.json +++ b/assets/slackreport.json @@ -3,7 +3,7 @@ { "fallback": "Plain-text summary of the attachment.", "color": "<% if (success) { %>good<% } else { %>danger<%} %>", - "author_name": "nf-core/epitopeprediction v${version} - ${runName}", + "author_name": "nf-core/epitopeprediction ${version} - ${runName}", "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", "fields": [ diff --git a/bin/check_requested_models.py b/bin/check_requested_models.py index 342505f..3a4a63b 100755 --- a/bin/check_requested_models.py +++ b/bin/check_requested_models.py @@ -2,15 +2,14 @@ # Written by Sabrina Krakau, Christopher Mohr and released under the MIT license. -import sys -import csv import argparse +import csv import logging +import sys import urllib.request from epytope.Core.Allele import Allele from epytope.Core.Peptide import Peptide -from epytope.IO import FileReader from epytope.EpitopePrediction import EpitopePredictorFactory # instantiate global logger object @@ -25,7 +24,7 @@ def read_peptide_input(filename): peptides = [] """expected columns (min required): id sequence""" - with open(filename, "r") as peptide_input: + with open(filename) as peptide_input: # enable listing of protein names for each peptide csv.field_size_limit(600000) reader = csv.DictReader(peptide_input, delimiter="\t") @@ -62,7 +61,7 @@ def __main__(): parser.add_argument("-v", "--versions", help=" File with used software versions.", required=True) args = parser.parse_args() selected_methods = [item.split("-")[0] if "mhcnuggets" not in item else item for item in args.tools.split(",")] - with open(args.versions, "r") as versions_file: + with open(args.versions) as versions_file: tool_version = [(row[0].split()[0], str(row[1])) for row in csv.reader(versions_file, delimiter=":")] # NOTE this needs to be updated, if a newer version will be available via epytope and should be used in the future tool_version.append(("syfpeithi", "1.0")) # how to handle this? @@ -78,7 +77,7 @@ def __main__(): if args.alleles.startswith("http"): alleles = [Allele(a) for a in urllib.request.urlopen(args.alleles).read().decode("utf-8").splitlines()] elif args.alleles.endswith(".txt"): - alleles = [Allele(a) for a in open(args.alleles, "r").read().splitlines()] + alleles = [Allele(a) for a in open(args.alleles).read().splitlines()] else: alleles = [Allele(a) for a in args.alleles.split(";")] diff --git a/bin/check_supported_models.py b/bin/check_supported_models.py index 8edc5ba..61c4801 100755 --- a/bin/check_supported_models.py +++ b/bin/check_supported_models.py @@ -2,9 +2,8 @@ # Written by Sabrina Krakau, Christopher Mohr and released under the MIT license. -import sys -import csv import argparse +import csv from epytope.EpitopePrediction import EpitopePredictorFactory @@ -30,7 +29,7 @@ def __main__(): # NOTE this needs to be updated manually, if other methods should be used in the future available_methods = ["syfpeithi", "mhcflurry", "mhcnuggets-class-1", "mhcnuggets-class-2"] - with open(args.versions, "r") as versions_file: + with open(args.versions) as versions_file: tool_version = [(row[0].split()[0], str(row[1])) for row in csv.reader(versions_file, delimiter=":")] # NOTE this needs to be updated, if a newer version will be available via Fred2 and should be used in the future tool_version.append(("syfpeithi", "1.0")) diff --git a/bin/epaa.py b/bin/epaa.py index b01e13c..8445fa8 100755 --- a/bin/epaa.py +++ b/bin/epaa.py @@ -1,31 +1,29 @@ #!/usr/bin/env python # Written by Christopher Mohr and released under the MIT license (2022). -import os -import sys -import logging -import csv -import re -import vcf import argparse -import urllib +import csv import itertools -import pandas as pd -import numpy as np -import epytope.Core.Generator as generator -import math import json -import urllib.request +import logging +import math +import os +import re +import sys +from datetime import datetime -from epytope.IO.MartsAdapter import MartsAdapter -from epytope.Core.Variant import Variant, VariationType, MutationSyntax +import epytope.Core.Generator as generator +import numpy as np +import pandas as pd +import vcf +from Bio import SeqUtils +from epytope.Core.Allele import Allele +from epytope.Core.Peptide import Peptide +from epytope.Core.Variant import MutationSyntax, Variant, VariationType from epytope.EpitopePrediction import EpitopePredictorFactory from epytope.IO.ADBAdapter import EIdentifierTypes +from epytope.IO.MartsAdapter import MartsAdapter from epytope.IO.UniProtAdapter import UniProtDB -from epytope.Core.Allele import Allele -from epytope.Core.Peptide import Peptide -from Bio import SeqUtils -from datetime import datetime __author__ = "Christopher Mohr" VERSION = "1.1" @@ -151,7 +149,7 @@ def read_vcf(filename, pass_only=True): SNPEFF_KEY = "ANN" variants = list() - with open(filename, "rt") as tsvfile: + with open(filename) as tsvfile: vcf_reader = vcf.Reader(tsvfile) variants = [r for r in vcf_reader] @@ -197,8 +195,7 @@ def read_vcf(filename, pass_only=True): SNP => seq[pos] = OBS (replace) INSERTION => seqp[pos:pos] = obs (insert at that position) - DELETION => s = slice(pos, pos+len(ref)) (create slice that will be removed) - del seq[s] (remove) + DELETION => s = slice(pos, pos+len(ref)) (create slice that will be removed) del seq[s] (remove) """ for alt in alternative_list: isHomozygous = determine_zygosity(record) @@ -319,12 +316,10 @@ def read_vcf(filename, pass_only=True): for format_key in format_list: if getattr(sample.data, format_key, None) is None: logger.warning( - "FORMAT entry {entry} not defined for {genotype}. Skipping.".format( - entry=format_key, genotype=sample.sample - ) + f"FORMAT entry {format_key} not defined for {sample.sample}. Skipping." ) continue - format_header = "{}.{}".format(sample.sample, format_key) + format_header = f"{sample.sample}.{format_key}" final_metadata_list.append(format_header) if isinstance(sample[format_key], list): format_value = ",".join([str(i) for i in sample[format_key]]) @@ -363,7 +358,7 @@ def read_peptide_input(filename): metadata = [] """expected columns (min required): id sequence""" - with open(filename, "r") as peptide_input: + with open(filename) as peptide_input: # enable listing of protein names for each peptide csv.field_size_limit(600000) reader = csv.DictReader(peptide_input, delimiter="\t") @@ -385,7 +380,7 @@ def read_protein_quant(filename): # protein id: sample1: intensity, sample2: intensity: intensities = {} - with open(filename, "r") as inp: + with open(filename) as inp: inpreader = csv.DictReader(inp, delimiter="\t") for row in inpreader: if "REV" in row["Protein IDs"]: @@ -406,7 +401,7 @@ def read_diff_expression_values(filename): # feature id: log2fold changes fold_changes = {} - with open(filename, "r") as inp: + with open(filename) as inp: inp.readline() for row in inp: values = row.strip().split("\t") @@ -420,7 +415,7 @@ def read_lig_ID_values(filename): # sequence: score median intensity intensities = {} - with open(filename, "r") as inp: + with open(filename) as inp: reader = csv.DictReader(inp, delimiter=",") for row in reader: seq = re.sub("[\(].*?[\)]", "", row["sequence"]) @@ -465,11 +460,11 @@ def create_gene_column_value(pep, pep_dictionary): def create_variant_pos_column_value(pep, pep_dictionary): - return ",".join(set(["{}".format(variant.genomePos) for variant in set(pep_dictionary[pep])])) + return ",".join(set([f"{variant.genomePos}" for variant in set(pep_dictionary[pep])])) def create_variant_chr_column_value(pep, pep_dictionary): - return ",".join(set(["{}".format(variant.chrom) for variant in set(pep_dictionary[pep])])) + return ",".join(set([f"{variant.chrom}" for variant in set(pep_dictionary[pep])])) def create_variant_type_column_value(pep, pep_dictionary): @@ -497,7 +492,7 @@ def create_metadata_column_value(pep, c, pep_dictionary): if len(variant.get_metadata(c)) != 0 ] ) - if len(meta) is 0: + if len(meta) == 0: return np.nan else: return ",".join(meta) @@ -512,7 +507,7 @@ def create_wt_seq_column_value(pep, wtseqs): if bool(transcript.vars) and "{}_{}".format(str(pep["sequence"]), transcript.transcript_id) in wtseqs ] ) - if len(wild_type) is 0: + if len(wild_type) == 0: return np.nan else: return ",".join(wild_type) @@ -571,13 +566,11 @@ def create_expression_column_value_for_result(row, dict, deseq, gene_id_lengths) ) ) logger.warning( - "FKPM value will be based on transcript length for {gene}. Because gene could not be found in the DB".format( - gene=t - ) + f"FKPM value will be based on transcript length for {t}. Because gene could not be found in the DB" ) else: values.append(np.nan) - values = ["{0:.2f}".format(value) for value in values] + values = [f"{value:.2f}" for value in values] return ",".join(values) @@ -591,7 +584,7 @@ def create_quant_column_value_for_result(row, dict, swissProtDict, key): values.append(math.log(int(dict[p][key]), 2)) else: values.append(int(dict[p][key])) - if len(values) is 0: + if len(values) == 0: return np.nan else: return ",".join(set([str(v) for v in values])) @@ -677,9 +670,9 @@ def generate_wt_seqs(peptides): wt = SeqUtils.seq1(m.groups()[0]) mut_seq[key] = wt if not_available: - wt_dict["{}_{}".format(str(x), t.transcript_id)] = np.nan + wt_dict[f"{str(x)}_{t.transcript_id}"] = np.nan elif variant_available: - wt_dict["{}_{}".format(str(x), t.transcript_id)] = "".join(mut_seq) + wt_dict[f"{str(x)}_{t.transcript_id}"] = "".join(mut_seq) return wt_dict @@ -1034,7 +1027,7 @@ def __main__(): parser.print_help() sys.exit("Provide at least one argument to epaa.py.") - filehandler = logging.FileHandler("{}_prediction.log".format(args.identifier)) + filehandler = logging.FileHandler(f"{args.identifier}_prediction.log") filehandler.setLevel(logging.DEBUG) filehandler.setFormatter(formatter) logger.addHandler(filehandler) @@ -1070,13 +1063,7 @@ def __main__(): transcriptProteinTable = ma.get_protein_ids_from_transcripts(transcripts, type=ID_SYSTEM_USED) # get the alleles - # TODO: remove this in PR of nf-validation - if args.alleles.startswith("http"): - alleles = [Allele(a) for a in urllib.request.urlopen(args.alleles).read().decode("utf-8").splitlines()] - elif args.alleles.endswith(".txt"): - alleles = [Allele(a) for a in open(args.alleles, "r").read().splitlines()] - else: - alleles = [Allele(a) for a in args.alleles.split(";")] + alleles = [Allele(a) for a in args.alleles.split(";")] # create protein db instance for filtering self-peptides up_db = UniProtDB("sp") @@ -1091,7 +1078,7 @@ def __main__(): up_db.read_seqs(args.reference_proteome) selected_methods = [item.split("-")[0] if "mhcnuggets" not in item else item for item in args.tools.split(",")] - with open(args.versions, "r") as versions_file: + with open(args.versions) as versions_file: tool_version = [(row[0].split()[0], str(row[1])) for row in csv.reader(versions_file, delimiter=":")] # NOTE this needs to be updated, if a newer version will be available via epytope and should be used in the future tool_version.append(("syfpeithi", "1.0")) @@ -1123,7 +1110,7 @@ def __main__(): thresholds.update({"netmhc": 2, "netmhcpan": 2, "netmhcii": 10, "netmhciipan": 5}) if args.tool_thresholds: - with open(args.tool_thresholds, "r") as json_file: + with open(args.tool_thresholds) as json_file: threshold_file = json.load(json_file) for tool, thresh in threshold_file.items(): if tool in thresholds.keys(): @@ -1228,7 +1215,7 @@ def __main__(): protein_quant = read_protein_quant(args.protein_quantification) first_entry = protein_quant[protein_quant.keys()[0]] for k in first_entry.keys(): - complete_df["{} log2 protein LFQ intensity".format(k)] = complete_df.apply( + complete_df[f"{k} log2 protein LFQ intensity"] = complete_df.apply( lambda row: create_quant_column_value_for_result(row, protein_quant, transcriptSwissProtMap, k), axis=1 ) # parse (differential) expression analysis results, annotate features (genes/transcripts) @@ -1237,7 +1224,7 @@ def __main__(): gene_id_lengths = {} col_name = "RNA expression (RPKM)" - with open(args.gene_reference, "r") as gene_list: + with open(args.gene_reference) as gene_list: for l in gene_list: ids = l.split("\t") gene_id_in_df = complete_df.iloc[1]["gene"] @@ -1280,7 +1267,7 @@ def __main__(): ) # write mutated protein sequences to fasta file if args.fasta_output and predictions_available: - with open("{}_prediction_proteins.fasta".format(args.identifier), "w") as protein_outfile: + with open(f"{args.identifier}_prediction_proteins.fasta", "w") as protein_outfile: for p in proteins: variants = [] for v in p.vars: @@ -1289,15 +1276,15 @@ def __main__(): cf = list(itertools.chain.from_iterable(c)) cds = ",".join([y.cdsMutationSyntax for y in set(cf)]) aas = ",".join([y.aaMutationSyntax for y in set(cf)]) - protein_outfile.write(">{}:{}:{}\n".format(p.transcript_id, aas, cds)) - protein_outfile.write("{}\n".format(str(p))) + protein_outfile.write(f">{p.transcript_id}:{aas}:{cds}\n") + protein_outfile.write(f"{str(p)}\n") complete_df["binder"] = complete_df[[col for col in complete_df.columns if "binder" in col]].any(axis=1) # write dataframe to tsv complete_df.fillna("") if predictions_available: - complete_df.to_csv("{}_prediction_result.tsv".format(args.identifier), "\t", index=False) + complete_df.to_csv(f"{args.identifier}_prediction_result.tsv", "\t", index=False) statistics["tool_thresholds"] = thresholds statistics["number_of_predictions"] = len(complete_df) @@ -1306,7 +1293,7 @@ def __main__(): statistics["number_of_unique_binders"] = list(set(binders)) statistics["number_of_unique_nonbinders"] = list(set(non_binders) - set(binders)) - with open("{}_report.json".format(args.identifier), "w") as json_out: + with open(f"{args.identifier}_report.json", "w") as json_out: json.dump(statistics, json_out) logger.info("Finished predictions at " + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) diff --git a/bin/gen_peptides.py b/bin/gen_peptides.py index 50613a0..d962ed3 100755 --- a/bin/gen_peptides.py +++ b/bin/gen_peptides.py @@ -1,13 +1,11 @@ #!/usr/bin/env python # Written by Sabrina Krakau, Christopher Mohr and released under the MIT license (2022). -import sys import argparse -import pandas as pd +import pandas as pd from Bio.SeqIO.FastaIO import SimpleFastaParser -from epytope.Core import Allele, Peptide, Protein, generate_peptides_from_proteins - +from epytope.Core import Protein, generate_peptides_from_proteins parser = argparse.ArgumentParser("Generating peptides from protein sequences.") parser.add_argument( diff --git a/bin/merge_jsons.py b/bin/merge_jsons.py index 75330ce..f6ebd34 100755 --- a/bin/merge_jsons.py +++ b/bin/merge_jsons.py @@ -2,10 +2,10 @@ # Written by Sabrina Krakau, Christopher Mohr, Gisela Gabernet and released under the MIT license (2022). +import argparse +import json import os import sys -import json -import argparse def __main__(): @@ -43,14 +43,14 @@ def combine_dicts(a, b): # read in json reports data = dict() if args.single_input: - with open(args.single_input, "r") as infile: + with open(args.single_input) as infile: json_content = json.load(infile) data = combine_dicts(data, json_content) else: for file in os.listdir(args.input): if file.endswith(".json"): - with open(os.path.join(args.input, file), "r") as infile: + with open(os.path.join(args.input, file)) as infile: json_content = json.load(infile) data = combine_dicts(data, json_content) @@ -69,7 +69,7 @@ def combine_dicts(a, b): data["number_of_predictions"] = sum(list(flatten(data["number_of_predictions"]))) data["number_of_variants"] = sum(list(flatten(data["number_of_variants"]))) - with open("{}_prediction_report.json".format(args.prefix), "w") as outfile: + with open(f"{args.prefix}_prediction_report.json", "w") as outfile: json.dump(data, outfile) diff --git a/bin/split_peptides.py b/bin/split_peptides.py index 5c96dad..4e31134 100755 --- a/bin/split_peptides.py +++ b/bin/split_peptides.py @@ -2,9 +2,8 @@ # Written by Sabrina Krakau, Christopher Mohr and released under the MIT license (2022). -import math import argparse - +import math parser = argparse.ArgumentParser("Split peptides input file.") parser.add_argument("-i", "--input", metavar="FILE", type=str, help="Input file containing peptides.") @@ -17,13 +16,13 @@ ) args = parser.parse_args() -with open(args.input, "r") as infile: +with open(args.input) as infile: tot_size = sum([1 for _ in infile]) - 1 n = int(min(math.ceil(float(tot_size) / args.min_size), args.max_chunks)) h = int(max(args.min_size, math.ceil(float(tot_size) / n))) -with open(args.input, "r") as infile: +with open(args.input) as infile: header = next(infile) for chunk in range(n): with open(args.output_base + ".chunk_" + str(chunk) + ".tsv", "w") as outfile: diff --git a/bin/split_vcf_by_variants.py b/bin/split_vcf_by_variants.py index 5c389f1..fade55e 100755 --- a/bin/split_vcf_by_variants.py +++ b/bin/split_vcf_by_variants.py @@ -2,14 +2,13 @@ # Written by Christopher Mohr and released under the MIT license (2022). import argparse -import logging import csv +import logging import os -from typing import Optional def determine_split_size(input_file, size): - with open(input_file, "r") as variants: + with open(input_file) as variants: input_lines = variants.readlines() num_variants = sum(1 for i in input_lines if not i.startswith("#")) if not size: @@ -57,7 +56,7 @@ def main(): metadata = "" var_group = "" - with open(args.input, "r") as input_file: + with open(args.input) as input_file: vcf_file = csv.reader(input_file, delimiter="\t") for line in vcf_file: diff --git a/conf/modules.config b/conf/modules.config index a89dd1a..3f1fec1 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -29,7 +29,7 @@ process { } withName: 'MULTIQC' { - ext.args = params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' + ext.args = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' } publishDir = [ path: { "${params.outdir}/multiqc" }, mode: params.publish_dir_mode, diff --git a/docs/images/nf-core-epitopeprediction_logo_dark.png b/docs/images/nf-core-epitopeprediction_logo_dark.png index 7c16270..f948b61 100644 Binary files a/docs/images/nf-core-epitopeprediction_logo_dark.png and b/docs/images/nf-core-epitopeprediction_logo_dark.png differ diff --git a/docs/images/nf-core-epitopeprediction_logo_light.png b/docs/images/nf-core-epitopeprediction_logo_light.png index 40ba75d..20530df 100644 Binary files a/docs/images/nf-core-epitopeprediction_logo_light.png and b/docs/images/nf-core-epitopeprediction_logo_light.png differ diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy index 01b8653..e248e4c 100755 --- a/lib/NfcoreTemplate.groovy +++ b/lib/NfcoreTemplate.groovy @@ -4,6 +4,7 @@ import org.yaml.snakeyaml.Yaml import groovy.json.JsonOutput +import nextflow.extension.FilesEx class NfcoreTemplate { @@ -141,12 +142,14 @@ class NfcoreTemplate { try { if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail + def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + sendmail_tf.withWriter { w -> w << sendmail_html } [ 'sendmail', '-t' ].execute() << sendmail_html log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" } catch (all) { // Catch failures and try with plaintext def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report.size() <= max_multiqc_email_size.toBytes() ) { + if ( mqc_report != null && mqc_report.size() <= max_multiqc_email_size.toBytes() ) { mail_cmd += [ '-A', mqc_report ] } mail_cmd.execute() << email_html @@ -155,14 +158,16 @@ class NfcoreTemplate { } // Write summary e-mail HTML to a file - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def output_hf = new File(output_d, "pipeline_report.html") + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - def output_tf = new File(output_d, "pipeline_report.txt") + FilesEx.copyTo(output_hf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.html"); + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } + FilesEx.copyTo(output_tf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.txt"); + output_tf.delete() } // @@ -227,15 +232,14 @@ class NfcoreTemplate { // Dump pipeline parameters in a json file // public static void dump_parameters(workflow, params) { - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') - def output_pf = new File(output_d, "params_${timestamp}.json") + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") def jsonStr = JsonOutput.toJson(params) - output_pf.text = JsonOutput.prettyPrint(jsonStr) + temp_pf.text = JsonOutput.prettyPrint(jsonStr) + + FilesEx.copyTo(temp_pf.toPath(), "${params.outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.delete() } // diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index e45309e..d54f98f 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -23,7 +23,7 @@ class WorkflowMain { // // Validate parameters and print summary to screen // - public static void initialise(workflow, params, log) { + public static void initialise(workflow, params, log, args) { // Print workflow version and exit on --version if (params.version) { @@ -34,6 +34,8 @@ class WorkflowMain { // Check that a -profile or Nextflow config has been provided to run the pipeline NfcoreTemplate.checkConfigProvided(workflow, log) + // Check that the profile doesn't contain spaces and doesn't end with a trailing comma + checkProfile(workflow.profile, args, log) // Check that conda channels are set-up correctly if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { @@ -59,4 +61,16 @@ class WorkflowMain { } return null } + + // + // Exit pipeline if --profile contains spaces + // + private static void checkProfile(profile, args, log) { + if (profile.endsWith(',')) { + Nextflow.error "Profile cannot end with a trailing comma. Please remove the comma from the end of the profile string.\nHint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." + } + if (args[0]) { + log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${args[0]}` has been detected.\n Hint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." + } + } } diff --git a/lib/nfcore_external_java_deps.jar b/lib/nfcore_external_java_deps.jar deleted file mode 100644 index 805c8bb..0000000 Binary files a/lib/nfcore_external_java_deps.jar and /dev/null differ diff --git a/main.nf b/main.nf index 3cb571d..19e10be 100644 --- a/main.nf +++ b/main.nf @@ -40,7 +40,7 @@ if (params.validate_params) { validateParameters() } -WorkflowMain.initialise(workflow, params, log) +WorkflowMain.initialise(workflow, params, log, args) /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/modules.json b/modules.json index 6aa6487..dac5465 100644 --- a/modules.json +++ b/modules.json @@ -7,22 +7,12 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "05c280924b6c768d484c7c443dad5e605c4ff4b4", - "installed_by": ["modules"] - }, - "fastqc": { - "branch": "master", - "git_sha": "a33464f205fa15305bfe268546f6607b6f4d4753", - "installed_by": ["modules"] - }, - "gunzip": { - "branch": "master", - "git_sha": "e06548bfa36ee31869b81041879dd6b3a83b1d57", + "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "a6e11ac655e744f7ebc724be669dd568ffdc0e80", + "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", "installed_by": ["modules"] } } diff --git a/modules/nf-core/custom/dumpsoftwareversions/environment.yml b/modules/nf-core/custom/dumpsoftwareversions/environment.yml new file mode 100644 index 0000000..9b3272b --- /dev/null +++ b/modules/nf-core/custom/dumpsoftwareversions/environment.yml @@ -0,0 +1,7 @@ +name: custom_dumpsoftwareversions +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::multiqc=1.19 diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf index c9d014b..f218761 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.15" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.15--pyhdfd78af_0' : - 'biocontainers/multiqc:1.15--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : + 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml index c32657d..5f15a5f 100644 --- a/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: @@ -16,7 +16,6 @@ input: type: file description: YML file containing software versions pattern: "*.yml" - output: - yml: type: file @@ -30,7 +29,9 @@ output: type: file description: File containing software versions pattern: "versions.yml" - authors: - "@drpatelh" - "@grst" +maintainers: + - "@drpatelh" + - "@grst" diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py index da03340..e55b8d4 100755 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -4,10 +4,11 @@ """Provide functions to merge multiple versions.yml files.""" -import yaml import platform from textwrap import dedent +import yaml + def _make_versions_html(versions): """Generate a tabular HTML output of all versions for MultiQC.""" diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test new file mode 100644 index 0000000..b1e1630 --- /dev/null +++ b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test @@ -0,0 +1,43 @@ +nextflow_process { + + name "Test Process CUSTOM_DUMPSOFTWAREVERSIONS" + script "../main.nf" + process "CUSTOM_DUMPSOFTWAREVERSIONS" + tag "modules" + tag "modules_nfcore" + tag "custom" + tag "dumpsoftwareversions" + tag "custom/dumpsoftwareversions" + + test("Should run without failures") { + when { + process { + """ + def tool1_version = ''' + TOOL1: + tool1: 0.11.9 + '''.stripIndent() + + def tool2_version = ''' + TOOL2: + tool2: 1.9 + '''.stripIndent() + + input[0] = Channel.of(tool1_version, tool2_version).collectFile() + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.versions, + file(process.out.mqc_yml[0]).readLines()[0..10], + file(process.out.yml[0]).readLines()[0..7] + ).match() + } + ) + } + } +} diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap new file mode 100644 index 0000000..5f59a93 --- /dev/null +++ b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap @@ -0,0 +1,33 @@ +{ + "Should run without failures": { + "content": [ + [ + "versions.yml:md5,76d454d92244589d32455833f7c1ba6d" + ], + [ + "data: \"\\n\\n \\n \\n \\n \\n \\n \\n \\n\\", + " \\n\\n\\n \\n \\n\\", + " \\ \\n\\n\\n\\n \\n \\", + " \\ \\n \\n\\n\\n\\n\\", + " \\n\\n \\n \\n\\", + " \\ \\n\\n\\n\\n\\n\\n \\n\\", + " \\ \\n \\n\\n\\n\\n\\", + " \\n\\n \\n \\n\\" + ], + [ + "CUSTOM_DUMPSOFTWAREVERSIONS:", + " python: 3.11.7", + " yaml: 5.4.1", + "TOOL1:", + " tool1: 0.11.9", + "TOOL2:", + " tool2: '1.9'", + "Workflow:" + ] + ], + "timestamp": "2024-01-09T23:01:18.710682" + } +} \ No newline at end of file diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml b/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml new file mode 100644 index 0000000..405aa24 --- /dev/null +++ b/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml @@ -0,0 +1,2 @@ +custom/dumpsoftwareversions: + - modules/nf-core/custom/dumpsoftwareversions/** diff --git a/modules/nf-core/fastqc/environment.yml b/modules/nf-core/fastqc/environment.yml new file mode 100644 index 0000000..1787b38 --- /dev/null +++ b/modules/nf-core/fastqc/environment.yml @@ -0,0 +1,7 @@ +name: fastqc +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::fastqc=0.12.1 diff --git a/modules/nf-core/fastqc/tests/main.nf.test b/modules/nf-core/fastqc/tests/main.nf.test new file mode 100644 index 0000000..1f21c66 --- /dev/null +++ b/modules/nf-core/fastqc/tests/main.nf.test @@ -0,0 +1,212 @@ +nextflow_process { + + name "Test Process FASTQC" + script "../main.nf" + process "FASTQC" + + tag "modules" + tag "modules_nfcore" + tag "fastqc" + + test("sarscov2 single-end [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [ id: 'test', single_end:true ], + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. + // looks like this:
Mon 2 Oct 2023
test.gz
+ // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 paired-end [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("") }, + { assert path(process.out.html[0][1][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 interleaved [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 paired-end [bam]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.sorted.bam', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/test_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/test_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 multiple [fastq]") { + + when { + process { + """ + input[0] = Channel.of([ + [id: 'test', single_end: false], // meta map + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test2_2.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1][0] ==~ ".*/test_1_fastqc.html" }, + { assert process.out.html[0][1][1] ==~ ".*/test_2_fastqc.html" }, + { assert process.out.html[0][1][2] ==~ ".*/test_3_fastqc.html" }, + { assert process.out.html[0][1][3] ==~ ".*/test_4_fastqc.html" }, + { assert process.out.zip[0][1][0] ==~ ".*/test_1_fastqc.zip" }, + { assert process.out.zip[0][1][1] ==~ ".*/test_2_fastqc.zip" }, + { assert process.out.zip[0][1][2] ==~ ".*/test_3_fastqc.zip" }, + { assert process.out.zip[0][1][3] ==~ ".*/test_4_fastqc.zip" }, + { assert path(process.out.html[0][1][0]).text.contains("") }, + { assert path(process.out.html[0][1][1]).text.contains("") }, + { assert path(process.out.html[0][1][2]).text.contains("") }, + { assert path(process.out.html[0][1][3]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 custom_prefix") { + + when { + process { + """ + input[0] = Channel.of([ + [ id:'mysample', single_end:true ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + + { assert process.out.html[0][1] ==~ ".*/mysample_fastqc.html" }, + { assert process.out.zip[0][1] ==~ ".*/mysample_fastqc.zip" }, + { assert path(process.out.html[0][1]).text.contains("") }, + + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 single-end [fastq] - stub") { + + options "-stub" + + when { + process { + """ + input[0] = Channel.of([ + [ id: 'test', single_end:true ], + [ file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true) ] + ]) + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out.html.collect { file(it[1]).getName() } + + process.out.zip.collect { file(it[1]).getName() } + + process.out.versions ).match() } + ) + } + } + +} diff --git a/modules/nf-core/fastqc/tests/main.nf.test.snap b/modules/nf-core/fastqc/tests/main.nf.test.snap new file mode 100644 index 0000000..5d624bb --- /dev/null +++ b/modules/nf-core/fastqc/tests/main.nf.test.snap @@ -0,0 +1,20 @@ +{ + "sarscov2 single-end [fastq] - stub": { + "content": [ + [ + "test.html", + "test.zip", + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "timestamp": "2024-01-17T18:40:57.254299" + }, + "versions": { + "content": [ + [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "timestamp": "2024-01-17T18:36:50.033627" + } +} \ No newline at end of file diff --git a/modules/nf-core/fastqc/tests/tags.yml b/modules/nf-core/fastqc/tests/tags.yml new file mode 100644 index 0000000..7834294 --- /dev/null +++ b/modules/nf-core/fastqc/tests/tags.yml @@ -0,0 +1,2 @@ +fastqc: + - modules/nf-core/fastqc/** diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml new file mode 100644 index 0000000..7625b75 --- /dev/null +++ b/modules/nf-core/multiqc/environment.yml @@ -0,0 +1,7 @@ +name: multiqc +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::multiqc=1.19 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 65d7dd0..1b9f7c4 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_single' - conda "bioconda::multiqc=1.15" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.15--pyhdfd78af_0' : - 'biocontainers/multiqc:1.15--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : + 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" @@ -25,12 +25,14 @@ process MULTIQC { def args = task.ext.args ?: '' def config = multiqc_config ? "--config $multiqc_config" : '' def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' + def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' """ multiqc \\ --force \\ $args \\ $config \\ $extra_config \\ + $logo \\ . cat <<-END_VERSIONS > versions.yml @@ -41,7 +43,7 @@ process MULTIQC { stub: """ - touch multiqc_data + mkdir multiqc_data touch multiqc_plots touch multiqc_report.html diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index f93b5ee..45a9bc3 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -1,5 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json -name: MultiQC +name: multiqc description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: - QC @@ -13,7 +12,6 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] - input: - multiqc_files: type: file @@ -31,7 +29,6 @@ input: type: file description: Optional logo file for MultiQC pattern: "*.{png}" - output: - report: type: file @@ -54,3 +51,8 @@ authors: - "@bunop" - "@drpatelh" - "@jfy133" +maintainers: + - "@abhi18av" + - "@bunop" + - "@drpatelh" + - "@jfy133" diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test new file mode 100644 index 0000000..d0438ed --- /dev/null +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -0,0 +1,83 @@ +nextflow_process { + + name "Test Process MULTIQC" + script "../main.nf" + process "MULTIQC" + tag "modules" + tag "modules_nfcore" + tag "multiqc" + + test("sarscov2 single-end [fastqc]") { + + when { + process { + """ + input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[1] = [] + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("versions") } + ) + } + + } + + test("sarscov2 single-end [fastqc] [config]") { + + when { + process { + """ + input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert process.out.report[0] ==~ ".*/multiqc_report.html" }, + { assert process.out.data[0] ==~ ".*/multiqc_data" }, + { assert snapshot(process.out.versions).match("versions") } + ) + } + } + + test("sarscov2 single-end [fastqc] - stub") { + + options "-stub" + + when { + process { + """ + input[0] = Channel.of([file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz_fastqc_zip'], checkIfExists: true)]) + input[1] = [] + input[2] = [] + input[3] = [] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.report.collect { file(it).getName() } + + process.out.data.collect { file(it).getName() } + + process.out.plots.collect { file(it).getName() } + + process.out.versions ).match() } + ) + } + + } +} diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap new file mode 100644 index 0000000..d37e730 --- /dev/null +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -0,0 +1,21 @@ +{ + "versions": { + "content": [ + [ + "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" + ] + ], + "timestamp": "2024-01-09T23:02:49.911994" + }, + "sarscov2 single-end [fastqc] - stub": { + "content": [ + [ + "multiqc_report.html", + "multiqc_data", + "multiqc_plots", + "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" + ] + ], + "timestamp": "2024-01-09T23:03:14.524346" + } +} \ No newline at end of file diff --git a/modules/nf-core/multiqc/tests/tags.yml b/modules/nf-core/multiqc/tests/tags.yml new file mode 100644 index 0000000..bea6c0d --- /dev/null +++ b/modules/nf-core/multiqc/tests/tags.yml @@ -0,0 +1,2 @@ +multiqc: + - modules/nf-core/multiqc/** diff --git a/nextflow.config b/nextflow.config index e1c459d..1755436 100644 --- a/nextflow.config +++ b/nextflow.config @@ -107,7 +107,7 @@ try { } // Load nf-core/epitopeprediction custom profiles from different institutions. -// Warning: Uncomment only if a pipeline-specific instititutional config already exists on nf-core/configs! +// Warning: Uncomment only if a pipeline-specific institutional config already exists on nf-core/configs! // try { // includeConfig "${params.custom_config_base}/pipeline/epitopeprediction.config" // } catch (Exception e) { @@ -118,6 +118,7 @@ profiles { dumpHashes = true process.beforeScript = 'echo $HOSTNAME' cleanup = false + nextflow.enable.configProcessNamesValidation = true } conda { conda.enabled = true @@ -126,6 +127,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + channels = ['conda-forge', 'bioconda', 'defaults'] apptainer.enabled = false } mamba { @@ -140,16 +142,16 @@ profiles { } docker { docker.enabled = true - docker.userEmulation = true conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + docker.runOptions = '-u $(id -u):$(id -g)' } arm { - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { singularity.enabled = true @@ -226,7 +228,7 @@ singularity.registry = 'quay.io' // Nextflow plugins plugins { - id 'nf-validation' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet } // Load igenomes.config if required @@ -249,6 +251,9 @@ env { // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] +// Disable process selector warnings by default. Use debug profile to enable warnings. +nextflow.enable.configProcessNamesValidation = false + def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true diff --git a/nextflow_schema.json b/nextflow_schema.json index e69966b..7f0a65b 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -203,7 +203,6 @@ "properties": { "external_tools_meta": { "type": "string", - "default": "${projectDir}/assets/external_tools_meta.json", "description": "Specifies the path to the JSON file with meta information on external prediction tools." }, "netmhc_system": { diff --git a/pyproject.toml b/pyproject.toml index 0d62beb..7d08e1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,13 @@ -# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Black. +# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff. # Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. -[tool.black] +[tool.ruff] line-length = 120 -target_version = ["py37", "py38", "py39", "py310"] +target-version = "py38" +select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] +cache-dir = "~/.cache/ruff" -[tool.isort] -profile = "black" -known_first_party = ["nf_core"] -multi_line_output = 3 +[tool.ruff.isort] +known-first-party = ["nf_core"] + +[tool.ruff.per-file-ignores] +"__init__.py" = ["E402", "F401"] diff --git a/workflows/epitopeprediction.nf b/workflows/epitopeprediction.nf index 00549dd..77dfec8 100644 --- a/workflows/epitopeprediction.nf +++ b/workflows/epitopeprediction.nf @@ -477,6 +477,13 @@ workflow.onComplete { } } +workflow.onError { + if (workflow.errorReport.contains("Process requirement exceeds available memory")) { + println("🛑 Default resources exceed availability 🛑 ") + println("💡 See here on how to configure pipeline: https://nf-co.re/docs/usage/configuration#tuning-workflow-resources 💡") + } +} + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ THE END
Process Name \\", + " \\ Software Version
CUSTOM_DUMPSOFTWAREVERSIONSpython3.11.7
yaml5.4.1
TOOL1tool10.11.9
TOOL2tool21.9
WorkflowNextflow
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls
File typeConventional base calls