fix: use virtool-workflow^6.0.0-alpha11

* Overhaul workflow to use `virtool-workflow^6.0.0`. * Cleanup tests and example data. * Source some tools from `workflow-tools`. * Improve SPAdes logging.
virtool · Jan 22, 2024 · 2f14e66 · 2f14e66
1 parent e551612
commit 2f14e66
Show file tree

Hide file tree

Showing 47 changed files with 1,456,770 additions and 3,864 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: commitlint
@@ -22,22 +22,33 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+      - name: Setup Docker
+        uses: docker/setup-buildx-action@v3
+      - name: Build
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          context: .
+          load: true
+          target: test
       - name: Test
-        run: docker build -t nuvs-test --target test . && docker run nuvs-test
+        run: docker run --rm -t ${{ steps.build.outputs.imageid }} pytest
   release:
     runs-on: ubuntu-22.04
     needs: [test, commitlint]
     if: github.event_name == 'push'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: 20
       - name: Install semantic-release
-        run: npm i semantic-release@v18.0.0 conventional-changelog-conventionalcommits@4.6.1
+        run: npm i semantic-release@v23.0.0 conventional-changelog-conventionalcommits@7.0.2
       - name: Release
         env:
           GITHUB_USERNAME: ${{ secrets.GH_USERNAME }}

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -16,13 +16,11 @@ jobs:
     if: github.repository_owner == 'Virtool'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Write VERSION file
         run: echo ${{ github.event.release.tag_name }} > VERSION
-      - name: Update pyproject.toml version
-        run: sed -i 's/0\.0\.0/${{ github.event.release.tag_name }}/' pyproject.toml
       - name: Login to Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ secrets.GH_USERNAME }}

diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,5 @@
 __pycache__
 .idea
 .pytest_cache
-/target/
-.vscode
+.vscode
+target
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,5 +1,7 @@
 repos:
-  - repo: https://github.com/psf/black
-    rev: 23.1.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.13
     hooks:
-      - id: black
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format
diff --git a/.releaserc b/.releaserc
@@ -1,15 +1,6 @@
 {
   "branches": [
-    "+([0-9])?(.{+([0-9]),x}).x",
     "main",
-    {
-      "name": "beta",
-      "prerelease": true
-    },
-    {
-      "name": "alpha",
-      "prerelease": true
-    }
   ],
   "plugins": [
     ["@semantic-release/commit-analyzer", {

diff --git a/.vscode/settings.json b/.vscode/settings.json
diff --git a/Dockerfile b/Dockerfile
@@ -1,13 +1,6 @@
 FROM debian:buster as prep
 WORKDIR /build
 RUN apt-get update && apt-get install -y cmake gcc g++ make unzip wget zlib1g-dev
-RUN wget https://zlib.net/pigz/pigz-2.8.tar.gz
-RUN tar -xvf pigz-2.8.tar.gz
-WORKDIR /build/pigz-2.8
-RUN make
-WORKDIR /build
-RUN wget https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v0.11.9.zip
-RUN unzip fastqc_v0.11.9.zip
 RUN wget https://github.com/BenLangmead/bowtie2/releases/download/v2.3.2/bowtie2-2.3.2-legacy-linux-x86_64.zip
 RUN unzip bowtie2-2.3.2-legacy-linux-x86_64.zip
 RUN mkdir bowtie2
@@ -17,52 +10,48 @@ RUN tar -xvf SPAdes-3.11.0-Linux.tar.gz
 RUN mv SPAdes-3.11.0-Linux spades
 RUN sed -i 's/import collections/import collections\nimport collections.abc/g' spades/share/spades/pyyaml3/constructor.py
 RUN sed -i 's/key, collections.Hashable/key, collections.abc.Hashable/g' spades/share/spades/pyyaml3/constructor.py
-RUN wget http://eddylab.org/software/hmmer/hmmer-3.2.1.tar.gz
-RUN tar -xf hmmer-3.2.1.tar.gz
-WORKDIR /build/hmmer-3.2.1
-RUN ./configure --prefix /build/hmmer
-RUN make
-RUN make install
-WORKDIR /build
-RUN wget https://github.com/relipmoc/skewer/archive/0.2.2.tar.gz
-RUN tar -xf 0.2.2.tar.gz
-WORKDIR /build/skewer-0.2.2
-RUN make
-RUN mv skewer /build
 
-FROM python:3.10-buster as base
+FROM python:3.10-bullseye as build
 WORKDIR /app
 COPY --from=prep /build/bowtie2/* /usr/local/bin/
-COPY --from=prep /build/FastQC /opt/fastqc
-COPY --from=prep /build/hmmer /opt/hmmer
-COPY --from=prep /build/pigz-2.8/pigz /usr/local/bin/pigz
-COPY --from=prep /build/skewer /usr/local/bin/
 COPY --from=prep /build/spades /opt/spades
-RUN chmod ugo+x /opt/fastqc/fastqc && \
-    ln -fs /opt/fastqc/fastqc /usr/local/bin/fastqc && \
-    for file in `ls /opt/hmmer/bin`; do ln -fs /opt/hmmer/bin/${file} /usr/local/bin/${file};  done
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends curl build-essential default-jre && \
-    rm -rf /var/lib/apt/lists/* && \
-    apt-get clean
-RUN apt-get update && apt-get install -y curl build-essential
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /opt/hmmer /opt/hmmer
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /usr/local/bin/skewer /usr/local/bin/
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /usr/local/bin/pigz /usr/local/bin/
+RUN apt-get update && apt-get install -y --no-install-recommends curl build-essential default-jre
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 RUN curl -sSL https://install.python-poetry.org | python -
-ENV PATH="/root/.cargo/bin:/root/.local/bin:/opt/spades/bin:${PATH}"
-RUN pip install --upgrade pip
-RUN pip install maturin==0.14.12
+ENV PATH="/root/.cargo/bin:/root/.local/bin:/opt/spades/bin:/opt/hmmer/bin/:${PATH}" \
+    POETRY_CACHE_DIR='/tmp/poetry_cache' \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1
 COPY src src
-COPY Cargo.toml Cargo.lock poetry.lock pyproject.toml workflow.py ./
-RUN maturin build --release
-RUN poetry export > requirements.txt
-RUN pip install -r requirements.txt
-RUN pip install /app/target/wheels/nuvs_rust*.whl
+COPY Cargo.toml Cargo.lock poetry.lock pyproject.toml ./
+RUN poetry install --only rust
+RUN poetry run maturin build --release
+RUN poetry remove nuvs-rust && poetry add target/wheels/*.whl && poetry install
+RUN poetry install --without dev --no-root && rm -rf $POETRY_CACHE_DIR
+
+FROM build as test
+ENV PATH="/root/.cargo/bin:/root/.local/bin:/opt/spades/bin:${PATH}" \
+    POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1
+RUN poetry install --with dev
+COPY example ./example
+COPY tests ./tests
+COPY workflow.py ./
+ENTRYPOINT ["poetry", "run"]
 
-FROM base as test
+FROM python:3.10-bullseye as base
 WORKDIR /app
-ENV PATH="/opt/spades/bin:${PATH}"
-RUN poetry export --with dev > requirements.txt
-RUN pip install -r requirements.txt
+COPY --from=prep /build/bowtie2/* /usr/local/bin/
+COPY --from=prep /build/spades /opt/spades
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /opt/hmmer /opt/hmmer
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /usr/local/bin/skewer /usr/local/bin/
+COPY --from=ghcr.io/virtool/workflow-tools:2.0.1 /usr/local/bin/pigz /usr/local/bin/
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:/opt/spades/bin:/opt/hmmer/bin:${PATH}"
+COPY --from=build /app/.venv /app/.venv
 COPY workflow.py ./
-COPY tests ./tests
-RUN pytest
diff --git a/docker-compose.yml b/docker-compose.yml
diff --git a/tests/data/hmms/profiles.hmm → example/hmms/profiles.hmm b/tests/data/hmms/profiles.hmm → example/hmms/profiles.hmm
diff --git a/tests/data/hmms/profiles.hmm.h3f → example/hmms/profiles.hmm.h3f b/tests/data/hmms/profiles.hmm.h3f → example/hmms/profiles.hmm.h3f
diff --git a/tests/data/hmms/profiles.hmm.h3i → example/hmms/profiles.hmm.h3i b/tests/data/hmms/profiles.hmm.h3i → example/hmms/profiles.hmm.h3i
diff --git a/tests/data/hmms/profiles.hmm.h3m → example/hmms/profiles.hmm.h3m b/tests/data/hmms/profiles.hmm.h3m → example/hmms/profiles.hmm.h3m
diff --git a/tests/data/hmms/profiles.hmm.h3p → example/hmms/profiles.hmm.h3p b/tests/data/hmms/profiles.hmm.h3p → example/hmms/profiles.hmm.h3p
diff --git a/tests/data/index/reference.1.bt2 → example/index/reference.1.bt2 b/tests/data/index/reference.1.bt2 → example/index/reference.1.bt2
diff --git a/tests/data/index/reference.2.bt2 → example/index/reference.2.bt2 b/tests/data/index/reference.2.bt2 → example/index/reference.2.bt2
diff --git a/tests/data/index/reference.3.bt2 → example/index/reference.3.bt2 b/tests/data/index/reference.3.bt2 → example/index/reference.3.bt2
diff --git a/tests/data/index/reference.4.bt2 → example/index/reference.4.bt2 b/tests/data/index/reference.4.bt2 → example/index/reference.4.bt2
diff --git a/tests/data/index/reference.rev.1.bt2 → example/index/reference.rev.1.bt2 b/tests/data/index/reference.rev.1.bt2 → example/index/reference.rev.1.bt2
diff --git a/tests/data/index/reference.rev.2.bt2 → example/index/reference.rev.2.bt2 b/tests/data/index/reference.rev.2.bt2 → example/index/reference.rev.2.bt2
diff --git a/example/orfs.fa b/example/orfs.fa
@@ -0,0 +1,13 @@
+>sequence_0.0
+MVAVRAPRRKRASATDLYKTCKAAGTCPPDVIPKIEGSTLADKILQWSGLGIFLGGLGIGTGTGSGGRTGYIPLGGGGRPSVVDIGPT
+RPPIIIEPVGPTEPSIVTLVEESSIIQSGAPIPTFSGGNGFELTTSSATTPAVLDITPSAGTVHVTSTNIQNPLYIEPPIDIPQAGEA
+SGHIFTTTSTAGTHSYEEIPMEVFASTNGTGLEPISSTPIPGIQRVSAPRLYSKAYQQVKVTDPNFIGNPSTFVTFDNPAYEPIDETL
+TYASSSTVAPDPDFLDIIALHRPALTSRKGTVRYSRLGQKATMKTRSGKQIGATVHYYHDISPIQSFAEHEEIELQPLHTSTHSSAPL
+FDIYADPDTVPSIHTPRMSYSPTTLPVPRYASNVFSSINTSTTNVTVPLSTSFELPVYSGSDIYTPTSSPTWPSLPPPPTTNLPAIVV
+HGDNYYLWPYIYLIHKRRKRMPYFFSDGFVAY
+>sequence_2.0
+MSSLVSETSNSEVGSQMESPGRGGQSIDAPSSSCFKVRARNLFLTYSKCNLTAVFLLEYISSLLKKYCPTYIYVAQEAHKDGSHHLHC
+IIQCSKYVRTTSAKFFDIKEFHPNVQNPRMPKKALSYCKKSPISEAEYGVFQEIKRPRKKKADAPSTKDAKMAEIIKSSTNKEDYLSM
+VRKSFPFDWATRLQQFQFSAESLFPSTPPPYVDPFGMPSQDTHPVIGAWLRDELYTDRSPTERRRSLYICGPTRTGKTSWARSLGSHN
+YWQHSVDFLHVIQNARYNVIDDIPFKFVPCWKGLVGSQKDITVNPKYGKKRLLSNGIPCIILVNEDEDWLQQMQPSQADWFNANAVVH
+YMYSGESFFEAL
diff --git a/example/sample/paired_1.fq.gz b/example/sample/paired_1.fq.gz
diff --git a/example/sample/paired_2.fq.gz b/example/sample/paired_2.fq.gz
diff --git a/example/sample/reads_1.fq.gz b/example/sample/reads_1.fq.gz
diff --git a/tests/data/scaffolds_u.fa → example/scaffolds.fasta b/tests/data/scaffolds_u.fa → example/scaffolds.fasta
diff --git a/tests/data/subtraction/subtraction.1.bt2 → example/subtraction/subtraction.1.bt2 b/tests/data/subtraction/subtraction.1.bt2 → example/subtraction/subtraction.1.bt2
diff --git a/tests/data/subtraction/subtraction.2.bt2 → example/subtraction/subtraction.2.bt2 b/tests/data/subtraction/subtraction.2.bt2 → example/subtraction/subtraction.2.bt2
diff --git a/tests/data/subtraction/subtraction.3.bt2 → example/subtraction/subtraction.3.bt2 b/tests/data/subtraction/subtraction.3.bt2 → example/subtraction/subtraction.3.bt2
diff --git a/tests/data/subtraction/subtraction.4.bt2 → example/subtraction/subtraction.4.bt2 b/tests/data/subtraction/subtraction.4.bt2 → example/subtraction/subtraction.4.bt2
diff --git a/tests/data/test.fq → example/subtraction/subtraction.fa.gz b/tests/data/test.fq → example/subtraction/subtraction.fa.gz
diff --git a/tests/data/subtraction/subtraction.rev.1.bt2 → example/subtraction/subtraction.rev.1.bt2 b/tests/data/subtraction/subtraction.rev.1.bt2 → example/subtraction/subtraction.rev.1.bt2
diff --git a/tests/data/subtraction/subtraction.rev.2.bt2 → example/subtraction/subtraction.rev.2.bt2 b/tests/data/subtraction/subtraction.rev.2.bt2 → example/subtraction/subtraction.rev.2.bt2
diff --git a/tests/data/unmapped_1.fq → example/unmapped_1.fq b/tests/data/unmapped_1.fq → example/unmapped_1.fq
diff --git a/tests/data/unmapped_2.fq → example/unmapped_2.fq b/tests/data/unmapped_2.fq → example/unmapped_2.fq
diff --git a/tests/data/unmapped_otus.fq → example/unmapped_otus.fq b/tests/data/unmapped_otus.fq → example/unmapped_otus.fq