fix: support int | str group ids

.github/workflows/ci.yml

@@ -10,28 +10,28 @@ on:
 
 jobs:
   commitlint:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: github.event_name == 'pull_request'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - uses: wagoid/commitlint-github-action@v4
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Test
         run: docker build --target test .
   release:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: test
     if: github.event_name == 'push'
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v2
         with:
-          node-version: 14
+          node-version: 18
       - name: Install semantic-release
         run: npm i [email protected] [email protected]
       - name: Release

.github/workflows/publish.yml

@@ -36,6 +36,7 @@ jobs:
       uses: docker/build-push-action@v4
       with:
         context: .
+        labels: ${{ steps.meta.outputs.labels }}
         push: true
         tags: ${{ steps.meta.outputs.tags }}
-        labels: ${{ steps.meta.outputs.labels }}
+        target: base

Dockerfile

@@ -1,11 +1,27 @@
-FROM virtool/workflow:5.3.0 as base
+FROM debian:buster as prep
+WORKDIR /build
+RUN apt-get update && apt-get install -y make gcc zlib1g-dev wget unzip
+RUN wget https://zlib.net/pigz/pigz-2.8.tar.gz && \
+    tar -xzvf pigz-2.8.tar.gz && \
+    cd pigz-2.8 && \
+    make
+RUN wget https://github.com/BenLangmead/bowtie2/releases/download/v2.3.2/bowtie2-2.3.2-legacy-linux-x86_64.zip && \
+    unzip bowtie2-2.3.2-legacy-linux-x86_64.zip && \
+    mkdir bowtie2 && \
+    cp bowtie2-2.3.2-legacy/bowtie2* bowtie2
+
+FROM python:3.10-buster as base
 WORKDIR /workflow
-COPY workflow.py utils.py /workflow/
+COPY --from=prep /build/bowtie2/* /usr/local/bin/
+COPY --from=prep /build/pigz-2.8/pigz /usr/local/bin/pigz
+COPY pyproject.toml poetry.lock utils.py workflow.py ./
+RUN curl -sSL https://install.python-poetry.org | python -
+ENV PATH="/root/.local/bin:${PATH}"
+RUN poetry export > requirements.txt
+RUN pip install -r requirements.txt
 
 FROM base as test
-WORKDIR /test
-RUN curl -sSL https://install.python-poetry.org | python3 -
-COPY ./pyproject.toml ./poetry.lock ./
-RUN poetry install
-COPY . .
-RUN ["poetry", "run", "pytest"]
+RUN poetry export  --with dev > requirements.txt
+RUN pip install -r requirements.txt
+COPY tests ./tests
+RUN pytest

pyproject.toml

@@ -1,27 +1,25 @@
 [tool.poetry]
 name = "workflow-build-index"
 version = "0.0.0"
-description = "The build index workflow for Virtool."
+description = "A workflow for building Virtool reference indexes"
 authors = ["Ian Boyes", "Blake Smith", "Tiensheng Sui"]
 license = "MIT"
-readme="README.md"
 classifiers=[
     "Programming Language:: Python:: 3.10",
 ]
-repository="https://github.com/virtool/workflow-build-index"
 
 [tool.poetry.dependencies]
 python = "~3.10"
-virtool-workflow = "^5.1.0"
-black = "^23.1.0"
-pre-commit = "^3.0.4"
+virtool-workflow = "^5.4.2"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^6.2.4"
-pytest-asyncio = "^0.15.1"
+black = "^23.9.1"
+pre-commit = "^3.4.0"
+pytest = "^7.4.2"
 pytest-aiohttp = "^0.3.0"
-pytest-regressions = "^2.2.0"
+pytest-asyncio = "^0.15.1"
 pytest-mock = "^3.6.1"
+pytest-regressions = "^2.2.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

utils.py

@@ -1,19 +1,18 @@
-import asyncio
 import gzip
-from typing import List, Iterable, Generator, Dict, Mapping
+from typing import Iterable, Generator, Mapping
 
 import aiofiles
 
-ISOLATE_KEYS = ["id", "source_type", "source_name", "default"]
+ISOLATE_KEYS = ("id", "source_type", "source_name", "default")
 
-OTU_KEYS = ["name", "abbreviation", "schema"]
+OTU_KEYS = ("name", "abbreviation", "schema")
 
-RIGHTS = ["build", "modify", "modify_otu", "remove"]
+RIGHTS = ("build", "modify", "modify_otu", "remove")
 
-SEQUENCE_KEYS = ["accession", "definition", "host", "sequence"]
+SEQUENCE_KEYS = ("accession", "definition", "host", "sequence")
 
 
-def extract_default_sequences(joined: Mapping[str, Mapping]) -> List[Mapping]:
+def extract_default_sequences(joined: Mapping[str, Mapping]) -> list[Mapping]:
     """Return a list of sequences from the default isolate of the passed joined otu document.
     :param joined: the joined otu document.
     :return: a list of sequences associated with the default isolate.
@@ -84,7 +83,7 @@ def get_sequences_from_patched_otus(
                 yield sequence
 
 
-def prepare_export_otus(otus: List[Dict]) -> List[Dict]:
+def prepare_export_otus(otus: list[dict]) -> list[dict]:
     cleaned = list()
 
     otu_keys = OTU_KEYS + ["_id"]
@@ -109,7 +108,7 @@ def prepare_export_otus(otus: List[Dict]) -> List[Dict]:
 
 
 def prepare_exportable_otu(
-    otu, otu_keys: List[str] = None, sequence_keys: List[str] = None
+    otu, otu_keys: list[str] = None, sequence_keys: list[str] = None
 ):
     otu_keys = otu_keys or OTU_KEYS
     sequence_keys = sequence_keys or SEQUENCE_KEYS

workflow.py

@@ -1,8 +1,8 @@
 import asyncio
-from typing import List
 
 from virtool_core.utils import compress_file
 from virtool_workflow import step, fixture, hooks
+from virtool_workflow.api.indexes import IndexProvider
 from virtool_workflow.data_model.indexes import WFIndex
 
 BOWTIE_LINE_ENDINGS = (
@@ -16,17 +16,17 @@
 
 
 @hooks.on_failure
-async def delete_index(index_provider):
+async def delete_index(index_provider: IndexProvider):
     await index_provider.delete()
 
 
 @fixture
-def index(indexes: List[WFIndex]):
+def index(indexes: list[WFIndex]) -> WFIndex:
     return indexes[0]
 
 
 @step
-async def bowtie_build(index, proc):
+async def bowtie_build(index: WFIndex, proc: int):
     """
     Build a Bowtie2 mapping index for the reference.
 
@@ -43,7 +43,7 @@ async def bowtie_build(index, proc):
 
 
 @step
-async def finalize(index):
+async def finalize(index: WFIndex):
     """Compress and save the new reference index files."""
     await asyncio.to_thread(
         compress_file,