Merge pull request #165 from sanger/develop

Develop to master

.github/workflows/generate_docs.yml

@@ -0,0 +1,70 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+permissions:
+  pages: write      # Allow writing to the GitHub Pages
+  id-token: write   # Allow OIDC token to be issued
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.11
+
+      - name: Get latest release tag and version
+        run: |
+          LATEST_RELEASE_TAG=$(curl --silent "https://api.github.com/repos/sanger/tol-lab-share/releases/latest" | jq -r '.tag_name')
+          VERSION=${LATEST_RELEASE_TAG#v}
+          echo "LATEST_RELEASE_TAG: $LATEST_RELEASE_TAG and VERSION: $VERSION"
+
+      - name: Install dependencies for docs generation
+        run: |
+          pip install pydoctor
+          pip install mkdocs
+          pip install mkdocs-material
+          pip install mkdocs-glightbox
+          pip install mkdocs-git-revision-date-localized-plugin
+          pip install mkdocs-table-reader-plugin
+
+      - name: Create rst files for docs and generate api-docs
+        run: |
+          mkdir doc
+          mkdir doc/api-docs
+          pydoctor \
+          --project-name=tol-lab-share \
+          --project-version=$VERSION \
+          --project-url=https://github.com/sanger/tol-lab-share/ \
+          --html-viewsource-base=https://github.com/sanger/tol-lab-share/tree/$LATEST_RELEASE_TAG \
+          --make-html \
+          --html-output=doc/api-docs \
+          --project-base-dir="." \
+          --docformat=google \
+          --intersphinx=https://docs.python.org/3/objects.inv \
+          ./tol_lab_share || true
+
+      - name: Create mkdocs documentation
+        run: |
+          mkdocs build -f documentation/mkdocs.yml
+          cp -r documentation/site/* doc/
+
+      - name: Upload artifact to GitHub Pages
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: doc
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: build  # The deploy job will only run if the build job is successful
+
+    steps:
+      - name: Deploy to GitHub Pages
+        uses: actions/deploy-pages@v4

.gitignore

@@ -79,6 +79,10 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/build
+docs/tol_lab_share.**.rst
+doc
+documentation/site
 
 # PyBuilder
 target/

.release-version

@@ -1 +1 @@
-2.2.0
+2.3.0

Pipfile

@@ -15,10 +15,19 @@ pytest-cov = "*"
 pytest-freezegun = "*"
 types-python-dateutil = "*"
 responses = "*"
+sphinx = "*"
+sphinx-autodoc-typehints = "*"
+sphinx-rtd-theme = "*"
+furo = "*"
+recommonmark = "*"
+markdown = "*"
+sphinx-adc-theme = "*"
+sphinx-theme-pd = "*"
+sphinx-pdj-theme = "*"
 
 [packages]
 colorlog = "~=6.7"
-more-itertools = "~=10.3"
+more-itertools = "~=10.5"
 python-dotenv = "~=1.0"
 requests = "~=2.31"
 slackclient = "~=2.9"

documentation/docs/aliquot-export.csv

@@ -0,0 +1,6 @@
+id,id_lims,aliquot_uuid,aliquot_type,source_type,source_barcode,sample_name,used_by_type,used_by_barcode,volume,concentration,last_updated,recorded_at,created_at,insert_size
+1,Traction,49aab538-b66a-45fd-ae20-3adf4eb56dec,primary,library,TRAC-2-10376,DTOL9987549,none,,17.60,17.00,"2024-08-12 12:29:30.000000","2024-08-12 12:29:30.813950","2024-08-12 12:29:30.814016",13258
+2,Traction,92769e66-7c4b-449f-973e-57a635084ed4,primary,library,TRAC-2-8773,DTOL14550201,none,,11.60,11.50,"2024-08-13 08:45:04.000000","2024-08-13 08:45:04.610399","2024-08-13 08:45:04.610429",8820
+3,Traction,b2dcfc97-9af8-458b-bbfd-f5eded25a598,derived,library,TRAC-2-9721,ToL_PacBio_HiFi012:E1,run,1021188000351120003320241115:1:A1,10.00,8.00,"2024-08-13 09:29:24.000000","2024-08-13 09:29:24.622711","2024-08-13 09:29:24.622748",NULL
+4,Traction,a10f428c-c32f-4ab5-9023-56c4b544eb3a,derived,library,TRAC-2-9722,ToL_PacBio_HiFi012:F1,run,1021188000351120003320241115:1:B1,13.50,19.40,"2024-08-13 09:29:24.000000","2024-08-13 09:29:24.691334","2024-08-13 09:29:24.691363",NULL
+5,Traction,2fa0cd25-b281-470a-acf2-88ec32d35b57,derived,library,TRAC-2-9723,ToL_PacBio_HiFi012:G1,run,1021188000351120003320241115:1:C1,10.00,12.40,"2024-08-13 09:29:24.000000","2024-08-13 09:29:24.790937","2024-08-13 09:29:24.790966",NULL

documentation/docs/index.md

@@ -0,0 +1,113 @@
+# Tol Lab Share
+
+RabbitMQ consumer for TOL input, and for Traction input in volume tracking.
+
+## Getting Started
+
+The following tools are required for development:
+
+- python (use pyenv or something similar to install the python version specified in the `Pipfile`)
+
+Use pyenv or something similar to install the version of python
+defined in the `Pipfile`:
+
+```bash
+brew install pyenv
+pyenv install <python_version>
+```
+
+Use pipenv to install the required python packages for the application and development:
+
+```bash
+pipenv install --dev
+```
+
+### Setting up with Docker
+
+If you want to setup a local development environment with Docker please check
+the instructions in [SETUP_DOCKER.md](https://github.com/sanger/tol-lab-share/blob/develop/SETUP_DOCKER.md)
+
+## Running
+
+1. Enter the python virtual environment using:
+
+    ```bash
+    pipenv shell
+    ```
+
+1. Run the app using:
+
+    ```bash
+    python main.py
+    ```
+
+## Testing
+
+Run the tests using pytest (flags are for verbose and exit early):
+
+```bash
+python -m pytest -vx
+```
+
+## Deployment
+
+This project uses a Docker image as the unit of deployment. Update `.release-version` with
+major/minor/patch. On merging a pull request into *develop* or *master*, a release will be created
+along with the Docker image associated to that release.
+
+## Snappy
+
+If when you install the dependencies and you see the following error:
+
+```stdout
+[pipenv.exceptions.InstallError]:       src/snappy/snappymodule.cc:33:10: fatal error: 'snappy-c.h' file not found
+[pipenv.exceptions.InstallError]:       #include <snappy-c.h>
+[pipenv.exceptions.InstallError]:                ^~~~~~~~~~~~
+[pipenv.exceptions.InstallError]:       1 error generated.
+[pipenv.exceptions.InstallError]:       error: command '/usr/bin/clang' failed with exit code 1
+[pipenv.exceptions.InstallError]:       [end of output]
+[pipenv.exceptions.InstallError]:
+[pipenv.exceptions.InstallError]:   note: This error originates from a subprocess, and is likely not a problem with pip.
+[pipenv.exceptions.InstallError]:   ERROR: Failed building wheel for python-snappy
+[pipenv.exceptions.InstallError]: ERROR: Could not build wheels for python-snappy, which is required to install pyproject.toml-based projects
+ERROR: Couldn't install package: {}
+ Package installation failed...
+```
+
+You need to install snappy:
+
+```bash
+brew install snappy
+```
+
+Ensure the `include` and `lib` directories of `homebrew` are set in environment variables.
+You might want to add these to your `~/.zshrc` file:
+
+```bash
+export CPPFLAGS="-I$(brew --prefix)/include"
+export LDFLAGS="-L$(brew --prefix)/lib"
+```
+
+## TOL Automated Manifest Process
+
+Following diagram discusses how automated manifests are received by `tol-lab-share` and published to Traction.
+
+![TOL Labware Production Flow - Architecture](https://github.com/sanger/tol-lab-share/assets/519327/5356846a-6d9b-4b8d-8ffb-af26d0776222)
+
+## Formatting, Type Checking and Linting
+
+Black is used as a formatter, to format code before committing:
+
+    black .
+
+Mypy is used as a type checker, to execute:
+
+    mypy .
+
+Flake8 is used for linting, to execute:
+
+    flake8
+
+## API Docs
+
+API Docs can be accessed via [https://sanger.github.io/tol-lab-share/api-docs/](https://sanger.github.io/tol-lab-share/api-docs/).

documentation/docs/javascripts/mathjax.js

@@ -0,0 +1,19 @@
+window.MathJax = {
+    tex: {
+      inlineMath: [["\\(", "\\)"]],
+      displayMath: [["\\[", "\\]"]],
+      processEscapes: true,
+      processEnvironments: true
+    },
+    options: {
+      ignoreHtmlClass: ".*|",
+      processHtmlClass: "arithmatex"
+    }
+  };
+
+  document$.subscribe(() => { 
+    MathJax.startup.output.clearCache()
+    MathJax.typesetClear()
+    MathJax.texReset()
+    MathJax.typesetPromise()
+  })

documentation/docs/javascripts/tablesort.js

@@ -0,0 +1,6 @@
+document$.subscribe(function() {
+    var tables = document.querySelectorAll("article table:not([class])")
+    tables.forEach(function(table) {
+      new Tablesort(table)
+    })
+  })

documentation/docs/lab-share-lib.md

@@ -0,0 +1,128 @@
+# lab-share-lib
+
+`lab-share-lib` is a separate [library](https://github.com/sanger/lab-share-lib) that encapsulates AMQP connectivity logic, message deserialization and schema validation. 
+Sections below are some discussions on how it encapsulates logic related to messaging infrastructure and schema validation.
+
+## Connectivity with Message Queues
+
+`tol-lab-share` establishes AMQP connections with a RabbitMQ broker. `lab-share-lib`, therefore, uses [`pika`](https://pika.readthedocs.io/en/stable/) library to manage connectivity logic and handle failures. 
+`pika` uses several callback functions to handle messaging logic, and these are captured in [`lab_share_lib/rabbit/async_consumer.py`](https://github.com/sanger/lab-share-lib/blob/6724ec3c5053b75bb2a33958621930d9bb876a31/lab_share_lib/rabbit/async_consumer.py) in `lab-share-lib`.
+
+```python linenums="1" title="Snippets on connectivity logic and callbacks with RabbitMQ broker"
+
+def start_consuming(self):
+    if self._channel:
+        LOGGER.info("Issuing consumer related RPC commands")
+        self.add_on_cancel_callback()
+        self._consumer_tag = self._channel.basic_consume(self._queue, self.on_message)
+        self.was_consuming = True
+        self.had_transient_error = False
+        self._consuming = True
+
+def on_message(self, channel, basic_deliver, properties, body):
+    LOGGER.info(f"Received message # {basic_deliver.delivery_tag}")
+    MESSAGE_LOGGER.info(f"Received message # {basic_deliver.delivery_tag} with body:  {body}")
+    delivery_tag = basic_deliver.delivery_tag
+
+    try:
+        should_ack_message = self._process_message(properties.headers, body)
+    except TransientRabbitError:
+        self.had_transient_error = True
+        raise
+
+    if should_ack_message:
+        LOGGER.info("Acknowledging message # %s", delivery_tag)
+        channel.basic_ack(delivery_tag)
+    else:
+        LOGGER.info("Rejecting message # %s", delivery_tag)
+        channel.basic_nack(delivery_tag, requeue=False)
+```
+
+`AsyncConsumer` is a class declared in `lab-share-lib` that contains all these logic related to AMQP connectivity. 
+`AsyncConsumer` is instantiated in a `BackgroundConsumer` thread, that is started when `tol-lab-share` is run at [main.py](https://github.com/sanger/tol-lab-share/blob/dce2e4441313791171922caaec8450e238a1e939/main.py).
+
+!!! note
+
+    Note that `AsyncConsumer` is instantiated at the overridden function`run` in `BackgroundConsumer`, and each `AsyncConsumer` listens to a certain queue. 
+    In `bring_stack_up` function at [`rabbit_stack.py`](https://github.com/sanger/lab-share-lib/blob/bef1588724a9449f1a33b78dbcc60160d77df129/lab_share_lib/rabbit/rabbit_stack.py), `BackgroundConsumer` objects are created, and the `run` function is invoked by calling `start()` method on the consumer thread. When `on_message` callback is triggered when a message is received to the queue, the `process_message` function which was handed over to the `BackgroundConsumer` class (and therefore `AsyncConsumer` class) is invoked.
+
+
+## Message Processors
+
+For each message type identified by the `subject`, processor objects that inherits from `BaseProcessor` are instantiated. 
+When a message is received with a certain subject in the message header from the queue, that message is processed by the corresponding processor.
+
+!!! note
+
+    A subject-to-processor mapping is declared through the class `RabbitConfig`. 
+    This is declared in `tol_lab_share/config/rabbit.py` and can **not** be updated dynamically through deployment configurations.
+
+    ```py linenums="1" title="rabbit.py"
+    RABBITMQ_SERVERS = [
+        RabbitConfig(
+            consumer_details=TOL_RABBIT_SERVER,
+            consumed_queue="tol.crud-operations",
+            message_subjects={
+                RABBITMQ_SUBJECT_CREATE_LABWARE: MessageSubjectConfig(
+                    processor=CreateLabwareProcessor, reader_schema_version="2"
+                ),
+                RABBITMQ_SUBJECT_UPDATE_LABWARE: MessageSubjectConfig(
+                    processor=UpdateLabwareProcessor, reader_schema_version="1"
+                ),
+            },
+            publisher_details=TOL_RABBIT_SERVER,
+        ),
+        RabbitConfig(
+            consumer_details=ISG_RABBIT_SERVER,
+            consumed_queue="tls.poolxp-export-to-traction",
+            message_subjects={
+                RABBITMQ_SUBJECT_BIOSCAN_POOL_XP_TO_TRACTION: MessageSubjectConfig(
+                    processor=BioscanPoolXpToTractionProcessor, reader_schema_version="1"
+                ),
+            },
+            publisher_details=ISG_RABBIT_SERVER,
+        ),
+        RabbitConfig(
+            consumer_details=ISG_RABBIT_SERVER,
+            consumed_queue="tls.volume-tracking",
+            message_subjects={
+                RABBITMQ_SUBJECT_CREATE_ALIQUOT_IN_MLWH: MessageSubjectConfig(
+                    processor=CreateAliquotProcessor, reader_schema_version="1"
+                ),
+            },
+            publisher_details=MLWH_RABBIT_SERVER,
+        ),
+    ]
+    ```
+
+    This maps the messages coming from servers to the processors based on the subjects declared in the message header.
+
+    <center>
+
+    | **Messaging Server** |            **Subject in the header**           |            **Processor**           |   **Published to**   |
+    |:--------------------:|:----------------------------------------------:|:----------------------------------:|:--------------------:|
+    | `TOL_RABBIT_SERVER`  | `RABBITMQ_SUBJECT_CREATE_LABWARE`              | `CreateLabwareProcessor`           | `TOL_RABBIT_SERVER`  |
+    | `TOL_RABBIT_SERVER`  | `RABBITMQ_SUBJECT_UPDATE_LABWARE`              | `UpdateLabwareProcessor`           | `TOL_RABBIT_SERVER`  |
+    | `ISG_RABBIT_SERVER`  | `RABBITMQ_SUBJECT_BIOSCAN_POOL_XP_TO_TRACTION` | `BioscanPoolXpToTractionProcessor` | `ISG_RABBIT_SERVER`  |
+    | `ISG_RABBIT_SERVER`  | `RABBITMQ_SUBJECT_CREATE_ALIQUOT_IN_MLWH`      | `CreateAliquotProcessor`           | `MLWH_RABBIT_SERVER` |
+
+    </center>
+
+    Each server identified by `consumer_details` and `publisher_details` are declared in `rabbit_servers.py` in `tol-lab-share`. 
+    The processors that inherit from `BaseProcessor` are in `tol_lab_share/processors` package.
+
+## Schema Registry
+
+`lab-share-lib` uses Python `requests` library to interact with RedPanda's API. 
+The schema responses are cached using `@lru_cache`, and will be re-fetched upon cache expiry and cache misses.
+Upon arrival of messages (i.e., invocation of `on_message` callback through `pika`) the message bytes are converted into a `RabbitMessage` instance, and `decode` function is called for each instance. The `decode` function uses the cached schemas, and will validate the message against the reader and writer schema versions. 
+Reader schema version for each message is configured in `rabbit.py` as noted above. 
+The writer schema is encoded into the message headers, which the `decode` function extracts and uses for validation. 
+
+!!! note
+
+    Therefore, schema validations occur _before_ application control returns to code in `tol-lab-share`. 
+    Schema validations are done via `lab-share-lib`. 
+    However, `lab-share-lib` and `tol-lab-share` are **not** two components; `tol-lab-share` is an up and running component while `lab-share-lib` provides necessary functions that enable `tol-lab-share` to perform its tasks.
+
+