ci.yml

name: CI

defaults:
  run:
    shell: bash

on:
  push:
    branches:
      - "**"
    tags:
      - "[0-9]+.[0-9]+.[0-9]+"
  pull_request:
    branches:
      - main

permissions:
  contents: read

jobs:
  pre-commit-hook:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      # Most of the steps in this workflow use Just: https://github.com/casey/just
      # Just can be used to run commands, it is similar to Make.
      # The just file is located in the root of the repository:  "justfile"
      - name: Install just
        run: ./ci/scripts/install-just.sh
      - uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Run pre-commit
        run: just lint
        shell: bash

  test-datastore:
    runs-on: ubuntu-latest
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install just
        run: ./ci/scripts/install-just.sh

      - name: Build docker containers
        run: just build

      - name: Run the unit test
        run: just unit

      - name: Start docker stack
        run: just services

      - name: Load the data into the database
        run: just ingest-load

      - name: Run the integration test
        run: just integration

      - name: Run the performance test
        run: just performance

      - name: Run the client test
        run: just client

      - name: Archive test artifacts
        uses: actions/upload-artifact@v4
        with:
          name: test-results-artifact
          path: |
            api/test/output/pytest-coverage.txt
            api/test/output/pytest.xml
            datastore/load-test/output/store_read_*.csv
            datastore/load-test/output/store_rw_*.csv

      - name: Print results
        run: |
          pip install csvkit
          echo "## Stats (READ ONLY)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/output/store_read_stats.csv >> $GITHUB_STEP_SUMMARY
          echo "## Failures (READ ONLY)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/output/store_read_failures.csv >> $GITHUB_STEP_SUMMARY
          echo "## Stats (WRITE + READ)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/output/store_rw_stats.csv >> $GITHUB_STEP_SUMMARY
          echo "## Failures (WRITE + READ)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/output/store_rw_failures.csv >> $GITHUB_STEP_SUMMARY

      - name: Cleanup
        if: always()
        run: just destroy

  test-ingest:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10"] # Add 3.11 back pybind11 bug is fixed
    steps:
      - name: Checkout the repo
        uses: actions/checkout@v4
      - name: Ubuntu setup
        run: sudo apt update && sudo apt install libeccodes-data rapidjson-dev pybind11-dev libssl-dev
      - name: Python Setup
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64

      - name: Install just
        run: ./ci/scripts/install-just.sh

      - name: Copy protobuf files
        run: just copy-proto

      - name: Run the unit test
        run: just ingest-unit

      - name: Archive test artifacts
        uses: actions/upload-artifact@v4
        with:
          name: ingest-test-results-artifact
          path: |
            ingest/test/output/pytest-coverage.txt
            ingest/test/output/pytest.xml

      - name: Cleanup
        if: always()
        run: just destroy

  publish-test-results:
    needs:
      - test-datastore
      - test-ingest
    runs-on: ubuntu-latest
    if: github.event_name != 'push' || github.event.ref_type != 'tag'
    permissions:
      contents: write
      issues: write
      pull-requests: write
    steps:
      - name: Download test-datastore results so that they can be published
        uses: actions/download-artifact@v4
        with:
          name: test-results-artifact
          path: ./artifacts/test-results

      - name: Download test-ingest results so that they can be published
        uses: actions/download-artifact@v4
        with:
          name: ingest-test-results-artifact
          path: ./artifacts/ingest-results

      - name: Comment coverage
        uses: MishaKav/pytest-coverage-comment@main
        with:
          title: Unit Test Coverage Report
          pytest-coverage-path: ./artifacts/test-results/api/test/output/pytest-coverage.txt
          multiple-files: |
            API Unit Tests, ./artifacts/test-results/api/test/output/pytest-coverage.txt, ./artifacts/test-results/api/test/output/pytest.xml
            Ingest Unit Tests, ./artifacts/ingest-results/pytest-coverage.txt, ./artifacts/ingest-results/pytest.xml