ci.yml

name: CI

defaults:
  run:
    shell: bash

on:
  push:
    branches:
      - "**"
    tags:
      - "[0-9]+.[0-9]+.[0-9]+"
  pull_request:
    branches:
      - main

permissions:
  contents: read

jobs:
  pre-commit-hook:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      # Most of the steps in this workflow use Just: https://github.com/casey/just
      # Just can be used to run commands, it is similar to Make.
      # The just file is located in the root of the repository:  "justfile"
      - name: Install just
        run: ./ci/scripts/install-just.sh
      - uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Run pre-commit
        run: just lint
        shell: bash

  test-datastore:
    runs-on: ubuntu-latest
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install just
        run: ./ci/scripts/install-just.sh

      - name: Build docker containers
        run: just build

      - name: Run the unit test
        run: just unit

      - name: Start docker stack
        run: just services

      - name: Load the data into the database
        run: just load

      - name: Run the integration test
        run: just integration

      - name: Run the performance test
        run: just performance

      - name: Run the client test
        run: just client

      - name: Archive test artifacts
        uses: actions/upload-artifact@v4
        with:
          name: test-results-artifact
          path: |
            api/test/output/pytest-coverage.txt
            api/test/output/pytest.xml
            datastore/load-test/output/store_read_*.csv
            datastore/load-test/output/store_rw_*.csv

      - name: Print results
        run: |
          pip install csvkit
          echo "## Stats (READ ONLY)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/output/store_read_stats.csv >> $GITHUB_STEP_SUMMARY
          echo "## Failures (READ ONLY)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/output/store_read_failures.csv >> $GITHUB_STEP_SUMMARY
          echo "## Stats (WRITE + READ)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/output/store_rw_stats.csv >> $GITHUB_STEP_SUMMARY
          echo "## Failures (WRITE + READ)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/output/store_rw_failures.csv >> $GITHUB_STEP_SUMMARY

      - name: Cleanup
        if: always()
        run: just destroy

  test-ingest:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10"] # Add 3.11 back pybind11 bug is fixed
    steps:
      - name: Checkout the repo
        uses: actions/checkout@v4
      - name: Ubuntu setup
        run: sudo apt update && sudo apt install libeccodes-data rapidjson-dev pybind11-dev libssl-dev
      - name: Python Setup
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          architecture: x64
      - name: Checkout Source
        uses: actions/checkout@v4

      - name: Install Dependencies
        run: |
          pip install --upgrade pip
          pip install pytest-timeout
          pip install pytest-cov
          pip install httpx
          pip install -r ./ingest/requirements.txt
          pip install ./ingest
          cd ./ingest && python3 api/generate_standard_name.py


      - name: Copy Protobuf file to api directory and build
        run: |
          mkdir ./ingest/protobuf
          cp ./protobuf/datastore.proto ./ingest/protobuf/datastore.proto
          python -m grpc_tools.protoc --proto_path=./ingest/protobuf --python_out=./ingest --grpc_python_out=./ingest ./ingest/protobuf/datastore.proto

      - name: Run Tests
        run: |
          cd ingest
          mkdir -p /tmp/metrics
          PROMETHEUS_MULTIPROC_DIR=/tmp/metrics python -m pytest -v --timeout=60

  publish-test-results:
    needs: test-datastore
    runs-on: ubuntu-latest
    permissions:
      contents: write
      issues: write
      pull-requests: write
    steps:
      - name: Download test results so that they can be published
        uses: actions/download-artifact@v4
        with:
          name: test-results-artifact
      - name: Comment coverage
        uses: MishaKav/pytest-coverage-comment@main
        with:
          pytest-coverage-path: api/test/output/pytest-coverage.txt
          coverage-path-prefix: api/test/output/
          title: API Unit Test Coverage Report
          hide-badge: true
          hide-report: false
          create-new-comment: false
          hide-comment: false
          report-only-changed-files: false
          remove-link-from-badge: false
          junitxml-path: api/test/output/pytest.xml
          junitxml-title: API Unit Test Coverage Summary