ci.yml

name: CI

on:
  push:
    branches:
      - "**"
    tags:
      - "[0-9]+.[0-9]+.[0-9]+"

permissions:
  contents: read

jobs:
  pre-commit-hook:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
      - name: Install pre-commit
        run: python -m pip install pre-commit
        shell: bash
      - name: Show pre-commit requirements
        run: python -m pip freeze --local
        shell: bash
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pre-commit
          key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Run pre-commit for all files
        run: pre-commit run --config './.pre-commit-config.yaml' --all-files --color=always --show-diff-on-failure
        shell: bash

  test-datastore:
    runs-on: ubuntu-latest
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3

      - uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Copy Protobuf files to Dockerfile directories
        run: ./ci/docker/copy-protobuf.sh

      - name: Build docker containers
        run: docker compose --profile test build

      - name: Start docker stack
        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose up -d --wait --wait-timeout 120

      - name: Test loader runs without errors
        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm loader

      - name: Integration test
        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm integration

      - name: Test client runs without errors
        run: DYNAMICTIME=false LOTIME=1000-01-01T00:00:00Z HITIME=9999-12-31T23:59:59Z docker compose run --rm client

      - name: Run load test (read only)
        run: |
          python --version
          pip install -r datastore/load-test/requirements.txt
          python -m grpc_tools.protoc --proto_path=./protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test
          cd datastore/load-test
          locust -f locustfile_read.py --headless -u 5 -r 10 --run-time 60 --only-summary --csv store_read

      - name: Run load test (write + read)
        run: |
          pip install -r datastore/load-test/requirements.txt
          python -m grpc_tools.protoc --proto_path=./protobuf datastore.proto --python_out=datastore/load-test --grpc_python_out=datastore/load-test
          cd datastore/load-test
          python schedule_write.py > schedule_write.log 2>&1 &
          locust -f locustfile_read.py --headless -u 5 -r 10 --run-time 60 --only-summary --csv store_rw
          kill %1
          echo Catting schedule_write output...
          cat schedule_write.log
          echo Done catting

      - name: Archive load test artifacts
        uses: actions/upload-artifact@v3
        with:
          name: performance
          path: |
            datastore/load-test/store_read_*.csv
            datastore/load-test/store_rw_*.csv

      - name: Print results
        run: |
          pip install csvkit
          echo "## Stats (READ ONLY)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/store_read_stats.csv >> $GITHUB_STEP_SUMMARY
          echo "## Failures (READ ONLY)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/store_read_failures.csv >> $GITHUB_STEP_SUMMARY
          echo "## Stats (WRITE + READ)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/store_rw_stats.csv >> $GITHUB_STEP_SUMMARY
          echo "## Failures (WRITE + READ)" >> $GITHUB_STEP_SUMMARY
          csvlook datastore/load-test/store_rw_failures.csv >> $GITHUB_STEP_SUMMARY

      - name: Cleanup
        if: always()
        run: docker compose down --volumes

# TODO: These tests don't currently work. Uncomment once this is resolved.
#  test-ingest:
#    runs-on: ubuntu-latest
#    strategy:
#      matrix:
#        python-version: ["3.10"] # Add 3.11 back pybind11 bug is fixed
#    steps:
#      - name: Checkout the repo
#        uses: actions/checkout@v3
#      - name: Ubuntu setup
#        run: sudo apt update && sudo apt install libeccodes-data	rapidjson-dev	pybind11-dev libssl-dev
#      - name: Python Setup
#        uses: actions/setup-python@v4
#        with:
#          python-version: ${{ matrix.python-version }}
#          architecture: x64
#      - name: Checkout Source
#        uses: actions/checkout@v3
#      - name: Install Dependencies
#        run: |
#          pip install --upgrade pip
#          pip install pytest-timeout
#          pip install pytest-cov
#          pip install ./ingest
#      - name: Run Tests
#        run: python -m pytest -v --timeout=60 ./ingest