Removed obsolete protobuf version #1594
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| defaults: | |
| run: | |
| shell: bash | |
| on: | |
| push: | |
| branches: | |
| - "**" | |
| tags: | |
| - "[0-9]+.[0-9]+.[0-9]+" | |
| pull_request: | |
| branches: | |
| - main | |
| permissions: | |
| contents: read | |
| jobs: | |
| pre-commit-hook: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| # Most of the steps in this workflow use Just: https://github.com/casey/just | |
| # Just can be used to run commands, it is similar to Make. | |
| # The just file is located in the root of the repository: "justfile" | |
| - name: Install just | |
| run: ./ci/scripts/install-just.sh | |
| - uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Run pre-commit | |
| run: just lint | |
| shell: bash | |
| test-datastore: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install just | |
| run: ./ci/scripts/install-just.sh | |
| - name: Build docker containers | |
| run: just build | |
| - name: Run the unit test | |
| run: just unit | |
| - name: Start docker stack | |
| run: just services | |
| - name: Load the data into the database | |
| run: just load | |
| - name: Run the integration test | |
| run: just integration | |
| - name: Run the performance test | |
| run: just performance | |
| - name: Run the client test | |
| run: just client | |
| - name: Archive test artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results-artifact | |
| path: | | |
| api/test/output/pytest-coverage.txt | |
| api/test/output/pytest.xml | |
| datastore/load-test/output/store_read_*.csv | |
| datastore/load-test/output/store_rw_*.csv | |
| - name: Print results | |
| run: | | |
| pip install csvkit | |
| echo "## Stats (READ ONLY)" >> $GITHUB_STEP_SUMMARY | |
| csvlook datastore/load-test/output/store_read_stats.csv >> $GITHUB_STEP_SUMMARY | |
| echo "## Failures (READ ONLY)" >> $GITHUB_STEP_SUMMARY | |
| csvlook datastore/load-test/output/store_read_failures.csv >> $GITHUB_STEP_SUMMARY | |
| echo "## Stats (WRITE + READ)" >> $GITHUB_STEP_SUMMARY | |
| csvlook datastore/load-test/output/store_rw_stats.csv >> $GITHUB_STEP_SUMMARY | |
| echo "## Failures (WRITE + READ)" >> $GITHUB_STEP_SUMMARY | |
| csvlook datastore/load-test/output/store_rw_failures.csv >> $GITHUB_STEP_SUMMARY | |
| - name: Cleanup | |
| if: always() | |
| run: just destroy | |
| test-ingest: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.10"] # Add 3.11 back pybind11 bug is fixed | |
| steps: | |
| - name: Checkout the repo | |
| uses: actions/checkout@v4 | |
| - name: Ubuntu setup | |
| run: sudo apt update && sudo apt install libeccodes-data rapidjson-dev pybind11-dev libssl-dev | |
| - name: Python Setup | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| architecture: x64 | |
| - name: Checkout Source | |
| uses: actions/checkout@v4 | |
| - name: Install Dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install pytest-timeout | |
| pip install pytest-cov | |
| pip install httpx | |
| pip install -r ./ingest/requirements.txt | |
| pip install ./ingest | |
| cd ./ingest && python3 api/generate_standard_name.py | |
| - name: Copy Protobuf file to api directory and build | |
| run: | | |
| mkdir ./ingest/protobuf | |
| cp ./protobuf/datastore.proto ./ingest/protobuf/datastore.proto | |
| python -m grpc_tools.protoc --proto_path=./ingest/protobuf --python_out=./ingest --grpc_python_out=./ingest ./ingest/protobuf/datastore.proto | |
| - name: Run Tests | |
| run: | | |
| cd ingest | |
| mkdir -p /tmp/metrics | |
| PROMETHEUS_MULTIPROC_DIR=/tmp/metrics python -m pytest -v --timeout=60 | |
| publish-test-results: | |
| needs: test-datastore | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| issues: write | |
| pull-requests: write | |
| steps: | |
| - name: Download test results so that they can be published | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: test-results-artifact | |
| - name: Comment coverage | |
| uses: MishaKav/pytest-coverage-comment@main | |
| with: | |
| pytest-coverage-path: api/test/output/pytest-coverage.txt | |
| coverage-path-prefix: api/test/output/ | |
| title: API Unit Test Coverage Report | |
| hide-badge: true | |
| hide-report: false | |
| create-new-comment: false | |
| hide-comment: false | |
| report-only-changed-files: false | |
| remove-link-from-badge: false | |
| junitxml-path: api/test/output/pytest.xml | |
| junitxml-title: API Unit Test Coverage Summary |