[COST-5007] move trino migration to a masu management command #7053
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| merge_group: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| sanity: | |
| name: Sanity | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Install Python | |
| uses: actions/[email protected] | |
| with: | |
| python-version: '3.9' | |
| - name: Run pre-commit checks | |
| uses: pre-commit/[email protected] | |
| env: | |
| SETUPTOOLS_USE_DISTUTILS: stdlib | |
| - name: Check clowdapp manifest | |
| run: bash .github/scripts/check_clowdapp.sh | |
| smokes-labeler: | |
| name: Smoke Test Label | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| - name: get Docker image files | |
| id: docker-files | |
| run: .github/scripts/files_require_smokes.sh >> docker-files.txt | |
| - name: add other required files | |
| id: add-files | |
| run: | | |
| echo .dockerignore >> docker-files.txt; | |
| echo Dockerfile >> docker-files.txt; | |
| echo pr_check.sh >> docker-files.txt; | |
| echo deploy/clowdapp.yaml >> docker-files.txt; | |
| - name: Show Dockerfiles | |
| run: cat docker-files.txt | |
| - name: Get changed files | |
| id: changed-files | |
| uses: tj-actions/[email protected] | |
| with: | |
| files_from_source_file: docker-files.txt | |
| - name: Set whether to run tests | |
| id: check-files | |
| run: | | |
| if [ ! -z "${{ steps.changed-files.outputs.all_changed_and_modified_files }}" ]; then | |
| echo "RUN_TESTS=true" >> $GITHUB_ENV | |
| fi | |
| - name: Setting smokes-required label | |
| if: env.RUN_TESTS == 'true' | |
| uses: actions/[email protected] | |
| continue-on-error: true | |
| with: | |
| script: | | |
| github.rest.issues.addLabels({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| labels: [ 'smokes-required' ] | |
| }) | |
| github.rest.issues.removeLabel({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'ok-to-skip-smokes' | |
| }) | |
| - name: Remove smokes-required label | |
| if: env.RUN_TESTS != 'true' | |
| uses: actions/[email protected] | |
| continue-on-error: true | |
| with: | |
| script: | | |
| github.rest.issues.addLabels({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| labels: [ 'ok-to-skip-smokes' ] | |
| }) | |
| github.rest.issues.removeLabel({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| name: 'smokes-required' | |
| }) | |
| changed-files: | |
| name: Detect changed files | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| run_tests: ${{ steps.check-files-or-fork.outputs.run_tests }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| - name: Get changed files | |
| id: changed-files | |
| uses: tj-actions/[email protected] | |
| with: | |
| files: | | |
| db_functions/** | |
| koku/** | |
| .github/postgres/** | |
| .github/scripts/check_migrations.sh | |
| .github/workflows/ci.yml | |
| Pipfile.lock | |
| - name: Check files or fork | |
| id: check-files-or-fork | |
| run: | | |
| if [ "${{ steps.changed-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event.pull_request.head.repo.full_name }}" != "project-koku/koku" ]; then | |
| echo "run_tests=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Show changed files | |
| run: | | |
| echo "Changed files:" | |
| for file in ${{ steps.changed-files.outputs.all_changed_and_modified_files }}; do | |
| echo " $file" | |
| done | |
| units: | |
| name: Units - ${{ matrix.python-version }} | |
| needs: changed-files | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 4 | |
| matrix: | |
| python-version: | |
| - '3.9' | |
| env: | |
| COMPOSE_FILE: .github/postgres/docker-compose.yaml | |
| steps: | |
| - name: Checkout | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| - name: Display build environment | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| run: printenv | |
| - name: Start the postgres DB | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| run: docker-compose up -d db | |
| - name: Set up Python ${{ matrix.python-version }} | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: actions/[email protected] | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install pipenv | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| run: sudo python3 -m pip install pipenv | |
| - name: Cache dependencies | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| id: cache-dependencies | |
| uses: actions/[email protected] | |
| with: | |
| path: | | |
| ~/.cache/pipenv | |
| ~/.local/share/virtualenvs | |
| key: ${{ runner.os }}-env-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}-${{ secrets.ACTIONS_CACHE_KEY_UUID }} | |
| - name: Install dependencies | |
| if: needs.changed-files.outputs.run_tests == 'true' && steps.cache-dependencies.outputs.cache-hit != 'true' | |
| run: | | |
| pipenv install --dev --ignore-pipfile --python ${{ matrix.python-version }} | |
| - name: Check migrations | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| run: bash .github/scripts/check_migrations.sh | |
| env: | |
| DATABASE_SERVICE_NAME: POSTGRES_SQL | |
| DATABASE_ENGINE: postgresql | |
| DATABASE_NAME: postgres | |
| DATABASE_USER: postgres | |
| DATABASE_PASSWORD: postgres | |
| POSTGRES_SQL_SERVICE_HOST: localhost | |
| POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }} | |
| PROMETHEUS_MULTIPROC_DIR: /tmp | |
| - name: Run unit tests | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| id: unit_tests_run | |
| run: pipenv run coverage run ./koku/manage.py test --noinput --verbosity 2 ./koku/ | |
| env: | |
| DATABASE_SERVICE_NAME: POSTGRES_SQL | |
| DATABASE_ENGINE: postgresql | |
| DATABASE_NAME: postgres | |
| DATABASE_USER: postgres | |
| DATABASE_PASSWORD: postgres | |
| POSTGRES_SQL_SERVICE_HOST: localhost | |
| POSTGRES_SQL_SERVICE_PORT: ${{ job.services.postgres.ports[5432] }} | |
| HIVE_DATABASE_PASSWORD: hivedbpw | |
| ACCOUNT_ENHANCED_METRICS: True | |
| PROMETHEUS_MULTIPROC_DIR: /tmp | |
| TRINO_DATE_STEP: 31 | |
| MIDDLEWARE_TIME_TO_LIVE: 0 | |
| ENHANCED_ORG_ADMIN: True | |
| ENABLE_S3_ARCHIVING: True | |
| - name: Show test status | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| id: show_test_status | |
| run: echo "Unit tests and migration check were '${{ steps.unit_tests_run.outcome }}'" | |
| - name: Convert coverage report to XML | |
| if: steps.unit_tests_run.outcome == 'success' | |
| run: pipenv run coverage xml | |
| - name: Upload test coverage file | |
| if: steps.unit_tests_run.outcome == 'success' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage | |
| path: coverage.xml | |
| coverage: | |
| name: Coverage | |
| needs: [changed-files,units] | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| # this checkout is required for the coverage report. If we don't do this, then | |
| # the uploaded report is invalid and codecov doesn't know how to process it. | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: actions/[email protected] | |
| with: | |
| fetch-depth: 0 | |
| - name: Download coverage result from units | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: coverage | |
| - name: Upload coverage to Codecov | |
| if: needs.changed-files.outputs.run_tests == 'true' | |
| uses: codecov/[email protected] | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| with: | |
| file: ./coverage.xml | |
| flags: unittests | |
| name: Python-${{ matrix.python-version}} | |
| fail_ci_if_error: true | |
| plugin: pycoverage # Only run one plugin even though we don't want any to run. | |
| - name: Set Codecov job status on skipped tests | |
| if: needs.changed-files.outputs.run_tests != 'true' | |
| env: | |
| GITHUB_TOKEN: "${{ github.token }}" | |
| run: | | |
| gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/patch" -f state="success" | |
| gh api "/repos/project-koku/koku/statuses/${{ github.event.pull_request.head.sha }}" -f description="codecov skipped because unit tests were not required" -f context="codecov/project" -f state="success" |