Microbatch: event_time ref + source filtering #1252
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # **what?** | |
| # Compares the schema of the dbt version of the given ref vs | |
| # the latest official schema releases found in schemas.getdbt.com. | |
| # If there are differences, the workflow will fail and upload the | |
| # diff as an artifact. The metadata team should be alerted to the change. | |
| # | |
| # **why?** | |
| # Reaction work may need to be done if artifact schema changes | |
| # occur so we want to proactively alert to it. | |
| # | |
| # **when?** | |
| # On pushes to `develop` and release branches. Manual runs are also enabled. | |
| name: Artifact Schema Check | |
| on: | |
| pull_request: | |
| types: [ opened, reopened, labeled, unlabeled, synchronize ] | |
| paths-ignore: [ '.changes/**', '.github/**', 'tests/**', '**.md', '**.yml' ] | |
| workflow_dispatch: | |
| # no special access is needed | |
| permissions: read-all | |
| env: | |
| LATEST_SCHEMA_PATH: ${{ github.workspace }}/new_schemas | |
| SCHEMA_DIFF_ARTIFACT: ${{ github.workspace }}/schema_changes.txt | |
| DBT_REPO_DIRECTORY: ${{ github.workspace }}/dbt | |
| SCHEMA_REPO_DIRECTORY: ${{ github.workspace }}/schemas.getdbt.com | |
| jobs: | |
| checking-schemas: | |
| name: "Checking schemas" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.8 | |
| - name: Checkout dbt repo | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.DBT_REPO_DIRECTORY }} | |
| - name: Check for changes in core/dbt/artifacts | |
| # https://github.com/marketplace/actions/paths-changes-filter | |
| uses: dorny/paths-filter@v3 | |
| id: check_artifact_changes | |
| with: | |
| filters: | | |
| artifacts_changed: | |
| - 'core/dbt/artifacts/**' | |
| list-files: shell | |
| working-directory: ${{ env.DBT_REPO_DIRECTORY }} | |
| - name: Succeed if no artifacts have changed | |
| if: steps.check_artifact_changes.outputs.artifacts_changed == 'false' | |
| run: | | |
| echo "No artifact changes found in core/dbt/artifacts. CI check passed." | |
| - name: Checkout schemas.getdbt.com repo | |
| if: steps.check_artifact_changes.outputs.artifacts_changed == 'true' | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: dbt-labs/schemas.getdbt.com | |
| ref: 'main' | |
| path: ${{ env.SCHEMA_REPO_DIRECTORY }} | |
| - name: Generate current schema | |
| if: steps.check_artifact_changes.outputs.artifacts_changed == 'true' | |
| run: | | |
| cd ${{ env.DBT_REPO_DIRECTORY }} | |
| python3 -m venv env | |
| source env/bin/activate | |
| pip install --upgrade pip | |
| pip install -r dev-requirements.txt -r editable-requirements.txt | |
| python scripts/collect-artifact-schema.py --path ${{ env.LATEST_SCHEMA_PATH }} | |
| # Copy generated schema files into the schemas.getdbt.com repo | |
| # Do a git diff to find any changes | |
| # Ignore any lines with date-like (yyyy-mm-dd) or version-like (x.y.z) changes | |
| - name: Compare schemas | |
| if: steps.check_artifact_changes.outputs.artifacts_changed == 'true' | |
| run: | | |
| cp -r ${{ env.LATEST_SCHEMA_PATH }}/dbt ${{ env.SCHEMA_REPO_DIRECTORY }} | |
| cd ${{ env.SCHEMA_REPO_DIRECTORY }} | |
| git diff -I='*[0-9]{4}-[0-9]{2}-[0-9]{2}' -I='*[0-9]+\.[0-9]+\.[0-9]+' --exit-code > ${{ env.SCHEMA_DIFF_ARTIFACT }} | |
| - name: Upload schema diff | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ failure() && steps.check_artifact_changes.outputs.artifacts_changed == 'true' }} | |
| with: | |
| name: 'schema_changes.txt' | |
| path: '${{ env.SCHEMA_DIFF_ARTIFACT }}' |