cdk_connectors_tests.yml

name: CDK Changes - Connectors Tests

concurrency:
  # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
  #
  # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
  #  ensuring that only one run per pull request is active at a time.
  #
  # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches.
  # This allows us to run multiple workflow dispatches in parallel.
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

on:
  workflow_dispatch:
  pull_request:
    types:
      - opened
      - synchronize
jobs:
  cdk_changes:
    runs-on: ubuntu-latest
    outputs:
      python_cdk: ${{ steps.changes.outputs.python_cdk }}
    permissions:
      statuses: write
    steps:
      - name: Checkout Airbyte
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v4
      - id: changes
        uses: dorny/paths-filter@v2
        with:
          filters: |
            python_cdk:
              - 'airbyte-cdk/python/airbyte_cdk/**'
              - 'airbyte-cdk/python/bin/**'
              - 'airbyte-cdk/python/poetry.lock'
              - 'airbyte-cdk/python/pyproject.toml'
      # The Connector CI Tests is a status check emitted by airbyte-ci
      # We make it pass once we have determined that there are no changes to the connectors
      - name: "Skip Connectors CI tests"
        if: steps.changes.outputs.python_cdk != 'true' && github.event_name == 'pull_request'
        run: |
          curl --request POST \
          --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
          --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
          --header 'content-type: application/json' \
          --data '{
            "state": "success",
            "context": "CDK Changes - Connectors Tests",
            "target_url": "${{ github.event.workflow_run.html_url }}"
            }' \

  connectors_ci:
    needs: cdk_changes
    # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR
    # Forked PRs are handled by the community_ci.yml workflow
    # If the condition is not met the job will be skipped (it will not fail)
    if: (github.event_name == 'pull_request' && needs.cdk_changes.outputs.python_cdk == 'true' && github.event.pull_request.head.repo.fork != true) || github.event_name == 'workflow_dispatch'
    name: Connectors CI
    runs-on: connector-test-large
    timeout-minutes: 360 # 6 hours
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v4
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
      - name: Fetch last commit id from remote branch [PULL REQUESTS]
        if: github.event_name == 'pull_request'
        id: fetch_last_commit_id_pr
        run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT
      - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH]
        if: github.event_name == 'workflow_dispatch'
        id: fetch_last_commit_id_wd
        run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT
      - name: Test connectors
        uses: ./.github/actions/run-airbyte-ci
        with:
          context: ${{ github.event_name == 'pull_request' && 'pull_request' || 'manual' }}
          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
          git_branch: ${{ github.head_ref }}
          git_revision: ${{ github.event_name == 'pull_request' && steps.fetch_last_commit_id_pr.outputs.commit_id || steps.fetch_last_commit_id_wd.outputs.commit_id }}
          github_token: ${{ env.PAT }}
          s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds)
          subcommand: "connectors --use-local-cdk --name source-shopify --name source-zendesk-support --execute-timeout=18000 test --global-status-check-context='CDK Changes - Connectors Tests'"