fix(Low-Code Concurrent CDK): Refactor the low-code AsyncRetriever to… #778
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow builds the python package. | |
| # When from from a release tags or a workflow dispatch, it also publishes to PyPI and DockerHub along | |
| # with bumping the Connector Builder pinned version. | |
| # Note: We may want to rename this file at some point. However, if we rename the workflow file name, | |
| # we have to also update the Trusted Publisher settings on PyPI. | |
| name: Packaging and Publishing | |
| on: | |
| push: | |
| tags: | |
| - 'v*' | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: "Note that this workflow is intended for prereleases. For public-facing stable releases, please use the GitHub Releases workflow instead: https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/RELEASES.md. If running this workflow from main or from a dev branch, please enter the desired version number here, for instance 1.2.3dev0 or 1.2.3rc1." | |
| required: false | |
| publish_to_pypi: | |
| description: "Publish to PyPI. If true, the workflow will publish to PyPI." | |
| type: boolean | |
| required: true | |
| default: true | |
| publish_to_dockerhub: | |
| description: "Publish to DockerHub. If true, the workflow will publish the SDM connector to DockerHub." | |
| type: boolean | |
| required: true | |
| default: true | |
| update_connector_builder: | |
| description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder." | |
| type: boolean | |
| required: true | |
| default: false | |
| jobs: | |
| build: | |
| name: Build Python Package | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Detect Release Tag Version | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| run: | | |
| DETECTED_VERSION=${{ github.ref_name }} | |
| echo "Version ref set to '${DETECTED_VERSION}'" | |
| # Remove the 'v' prefix if it exists | |
| DETECTED_VERSION="${DETECTED_VERSION#v}" | |
| echo "Setting version to '$DETECTED_VERSION'" | |
| echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV | |
| - name: Validate and set VERSION from git ref ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }}) | |
| id: set_version | |
| run: | | |
| INPUT_VERSION=${{ github.event.inputs.version }} | |
| echo "Version input set to '${INPUT_VERSION}'" | |
| # Exit with success if both detected and input versions are empty | |
| if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then | |
| echo "No version detected or input. Will publish to SHA tag instead." | |
| echo 'VERSION=' >> $GITHUB_ENV | |
| exit 0 | |
| fi | |
| # Remove the 'v' prefix if it exists | |
| INPUT_VERSION="${INPUT_VERSION#v}" | |
| # Fail if detected version is non-empty and different from the input version | |
| if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then | |
| echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'." | |
| exit 1 | |
| fi | |
| # Set the version to the input version if non-empty, otherwise the detected version | |
| VERSION="${INPUT_VERSION:-$DETECTED_VERSION}" | |
| # Fail if the version is still empty | |
| if [ -z "$VERSION" ]; then | |
| echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." | |
| exit 1 | |
| fi | |
| echo "Setting version to '$VERSION'" | |
| echo "VERSION=${VERSION}" >> $GITHUB_ENV | |
| echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT | |
| # Check if version is a prerelease version (will not tag 'latest') | |
| if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| echo "IS_PRERELEASE=false" >> $GITHUB_ENV | |
| echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "IS_PRERELEASE=true" >> $GITHUB_ENV | |
| echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT | |
| fi | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: hynek/build-and-inspect-python-package@v2 | |
| env: | |
| # Pass in the evaluated version from the previous step | |
| # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables | |
| POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: Packages-${{ github.run_id }} | |
| path: | | |
| /tmp/baipp/dist/*.whl | |
| /tmp/baipp/dist/*.tar.gz | |
| outputs: | |
| VERSION: ${{ steps.set_version.outputs.VERSION }} | |
| IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }} | |
| publish_cdk: | |
| name: Publish CDK version to PyPI | |
| runs-on: ubuntu-latest | |
| needs: [build] | |
| permissions: | |
| id-token: write | |
| contents: write | |
| environment: | |
| name: PyPi | |
| url: https://pypi.org/p/airbyte-cdk/ | |
| if: > | |
| (github.event_name == 'push' && | |
| startsWith(github.ref, 'refs/tags/v')) || | |
| (github.event_name == 'workflow_dispatch' && | |
| (github.event.inputs.publish_to_pypi == 'true' || | |
| github.event.inputs.update_connector_builder == 'true' | |
| )) | |
| env: | |
| VERSION: ${{ needs.build.outputs.VERSION }} | |
| IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: Packages-${{ github.run_id }} | |
| path: dist | |
| - name: Upload wheel to release | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: dist/*.whl | |
| tag: ${{ github.ref }} | |
| overwrite: true | |
| file_glob: true | |
| - name: Publish to PyPI | |
| if: > | |
| (github.event_name == 'push' && | |
| startsWith(github.ref, 'refs/tags/v') | |
| ) || github.event.inputs.publish_to_pypi == 'true' | |
| uses: pypa/[email protected] | |
| publish_sdm: | |
| name: Publish SDM to DockerHub | |
| if: > | |
| (github.event_name == 'push' && | |
| startsWith(github.ref, 'refs/tags/v')) || | |
| (github.event_name == 'workflow_dispatch' && | |
| github.event.inputs.publish_to_dockerhub == 'true' | |
| ) | |
| runs-on: ubuntu-latest | |
| needs: [build] | |
| environment: | |
| name: DockerHub | |
| url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags | |
| env: | |
| VERSION: ${{ needs.build.outputs.VERSION }} | |
| IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # We need to download the build artifact again because the previous job was on a different runner | |
| - name: Download Build Artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: Packages-${{ github.run_id }} | |
| path: dist | |
| - name: Set up QEMU for multi-platform builds | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )" | |
| if: env.VERSION != '' | |
| run: | | |
| tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" | |
| if [ -z "$tag" ]; then | |
| echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." | |
| exit 1 | |
| fi | |
| echo "Checking if tag '$tag' exists on DockerHub..." | |
| if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then | |
| echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite." | |
| exit 1 | |
| fi | |
| echo "No existing tag '$tag' found. Proceeding with publish." | |
| - name: "Build and push (sha tag: '${{ github.sha }}')" | |
| # Only run if the version is not set | |
| if: env.VERSION == '' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: | | |
| airbyte/source-declarative-manifest:${{ github.sha }} | |
| - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})" | |
| # Only run if the version is set | |
| if: env.VERSION != '' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: | | |
| airbyte/source-declarative-manifest:${{ env.VERSION }} | |
| - name: Build and push ('latest' tag) | |
| # Only run if version is set and IS_PRERELEASE is false | |
| if: env.VERSION != '' && env.IS_PRERELEASE == 'false' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: | | |
| airbyte/source-declarative-manifest:latest | |
| update-connector-builder: | |
| # Create a PR against the Builder, to update the CDK version that it uses. | |
| # In the future, Builder may use the SDM docker image instead of the Python CDK package. | |
| name: Bump Connector Builder CDK version | |
| environment: | |
| name: Connector Builder | |
| url: https://github.com/airbytehq/airbyte-platform-internal/pulls?q=is%3Apr+automatic-cdk-release+ | |
| needs: | |
| - build | |
| - publish_cdk | |
| - publish_sdm | |
| if: > | |
| (github.event_name == 'push' && | |
| needs.build.outputs.IS_PRERELEASE == 'false' && | |
| startsWith(github.ref, 'refs/tags/v')) || | |
| (github.event_name == 'workflow_dispatch' && | |
| github.event.inputs.update_connector_builder == 'true') | |
| env: | |
| VERSION: ${{ needs.build.outputs.VERSION }} | |
| IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Checkout Airbyte Platform Internal | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: airbytehq/airbyte-platform-internal | |
| token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| - name: Update Builder's CDK version to ${{ env.VERSION }} | |
| # PyPI servers aren't immediately updated so we may need to retry a few times. | |
| uses: nick-fields/retry@v3 | |
| with: | |
| shell: bash | |
| max_attempts: 5 | |
| retry_wait_seconds: 30 | |
| timeout_minutes: 7 | |
| command: | | |
| set -euo pipefail | |
| PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) | |
| sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "oss/airbyte-connector-builder-server/Dockerfile" | |
| sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "cloud/airbyte-connector-builder-server-wrapped/Dockerfile" | |
| sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in | |
| echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION | |
| cd oss/airbyte-connector-builder-server | |
| python -m pip install --no-cache-dir pip-tools | |
| pip-compile --upgrade | |
| - name: Create Pull Request | |
| id: create-pull-request | |
| uses: peter-evans/create-pull-request@v7 | |
| with: | |
| token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| commit-message: "chore: update CDK version following release" | |
| title: "chore: update CDK version following release" | |
| body: This is an automatically generated PR triggered by a CDK release | |
| branch: automatic-cdk-release | |
| base: master | |
| delete-branch: true | |
| - name: Post success to Slack channel dev-connectors-extensibility | |
| uses: slackapi/[email protected] | |
| continue-on-error: true | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} | |
| with: | |
| channel-id: C04J1M66D8B | |
| # Channel: #dev-connectors-extensibility-releases | |
| # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B | |
| payload: | | |
| { | |
| "text": "A new version of Python CDK has been released!", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Python CDK `v${{ env.VERSION }}` has been <https://github.com/airbytehq/airbyte-python-cdk/releases|released>\n\n" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" | |
| } | |
| } | |
| ] | |
| } | |
| - name: Post failure to Slack channel dev-connectors-extensibility | |
| if: failure() | |
| uses: slackapi/[email protected] | |
| continue-on-error: true | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} | |
| with: | |
| channel-id: C04J1M66D8B | |
| # Channel: #dev-connectors-extensibility-releases | |
| # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B | |
| payload: | | |
| { | |
| "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated", | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "Python CDK `v${{ env.VERSION }}` has been <https://github.com/airbytehq/airbyte-python-cdk/releases|released>\n\n" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": ":warning: Could not automatically create a PR for Connector Builder>\n" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n" | |
| } | |
| } | |
| ] | |
| } |