pypi_publish.yml

# This workflow builds the python package.

# When from from a release tags or a workflow dispatch, it also publishes to PyPI and DockerHub along
# with bumping the Connector Builder pinned version.
# Note: We may want to rename this file at some point. However, if we rename the workflow file name,
# we have to also update the Trusted Publisher settings on PyPI.

name: Packaging and Publishing

on:
  push:
    tags:
      - 'v*'
  workflow_dispatch:
    inputs:
      version:
        description: "Note that this workflow is intended for prereleases. For public-facing stable releases, please use the GitHub Releases workflow instead: https://github.com/airbytehq/airbyte-python-cdk/blob/main/docs/RELEASES.md. If running this workflow from main or from a dev branch, please enter the desired version number here, for instance 1.2.3dev0 or 1.2.3rc1."
        required: false
      publish_to_pypi:
        description: "Publish to PyPI. If true, the workflow will publish to PyPI."
        type: boolean
        required: true
        default: true
      publish_to_dockerhub:
        description: "Publish to DockerHub. If true, the workflow will publish the SDM connector to DockerHub."
        type: boolean
        required: true
        default: true
      update_connector_builder:
        description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder."
        type: boolean
        required: true
        default: false

jobs:
  build:
    name: Build Python Package
    runs-on: ubuntu-latest
    steps:
      - name: Detect Release Tag Version
        if: startsWith(github.ref, 'refs/tags/v')
        run: |
          DETECTED_VERSION=${{ github.ref_name }}
          echo "Version ref set to '${DETECTED_VERSION}'"
          # Remove the 'v' prefix if it exists
          DETECTED_VERSION="${DETECTED_VERSION#v}"
          echo "Setting version to '$DETECTED_VERSION'"
          echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV

      - name: Validate and set VERSION from git ref ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }})
        id: set_version
        run: |
          INPUT_VERSION=${{ github.event.inputs.version }}
          echo "Version input set to '${INPUT_VERSION}'"
          # Exit with success if both detected and input versions are empty
          if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then
            echo "No version detected or input. Will publish to SHA tag instead."
            echo 'VERSION=' >> $GITHUB_ENV
            exit 0
          fi
          # Remove the 'v' prefix if it exists
          INPUT_VERSION="${INPUT_VERSION#v}"
          # Fail if detected version is non-empty and different from the input version
          if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then
            echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'."
            exit 1
          fi
          # Set the version to the input version if non-empty, otherwise the detected version
          VERSION="${INPUT_VERSION:-$DETECTED_VERSION}"
          # Fail if the version is still empty
          if [ -z "$VERSION" ]; then
            echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
            exit 1
          fi
          echo "Setting version to '$VERSION'"
          echo "VERSION=${VERSION}" >> $GITHUB_ENV
          echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
          # Check if version is a prerelease version (will not tag 'latest')
          if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
            echo "IS_PRERELEASE=false" >> $GITHUB_ENV
            echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT
          else
            echo "IS_PRERELEASE=true" >> $GITHUB_ENV
            echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT
          fi

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: hynek/build-and-inspect-python-package@v2
        env:
          # Pass in the evaluated version from the previous step
          # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables
          POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}}

      - uses: actions/upload-artifact@v4
        with:
          name: Packages-${{ github.run_id }}
          path: |
            /tmp/baipp/dist/*.whl
            /tmp/baipp/dist/*.tar.gz

    outputs:
      VERSION: ${{ steps.set_version.outputs.VERSION }}
      IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }}

  publish_cdk:
    name: Publish CDK version to PyPI
    runs-on: ubuntu-latest
    needs: [build]
    permissions:
      id-token: write
      contents: write
    environment:
      name: PyPi
      url: https://pypi.org/p/airbyte-cdk/
    if: >
      (github.event_name == 'push' &&
       startsWith(github.ref, 'refs/tags/v')) ||
      (github.event_name == 'workflow_dispatch' &&
       (github.event.inputs.publish_to_pypi == 'true' ||
        github.event.inputs.update_connector_builder == 'true'
      ))
    env:
      VERSION: ${{ needs.build.outputs.VERSION }}
      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: Packages-${{ github.run_id }}
          path: dist

      - name: Upload wheel to release
        if: startsWith(github.ref, 'refs/tags/v')
        uses: svenstaro/upload-release-action@v2
        with:
          repo_token: ${{ secrets.GITHUB_TOKEN }}
          file: dist/*.whl
          tag: ${{ github.ref }}
          overwrite: true
          file_glob: true

      - name: Publish to PyPI
        if: >
          (github.event_name == 'push' &&
           startsWith(github.ref, 'refs/tags/v')
          ) || github.event.inputs.publish_to_pypi == 'true'
        uses: pypa/gh-action-pypi-publish@v1.12.3

  publish_sdm:
    name: Publish SDM to DockerHub
    if: >
      (github.event_name == 'push' &&
       startsWith(github.ref, 'refs/tags/v')) ||
      (github.event_name == 'workflow_dispatch' &&
       github.event.inputs.publish_to_dockerhub == 'true'
      )
    runs-on: ubuntu-latest
    needs: [build]
    environment:
      name: DockerHub
      url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags
    env:
      VERSION: ${{ needs.build.outputs.VERSION }}
      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # We need to download the build artifact again because the previous job was on a different runner
      - name: Download Build Artifact
        uses: actions/download-artifact@v4
        with:
          name: Packages-${{ github.run_id }}
          path: dist

      - name: Set up QEMU for multi-platform builds
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_PASSWORD }}

      - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
        if: env.VERSION != ''
        run: |
          tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
          if [ -z "$tag" ]; then
            echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
            exit 1
          fi
          echo "Checking if tag '$tag' exists on DockerHub..."
          if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
            echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
            exit 1
          fi
          echo "No existing tag '$tag' found. Proceeding with publish."

      - name: "Build and push (sha tag: '${{ github.sha }}')"
        # Only run if the version is not set
        if: env.VERSION == ''
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            airbyte/source-declarative-manifest:${{ github.sha }}

      - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
        # Only run if the version is set
        if: env.VERSION != ''
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            airbyte/source-declarative-manifest:${{ env.VERSION }}

      - name: Build and push ('latest' tag)
        # Only run if version is set and IS_PRERELEASE is false
        if: env.VERSION != '' && env.IS_PRERELEASE == 'false'
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: |
            airbyte/source-declarative-manifest:latest

  update-connector-builder:
    # Create a PR against the Builder, to update the CDK version that it uses.
    # In the future, Builder may use the SDM docker image instead of the Python CDK package.
    name: Bump Connector Builder CDK version
    environment:
      name: Connector Builder
      url: https://github.com/airbytehq/airbyte-platform-internal/pulls?q=is%3Apr+automatic-cdk-release+
    needs:
      - build
      - publish_cdk
      - publish_sdm
    if: >
      (github.event_name == 'push' &&
       needs.build.outputs.IS_PRERELEASE == 'false' &&
       startsWith(github.ref, 'refs/tags/v')) ||
      (github.event_name == 'workflow_dispatch' &&
       github.event.inputs.update_connector_builder == 'true')
    env:
      VERSION: ${{ needs.build.outputs.VERSION }}
      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Checkout Airbyte Platform Internal
        uses: actions/checkout@v4
        with:
          repository: airbytehq/airbyte-platform-internal
          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
      - name: Update Builder's CDK version to ${{ env.VERSION }}
        # PyPI servers aren't immediately updated so we may need to retry a few times.
        uses: nick-fields/retry@v3
        with:
          shell: bash
          max_attempts: 5
          retry_wait_seconds: 30
          timeout_minutes: 7
          command: |
            set -euo pipefail
            PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION)
            sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "oss/airbyte-connector-builder-server/Dockerfile"
            sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "cloud/airbyte-connector-builder-server-wrapped/Dockerfile"
            sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in
            echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION
            cd oss/airbyte-connector-builder-server
            python -m pip install --no-cache-dir pip-tools
            pip-compile --upgrade
      - name: Create Pull Request
        id: create-pull-request
        uses: peter-evans/create-pull-request@v7
        with:
          token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
          commit-message: "chore: update CDK version following release"
          title: "chore: update CDK version following release"
          body: This is an automatically generated PR triggered by a CDK release
          branch: automatic-cdk-release
          base: master
          delete-branch: true
      - name: Post success to Slack channel dev-connectors-extensibility
        uses: slackapi/slack-github-action@v1.23.0
        continue-on-error: true
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          channel-id: C04J1M66D8B
          # Channel: #dev-connectors-extensibility-releases
          # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B
          payload: |
            {
                "text": "A new version of Python CDK has been released!",
                "blocks": [
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "Python CDK `v${{ env.VERSION }}` has been <https://github.com/airbytehq/airbyte-python-cdk/releases|released>\n\n"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
                        }
                    }
                ]
            }
      - name: Post failure to Slack channel dev-connectors-extensibility
        if: failure()
        uses: slackapi/slack-github-action@v1.23.0
        continue-on-error: true
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          channel-id: C04J1M66D8B
          # Channel: #dev-connectors-extensibility-releases
          # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B
          payload: |
            {
                "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated",
                "blocks": [
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "Python CDK `v${{ env.VERSION }}` has been <https://github.com/airbytehq/airbyte-python-cdk/releases|released>\n\n"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": ":warning: Could not automatically create a PR for Connector Builder>\n"
                        }
                    },
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
                        }
                    }
                ]
            }