fix: combined publish flow for CDK/SDM (#77)

Co-authored-by: Aaron Steers <[email protected]>

.github/workflows/pypi_publish.yml

@@ -1,37 +1,50 @@
-name: Python Packaging
+name: Packaging and Publishing
 
 on:
   push:
-
   workflow_dispatch:
+    inputs:
+      version:
+        description: "The version to publish, ie 1.0.0 or 1.0.0-dev1"
+        required: true
 
 jobs:
   build:
-    name: Build
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ github.event_name == 'workflow_dispatch' && format('v{0}', github.event.inputs.version) || github.ref }}
+
       - uses: hynek/build-and-inspect-python-package@v2
 
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Packages-${{ github.run_id }}
+          path: |
+            /tmp/baipp/dist/*.whl
+            /tmp/baipp/dist/*.tar.gz
+
   publish:
-    name: Publish to PyPI
+    name: Publish CDK version to PyPI
     runs-on: ubuntu-latest
     needs: [build]
     permissions:
-      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
-      contents: write # Needed to upload artifacts to the release
+      id-token: write
+      contents: write
     environment:
-      name: PyPI
-      url: "https://pypi.org/p/airbyte-cdk"
-    if: startsWith(github.ref, 'refs/tags/v')
+      name: PyPi
+      url: https://pypi.org/p/airbyte
+    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/download-artifact@v4
         with:
-          name: Packages
+          name: Packages-${{ github.run_id }}
           path: dist
-      - name: Attach Wheel to GitHub Release
+
+      - name: Upload wheel to release
+        if: startsWith(github.ref, 'refs/tags/v')
         uses: svenstaro/upload-release-action@v2
         with:
           repo_token: ${{ secrets.GITHUB_TOKEN }}
@@ -40,8 +53,72 @@ jobs:
           overwrite: true
           file_glob: true
 
-      - name: Publish to PyPI (${{vars.PYPI_PUBLISH_URL}})
-        uses: pypa/[email protected]
+      - name: Publish to PyPI
+        uses: pypa/[email protected]
+
+  publish_sdm:
+    name: Publish SDM to DockerHub
+    if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    needs: [publish]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set Version (workflow_dispatch)
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          echo "Version set to ${{ github.event.inputs.version }}"
+          echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+
+      - name: Set Version (tag)
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          echo "Version set to ${{ github.ref_name }}"
+          echo "VERSION=${{ github.ref_name }}" >> $GITHUB_ENV
+
+      # We need to download the build artifact again because the previous job was on a different runner
+      - name: Download Build Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: Packages-${{ github.run_id }}
+          path: dist
+
+      - name: Set up QEMU for multi-platform builds
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Check for existing tag
+        run: |
+          tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
+          if [ -z "$tag" ]; then
+            echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
+            exit 1
+          fi
+          echo "Checking if tag '$tag' exists on DockerHub..."
+          if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
+            echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
+            exit 1
+          fi
+          echo "No existing tag '$tag' found. Proceeding with publish."
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
         with:
-          # Can be toggled at the repository level between `https://upload.pypi.org/legacy/` and `https://test.pypi.org/legacy/`
-          repository-url: ${{vars.PYPI_PUBLISH_URL}}
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            airbyte/source-declarative-manifest:latest
+            airbyte/source-declarative-manifest:${{ env.VERSION }}
+            airbyte/source-declarative-manifest:${{ github.sha }}

Dockerfile

@@ -4,6 +4,7 @@ WORKDIR /airbyte/integration_code
 
 # Copy project files needed for build
 COPY pyproject.toml poetry.lock README.md ./
+COPY dist/*.whl ./dist/
 
 # Install dependencies - ignore keyring warnings
 RUN poetry config virtualenvs.create false \
@@ -13,6 +14,6 @@ RUN poetry config virtualenvs.create false \
 COPY airbyte_cdk ./airbyte_cdk
 
 # Build and install the package
-RUN poetry build && pip install dist/*.whl
+RUN pip install dist/*.whl
 
 ENTRYPOINT ["poetry", "run", "source-declarative-manifest"]