Skip to content

Commit

Permalink
CI: move SDM image publish to dedicated step (#83)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aaronsteers
aaronsteers authored Nov 22, 2024
1 parent 2368b59 commit e52bde4
Show file tree
Hide file tree
Showing 2 changed files with 180 additions and 78 deletions.
180 changes: 180 additions & 0 deletions .github/workflows/publish_sdm_connector.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
# This flow publishes the Source-Declarative-Manifest (SDM)
# connector to DockerHub as a Docker image.

name: Publish SDM Connector

on:
workflow_dispatch:
inputs:
version:
description:
The version to publish, ie 1.0.0 or 1.0.0-dev1.
If omitted, and if run from a release branch, the version will be
inferred from the git tag.
If omitted, and if run from a non-release branch, then only a SHA-based
Docker tag will be created.
required: false
dry_run:
description: If true, the workflow will not push to DockerHub.
type: boolean
required: false
default: false

jobs:
build:
runs-on: ubuntu-latest
steps:

- name: Detect Release Tag Version
if: startsWith(github.ref, 'refs/tags/v')
run: |
DETECTED_VERSION=${{ github.ref_name }}
echo "Version ref set to '${DETECTED_VERSION}'"
# Remove the 'v' prefix if it exists
DETECTED_VERSION="${DETECTED_VERSION#v}"
echo "Setting version to '$DETECTED_VERSION'"
echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV
- name: Validate and set VERSION from tag ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }})
id: set_version
if: github.event_name == 'workflow_dispatch'
run: |
INPUT_VERSION=${{ github.event.inputs.version }}
echo "Version input set to '${INPUT_VERSION}'"
# Exit with success if both detected and input versions are empty
if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then
echo "No version detected or input. Will publish to SHA tag instead."
echo 'VERSION=""' >> $GITHUB_ENV
exit 0
fi
# Remove the 'v' prefix if it exists
INPUT_VERSION="${INPUT_VERSION#v}"
# Fail if detected version is non-empty and different from the input version
if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then
echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'."
exit 1
fi
# Set the version to the input version if non-empty, otherwise the detected version
VERSION="${INPUT_VERSION:-$DETECTED_VERSION}"
# Fail if the version is still empty
if [ -z "$VERSION" ]; then
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
exit 1
fi
echo "Setting version to '$VERSION'"
echo "VERSION=${VERSION}" >> $GITHUB_ENV
echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
# Check if version is a prerelease version (will not tag 'latest')
if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "IS_PRERELEASE=false" >> $GITHUB_ENV
echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT
else
echo "IS_PRERELEASE=true" >> $GITHUB_ENV
echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT
fi
- uses: actions/checkout@v4
with:
fetch-depth: 0

- uses: hynek/build-and-inspect-python-package@v2
name: Build package with version ref '${{ env.VERSION }}'
env:
# Pass in the evaluated version from the previous step
# More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables
POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION }}

- uses: actions/upload-artifact@v4
with:
name: Packages-${{ github.run_id }}
path: |
/tmp/baipp/dist/*.whl
/tmp/baipp/dist/*.tar.gz
outputs:
VERSION: ${{ steps.set_version.outputs.VERSION }}
IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }}

publish_sdm:
name: Publish SDM to DockerHub
if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
needs: [build]
environment:
name: DockerHub
url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags
env:
VERSION: ${{ needs.build.outputs.VERSION }}
IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

# We need to download the build artifact again because the previous job was on a different runner
- name: Download Build Artifact
uses: actions/download-artifact@v4
with:
name: Packages-${{ github.run_id }}
path: dist

- name: Set up QEMU for multi-platform builds
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_PASSWORD }}

- name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
if: env.VERSION != ''
run: |
tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
if [ -z "$tag" ]; then
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
exit 1
fi
echo "Checking if tag '$tag' exists on DockerHub..."
if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
exit 1
fi
echo "No existing tag '$tag' found. Proceeding with publish."
- name: Build and push (sha tag)
# Only run if the version is not set
if: env.VERSION == '' && github.event.inputs.dry_run == 'false'
uses: docker/build-push-action@v5
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: |
airbyte/source-declarative-manifest:${{ github.sha }}
- name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
# Only run if the version is set
if: env.VERSION != '' && github.event.inputs.dry_run == 'false'
uses: docker/build-push-action@v5
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: |
airbyte/source-declarative-manifest:${{ env.VERSION }}
- name: Build and push ('latest' tag)
# Only run if version is set and IS_PRERELEASE is false
if: env.VERSION != '' && env.IS_PRERELEASE == 'false' && github.event.inputs.dry_run == 'false'
uses: docker/build-push-action@v5
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: |
airbyte/source-declarative-manifest:latest
78 changes: 0 additions & 78 deletions .github/workflows/pypi_publish.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,81 +59,3 @@ jobs:

- name: Publish to PyPI
uses: pypa/[email protected]

# publish_sdm:
# name: Publish SDM to DockerHub
# if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
# runs-on: ubuntu-latest
# needs: [publish]
# environment:
# name: DockerHub
# url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags

# steps:
# - uses: actions/checkout@v4
# with:
# fetch-depth: 0

# - name: Set Version (workflow_dispatch)
# if: github.event_name == 'workflow_dispatch'
# run: |
# VERSION=${{ github.event.inputs.version }}
# echo "Version input set to '${VERSION}'"
# # Remove the 'v' prefix if it exists
# VERSION=${VERSION#v}
# echo "Setting version to '$VERSION'"
# echo "VERSION=${VERSION}" >> $GITHUB_ENV

# - name: Set Version (tag)
# if: startsWith(github.ref, 'refs/tags/v')
# run: |
# VERSION=${{ github.ref_name }}
# echo "Version ref set to '${VERSION}'"
# # Remove the 'v' prefix if it exists
# VERSION=${VERSION#v}
# echo "Setting version to '$VERSION'"
# echo "VERSION=${VERSION}" >> $GITHUB_ENV

# # We need to download the build artifact again because the previous job was on a different runner
# - name: Download Build Artifact
# uses: actions/download-artifact@v4
# with:
# name: Packages-${{ github.run_id }}
# path: dist

# - name: Set up QEMU for multi-platform builds
# uses: docker/setup-qemu-action@v3

# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v3

# - name: Login to Docker Hub
# uses: docker/login-action@v3
# with:
# username: ${{ secrets.DOCKER_HUB_USERNAME }}
# password: ${{ secrets.DOCKER_HUB_PASSWORD }}

# - name: Check for existing tag
# run: |
# tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
# if [ -z "$tag" ]; then
# echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
# exit 1
# fi
# echo "Checking if tag '$tag' exists on DockerHub..."
# if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
# echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
# exit 1
# fi
# echo "No existing tag '$tag' found. Proceeding with publish."

# - name: Build and push
# uses: docker/build-push-action@v5
# with:
# context: .
# platforms: linux/amd64,linux/arm64
# push: true
# tags: |
# airbyte/source-declarative-manifest:latest
# airbyte/source-declarative-manifest:${{ env.VERSION }}
# airbyte/source-declarative-manifest:${{ github.sha }}

0 comments on commit e52bde4

Please sign in to comment.