From e52bde4bd78e56a08ee00b1aa193be57b9f649f7 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 22 Nov 2024 11:38:43 -0800 Subject: [PATCH] CI: move SDM image publish to dedicated step (#83) --- .github/workflows/publish_sdm_connector.yml | 180 ++++++++++++++++++++ .github/workflows/pypi_publish.yml | 78 --------- 2 files changed, 180 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/publish_sdm_connector.yml diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml new file mode 100644 index 00000000..c1b8f81c --- /dev/null +++ b/.github/workflows/publish_sdm_connector.yml @@ -0,0 +1,180 @@ +# This flow publishes the Source-Declarative-Manifest (SDM) +# connector to DockerHub as a Docker image. + +name: Publish SDM Connector + +on: + workflow_dispatch: + inputs: + version: + description: + The version to publish, ie 1.0.0 or 1.0.0-dev1. + If omitted, and if run from a release branch, the version will be + inferred from the git tag. + If omitted, and if run from a non-release branch, then only a SHA-based + Docker tag will be created. + required: false + dry_run: + description: If true, the workflow will not push to DockerHub. + type: boolean + required: false + default: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + + - name: Detect Release Tag Version + if: startsWith(github.ref, 'refs/tags/v') + run: | + DETECTED_VERSION=${{ github.ref_name }} + echo "Version ref set to '${DETECTED_VERSION}'" + # Remove the 'v' prefix if it exists + DETECTED_VERSION="${DETECTED_VERSION#v}" + echo "Setting version to '$DETECTED_VERSION'" + echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV + + - name: Validate and set VERSION from tag ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }}) + id: set_version + if: github.event_name == 'workflow_dispatch' + run: | + INPUT_VERSION=${{ github.event.inputs.version }} + echo "Version input set to '${INPUT_VERSION}'" + # Exit with success if both detected and input versions are empty + if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then + echo "No version detected or input. Will publish to SHA tag instead." + echo 'VERSION=""' >> $GITHUB_ENV + exit 0 + fi + # Remove the 'v' prefix if it exists + INPUT_VERSION="${INPUT_VERSION#v}" + # Fail if detected version is non-empty and different from the input version + if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then + echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'." + exit 1 + fi + # Set the version to the input version if non-empty, otherwise the detected version + VERSION="${INPUT_VERSION:-$DETECTED_VERSION}" + # Fail if the version is still empty + if [ -z "$VERSION" ]; then + echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." + exit 1 + fi + echo "Setting version to '$VERSION'" + echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT + # Check if version is a prerelease version (will not tag 'latest') + if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "IS_PRERELEASE=false" >> $GITHUB_ENV + echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT + else + echo "IS_PRERELEASE=true" >> $GITHUB_ENV + echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT + fi + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: hynek/build-and-inspect-python-package@v2 + name: Build package with version ref '${{ env.VERSION }}' + env: + # Pass in the evaluated version from the previous step + # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables + POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION }} + + - uses: actions/upload-artifact@v4 + with: + name: Packages-${{ github.run_id }} + path: | + /tmp/baipp/dist/*.whl + /tmp/baipp/dist/*.tar.gz + outputs: + VERSION: ${{ steps.set_version.outputs.VERSION }} + IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }} + + publish_sdm: + name: Publish SDM to DockerHub + if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + needs: [build] + environment: + name: DockerHub + url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags + env: + VERSION: ${{ needs.build.outputs.VERSION }} + IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # We need to download the build artifact again because the previous job was on a different runner + - name: Download Build Artifact + uses: actions/download-artifact@v4 + with: + name: Packages-${{ github.run_id }} + path: dist + + - name: Set up QEMU for multi-platform builds + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )" + if: env.VERSION != '' + run: | + tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" + if [ -z "$tag" ]; then + echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." + exit 1 + fi + echo "Checking if tag '$tag' exists on DockerHub..." + if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then + echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite." + exit 1 + fi + echo "No existing tag '$tag' found. Proceeding with publish." + + - name: Build and push (sha tag) + # Only run if the version is not set + if: env.VERSION == '' && github.event.inputs.dry_run == 'false' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:${{ github.sha }} + + - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})" + # Only run if the version is set + if: env.VERSION != '' && github.event.inputs.dry_run == 'false' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:${{ env.VERSION }} + + + - name: Build and push ('latest' tag) + # Only run if version is set and IS_PRERELEASE is false + if: env.VERSION != '' && env.IS_PRERELEASE == 'false' && github.event.inputs.dry_run == 'false' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:latest diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 09b51937..41329ce6 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -59,81 +59,3 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@v1.10.3 - - # publish_sdm: - # name: Publish SDM to DockerHub - # if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' - # runs-on: ubuntu-latest - # needs: [publish] - # environment: - # name: DockerHub - # url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags - - # steps: - # - uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - - # - name: Set Version (workflow_dispatch) - # if: github.event_name == 'workflow_dispatch' - # run: | - # VERSION=${{ github.event.inputs.version }} - # echo "Version input set to '${VERSION}'" - # # Remove the 'v' prefix if it exists - # VERSION=${VERSION#v} - # echo "Setting version to '$VERSION'" - # echo "VERSION=${VERSION}" >> $GITHUB_ENV - - # - name: Set Version (tag) - # if: startsWith(github.ref, 'refs/tags/v') - # run: | - # VERSION=${{ github.ref_name }} - # echo "Version ref set to '${VERSION}'" - # # Remove the 'v' prefix if it exists - # VERSION=${VERSION#v} - # echo "Setting version to '$VERSION'" - # echo "VERSION=${VERSION}" >> $GITHUB_ENV - - # # We need to download the build artifact again because the previous job was on a different runner - # - name: Download Build Artifact - # uses: actions/download-artifact@v4 - # with: - # name: Packages-${{ github.run_id }} - # path: dist - - # - name: Set up QEMU for multi-platform builds - # uses: docker/setup-qemu-action@v3 - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - # - name: Login to Docker Hub - # uses: docker/login-action@v3 - # with: - # username: ${{ secrets.DOCKER_HUB_USERNAME }} - # password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - # - name: Check for existing tag - # run: | - # tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" - # if [ -z "$tag" ]; then - # echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." - # exit 1 - # fi - # echo "Checking if tag '$tag' exists on DockerHub..." - # if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then - # echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite." - # exit 1 - # fi - # echo "No existing tag '$tag' found. Proceeding with publish." - - # - name: Build and push - # uses: docker/build-push-action@v5 - # with: - # context: . - # platforms: linux/amd64,linux/arm64 - # push: true - # tags: | - # airbyte/source-declarative-manifest:latest - # airbyte/source-declarative-manifest:${{ env.VERSION }} - # airbyte/source-declarative-manifest:${{ github.sha }}