diff --git a/.github/workflows/publish-manually.yml b/.github/workflows/publish-manually.yml new file mode 100644 index 00000000..18725dfa --- /dev/null +++ b/.github/workflows/publish-manually.yml @@ -0,0 +1,389 @@ +# Imported from monorepo +name: Publish Python CDK Manually +on: + workflow_dispatch: + inputs: + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + required: false + default: master + release-type: + type: choice + description: "Choose the type of version upgrade : major|minor|patch" + options: + - none + - major + - minor + - patch + required: true + changelog-message: + description: "Changelog message to be added to CHANGELOG.md" + required: false + +concurrency: + group: publish-airbyte-cdk + cancel-in-progress: false + +jobs: + build-cdk: + runs-on: ubuntu-latest + steps: + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install Poetry + id: install_poetry + uses: snok/install-poetry@v1 + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Install Dependencies + id: install_dependencies + working-directory: airbyte-cdk/python + run: poetry install + - name: Build CDK Package + working-directory: airbyte-cdk/python + run: poetry run poe build + - name: Post failure to Slack channel dev-connectors-extensibility + if: ${{ failure() }} + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + with: + channel-id: C04J1M66D8B + payload: | + { + "text": "Error during `build-cdk` while publishing Python CDK!", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Error while publishing Python CDK!" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + + bump-version: + needs: build-cdk + if: github.event.inputs.release-type != 'none' + runs-on: ubuntu-latest + outputs: + new_cdk_version: ${{ steps.bumpversion.outputs.NEW_VERSION }} + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install Poetry + id: install_poetry + uses: snok/install-poetry@v1 + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} # This token is what allows us to commit directly to master + - name: "Publish Python CDK: bump Poetry package version" + id: bumpversion + run: | + cd airbyte-cdk/python + # Bump package version + poetry version ${{ github.event.inputs.release-type }} + new_version="$(poetry version -s)" + awk -v NEW_VERSION="$new_version" -v CHANGELOG_MESSAGE="${{ github.event.inputs.changelog-message }}" 'NR==3{print "## " NEW_VERSION "\n" CHANGELOG_MESSAGE "\n"}1' CHANGELOG.md > tmp && mv tmp CHANGELOG.md + echo NEW_VERSION=$new_version >> $GITHUB_OUTPUT + - name: Commit and Push Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + file_pattern: airbyte-cdk/python/pyproject.toml airbyte-cdk/python/CHANGELOG.md + commit_message: 🤖 ${{ github.event.inputs.release-type }} bump Python CDK to version ${{ steps.bumpversion.outputs.NEW_VERSION }} + commit_user_name: Octavia Squidington III + commit_user_email: octavia-squidington-iii@users.noreply.github.com + - name: Post failure to Slack channel dev-connectors-extensibility + if: ${{ failure() }} + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + with: + channel-id: C04J1M66D8B + payload: | + { + "text": "Error during `bump-version` while publishing Python CDK!", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Error while publishing Python CDK!" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + + publish-cdk: + name: Publish Python CDK to PyPi + needs: bump-version + runs-on: ubuntu-latest + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Build and publish to pypi + uses: JRubics/poetry-publish@v2.0 + with: + pypi_token: ${{ secrets.PYPI_TOKEN }} + python_version: "3.10" + package_directory: "airbyte-cdk/python" + - name: Post failure to Slack channel dev-connectors-extensibility + if: ${{ failure() }} + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + with: + channel-id: C04J1M66D8B + payload: | + { + "text": "Error during `publish-cdk` while publishing Python CDK!", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Error while publishing Python CDK!" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + + bump-manifest-source: + name: Bump CDK dependency of source-declarative-manifest + needs: + - bump-version + - publish-cdk + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install Poetry + id: install_poetry + uses: snok/install-poetry@v1 + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} # This token is what allows us to commit directly to master + - name: Bump CDK dependency of source-declarative-manifest + timeout-minutes: 10 + run: | + cd airbyte-integrations/connectors/source-declarative-manifest + echo "Attempting to pull the newly published version of airbyte-cdk." + while true; do + # --no-cache to force looking for the new version + poetry add airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}} --no-cache && break + # Loop to wait for the new version to be available to poetry + echo "Couldn't add new dependency. This is normal if the dependency could not (yet) be found. Retrying in 10 seconds..." + sleep 10 + done + echo "Successfully updated the CDK dependency of source-declarative-manifest to ${{needs.bump-version.outputs.new_cdk_version}}" + - name: Bump version of source-declarative-manifest + uses: ./.github/actions/run-airbyte-ci + with: + context: "master" # TODO: figure out why changing this yells with `The ci_gcs_credentials was not set on this PipelineContext.` + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_5 }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} + spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + # There is no pull request number as we do this manually, so will just reference when we started doing it manually for now + subcommand: "connectors --concurrency=1 --execute-timeout=3600 --name=source-declarative-manifest bump-version version:${{needs.bump-version.outputs.new_cdk_version}} 'Bump CDK version to ${{needs.bump-version.outputs.new_cdk_version}}' --pr-number=36501" + python_registry_token: ${{ secrets.PYPI_TOKEN }} + - name: Commit and Push Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + file_pattern: docs/integrations/sources/low-code.md airbyte-integrations/connectors/source-declarative-manifest/* + commit_message: 🤖 Cut version ${{needs.bump-version.outputs.new_cdk_version}} of source-declarative-manifest + commit_user_name: Octavia Squidington III + commit_user_email: octavia-squidington-iii@users.noreply.github.com + - name: Post failure to Slack channel dev-connectors-extensibility + if: ${{ failure() }} + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + with: + channel-id: C04J1M66D8B + payload: | + { + "text": ":warning: A new version of Python CDK has been released but `source-declarative-manifest` and Connector Builder haven't been automatically updated", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":warning: Could not bump version of `source-declarative-manifest`.>\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + + update-connector-builder: + needs: + - bump-version + - bump-manifest-source + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Checkout Airbyte Platform Internal + uses: actions/checkout@v3 + with: + repository: airbytehq/airbyte-platform-internal + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + - name: Update CDK version + run: | + PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) + sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/Dockerfile + sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" cloud/airbyte-connector-builder-server-wrapped/Dockerfile + sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/requirements.in + echo ${{needs.bump-version.outputs.new_cdk_version}} > oss/airbyte-connector-builder-resources/CDK_VERSION + cd oss/airbyte-connector-builder-server + pip install pip-tools + pip-compile --upgrade + - name: Create Pull Request + id: create-pull-request + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + commit-message: "chore: update CDK version following release" + title: "chore: update CDK version following release" + body: This is an automatically generated PR triggered by a CDK release + branch: automatic-cdk-release + base: master + delete-branch: true + - name: Post success to Slack channel dev-connectors-extensibility + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + with: + channel-id: C04J1M66D8B + payload: | + { + "text": "A new version of Python CDK has been released!", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + - name: Post failure to Slack channel dev-connectors-extensibility + if: ${{ failure() }} + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + with: + channel-id: C04J1M66D8B + payload: | + { + "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":warning: Could not automatically create a PR for Connector Builder>\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}