Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci(draft): add back manual publish workflow from monorepo [do not merge] #41

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

ci(draft): add back manual publish workflow from monorepo [do not merge] #41

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
389 changes: 389 additions & 0 deletions .github/workflows/publish-manually.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,389 @@
# Imported from monorepo
name: Publish Python CDK Manually
on:
workflow_dispatch:
inputs:
repo:
description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
required: false
default: "airbytehq/airbyte"
gitref:
description: "The git ref to check out from the specified repository."
required: false
default: master
release-type:
type: choice
description: "Choose the type of version upgrade : major|minor|patch"
options:
- none
- major
- minor
- patch
required: true
changelog-message:
description: "Changelog message to be added to CHANGELOG.md"
required: false

concurrency:
group: publish-airbyte-cdk
cancel-in-progress: false

jobs:
build-cdk:
runs-on: ubuntu-latest
steps:
- name: Install Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install Poetry
id: install_poetry
uses: snok/install-poetry@v1
- name: Checkout Airbyte
uses: actions/checkout@v3
with:
repository: ${{ github.event.inputs.repo }}
ref: ${{ github.event.inputs.gitref }}
- name: Install Dependencies
id: install_dependencies
working-directory: airbyte-cdk/python
run: poetry install
- name: Build CDK Package
working-directory: airbyte-cdk/python
run: poetry run poe build
- name: Post failure to Slack channel dev-connectors-extensibility
if: ${{ failure() }}
uses: slackapi/[email protected]
continue-on-error: true
with:
channel-id: C04J1M66D8B
payload: |
{
"text": "Error during `build-cdk` while publishing Python CDK!",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "Error while publishing Python CDK!"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
}
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}

bump-version:
needs: build-cdk
if: github.event.inputs.release-type != 'none'
runs-on: ubuntu-latest
outputs:
new_cdk_version: ${{ steps.bumpversion.outputs.NEW_VERSION }}
steps:
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install Poetry
id: install_poetry
uses: snok/install-poetry@v1
- name: Checkout Airbyte
uses: actions/checkout@v3
with:
repository: ${{ github.event.inputs.repo }}
ref: ${{ github.event.inputs.gitref }}
token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} # This token is what allows us to commit directly to master
- name: "Publish Python CDK: bump Poetry package version"
id: bumpversion
run: |
cd airbyte-cdk/python
# Bump package version
poetry version ${{ github.event.inputs.release-type }}
new_version="$(poetry version -s)"
awk -v NEW_VERSION="$new_version" -v CHANGELOG_MESSAGE="${{ github.event.inputs.changelog-message }}" 'NR==3{print "## " NEW_VERSION "\n" CHANGELOG_MESSAGE "\n"}1' CHANGELOG.md > tmp && mv tmp CHANGELOG.md
echo NEW_VERSION=$new_version >> $GITHUB_OUTPUT
- name: Commit and Push Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
file_pattern: airbyte-cdk/python/pyproject.toml airbyte-cdk/python/CHANGELOG.md
commit_message: 🤖 ${{ github.event.inputs.release-type }} bump Python CDK to version ${{ steps.bumpversion.outputs.NEW_VERSION }}
commit_user_name: Octavia Squidington III
commit_user_email: [email protected]
- name: Post failure to Slack channel dev-connectors-extensibility
if: ${{ failure() }}
uses: slackapi/[email protected]
continue-on-error: true
with:
channel-id: C04J1M66D8B
payload: |
{
"text": "Error during `bump-version` while publishing Python CDK!",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "Error while publishing Python CDK!"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
}
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}

publish-cdk:
name: Publish Python CDK to PyPi
needs: bump-version
runs-on: ubuntu-latest
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
with:
repository: ${{ github.event.inputs.repo }}
ref: ${{ github.event.inputs.gitref }}
- name: Build and publish to pypi
uses: JRubics/[email protected]
with:
pypi_token: ${{ secrets.PYPI_TOKEN }}
python_version: "3.10"
package_directory: "airbyte-cdk/python"
- name: Post failure to Slack channel dev-connectors-extensibility
if: ${{ failure() }}
uses: slackapi/[email protected]
continue-on-error: true
with:
channel-id: C04J1M66D8B
payload: |
{
"text": "Error during `publish-cdk` while publishing Python CDK!",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "Error while publishing Python CDK!"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
}
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}

bump-manifest-source:
name: Bump CDK dependency of source-declarative-manifest
needs:
- bump-version
- publish-cdk
runs-on: ubuntu-latest
steps:
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install Poetry
id: install_poetry
uses: snok/install-poetry@v1
- name: Checkout Airbyte
uses: actions/checkout@v3
with:
repository: ${{ github.event.inputs.repo }}
ref: ${{ github.event.inputs.gitref }}
token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} # This token is what allows us to commit directly to master
- name: Bump CDK dependency of source-declarative-manifest
timeout-minutes: 10
run: |
cd airbyte-integrations/connectors/source-declarative-manifest
echo "Attempting to pull the newly published version of airbyte-cdk."
while true; do
# --no-cache to force looking for the new version
poetry add airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}} --no-cache && break
# Loop to wait for the new version to be available to poetry
echo "Couldn't add new dependency. This is normal if the dependency could not (yet) be found. Retrying in 10 seconds..."
sleep 10
done
echo "Successfully updated the CDK dependency of source-declarative-manifest to ${{needs.bump-version.outputs.new_cdk_version}}"
- name: Bump version of source-declarative-manifest
uses: ./.github/actions/run-airbyte-ci
with:
context: "master" # TODO: figure out why changing this yells with `The ci_gcs_credentials was not set on this PipelineContext.`
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_5 }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
github_token: ${{ secrets.GITHUB_TOKEN }}
metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
# There is no pull request number as we do this manually, so will just reference when we started doing it manually for now
subcommand: "connectors --concurrency=1 --execute-timeout=3600 --name=source-declarative-manifest bump-version version:${{needs.bump-version.outputs.new_cdk_version}} 'Bump CDK version to ${{needs.bump-version.outputs.new_cdk_version}}' --pr-number=36501"
python_registry_token: ${{ secrets.PYPI_TOKEN }}
- name: Commit and Push Changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
file_pattern: docs/integrations/sources/low-code.md airbyte-integrations/connectors/source-declarative-manifest/*
commit_message: 🤖 Cut version ${{needs.bump-version.outputs.new_cdk_version}} of source-declarative-manifest
commit_user_name: Octavia Squidington III
commit_user_email: [email protected]
- name: Post failure to Slack channel dev-connectors-extensibility
if: ${{ failure() }}
uses: slackapi/[email protected]
continue-on-error: true
with:
channel-id: C04J1M66D8B
payload: |
{
"text": ":warning: A new version of Python CDK has been released but `source-declarative-manifest` and Connector Builder haven't been automatically updated",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":warning: Could not bump version of `source-declarative-manifest`.>\n"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
}
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}

update-connector-builder:
needs:
- bump-version
- bump-manifest-source
runs-on: ubuntu-latest
steps:
- uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Checkout Airbyte Platform Internal
uses: actions/checkout@v3
with:
repository: airbytehq/airbyte-platform-internal
token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
- name: Update CDK version
run: |
PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION)
sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/Dockerfile
sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" cloud/airbyte-connector-builder-server-wrapped/Dockerfile
sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/requirements.in
echo ${{needs.bump-version.outputs.new_cdk_version}} > oss/airbyte-connector-builder-resources/CDK_VERSION
cd oss/airbyte-connector-builder-server
pip install pip-tools
pip-compile --upgrade
- name: Create Pull Request
id: create-pull-request
uses: peter-evans/create-pull-request@v6
with:
token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
commit-message: "chore: update CDK version following release"
title: "chore: update CDK version following release"
body: This is an automatically generated PR triggered by a CDK release
branch: automatic-cdk-release
base: master
delete-branch: true
- name: Post success to Slack channel dev-connectors-extensibility
uses: slackapi/[email protected]
continue-on-error: true
with:
channel-id: C04J1M66D8B
payload: |
{
"text": "A new version of Python CDK has been released!",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
}
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
- name: Post failure to Slack channel dev-connectors-extensibility
if: ${{ failure() }}
uses: slackapi/[email protected]
continue-on-error: true
with:
channel-id: C04J1M66D8B
payload: |
{
"text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":warning: Could not automatically create a PR for Connector Builder>\n"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
}
}
]
}
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
Loading