Skip to content

fix: add datasources.config for airbyte-workload-launcher to fix breaking JDBC connection #7226

fix: add datasources.config for airbyte-workload-launcher to fix breaking JDBC connection

fix: add datasources.config for airbyte-workload-launcher to fix breaking JDBC connection #7226

Workflow file for this run

# The goal of this build is to make sure that OSS contributors can build the project and run the tests
# so that they can develop locally. It is NOT a release verification. As such, we just run build and
# unit test. No additional acceptance test, etc.
name: Airbyte Platform OSS Developer Build
env:
S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
on:
#ability to start task manually in Web UI
workflow_dispatch:
inputs:
debug_mode:
description: "Enable or disable tmate session for debug during helm ac tests"
type: choice
default: "false"
options:
- "true"
- "false"
required: false
push:
branches:
- main
pull_request:
permissions: write-all
jobs:
ensure-images-exist:
name: "Ensure all required Docker images exist on Dockerhub"
timeout-minutes: 10
runs-on: ubuntu-latest
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Check images exist
run: ./tools/bin/check_images_exist.sh all
# The output of this job is used to trigger the following builds.
changes:
name: "Detect Modified Files"
# The filtering action does not deal with well scheduled events so skip to avoid errors.
# See https://github.com/dorny/paths-filter/issues/100 for more info.
# This is okay this workflow is only scheduled on main, where we want to build everything
# so filtering is not required. Use always() in each start block to force the start task.
if: github.event_name != 'schedule'
runs-on: ubuntu-latest
outputs:
backend: ${{ steps.filter.outputs.backend }}
build: ${{ steps.filter.outputs.build }}
cli: ${{ steps.filter.outputs.cli }}
db: ${{ steps.filter.outputs.db }}
frontend: ${{ steps.filter.outputs.frontend }}
frontendServers: ${{ steps.changes.outputs.frontendServers }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
id: filter
with:
# Note, the following glob expression within a filters are ORs.
filters: |
backend:
- 'airbyte-!(cdk|integrations|webapp)/**'
- 'airbyte-integrations/connectors/(destination-jdbc|destination-postgres|source-jdbc|source-postgres)/**'
- 'airbyte-config/init/src/main/resources/seed/(source|destination)_definitions.yaml'
- '(charts|kube)/**'
build:
- '.github/**'
- 'buildSrc/**'
- 'tools/**'
- '*.gradle'
- 'deps.toml'
cli:
- 'airbyte-api/**'
db:
- 'airbyte-db/**'
frontend:
- 'airbyte-api/server-api/src/main/openapi/config.yaml'
- 'oss/airbyte-api/commons/src/main/openapi/cloud-config.yaml'
- 'airbyte-connector-builder-resources/CDK_VERSION'
- 'airbyte-connector-builder-server/src/main/openapi/openapi.yaml'
- 'airbyte-webapp/**'
frontendServers:
- 'airbyte-(server|data|connector-builder-server|featureflag|commons|commons-!(worker|temporal*))/**'
# Uncomment to debug.
# changes-output:
# name: "Debug Change Detection Logic"
# needs: changes
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - run: |
# echo '${{ toJSON(needs) }}'
platform-build:
name: "Platform: Build"
# At the time of writing, using the beefier AWS runners made this step take 7 minutes instead of 10 minutes
# on the stock runners. Since no developer iteration loop depends on this, we use the stock gh runners. If
# we need to bring back the AWS runners, checkout 98f3e85a9ea2634298653642b18ac38fd1b85929 as a reference.
runs-on: ubuntu-latest
needs:
- changes
# Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on master.
if: |
needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/main'
|| (always() && needs.changes.outputs.frontendServers == 'true')
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Cache Build Artifacts
uses: ./.github/actions/cache-build-artifacts
with:
cache-key: ${{ secrets.CACHE_VERSION }}
- uses: actions/setup-java@v3
with:
distribution: "zulu"
java-version: "21"
- uses: actions/setup-node@v3
with:
node-version: "lts/*"
- name: Set up CI Gradle Properties
run: |
mkdir -p ~/.gradle/
cat > ~/.gradle/gradle.properties <<EOF
org.gradle.jvmargs=-Xmx8g -Xss4m \
--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
org.gradle.workers.max=8
org.gradle.vfs.watch=false
EOF
- name: Format
uses: Wandalen/[email protected]
with:
command: ./gradlew format --scan --info --stacktrace
attempt_limit: 3
attempt_delay: 5000 # in ms
- name: Ensure no file change
run: git --no-pager diff && test -z "$(git --no-pager diff)"
- name: Build
uses: Wandalen/[email protected]
with:
# todo (cgardens): temporarily skipping airbyte-commons-worker because one of the tests uses too much memory. we are
# adjusting the test to use a more reasonable amount of memory. https://github.com/airbytehq/oncall/issues/4415
# command: ./gradlew build javadoc --scan
command: ./gradlew build -x :oss:airbyte-commons-worker:test javadoc --scan
attempt_limit: 3
attempt_delay: 5000 # in ms
- name: Check documentation
if: success() && github.ref == 'refs/heads/main'
run: ./tools/site/link_checker.sh check_docs
# This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
# - name: Get Docker Space
# run: docker run --rm busybox df -h
notify-failure-slack-channel:
name: "Notify Slack Channel on Build Failures"
runs-on: ubuntu-latest
needs:
- platform-build
if: ${{ failure() && github.ref == 'refs/heads/main' }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Match GitHub User to Slack User
id: match-github-to-slack-user
uses: ./.github/actions/match-github-to-slack-user
env:
AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Publish to OSS Build Failure Slack Channel
uses: abinoda/slack-action@master
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
with:
args: >-
{\"channel\":\"C03BEADRPNY\", \"blocks\":[
{\"type\":\"divider\"},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS main failed! :bangbang: \n\n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
{\"type\":\"divider\"}]}
notify-failure-slack-channel-fixed-broken-build:
name: "Notify Slack Channel on Build Fixes"
runs-on: ubuntu-latest
needs:
- platform-build
if: ${{ success() && github.ref == 'refs/heads/main' }}
steps:
- name: Get Previous Workflow Status
uses: Mercymeilya/[email protected]
id: last_status
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
# To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed.
- name: Publish Build Fixed Message to OSS Build Failure Slack Channel
if: ${{ steps.last_status.outputs.last_status == 'failure' }}
uses: abinoda/slack-action@master
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
with:
args: >-
{\"channel\":\"C03BEADRPNY\", \"blocks\":[
{\"type\":\"divider\"},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Main Fixed! :white_check_mark: \n\n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}},
{\"type\":\"divider\"}]}