gradle.yml

# The goal of this build is to make sure that OSS contributors can build the project and run the tests
# so that they can develop locally. It is NOT a release verification. As such, we just run build and
# unit test. No additional acceptance test, etc.
name: Airbyte Platform OSS Developer Build

env:
  S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
  S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}

on:
  #ability to start task manually in Web UI
  workflow_dispatch:
    inputs:
      debug_mode:
        description: "Enable or disable tmate session for debug during helm ac tests"
        type: choice
        default: "false"
        options:
          - "true"
          - "false"
        required: false
  push:
    branches:
      - main
  pull_request:

permissions: write-all

jobs:
  ensure-images-exist:
    name: "Ensure all required Docker images exist on Dockerhub"
    timeout-minutes: 10
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3

      - name: Check images exist
        run: ./tools/bin/check_images_exist.sh all
  # The output of this job is used to trigger the following builds.
  changes:
    name: "Detect Modified Files"
    # The filtering action does not deal with well scheduled events so skip to avoid errors.
    # See https://github.com/dorny/paths-filter/issues/100 for more info.
    # This is okay this workflow is only scheduled on main, where we want to build everything
    # so filtering is not required. Use always() in each start block to force the start task.
    if: github.event_name != 'schedule'
    runs-on: ubuntu-latest
    outputs:
      backend: ${{ steps.filter.outputs.backend }}
      build: ${{ steps.filter.outputs.build }}
      cli: ${{ steps.filter.outputs.cli }}
      db: ${{ steps.filter.outputs.db }}
      frontend: ${{ steps.filter.outputs.frontend }}
      frontendServers: ${{ steps.changes.outputs.frontendServers }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          # Note, the following glob expression within a filters are ORs.
          filters: |
            backend:
              - 'airbyte-!(cdk|integrations|webapp)/**'
              - 'airbyte-integrations/connectors/(destination-jdbc|destination-postgres|source-jdbc|source-postgres)/**'
              - 'airbyte-config/init/src/main/resources/seed/(source|destination)_definitions.yaml'
              - '(charts|kube)/**'
            build:
              - '.github/**'
              - 'buildSrc/**'
              - 'tools/**'
              - '*.gradle'
              - 'deps.toml'
            cli:
              - 'airbyte-api/**'
            db:
              - 'airbyte-db/**'
            frontend:
              - 'airbyte-api/server-api/src/main/openapi/config.yaml'
              - 'oss/airbyte-api/commons/src/main/openapi/cloud-config.yaml'
              - 'airbyte-connector-builder-resources/CDK_VERSION'
              - 'airbyte-connector-builder-server/src/main/openapi/openapi.yaml'
              - 'airbyte-webapp/**'
            frontendServers:
              - 'airbyte-(server|data|connector-builder-server|featureflag|commons|commons-!(worker|temporal*))/**'

  # Uncomment to debug.
  #  changes-output:
  #    name: "Debug Change Detection Logic"
  #    needs: changes
  #    runs-on: ubuntu-latest
  #    steps:
  #      - uses: actions/checkout@v3
  #      - run: |
  #          echo '${{ toJSON(needs) }}'

  platform-build:
    name: "Platform: Build"
    # At the time of writing, using the beefier AWS runners made this step take 7 minutes instead of 10 minutes
    # on the stock runners. Since no developer iteration loop depends on this, we use the stock gh runners. If
    # we need to bring back the AWS runners, checkout 98f3e85a9ea2634298653642b18ac38fd1b85929 as a reference.
    runs-on: ubuntu-latest
    needs:
      - changes
    # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on master.
    if: |
      needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/main'
      || (always() && needs.changes.outputs.frontendServers == 'true')
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}

      - uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: "21"

      - uses: actions/setup-node@v3
        with:
          node-version: "lts/*"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m \
            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Format
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: ./gradlew format --scan --info --stacktrace
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Ensure no file change
        run: git --no-pager diff && test -z "$(git --no-pager diff)"

      - name: Build
        uses: Wandalen/wretry.action@v1.0.36
        with:
          # todo (cgardens): temporarily skipping airbyte-commons-worker because one of the tests uses too much memory. we are
          # adjusting the test to use a more reasonable amount of memory. https://github.com/airbytehq/oncall/issues/4415
          # command: ./gradlew build javadoc --scan
          command: ./gradlew build -x :oss:airbyte-commons-worker:test javadoc --scan
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Check documentation
        if: success() && github.ref == 'refs/heads/main'
        run: ./tools/site/link_checker.sh check_docs

      #       This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
      #      - name: Get Docker Space
      #        run: docker run --rm busybox df -h

  notify-failure-slack-channel:
    name: "Notify Slack Channel on Build Failures"
    runs-on: ubuntu-latest
    needs:
      - platform-build
    if: ${{ failure() && github.ref == 'refs/heads/main' }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Match GitHub User to Slack User
        id: match-github-to-slack-user
        uses: ./.github/actions/match-github-to-slack-user
        env:
          AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Publish to OSS Build Failure Slack Channel
        uses: abinoda/slack-action@master
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          args: >-
            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
            {\"type\":\"divider\"},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS main failed! :bangbang: \n\n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
            {\"type\":\"divider\"}]}

  notify-failure-slack-channel-fixed-broken-build:
    name: "Notify Slack Channel on Build Fixes"
    runs-on: ubuntu-latest
    needs:
      - platform-build
    if: ${{ success() && github.ref == 'refs/heads/main' }}
    steps:
      - name: Get Previous Workflow Status
        uses: Mercymeilya/last-workflow-status@v0.3
        id: last_status
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
      # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed.
      - name: Publish Build Fixed Message to OSS Build Failure Slack Channel
        if: ${{ steps.last_status.outputs.last_status == 'failure' }}
        uses: abinoda/slack-action@master
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          args: >-
            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
            {\"type\":\"divider\"},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Main Fixed! :white_check_mark: \n\n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}},
            {\"type\":\"divider\"}]}