gradle.yml

name: Airbyte Platform CI

env:
  S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
  S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}

on:
  #ability to start task manually in Web UI
  workflow_dispatch:
    inputs:
      debug_mode:
        description: "Enable or disable tmate session for debug during helm ac tests"
        type: choice
        default: "false"
        options:
          - "true"
          - "false"
        required: false
  push:
    branches:
      - main
  pull_request:

permissions: write-all

jobs:
  # COMMON TASKS
  ensure-images-exist:
    name: "Ensure all required Docker images exist on Dockerhub"
    timeout-minutes: 10
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3

      - name: Check images exist
        run: ./tools/bin/check_images_exist.sh all
  # The output of this job is used to trigger the following builds.
  changes:
    name: "Detect Modified Files"
    # The filtering action does not deal with well scheduled events so skip to avoid errors.
    # See https://github.com/dorny/paths-filter/issues/100 for more info.
    # This is okay this workflow is only scheduled on main, where we want to build everything
    # so filtering is not required. Use always() in each start block to force the start task.
    if: github.event_name != 'schedule'
    runs-on: ubuntu-latest
    outputs:
      backend: ${{ steps.filter.outputs.backend }}
      build: ${{ steps.filter.outputs.build }}
      cli: ${{ steps.filter.outputs.cli }}
      db: ${{ steps.filter.outputs.db }}
      frontend: ${{ steps.filter.outputs.frontend }}
      frontendServers: ${{ steps.changes.outputs.frontendServers }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          # Note, the following glob expression within a filters are ORs.
          filters: |
            backend:
              - 'airbyte-!(cdk|integrations|webapp)/**'
              - 'airbyte-integrations/connectors/(destination-jdbc|destination-postgres|source-jdbc|source-postgres)/**'
              - 'airbyte-config/init/src/main/resources/seed/(source|destination)_definitions.yaml'
              - 'docker-compose*.yaml'
              - '(charts|kube)/**'
            build:
              - '.github/**'
              - 'buildSrc/**'
              - 'tools/**'
              - '*.gradle'
              - 'deps.toml'
            cli:
              - 'airbyte-api/**'
            db:
              - 'airbyte-db/**'
            frontend:
              - 'airbyte-api/src/main/openapi/config.yaml'
              - 'oss/airbyte-api/src/main/openapi/cloud-config.yaml'
              - 'airbyte-connector-builder-resources/CDK_VERSION'
              - 'airbyte-connector-builder-server/src/main/openapi/openapi.yaml'
              - 'airbyte-webapp/**'
            frontendServers:
              - 'airbyte-(server|data|connector-builder-server|commons|commons-!(worker|temporal*))/**'

  # Uncomment to debug.
  #  changes-output:
  #    name: "Debug Change Detection Logic"
  #    needs: changes
  #    runs-on: ubuntu-latest
  #    steps:
  #      - uses: actions/checkout@v3
  #      - run: |
  #          echo '${{ toJSON(needs) }}'

  ## BUILDS
  ## Frontend Test
  # In case of self-hosted EC2 errors, remove this block.
  start-frontend-runner:
    name: "Frontend: Start EC2 Runner"
    needs:
      - changes
    # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on main.
    if: |
      needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/main'
      || (always() && needs.changes.outputs.backend == 'true')
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ env.PAT }}
  frontend-build:
    name: "Frontend: Build"
    needs:
      - start-frontend-runner
    runs-on: ${{ needs.start-frontend-runner.outputs.label }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
        # We need to fetch at least one more commmit for the Chromatic action not to fail
        # but since we don't do screenshot comparison we don't need to fetch the full history.
        with:
          fetch-depth: 2

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: "21"

      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Build :airbyte-webapp
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: ./gradlew --no-daemon :airbyte-webapp:build --scan
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Publish Storybook to Chromatic
        uses: chromaui/action@v1
        with:
          projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
          token: ${{ secrets.GITHUB_TOKEN }}
          workingDir: ./airbyte-webapp/
          storybookBuildDir: build/storybook/
          autoAcceptChanges: true
          exitOnceUploaded: true

  # In case of self-hosted EC2 errors, remove this block.
  stop-frontend-runner:
    name: "Frontend: Stop Runner"
    timeout-minutes: 10
    needs:
      - start-frontend-runner # required to get output from the start-runner job
      - frontend-build # required to wait when then build job is done
    runs-on: ubuntu-latest
    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
    # Thus, we check for skipped here.
    if: ${{ always() && needs.start-frontend-runner.result != 'skipped'}}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Stop EC2 runner
        uses: airbytehq/ec2-github-runner@MSGv0.0.5
        with:
          mode: stop
          github-token: ${{ env.PAT }}
          label: ${{ needs.start-frontend-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-frontend-runner.outputs.ec2-instance-id }}

  ## Frontend Test
  # In case of self-hosted EC2 errors, remove this block.
  start-frontend-e2e-runner:
    name: "Frontend E2E Tests: Start EC2 Runner"
    needs:
      - changes
    # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on master.
    if: |
      needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/main'
      || (always() && needs.changes.outputs.frontendServers == 'true')
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ env.PAT }}

  frontend-e2e-test:
    name: "Frontend: Run End-to-End Tests"
    needs:
      - start-frontend-e2e-runner # required to have runner started
    runs-on: ${{ needs.start-frontend-e2e-runner.outputs.label }} # run the job on the newly created runner
    timeout-minutes: 120
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: "21"

      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Build Platform Docker Images
        run: ./gradlew --no-daemon assemble --scan

      - name: Run End-to-End Frontend Tests
        env:
          CYPRESS_WEBAPP_KEY: ${{ secrets.CYPRESS_WEBAPP_KEY }}
          CYPRESS_CACHE_FOLDER: "~/.cache/Cypress"
        run: ./tools/bin/e2e_test.sh

  # In case of self-hosted EC2 errors, remove this block.
  stop-frontend-e2e-runner:
    name: "Frontend E2E Tests: Stop Runner"
    timeout-minutes: 10
    needs:
      - start-frontend-e2e-runner # required to get output from the start-runner job
      - frontend-e2e-test # required to wait when the e2e-test job is done
    runs-on: ubuntu-latest
    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
    # Thus, we check for skipped here.
    if: ${{ always() && needs.start-frontend-e2e-runner.result != 'skipped'}}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Stop EC2 runner
        uses: airbytehq/ec2-github-runner@MSGv0.0.5
        with:
          mode: stop
          github-token: ${{ env.PAT }}
          label: ${{ needs.start-frontend-e2e-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-frontend-e2e-runner.outputs.ec2-instance-id }}

  ## FOLLOWING BUILDS ARE ALL PLATFORM BUILDS.

  # Main Platform
  # In case of self-hosted EC2 errors, remove this block.
  start-platform-build-runner:
    name: "Platform: Start Build EC2 Runner"
    needs:
      - changes
    # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on main.
    if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/main')
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ env.PAT }}
  platform-build:
    name: "Platform: Build"
    # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
    needs: start-platform-build-runner # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner
    timeout-minutes: 90
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: "21"

      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - uses: actions/setup-node@v3
        with:
          node-version: "lts/*"

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m \
            --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Format
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: ./gradlew format --scan --info --stacktrace
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Ensure no file change
        run: git --no-pager diff && test -z "$(git --no-pager diff)"

      - name: Build
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: ./gradlew build javadoc --scan
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Add coverage to PR
        if: github.event_name == 'pull_request'
        id: jacoco
        uses: madrapps/jacoco-report@v1.3
        with:
          # todo(mfsiega-airbyte): when v1.4 is released, use wildcards per https://github.com/Madrapps/jacoco-report/issues/20.
          paths: |
            ${{ github.workspace }}/airbyte-analytics/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-api/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-bootloader/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-commons/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-commons-protocol/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-commons-temporal/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-commons-worker/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-config/init/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-config/specs/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-config/config-persistence/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-config/config-models/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-container-orchestrator/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-db/db-lib/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-json-validation/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-metrics/reporter/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-metrics/metrics-lib/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-notification/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-oauth/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-persistence/job-persistence/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-server/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-test-utils/build/reports/jacoco/test/jacocoTestReport.xml,
            ${{ github.workspace }}/airbyte-workers/build/reports/jacoco/test/jacocoTestReport.xml
          token: ${{ secrets.GITHUB_TOKEN }}
          min-coverage-overall: 25
          min-coverage-changed-files: 60
          title: Airbyte Code Coverage
          update-comment: true
          debug-mode: false

      - name: Test if Seed spec is updated
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: ./gradlew format && git --no-pager diff && test -z "$(git --no-pager diff)"
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      # todo (cgardens) - scope by platform.
      - name: Check documentation
        if: success() && github.ref == 'refs/heads/main'
        run: ./tools/site/link_checker.sh check_docs

      #       This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
      #      - name: Get Docker Space
      #        run: docker run --rm busybox df -h

      - name: Run End-to-End Acceptance Tests
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: ./tools/bin/acceptance_test.sh
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - uses: actions/setup-python@v4
        if: always()
        with:
          python-version: "3.9"

      - name: Generate Test Report
        uses: dorny/test-reporter@v1
        if: always() # run this step even if previous step failed
        with:
          name: Platform Test Report with Docker E2E Test
          # Specify top-level and second-level modules. Note there cannot be a space between the comma.
          path: "/actions-runner/_work/airbyte-platform/airbyte-platform/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte-platform/airbyte-platform/*/*/build/test-results/*/*.xml"
          reporter: java-junit
          fail-on-error: "false"
        env:
          NODE_OPTIONS: '--max_old_space_size=8192'

  # In case of self-hosted EC2 errors, remove this block.
  stop-platform-build-runner:
    name: "Platform: Stop Build EC2 Runner"
    timeout-minutes: 10
    needs:
      - start-platform-build-runner # required to get output from the start-runner job
      - platform-build # required to wait when the main job is done
    runs-on: ubuntu-latest
    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
    # Thus, we check for skipped here.
    if: ${{ always() && needs.start-platform-build-runner.result != 'skipped'}}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Stop EC2 runner
        uses: airbytehq/ec2-github-runner@MSGv0.0.5
        with:
          mode: stop
          github-token: ${{ env.PAT }}
          label: ${{ needs.start-platform-build-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }}

  # ## Helm Acceptance Tests
  # # Docker acceptance tests run as part of the build job.
  # # In case of self-hosted EC2 errors, remove this block.
  start-helm-acceptance-test-runner:
    name: "Platform: Start Helm Acceptance Test Runner"
    needs:
      - changes
    # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on main.
    if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/main')
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
          # ec2-image-id: ami-0c1a9bc22624339d8
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ env.PAT }}
          iam-role-name: ${{ secrets.AWS_ASSUME_ROLE_NAME }}

  helm-acceptance-test:
    name: "Platform: Acceptance Tests (Helm)"
    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
    needs: [start-helm-acceptance-test-runner] # required to start the main job when the runner is ready
    runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
    # this is the label of the runner
    timeout-minutes: 90
    defaults:
      run:
        working-directory: "/actions-runner/_work/airbyte-platform/airbyte-platform"
    env:
      HOME: /actions-runner/_work/airbyte-platform/airbyte-platform
      GITHUB_WORKSPACE: /actions-runner/_work/airbyte-platform/airbyte-platform

    steps:
      - name: Fix EC-2 Runner
        run: |
          mkdir -p /actions-runner/_work/airbyte-platform/airbyte-platform && mkdir -p /actions-runner/_work/airbyte-platform/airbyte-platform/.kube

      - name: Checkout Airbyte
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: Delete default old docker and replace it with a new one
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: |
            sudo apt-get remove docker.io || sudo apt-get remove docker
            curl -fsSL https://get.docker.com | bash -
          attempt_limit: 3
          attempt_delay: 2000 # in ms

      - name: Cache Build Artifacts
        uses: ./.github/actions/cache-build-artifacts
        with:
          cache-key: ${{ secrets.CACHE_VERSION }}
          cache-python: "false"

      - uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - uses: actions/setup-java@v1
        with:
          java-version: "21"

      - name: Install unzip
        shell: bash
        run: sudo apt-get update && sudo apt-get install -y unzip

      - uses: azure/setup-helm@v3
        with:
          version: "latest"
          token: ${{ secrets.GITHUB_TOKEN }}
        id: install

      - uses: azure/setup-kubectl@v3
        with:
          version: "v1.21.2" # default is latest stable

      - name: Install tmate and it's dependencies
        if: inputs.debug_mode == 'true'
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: |
            sudo apt-get -y install tmate
          attempt_limit: 3
          attempt_delay: 2000 # in ms

      - name: Start tmate session in background
        if: inputs.debug_mode == 'true'
        shell: bash
        run: |
          tmate -S /tmp/tmate.sock new-session -d               # Launch tmate in a headless mode
          tmate -S /tmp/tmate.sock wait tmate-ready             # Blocks until the SSH connection is established
          tmate -S /tmp/tmate.sock display -p '#{tmate_ssh}'    # Prints the SSH connection string
          tmate -S /tmp/tmate.sock display -p '#{tmate_ssh_ro}' # Prints the read-only SSH connection string
          tmate -S /tmp/tmate.sock display -p '#{tmate_web}'    # Prints the web connection string
          tmate -S /tmp/tmate.sock display -p '#{tmate_web_ro}' # Prints the read-only web connection string

      - name: Start a local k8s cluster
        uses: jupyterhub/action-k3s-helm@v3
        with:
          k3s-version: v1.21.2+k3s1
          helm-version: v3.9.4
          extra-setup-args: --docker --kube-apiserver-arg service-node-port-range=6000-32767 --kubelet-arg=cgroup-driver=systemd

      - name: Try the cluster !
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: kubectl get pods -A
          attempt_limit: 3
          attempt_delay: 3000 # in ms

      - name: Set up CI Gradle Properties
        run: |
          mkdir -p ~/.gradle/
          cat > ~/.gradle/gradle.properties <<EOF
          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
          org.gradle.workers.max=8
          org.gradle.vfs.watch=false
          EOF

      - name: Build Platform Docker Images
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: pwd && ./gradlew assemble -x test --scan
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Run Helm End-to-End Acceptance Tests
        env:
          # USER: root
          # HOME: /home/runner
          # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
          WORKFLOW_RUN_ID: ${{ github.run_id }}
          AWS_S3_BUCKET: airbyte-ci-ac-tests-logs
          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
        timeout-minutes: 40
        uses: Wandalen/wretry.action@v1.0.36
        with:
          command: CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
          attempt_limit: 3
          attempt_delay: 5000 # in ms

      - name: Generate Test Report
        uses: dorny/test-reporter@v1
        if: always() # run this step even if previous step failed
        with:
          name: Platform Helm E2E Test Report
          path: "./*/build/test-results/*/*.xml"
          reporter: java-junit
          fail-on-error: "false"
        env:
          NODE_OPTIONS: '--max_old_space_size=8192'

      - name: "Display logs of k3s"
        if: failure()
        shell: bash
        run: |
          journalctl -xeu k3s.service

  # # In case of self-hosted EC2 errors, remove this block.
  stop-helm-acceptance-test-runner:
    name: "Platform: Stop Helm Acceptance Test EC2 Runner"
    timeout-minutes: 10
    needs:
      - start-helm-acceptance-test-runner # required to get output from the start-runner job
      - helm-acceptance-test # required to wait when the main job is done
    runs-on: ubuntu-latest
    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
    # Thus, we check for skipped here.
    if: ${{ always() && needs.start-helm-acceptance-test-runner.result != 'skipped'}}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Stop EC2 runner
        uses: airbytehq/ec2-github-runner@MSGv0.0.5
        with:
          mode: stop
          github-token: ${{ env.PAT }}
          label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }}

  notify-failure-slack-channel:
    name: "Notify Slack Channel on Build Failures"
    runs-on: ubuntu-latest
    needs:
      - frontend-build
      - platform-build
      # Todo: Kyryl turn this on.
      # - helm-acceptance-test
    if: ${{ failure() && github.ref == 'refs/heads/main' }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Match GitHub User to Slack User
        id: match-github-to-slack-user
        uses: ./.github/actions/match-github-to-slack-user
        env:
          AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }}
          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Publish to OSS Build Failure Slack Channel
        uses: abinoda/slack-action@master
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          args: >-
            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
            {\"type\":\"divider\"},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS main failed! :bangbang: \n\n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
            {\"type\":\"divider\"}]}

  notify-failure-slack-channel-fixed-broken-build:
    name: "Notify Slack Channel on Build Fixes"
    runs-on: ubuntu-latest
    needs:
      - frontend-build
      - platform-build
      # Todo: Kyryl turn this on.
      # - helm-acceptance-test
    if: ${{ success() && github.ref == 'refs/heads/main' }}
    steps:
      - name: Get Previous Workflow Status
        uses: Mercymeilya/last-workflow-status@v0.3
        id: last_status
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
      # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed.
      - name: Publish Build Fixed Message to OSS Build Failure Slack Channel
        if: ${{ steps.last_status.outputs.last_status == 'failure' }}
        uses: abinoda/slack-action@master
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
        with:
          args: >-
            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
            {\"type\":\"divider\"},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Main Fixed! :white_check_mark: \n\n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}},
            {\"type\":\"divider\"}]}