Bump helm chart version reference to 0.58.5 #4996
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Airbyte Platform CI | |
| env: | |
| S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| on: | |
| #ability to start task manually in Web UI | |
| workflow_dispatch: | |
| inputs: | |
| debug_mode: | |
| description: "Enable or disable tmate session for debug during helm ac tests" | |
| type: choice | |
| default: "false" | |
| options: | |
| - "true" | |
| - "false" | |
| required: false | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| permissions: write-all | |
| jobs: | |
| # COMMON TASKS | |
| ensure-images-exist: | |
| name: "Ensure all required Docker images exist on Dockerhub" | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check images exist | |
| run: ./tools/bin/check_images_exist.sh all | |
| # The output of this job is used to trigger the following builds. | |
| changes: | |
| name: "Detect Modified Files" | |
| # The filtering action does not deal with well scheduled events so skip to avoid errors. | |
| # See https://github.com/dorny/paths-filter/issues/100 for more info. | |
| # This is okay this workflow is only scheduled on main, where we want to build everything | |
| # so filtering is not required. Use always() in each start block to force the start task. | |
| if: github.event_name != 'schedule' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| backend: ${{ steps.filter.outputs.backend }} | |
| build: ${{ steps.filter.outputs.build }} | |
| cli: ${{ steps.filter.outputs.cli }} | |
| db: ${{ steps.filter.outputs.db }} | |
| frontend: ${{ steps.filter.outputs.frontend }} | |
| frontendServers: ${{ steps.changes.outputs.frontendServers }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| # Note, the following glob expression within a filters are ORs. | |
| filters: | | |
| backend: | |
| - 'airbyte-!(cdk|integrations|webapp)/**' | |
| - 'airbyte-integrations/connectors/(destination-jdbc|destination-postgres|source-jdbc|source-postgres)/**' | |
| - 'airbyte-config/init/src/main/resources/seed/(source|destination)_definitions.yaml' | |
| - 'docker-compose*.yaml' | |
| - '(charts|kube)/**' | |
| build: | |
| - '.github/**' | |
| - 'buildSrc/**' | |
| - 'tools/**' | |
| - '*.gradle' | |
| - 'deps.toml' | |
| cli: | |
| - 'airbyte-api/**' | |
| db: | |
| - 'airbyte-db/**' | |
| frontend: | |
| - 'airbyte-api/src/main/openapi/config.yaml' | |
| - 'oss/airbyte-api/src/main/openapi/cloud-config.yaml' | |
| - 'airbyte-connector-builder-resources/CDK_VERSION' | |
| - 'airbyte-connector-builder-server/src/main/openapi/openapi.yaml' | |
| - 'airbyte-webapp/**' | |
| frontendServers: | |
| - 'airbyte-(server|data|connector-builder-server|commons|commons-!(worker|temporal*))/**' | |
| # Uncomment to debug. | |
| # changes-output: | |
| # name: "Debug Change Detection Logic" | |
| # needs: changes | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # - run: | | |
| # echo '${{ toJSON(needs) }}' | |
| ## BUILDS | |
| ## Frontend Test | |
| # In case of self-hosted EC2 errors, remove this block. | |
| start-frontend-runner: | |
| name: "Frontend: Start EC2 Runner" | |
| needs: | |
| - changes | |
| # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on main. | |
| if: | | |
| needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/main' | |
| || (always() && needs.changes.outputs.backend == 'true') | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Start AWS Runner | |
| id: start-ec2-runner | |
| uses: ./.github/actions/start-aws-runner | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| github-token: ${{ env.PAT }} | |
| frontend-build: | |
| name: "Frontend: Build" | |
| needs: | |
| - start-frontend-runner | |
| runs-on: ${{ needs.start-frontend-runner.outputs.label }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| # We need to fetch at least one more commmit for the Chromatic action not to fail | |
| # but since we don't do screenshot comparison we don't need to fetch the full history. | |
| with: | |
| fetch-depth: 2 | |
| - name: Cache Build Artifacts | |
| uses: ./.github/actions/cache-build-artifacts | |
| with: | |
| cache-key: ${{ secrets.CACHE_VERSION }} | |
| cache-python: "false" | |
| - uses: actions/setup-java@v3 | |
| with: | |
| distribution: "zulu" | |
| java-version: "21" | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - name: Set up CI Gradle Properties | |
| run: | | |
| mkdir -p ~/.gradle/ | |
| cat > ~/.gradle/gradle.properties <<EOF | |
| org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED | |
| org.gradle.workers.max=8 | |
| org.gradle.vfs.watch=false | |
| EOF | |
| - name: Build :airbyte-webapp | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: ./gradlew --no-daemon :airbyte-webapp:build --scan | |
| attempt_limit: 3 | |
| attempt_delay: 5000 # in ms | |
| - name: Publish Storybook to Chromatic | |
| uses: chromaui/action@v1 | |
| with: | |
| projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| workingDir: ./airbyte-webapp/ | |
| storybookBuildDir: build/storybook/ | |
| autoAcceptChanges: true | |
| exitOnceUploaded: true | |
| # In case of self-hosted EC2 errors, remove this block. | |
| stop-frontend-runner: | |
| name: "Frontend: Stop Runner" | |
| timeout-minutes: 10 | |
| needs: | |
| - start-frontend-runner # required to get output from the start-runner job | |
| - frontend-build # required to wait when then build job is done | |
| runs-on: ubuntu-latest | |
| # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. | |
| # Thus, we check for skipped here. | |
| if: ${{ always() && needs.start-frontend-runner.result != 'skipped'}} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Stop EC2 runner | |
| uses: airbytehq/[email protected] | |
| with: | |
| mode: stop | |
| github-token: ${{ env.PAT }} | |
| label: ${{ needs.start-frontend-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-frontend-runner.outputs.ec2-instance-id }} | |
| ## Frontend Test | |
| # In case of self-hosted EC2 errors, remove this block. | |
| start-frontend-e2e-runner: | |
| name: "Frontend E2E Tests: Start EC2 Runner" | |
| needs: | |
| - changes | |
| # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on master. | |
| if: | | |
| needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/main' | |
| || (always() && needs.changes.outputs.frontendServers == 'true') | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Start AWS Runner | |
| id: start-ec2-runner | |
| uses: ./.github/actions/start-aws-runner | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| github-token: ${{ env.PAT }} | |
| frontend-e2e-test: | |
| name: "Frontend: Run End-to-End Tests" | |
| needs: | |
| - start-frontend-e2e-runner # required to have runner started | |
| runs-on: ${{ needs.start-frontend-e2e-runner.outputs.label }} # run the job on the newly created runner | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Cache Build Artifacts | |
| uses: ./.github/actions/cache-build-artifacts | |
| with: | |
| cache-key: ${{ secrets.CACHE_VERSION }} | |
| cache-python: "false" | |
| - uses: actions/setup-java@v3 | |
| with: | |
| distribution: "zulu" | |
| java-version: "21" | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - name: Set up CI Gradle Properties | |
| run: | | |
| mkdir -p ~/.gradle/ | |
| cat > ~/.gradle/gradle.properties <<EOF | |
| org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED | |
| org.gradle.workers.max=8 | |
| org.gradle.vfs.watch=false | |
| EOF | |
| - name: Build Platform Docker Images | |
| run: ./gradlew --no-daemon assemble --scan | |
| - name: Run End-to-End Frontend Tests | |
| env: | |
| CYPRESS_WEBAPP_KEY: ${{ secrets.CYPRESS_WEBAPP_KEY }} | |
| CYPRESS_CACHE_FOLDER: "~/.cache/Cypress" | |
| run: ./tools/bin/e2e_test.sh | |
| # In case of self-hosted EC2 errors, remove this block. | |
| stop-frontend-e2e-runner: | |
| name: "Frontend E2E Tests: Stop Runner" | |
| timeout-minutes: 10 | |
| needs: | |
| - start-frontend-e2e-runner # required to get output from the start-runner job | |
| - frontend-e2e-test # required to wait when the e2e-test job is done | |
| runs-on: ubuntu-latest | |
| # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. | |
| # Thus, we check for skipped here. | |
| if: ${{ always() && needs.start-frontend-e2e-runner.result != 'skipped'}} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Stop EC2 runner | |
| uses: airbytehq/[email protected] | |
| with: | |
| mode: stop | |
| github-token: ${{ env.PAT }} | |
| label: ${{ needs.start-frontend-e2e-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-frontend-e2e-runner.outputs.ec2-instance-id }} | |
| ## FOLLOWING BUILDS ARE ALL PLATFORM BUILDS. | |
| # Main Platform | |
| # In case of self-hosted EC2 errors, remove this block. | |
| start-platform-build-runner: | |
| name: "Platform: Start Build EC2 Runner" | |
| needs: | |
| - changes | |
| # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on main. | |
| if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/main') | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Start AWS Runner | |
| id: start-ec2-runner | |
| uses: ./.github/actions/start-aws-runner | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| github-token: ${{ env.PAT }} | |
| platform-build: | |
| name: "Platform: Build" | |
| # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. | |
| needs: start-platform-build-runner # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Cache Build Artifacts | |
| uses: ./.github/actions/cache-build-artifacts | |
| with: | |
| cache-key: ${{ secrets.CACHE_VERSION }} | |
| cache-python: "false" | |
| - uses: actions/setup-java@v3 | |
| with: | |
| distribution: "zulu" | |
| java-version: "21" | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: "lts/*" | |
| - name: Set up CI Gradle Properties | |
| run: | | |
| mkdir -p ~/.gradle/ | |
| cat > ~/.gradle/gradle.properties <<EOF | |
| org.gradle.jvmargs=-Xmx8g -Xss4m \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED | |
| org.gradle.workers.max=8 | |
| org.gradle.vfs.watch=false | |
| EOF | |
| - name: Format | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: ./gradlew format --scan --info --stacktrace | |
| attempt_limit: 3 | |
| attempt_delay: 5000 # in ms | |
| - name: Ensure no file change | |
| run: git --no-pager diff && test -z "$(git --no-pager diff)" | |
| - name: Build | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: ./gradlew build javadoc --scan | |
| attempt_limit: 3 | |
| attempt_delay: 5000 # in ms | |
| - name: Add coverage to PR | |
| if: github.event_name == 'pull_request' | |
| id: jacoco | |
| uses: madrapps/[email protected] | |
| with: | |
| # todo(mfsiega-airbyte): when v1.4 is released, use wildcards per https://github.com/Madrapps/jacoco-report/issues/20. | |
| paths: | | |
| ${{ github.workspace }}/airbyte-analytics/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-api/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-bootloader/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-commons/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-commons-protocol/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-commons-temporal/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-commons-worker/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-config/init/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-config/specs/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-config/config-persistence/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-config/config-models/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-container-orchestrator/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-db/db-lib/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-json-validation/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-metrics/reporter/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-metrics/metrics-lib/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-notification/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-oauth/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-persistence/job-persistence/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-server/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-test-utils/build/reports/jacoco/test/jacocoTestReport.xml, | |
| ${{ github.workspace }}/airbyte-workers/build/reports/jacoco/test/jacocoTestReport.xml | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| min-coverage-overall: 25 | |
| min-coverage-changed-files: 60 | |
| title: Airbyte Code Coverage | |
| update-comment: true | |
| debug-mode: false | |
| - name: Test if Seed spec is updated | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: ./gradlew format && git --no-pager diff && test -z "$(git --no-pager diff)" | |
| attempt_limit: 3 | |
| attempt_delay: 5000 # in ms | |
| # todo (cgardens) - scope by platform. | |
| - name: Check documentation | |
| if: success() && github.ref == 'refs/heads/main' | |
| run: ./tools/site/link_checker.sh check_docs | |
| # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use. | |
| # - name: Get Docker Space | |
| # run: docker run --rm busybox df -h | |
| - name: Run End-to-End Acceptance Tests | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: ./tools/bin/acceptance_test.sh | |
| attempt_limit: 3 | |
| attempt_delay: 5000 # in ms | |
| - uses: actions/setup-python@v4 | |
| if: always() | |
| with: | |
| python-version: "3.9" | |
| - name: Generate Test Report | |
| uses: dorny/test-reporter@v1 | |
| if: always() # run this step even if previous step failed | |
| with: | |
| name: Platform Test Report with Docker E2E Test | |
| # Specify top-level and second-level modules. Note there cannot be a space between the comma. | |
| path: "/actions-runner/_work/airbyte-platform/airbyte-platform/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte-platform/airbyte-platform/*/*/build/test-results/*/*.xml" | |
| reporter: java-junit | |
| fail-on-error: "false" | |
| env: | |
| NODE_OPTIONS: '--max_old_space_size=8192' | |
| # In case of self-hosted EC2 errors, remove this block. | |
| stop-platform-build-runner: | |
| name: "Platform: Stop Build EC2 Runner" | |
| timeout-minutes: 10 | |
| needs: | |
| - start-platform-build-runner # required to get output from the start-runner job | |
| - platform-build # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. | |
| # Thus, we check for skipped here. | |
| if: ${{ always() && needs.start-platform-build-runner.result != 'skipped'}} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Stop EC2 runner | |
| uses: airbytehq/[email protected] | |
| with: | |
| mode: stop | |
| github-token: ${{ env.PAT }} | |
| label: ${{ needs.start-platform-build-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }} | |
| # ## Helm Acceptance Tests | |
| # # Docker acceptance tests run as part of the build job. | |
| # # In case of self-hosted EC2 errors, remove this block. | |
| start-helm-acceptance-test-runner: | |
| name: "Platform: Start Helm Acceptance Test Runner" | |
| needs: | |
| - changes | |
| # Because scheduled builds on main require us to skip the changes job. Use always() to force this to run on main. | |
| if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/main') | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| outputs: | |
| label: ${{ steps.start-ec2-runner.outputs.label }} | |
| ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Start AWS Runner | |
| id: start-ec2-runner | |
| uses: ./.github/actions/start-aws-runner | |
| with: | |
| # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat | |
| # ec2-image-id: ami-0c1a9bc22624339d8 | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| github-token: ${{ env.PAT }} | |
| iam-role-name: ${{ secrets.AWS_ASSUME_ROLE_NAME }} | |
| helm-acceptance-test: | |
| name: "Platform: Acceptance Tests (Helm)" | |
| # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. | |
| needs: [start-helm-acceptance-test-runner] # required to start the main job when the runner is ready | |
| runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner | |
| # this is the label of the runner | |
| timeout-minutes: 90 | |
| defaults: | |
| run: | |
| working-directory: "/actions-runner/_work/airbyte-platform/airbyte-platform" | |
| env: | |
| HOME: /actions-runner/_work/airbyte-platform/airbyte-platform | |
| GITHUB_WORKSPACE: /actions-runner/_work/airbyte-platform/airbyte-platform | |
| steps: | |
| - name: Fix EC-2 Runner | |
| run: | | |
| mkdir -p /actions-runner/_work/airbyte-platform/airbyte-platform && mkdir -p /actions-runner/_work/airbyte-platform/airbyte-platform/.kube | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Delete default old docker and replace it with a new one | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: | | |
| sudo apt-get remove docker.io || sudo apt-get remove docker | |
| curl -fsSL https://get.docker.com | bash - | |
| attempt_limit: 3 | |
| attempt_delay: 2000 # in ms | |
| - name: Cache Build Artifacts | |
| uses: ./.github/actions/cache-build-artifacts | |
| with: | |
| cache-key: ${{ secrets.CACHE_VERSION }} | |
| cache-python: "false" | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - uses: actions/setup-java@v1 | |
| with: | |
| java-version: "21" | |
| - name: Install unzip | |
| shell: bash | |
| run: sudo apt-get update && sudo apt-get install -y unzip | |
| - uses: azure/setup-helm@v3 | |
| with: | |
| version: "latest" | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| id: install | |
| - uses: azure/setup-kubectl@v3 | |
| with: | |
| version: "v1.21.2" # default is latest stable | |
| - name: Install tmate and it's dependencies | |
| if: inputs.debug_mode == 'true' | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: | | |
| sudo apt-get -y install tmate | |
| attempt_limit: 3 | |
| attempt_delay: 2000 # in ms | |
| - name: Start tmate session in background | |
| if: inputs.debug_mode == 'true' | |
| shell: bash | |
| run: | | |
| tmate -S /tmp/tmate.sock new-session -d # Launch tmate in a headless mode | |
| tmate -S /tmp/tmate.sock wait tmate-ready # Blocks until the SSH connection is established | |
| tmate -S /tmp/tmate.sock display -p '#{tmate_ssh}' # Prints the SSH connection string | |
| tmate -S /tmp/tmate.sock display -p '#{tmate_ssh_ro}' # Prints the read-only SSH connection string | |
| tmate -S /tmp/tmate.sock display -p '#{tmate_web}' # Prints the web connection string | |
| tmate -S /tmp/tmate.sock display -p '#{tmate_web_ro}' # Prints the read-only web connection string | |
| - name: Start a local k8s cluster | |
| uses: jupyterhub/action-k3s-helm@v3 | |
| with: | |
| k3s-version: v1.21.2+k3s1 | |
| helm-version: v3.9.4 | |
| extra-setup-args: --docker --kube-apiserver-arg service-node-port-range=6000-32767 --kubelet-arg=cgroup-driver=systemd | |
| - name: Try the cluster ! | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: kubectl get pods -A | |
| attempt_limit: 3 | |
| attempt_delay: 3000 # in ms | |
| - name: Set up CI Gradle Properties | |
| run: | | |
| mkdir -p ~/.gradle/ | |
| cat > ~/.gradle/gradle.properties <<EOF | |
| org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \ | |
| --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED | |
| org.gradle.workers.max=8 | |
| org.gradle.vfs.watch=false | |
| EOF | |
| - name: Build Platform Docker Images | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: pwd && ./gradlew assemble -x test --scan | |
| attempt_limit: 3 | |
| attempt_delay: 5000 # in ms | |
| - name: Run Helm End-to-End Acceptance Tests | |
| env: | |
| # USER: root | |
| # HOME: /home/runner | |
| # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS | |
| AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} | |
| WORKFLOW_RUN_ID: ${{ github.run_id }} | |
| AWS_S3_BUCKET: airbyte-ci-ac-tests-logs | |
| SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} | |
| SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} | |
| timeout-minutes: 40 | |
| uses: Wandalen/[email protected] | |
| with: | |
| command: CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh | |
| attempt_limit: 3 | |
| attempt_delay: 5000 # in ms | |
| - name: Generate Test Report | |
| uses: dorny/test-reporter@v1 | |
| if: always() # run this step even if previous step failed | |
| with: | |
| name: Platform Helm E2E Test Report | |
| path: "./*/build/test-results/*/*.xml" | |
| reporter: java-junit | |
| fail-on-error: "false" | |
| env: | |
| NODE_OPTIONS: '--max_old_space_size=8192' | |
| - name: "Display logs of k3s" | |
| if: failure() | |
| shell: bash | |
| run: | | |
| journalctl -xeu k3s.service | |
| # # In case of self-hosted EC2 errors, remove this block. | |
| stop-helm-acceptance-test-runner: | |
| name: "Platform: Stop Helm Acceptance Test EC2 Runner" | |
| timeout-minutes: 10 | |
| needs: | |
| - start-helm-acceptance-test-runner # required to get output from the start-runner job | |
| - helm-acceptance-test # required to wait when the main job is done | |
| runs-on: ubuntu-latest | |
| # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped. | |
| # Thus, we check for skipped here. | |
| if: ${{ always() && needs.start-helm-acceptance-test-runner.result != 'skipped'}} | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-2 | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ | |
| ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} | |
| - name: Stop EC2 runner | |
| uses: airbytehq/[email protected] | |
| with: | |
| mode: stop | |
| github-token: ${{ env.PAT }} | |
| label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} | |
| ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }} | |
| notify-failure-slack-channel: | |
| name: "Notify Slack Channel on Build Failures" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - frontend-build | |
| - platform-build | |
| # Todo: Kyryl turn this on. | |
| # - helm-acceptance-test | |
| if: ${{ failure() && github.ref == 'refs/heads/main' }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: Match GitHub User to Slack User | |
| id: match-github-to-slack-user | |
| uses: ./.github/actions/match-github-to-slack-user | |
| env: | |
| AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} | |
| GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Publish to OSS Build Failure Slack Channel | |
| uses: abinoda/slack-action@master | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} | |
| with: | |
| args: >- | |
| {\"channel\":\"C03BEADRPNY\", \"blocks\":[ | |
| {\"type\":\"divider\"}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS main failed! :bangbang: \n\n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}}, | |
| {\"type\":\"divider\"}]} | |
| notify-failure-slack-channel-fixed-broken-build: | |
| name: "Notify Slack Channel on Build Fixes" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - frontend-build | |
| - platform-build | |
| # Todo: Kyryl turn this on. | |
| # - helm-acceptance-test | |
| if: ${{ success() && github.ref == 'refs/heads/main' }} | |
| steps: | |
| - name: Get Previous Workflow Status | |
| uses: Mercymeilya/[email protected] | |
| id: last_status | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed. | |
| - name: Publish Build Fixed Message to OSS Build Failure Slack Channel | |
| if: ${{ steps.last_status.outputs.last_status == 'failure' }} | |
| uses: abinoda/slack-action@master | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} | |
| with: | |
| args: >- | |
| {\"channel\":\"C03BEADRPNY\", \"blocks\":[ | |
| {\"type\":\"divider\"}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Main Fixed! :white_check_mark: \n\n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, | |
| {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}}, | |
| {\"type\":\"divider\"}]} |