feat(core): flow SLA #13237
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Main | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| tags: | |
| - v* | |
| pull_request: | |
| branches: | |
| - develop | |
| repository_dispatch: | |
| types: [ rebuild ] | |
| workflow_dispatch: | |
| inputs: | |
| skip-test: | |
| description: 'Skip test' | |
| type: choice | |
| required: true | |
| default: 'false' | |
| options: | |
| - "true" | |
| - "false" | |
| plugin-version: | |
| description: 'Plugin version' | |
| required: false | |
| type: string | |
| default: "LATEST" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| JAVA_VERSION: '21' | |
| DOCKER_APT_PACKAGES: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip | |
| DOCKER_PYTHON_LIBRARIES: kestra | |
| PLUGIN_VERSION: ${{ github.event.inputs.plugin-version != null && github.event.inputs.plugin-version || 'LATEST' }} | |
| jobs: | |
| build-artifacts: | |
| name: Build Artifacts | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| outputs: | |
| docker-tag: ${{ steps.vars.outputs.tag }} | |
| docker-artifact-name: ${{ steps.vars.outputs.artifact }} | |
| plugins: ${{ steps.plugins-list.outputs.plugins }} | |
| steps: | |
| # Checkout | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Checkout GitHub Actions | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: kestra-io/actions | |
| path: actions | |
| ref: main | |
| # Setup build | |
| - uses: ./actions/.github/actions/setup-build | |
| id: build | |
| with: | |
| java-enabled: true | |
| node-enabled: true | |
| caches-enabled: true | |
| # Get Plugins List | |
| - name: Get Plugins List | |
| uses: ./.github/actions/plugins-list | |
| if: "!startsWith(github.ref, 'refs/tags/v')" | |
| id: plugins-list | |
| with: | |
| plugin-version: ${{ env.PLUGIN_VERSION }} | |
| # Set Plugins List | |
| - name: Set Plugin List | |
| id: plugins | |
| if: "!startsWith(github.ref, 'refs/tags/v')" | |
| run: | | |
| PLUGINS="${{ steps.plugins-list.outputs.plugins }}" | |
| TAG=${GITHUB_REF#refs/*/} | |
| if [[ $TAG = "master" || $TAG == v* ]]; then | |
| echo "plugins=$PLUGINS" >> $GITHUB_OUTPUT | |
| else | |
| echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots $PLUGINS" >> $GITHUB_OUTPUT | |
| fi | |
| # Build | |
| - name: Build with Gradle | |
| run: | | |
| ./gradlew executableJar | |
| - name: Copy exe to image | |
| run: | | |
| cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra | |
| # Docker Tag | |
| - name: Set up Vars | |
| id: vars | |
| run: | | |
| TAG=${GITHUB_REF#refs/*/} | |
| if [[ $TAG = "master" ]] | |
| then | |
| TAG="latest"; | |
| elif [[ $TAG = "develop" ]] | |
| then | |
| TAG="develop"; | |
| elif [[ $TAG = v* ]] | |
| then | |
| TAG="${TAG}"; | |
| else | |
| TAG="build-${{ github.run_id }}"; | |
| fi | |
| echo "tag=${TAG}" >> $GITHUB_OUTPUT | |
| echo "artifact=docker-kestra-${TAG}" >> $GITHUB_OUTPUT | |
| # Docker setup | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| # Docker Build | |
| - name: Build & Export Docker Image | |
| uses: docker/build-push-action@v6 | |
| if: "!startsWith(github.ref, 'refs/tags/v')" | |
| with: | |
| context: . | |
| push: false | |
| file: Dockerfile | |
| tags: | | |
| kestra/kestra:${{ steps.vars.outputs.tag }} | |
| build-args: | | |
| KESTRA_PLUGINS=${{ steps.plugins.outputs.plugins }} | |
| APT_PACKAGES=${{ env.DOCKER_APT_PACKAGES }} | |
| PYTHON_LIBRARIES=${{ env.DOCKER_PYTHON_LIBRARIES }} | |
| outputs: type=docker,dest=/tmp/${{ steps.vars.outputs.artifact }}.tar | |
| # Upload artifacts | |
| - name: Upload JAR | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: jar | |
| path: build/libs/ | |
| - name: Upload Executable | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: exe | |
| path: build/executable/ | |
| - name: Upload Docker | |
| uses: actions/upload-artifact@v4 | |
| if: "!startsWith(github.ref, 'refs/tags/v')" | |
| with: | |
| name: ${{ steps.vars.outputs.artifact }} | |
| path: /tmp/${{ steps.vars.outputs.artifact }}.tar | |
| # Run Reusable Workflow from QA repository | |
| check-e2e: | |
| name: Check E2E Tests | |
| needs: build-artifacts | |
| if: ${{ (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') && !startsWith(github.ref, 'refs/tags/v') }} | |
| uses: ./.github/workflows/e2e.yml | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| backends: ["postgres"] | |
| with: | |
| tags: oss | |
| docker-artifact-name: ${{ needs.build-artifacts.outputs.docker-artifact-name }} | |
| docker-image-tag: kestra/kestra:${{ needs.build-artifacts.outputs.docker-tag }} | |
| backend: ${{ matrix.backends }} | |
| secrets: | |
| GITHUB_AUTH_TOKEN: ${{ secrets.GH_PERSONAL_TOKEN }} | |
| GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} | |
| check: | |
| env: | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} | |
| name: Check & Publish | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| # Checkout | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Checkout GitHub Actions | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: kestra-io/actions | |
| path: actions | |
| ref: main | |
| # Setup build | |
| - uses: ./actions/.github/actions/setup-build | |
| id: build | |
| with: | |
| java-enabled: true | |
| node-enabled: true | |
| python-enabled: true | |
| caches-enabled: true | |
| # Services | |
| - name: Build the docker-compose stack | |
| run: docker compose -f docker-compose-ci.yml up -d | |
| if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }} | |
| # Gradle check | |
| - name: Build with Gradle | |
| if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }} | |
| env: | |
| GOOGLE_SERVICE_ACCOUNT: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} | |
| run: | | |
| echo $GOOGLE_SERVICE_ACCOUNT | base64 -d > ~/.gcp-service-account.json | |
| export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.gcp-service-account.json | |
| ./gradlew check javadoc --parallel | |
| # Sonar | |
| - name: Analyze with Sonar | |
| if: ${{ env.SONAR_TOKEN != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} | |
| run: ./gradlew sonar --info | |
| # Allure check | |
| - name: Auth to Google Cloud | |
| id: auth | |
| if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 }} | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: '${{ secrets.GOOGLE_SERVICE_ACCOUNT }}' | |
| - uses: rlespinasse/github-slug-action@v5 | |
| - name: Publish allure report | |
| uses: andrcuns/[email protected] | |
| if: ${{ always() && env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} | |
| env: | |
| GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| JAVA_HOME: /usr/lib/jvm/default-jvm/ | |
| with: | |
| storageType: gcs | |
| resultsGlob: "**/build/allure-results" | |
| bucket: internal-kestra-host | |
| baseUrl: "https://internal.kestra.io" | |
| prefix: ${{ format('{0}/{1}', github.repository, 'allure/java') }} | |
| copyLatest: true | |
| ignoreMissingResults: true | |
| # Jacoco | |
| - name: 'Set up Cloud SDK' | |
| if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} | |
| uses: 'google-github-actions/setup-gcloud@v2' | |
| - name: 'Copy jacoco files' | |
| if: ${{ env.GOOGLE_SERVICE_ACCOUNT != 0 && (github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '') }} | |
| run: | | |
| mv build/reports/jacoco/testCodeCoverageReport build/reports/jacoco/test/ | |
| mv build/reports/jacoco/test/testCodeCoverageReport.xml build/reports/jacoco/test/jacocoTestReport.xml | |
| gsutil -m rsync -d -r build/reports/jacoco/test/ gs://internal-kestra-host/${{ format('{0}/{1}', github.repository, 'jacoco') }} | |
| # report test | |
| - name: Test Report | |
| uses: mikepenz/action-junit-report@v5 | |
| if: success() || failure() | |
| with: | |
| report_paths: '**/build/test-results/**/TEST-*.xml' | |
| # Codecov | |
| - uses: codecov/codecov-action@v5 | |
| if: ${{ github.event.inputs.skip-test == 'false' || github.event.inputs.skip-test == '' }} | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| release: | |
| name: Github Release | |
| runs-on: ubuntu-latest | |
| needs: [ check, check-e2e ] | |
| if: | | |
| always() && | |
| startsWith(github.ref, 'refs/tags/v') && | |
| needs.check.result == 'success' && | |
| (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success') | |
| steps: | |
| # Download Exec | |
| - name: Download executable | |
| uses: actions/download-artifact@v4 | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| with: | |
| name: exe | |
| path: build/executable | |
| # GitHub Release | |
| - name: Create GitHub release | |
| uses: "marvinpinto/action-automatic-releases@latest" | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| continue-on-error: true | |
| with: | |
| repo_token: "${{ secrets.GITHUB_TOKEN }}" | |
| prerelease: false | |
| files: | | |
| build/executable/* | |
| docker: | |
| name: Publish Docker | |
| runs-on: ubuntu-latest | |
| needs: [build-artifacts, check, check-e2e] | |
| if: github.ref == 'refs/heads/develop' | |
| strategy: | |
| matrix: | |
| image: | |
| - tag: ${{needs.build-artifacts.outputs.docker-tag}}-no-plugins | |
| packages: "" | |
| python-libraries: "" | |
| - tag: ${{needs.build-artifacts.outputs.docker-tag}} | |
| plugins: ${{needs.build-artifacts.outputs.plugins}} | |
| packages: python3 python3-venv python-is-python3 python3-pip nodejs npm curl zip unzip | |
| python-libraries: kestra | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # Docker setup | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| # Docker Login | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| # Vars | |
| - name: Set image name | |
| id: vars | |
| run: | | |
| TAG=${GITHUB_REF#refs/*/} | |
| if [[ $TAG = "master" || $TAG == v* ]]; then | |
| echo "plugins=${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "plugins=--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots ${{ matrix.image.plugins }}" >> $GITHUB_OUTPUT | |
| fi | |
| # Build Docker Image | |
| - name: Download executable | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: exe | |
| path: build/executable | |
| - name: Copy exe to image | |
| run: | | |
| cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra | |
| # Docker Build and push | |
| - name: Build Docker Image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| tags: kestra/kestra:${{ matrix.image.tag }} | |
| platforms: linux/amd64,linux/arm64 | |
| build-args: | | |
| KESTRA_PLUGINS=${{ steps.vars.outputs.plugins }} | |
| APT_PACKAGES=${{matrix.image.packages}} | |
| PYTHON_LIBRARIES=${{matrix.image.python-libraries}} | |
| maven: | |
| name: Publish to Maven | |
| runs-on: ubuntu-latest | |
| needs: [check, check-e2e] | |
| if: | | |
| always() && | |
| github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v') && | |
| needs.check.result == 'success' && | |
| (needs.check-e2e.result == 'skipped' || needs.check-e2e.result == 'success') | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # Checkout GitHub Actions | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: kestra-io/actions | |
| path: actions | |
| ref: main | |
| # Setup build | |
| - uses: ./actions/.github/actions/setup-build | |
| id: build | |
| with: | |
| java-enabled: true | |
| node-enabled: true | |
| caches-enabled: true | |
| # Publish | |
| - name: Publish package to Sonatype | |
| if: github.ref == 'refs/heads/develop' | |
| env: | |
| ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }} | |
| ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }} | |
| SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }} | |
| SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }} | |
| SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }} | |
| run: | | |
| mkdir -p ~/.gradle/ | |
| echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties | |
| echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties | |
| echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties | |
| echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg | |
| ./gradlew publishToSonatype | |
| # Release | |
| - name: Release package to Maven Central | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| env: | |
| ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }} | |
| ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }} | |
| SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }} | |
| SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }} | |
| SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }} | |
| run: | | |
| echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties | |
| echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties | |
| echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties | |
| echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg | |
| ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository | |
| end: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - check-e2e | |
| - check | |
| - maven | |
| - docker | |
| - release | |
| if: always() | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| steps: | |
| # Update | |
| - name: Update internal | |
| uses: benc-uk/workflow-dispatch@v1 | |
| if: github.ref == 'refs/heads/develop' && needs.docker.result == 'success' | |
| with: | |
| workflow: oss-build.yml | |
| repo: kestra-io/infra | |
| ref: master | |
| token: ${{ secrets.GH_PERSONAL_TOKEN }} | |
| # Slack | |
| - name: Slack notification | |
| uses: Gamesight/slack-workflow-status@master | |
| if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }} | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| name: GitHub Actions | |
| icon_emoji: ':github-actions:' | |
| channel: 'C02DQ1A7JLR' # _int_git channel |