From 8ccc61d5b71e4572d7187c327a7d916e3326583d Mon Sep 17 00:00:00 2001 From: ytimocin Date: Tue, 25 Jun 2024 00:28:56 +0300 Subject: [PATCH] Splitting functional tests as cloud and non-cloud Signed-off-by: ytimocin --- ...l-test.yaml => functional-test-cloud.yaml} | 78 +- .../workflows/functional-test-noncloud.yaml | 693 ++++++++++++++++++ build/test.mk | 12 +- .../running-functional-tests.md | 43 +- test/executeFunctionalTest.sh | 22 +- 5 files changed, 773 insertions(+), 75 deletions(-) rename .github/workflows/{functional-test.yaml => functional-test-cloud.yaml} (95%) create mode 100644 .github/workflows/functional-test-noncloud.yaml diff --git a/.github/workflows/functional-test.yaml b/.github/workflows/functional-test-cloud.yaml similarity index 95% rename from .github/workflows/functional-test.yaml rename to .github/workflows/functional-test-cloud.yaml index bcf12e2b914..0cabaffb130 100644 --- a/.github/workflows/functional-test.yaml +++ b/.github/workflows/functional-test-cloud.yaml @@ -4,7 +4,7 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software @@ -14,11 +14,11 @@ # limitations under the License. # ------------------------------------------------------------ -name: Functional tests +name: Functional tests - Cloud permissions: id-token: write # Required for requesting the JWT - contents: read # Required for listing the commits + contents: read # Required for listing the commits packages: write # Required for uploading the package on: @@ -33,31 +33,31 @@ on: repository_dispatch: types: [de-functional-test] workflow_run: - workflows: ['Approve Functional Tests'] + workflows: ["Approve Functional Tests"] types: - completed env: # Go version - GOVER: '1.22.2' + GOVER: "1.22.2" GOPROXY: https://proxy.golang.org # gotestsum version - see: https://github.com/gotestyourself/gotestsum GOTESTSUM_VER: 1.10.0 # Helm version - HELM_VER: 'v3.12.0' + HELM_VER: "v3.12.0" # KinD cluster version - KIND_VER: 'v0.20.0' + KIND_VER: "v0.20.0" # Dapr version - DAPR_VER: '1.12.0' - DAPR_DASHBOARD_VER: '0.14.0' + DAPR_VER: "1.12.0" + DAPR_DASHBOARD_VER: "0.14.0" # Kubectl version - KUBECTL_VER: 'v1.25.0' + KUBECTL_VER: "v1.25.0" # Azure Keyvault CSI driver chart version - AZURE_KEYVAULT_CSI_DRIVER_VER: '1.4.2' + AZURE_KEYVAULT_CSI_DRIVER_VER: "1.4.2" # Azure workload identity webhook chart version - AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER: '1.1.0' + AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER: "1.1.0" # Container registry for storing container images CONTAINER_REGISTRY: ghcr.io/radius-project/dev # Container registry for storing Bicep recipe artifacts @@ -71,11 +71,11 @@ env: # The Radius helm chart location. RADIUS_CHART_LOCATION: deploy/Chart/ # The region for AWS resources - AWS_REGION: 'us-west-2' + AWS_REGION: "us-west-2" # The AWS account ID - AWS_ACCOUNT_ID: '${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}' + AWS_ACCOUNT_ID: "${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}" # The current GitHub action link - ACTION_LINK: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}' + ACTION_LINK: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" # Server where terraform test modules are deployed TF_RECIPE_MODULE_SERVER_URL: "http://tf-module-server.radius-test-tf-module-server.svc.cluster.local" # The functional test GitHub app id @@ -91,8 +91,8 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'repository_dispatch' || (github.event_name == 'schedule' && github.repository == 'radius-project/radius') || github.event_name == 'workflow_run' env: - DE_IMAGE: 'ghcr.io/radius-project/deployment-engine' - DE_TAG: 'latest' + DE_IMAGE: "ghcr.io/radius-project/deployment-engine" + DE_TAG: "latest" outputs: REL_VERSION: ${{ steps.gen-id.outputs.REL_VERSION }} UNIQUE_ID: ${{ steps.gen-id.outputs.UNIQUE_ID }} @@ -106,7 +106,7 @@ jobs: - name: Get GitHub app token uses: tibdex/github-app-token@v2 id: get_installation_token - with: + with: app_id: ${{ env.FUNCTIONAL_TEST_APP_ID }} private_key: ${{ secrets.FUNCTIONAL_TEST_APP_PRIVATE_KEY }} - name: Set up checkout target (scheduled) @@ -117,16 +117,16 @@ jobs: - name: Set up checkout target (pull_request) if: github.event_name == 'pull_request' run: | - echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV - echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV - echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV + echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV + echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV + echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV - name: Use custom actions uses: actions/checkout@v4 - - name: 'Download PR data artifacts' + - name: "Download PR data artifacts" if: github.event_name == 'workflow_run' uses: ./.github/actions/download-pr-data-artifact id: get-pr-number - - name: 'Set PR context (workflow_run)' + - name: "Set PR context (workflow_run)" if: github.event_name == 'workflow_run' uses: actions/github-script@v7 with: @@ -175,7 +175,7 @@ jobs: fi UNIQUE_ID=func$(echo $BASE_STR | sha1sum | head -c 10) echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_ENV - + # Set output variables to be used in the other jobs echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_OUTPUT echo "UNIQUE_ID=${UNIQUE_ID}" >> $GITHUB_OUTPUT @@ -194,7 +194,7 @@ jobs: header: teststatus-${{ github.run_id }} number: ${{ env.PR_NUMBER }} hide: true - hide_classify: 'OUTDATED' + hide_classify: "OUTDATED" message: | ## Radius functional test overview @@ -223,7 +223,7 @@ jobs: * deployment-engine test image location: `${{ env.DE_IMAGE }}:${{ env.DE_TAG }}` - + ## Test Status - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -324,11 +324,11 @@ jobs: fail-fast: true matrix: os: [ubuntu-latest] - name: [ucp,kubernetes,shared,msgrp,daprrp,samples,cli] + name: [corerp-cloud, ucp-cloud] include: # datastorerp functional tests need the larger VM. - os: ubuntu-latest-m - name: datastoresrp + name: datastoresrp-cloud runs-on: ${{ matrix.os }} env: UNIQUE_ID: ${{ needs.build.outputs.UNIQUE_ID }} @@ -345,14 +345,14 @@ jobs: - name: Get GitHub app token uses: tibdex/github-app-token@v2 id: get_installation_token - with: + with: app_id: ${{ env.FUNCTIONAL_TEST_APP_ID }} private_key: ${{ secrets.FUNCTIONAL_TEST_APP_PRIVATE_KEY }} - uses: LouisBrunner/checks-action@v2.0.0 if: always() with: token: ${{ steps.get_installation_token.outputs.token }} - name: 'Functional Test Run' + name: "Functional Test Run" status: in_progress repo: ${{ github.repository }} sha: ${{ env.CHECKOUT_REF }} @@ -577,15 +577,15 @@ jobs: BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }} BICEP_RECIPE_TAG_VERSION: ${{ env.BICEP_RECIPE_TAG_VERSION }} GH_TOKEN: ${{ steps.get_installation_token.outputs.token }} - GOTESTSUM_OPTS: '--junitfile ./dist/functional_test/results.xml' + GOTESTSUM_OPTS: "--junitfile ./dist/functional_test/results.xml" - name: Process Functional Test Results uses: ./.github/actions/process-test-results # In case of failure, upload functional_test_results to artifacts so that they are not erased by subsequent runs. if: failure() && github.repository == 'radius-project/radius' with: - test_group_name: 'Functional Tests - ${{ matrix.name }}' - artifact_name: 'functional_test_results_${{ matrix.name }}' - result_directory: 'dist/functional_test/' + test_group_name: "Functional Tests - ${{ matrix.name }}" + artifact_name: "functional_test_results_${{ matrix.name }}" + result_directory: "dist/functional_test/" - uses: azure/setup-kubectl@v4 if: always() with: @@ -620,7 +620,7 @@ jobs: echo "Pod logs saved to recipes/pod-logs/" # Get kubernetes events and save to file kubectl get events -n $namespace > recipes/pod-logs/events.txt - - name: Upload Terraform recipe publishing logs + - name: Upload Terraform recipe publishing logs uses: actions/upload-artifact@v4 if: always() with: @@ -676,7 +676,7 @@ jobs: - name: Get GitHub app token uses: tibdex/github-app-token@v2 id: get_installation_token - with: + with: app_id: ${{ env.FUNCTIONAL_TEST_APP_ID }} private_key: ${{ secrets.FUNCTIONAL_TEST_APP_PRIVATE_KEY }} - name: Get tests job status @@ -703,7 +703,7 @@ jobs: if: always() with: token: ${{ steps.get_installation_token.outputs.token }} - name: 'Functional Test Run' + name: "Functional Test Run" repo: ${{ github.repository }} sha: ${{ env.CHECKOUT_REF }} status: completed @@ -729,7 +729,7 @@ jobs: event: 'schedule', per_page: 10 }); - + failureCount = 1; for (const run of response.data.workflow_runs) { if (run.conclusion === 'failure') { @@ -751,4 +751,4 @@ jobs: title: `Scheduled functional test failed - Run ID: ${context.runId}`, labels: ['bug', 'test-failure'], body: `## Bug information \n\nThis bug is generated automatically if the scheduled functional test fails at least ${process.env.ISSUE_CREATE_THRESHOLD} times in a row. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).` - }) \ No newline at end of file + }) diff --git a/.github/workflows/functional-test-noncloud.yaml b/.github/workflows/functional-test-noncloud.yaml new file mode 100644 index 00000000000..c18ba4beea6 --- /dev/null +++ b/.github/workflows/functional-test-noncloud.yaml @@ -0,0 +1,693 @@ +# ------------------------------------------------------------ +# Copyright 2023 The Radius Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------ + +name: Functional tests - Non-Cloud + +permissions: + id-token: write # Required for requesting the JWT + contents: read # Required for listing the commits + packages: write # Required for uploading the package + +on: + # Enable manual trigger + workflow_dispatch: + schedule: + # Run every 4 hours on weekdays. + - cron: "30 0,4,8,12,16,20 * * 1-5" + # Run every 12 hours on weekends. + - cron: "30 0,12 * * 0,6" + # Dispatch on external events + repository_dispatch: + types: [de-functional-test] + push: + branches: + - main + - release/* + tags: + - v* + pull_request: + branches: + - main + - features/* + - release/* + +env: + # Go version + GOVER: "1.22.2" + # Go proxy + GOPROXY: https://proxy.golang.org + # gotestsum version - see: https://github.com/gotestyourself/gotestsum + GOTESTSUM_VER: 1.10.0 + # Helm version + HELM_VER: "v3.12.0" + # KinD cluster version + KIND_VER: "v0.20.0" + # Dapr version + DAPR_VER: "1.12.0" + # Dapr dashboard version + DAPR_DASHBOARD_VER: "0.14.0" + # Kubectl version + KUBECTL_VER: "v1.25.0" + # Azure Keyvault CSI driver chart version + AZURE_KEYVAULT_CSI_DRIVER_VER: "1.4.2" + # Azure workload identity webhook chart version + AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER: "1.1.0" + # Container registry for storing container images + CONTAINER_REGISTRY: ghcr.io/radius-project/dev + # Container registry for storing Bicep recipe artifacts + BICEP_RECIPE_REGISTRY: ghcr.io/radius-project/dev + # The radius functional test timeout + FUNCTIONALTEST_TIMEOUT: 30m + # The base directory for storing test logs + RADIUS_CONTAINER_LOG_BASE: dist/container_logs + # The Radius helm chart location. + RADIUS_CHART_LOCATION: deploy/Chart/ + # The current GitHub action link + ACTION_LINK: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + # Server where terraform test modules are deployed + TF_RECIPE_MODULE_SERVER_URL: "http://tf-module-server.radius-test-tf-module-server.svc.cluster.local" + # The functional test GitHub app id + FUNCTIONAL_TEST_APP_ID: 425843 + # Private Git repository where terraform module for testing is stored. + TF_RECIPE_PRIVATE_GIT_SOURCE: "git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis" + # The number of failed tests to report. + ISSUE_CREATE_THRESHOLD: 2 + +jobs: + build: + name: Build Radius for test + runs-on: ubuntu-latest + env: + DE_IMAGE: "ghcr.io/radius-project/deployment-engine" + DE_TAG: "latest" + outputs: + REL_VERSION: ${{ steps.gen-id.outputs.REL_VERSION }} + UNIQUE_ID: ${{ steps.gen-id.outputs.UNIQUE_ID }} + PR_NUMBER: ${{ steps.gen-id.outputs.PR_NUMBER }} + CHECKOUT_REPO: ${{ steps.gen-id.outputs.CHECKOUT_REPO }} + CHECKOUT_REF: ${{ steps.gen-id.outputs.CHECKOUT_REF }} + RAD_CLI_ARTIFACT_NAME: ${{ steps.gen-id.outputs.RAD_CLI_ARTIFACT_NAME }} + DE_IMAGE: ${{ steps.gen-id.outputs.DE_IMAGE }} + DE_TAG: ${{ steps.gen-id.outputs.DE_TAG }} + steps: + - name: Get GitHub app token + uses: tibdex/github-app-token@v2 + id: get_installation_token + with: + app_id: ${{ env.FUNCTIONAL_TEST_APP_ID }} + private_key: ${{ secrets.FUNCTIONAL_TEST_APP_PRIVATE_KEY }} + - name: Set up checkout target (scheduled) + if: github.event_name == 'schedule' + run: | + echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV + echo "CHECKOUT_REF=refs/heads/main" >> $GITHUB_ENV + - name: Set up checkout target (pull_request) + if: github.event_name == 'pull_request' + run: | + echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV + echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV + echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV + - name: Use custom actions + uses: actions/checkout@v4 + - name: Set up checkout target (pull_request) + if: github.event_name == 'pull_request' + run: | + echo "CHECKOUT_REPO=${{ github.event.pull_request.head.repo.full_name }}" >> $GITHUB_ENV + echo "CHECKOUT_REF=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV + - name: Set DE image and tag (repository_dispatch from de-functional-test) + if: github.event_name == 'repository_dispatch' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }} + script: | + const clientPayload = context.payload.client_payload; + if (clientPayload && clientPayload.event_type === `de-functional-test`) { + var fs = require('fs'); + // Set environment variables + fs.appendFileSync(process.env.GITHUB_ENV, + `DE_IMAGE=${clientPayload.de_image}\n`+ + `DE_TAG=${clientPayload.de_tag}\n`+ + `CHECKOUT_REPO=${{ github.repository }}\n`+ + `CHECKOUT_REF=refs/heads/main` + ); + } + - name: Check out code + uses: actions/checkout@v4 + with: + repository: ${{ env.CHECKOUT_REPO }} + ref: ${{ env.CHECKOUT_REF }} + - name: Set up Go ${{ env.GOVER }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GOVER }} + - name: Generate ID for release + id: gen-id + run: | + BASE_STR="RADIUS|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}" + if [ "$GITHUB_EVENT_NAME" == "schedule" ]; then + # Add run number to randomize unique id for scheduled runs. + BASE_STR="${GITHUB_RUN_NUMBER}|${BASE_STR}" + fi + UNIQUE_ID=func$(echo $BASE_STR | sha1sum | head -c 10) + echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_ENV + + # Set output variables to be used in the other jobs + echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_OUTPUT + echo "UNIQUE_ID=${UNIQUE_ID}" >> $GITHUB_OUTPUT + echo "CHECKOUT_REPO=${{ env.CHECKOUT_REPO }}" >> $GITHUB_OUTPUT + echo "CHECKOUT_REF=${{ env.CHECKOUT_REF }}" >> $GITHUB_OUTPUT + echo "RAD_CLI_ARTIFACT_NAME=rad_cli_linux_amd64" >> $GITHUB_OUTPUT + echo "PR_NUMBER=${{ env.PR_NUMBER }}" >> $GITHUB_OUTPUT + echo "DE_IMAGE=${{ env.DE_IMAGE }}" >> $GITHUB_OUTPUT + echo "DE_TAG=${{ env.DE_TAG }}" >> $GITHUB_OUTPUT + - uses: marocchino/sticky-pull-request-comment@v2 + if: env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + hide: true + hide_classify: "OUTDATED" + message: | + ## Radius functional test overview + + :mag: **[Go to test action run](${{ env.ACTION_LINK }})** + + | Name | Value | + |------|-------| + |**Repository** | ${{ steps.gen-id.outputs.CHECKOUT_REPO }} | + |**Commit ref** | ${{ steps.gen-id.outputs.CHECKOUT_REF }} | + |**Unique ID** | ${{ steps.gen-id.outputs.UNIQUE_ID }} | + |**Image tag** | ${{ steps.gen-id.outputs.REL_VERSION }} | + +
+ Click here to see the list of tools in the current test run + + * gotestsum ${{ env.GOTESTSUM_VER }} + * KinD: ${{ env.KIND_VER }} + * Dapr: ${{ env.DAPR_VER }} + * Azure KeyVault CSI driver: ${{ env.AZURE_KEYVAULT_CSI_DRIVER_VER }} + * Azure Workload identity webhook: ${{ env.AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER }} + * Bicep recipe location `${{ env.BICEP_RECIPE_REGISTRY }}/test/testrecipes/test-bicep-recipes/:${{ env.REL_VERSION }}` + * Terraform recipe location `${{ env.TF_RECIPE_MODULE_SERVER_URL }}/.zip` (in cluster) + * applications-rp test image location: `${{ env.CONTAINER_REGISTRY }}/applications-rp:${{ env.REL_VERSION }}` + * controller test image location: `${{ env.CONTAINER_REGISTRY }}/controller:${{ env.REL_VERSION }}` + * ucp test image location: `${{ env.CONTAINER_REGISTRY }}/ucpd:${{ env.REL_VERSION }}` + * deployment-engine test image location: `${{ env.DE_IMAGE }}:${{ env.DE_TAG }}` + +
+ + ## Test Status + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: marocchino/sticky-pull-request-comment@v2 + if: env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :hourglass: Building Radius and pushing container images for functional tests... + - name: Build and Push container images + run: | + make build && make docker-build && make docker-push + env: + DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }} + DOCKER_TAG_VERSION: ${{ env.REL_VERSION }} + - name: Upload CLI binary + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.gen-id.outputs.RAD_CLI_ARTIFACT_NAME }} + path: | + ./dist/linux_amd64/release/rad + - uses: marocchino/sticky-pull-request-comment@v2 + if: success() && env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :white_check_mark: Container images build succeeded + - uses: marocchino/sticky-pull-request-comment@v2 + if: failure() && env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :x: Container images build failed + - uses: marocchino/sticky-pull-request-comment@v2 + if: env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :hourglass: Publishing Bicep Recipes for functional tests... + - name: Publish Bicep Test Recipes + run: | + mkdir ./bin + cp ./dist/linux_amd64/release/rad ./bin/rad + chmod +x ./bin/rad + export PATH=$GITHUB_WORKSPACE/bin:$PATH + which rad || { echo "cannot find rad"; exit 1; } + rad bicep download + rad version + make publish-test-bicep-recipes + env: + BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }} + BICEP_RECIPE_TAG_VERSION: ${{ env.REL_VERSION }} + - uses: marocchino/sticky-pull-request-comment@v2 + if: success() && env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :white_check_mark: Recipe publishing succeeded + - uses: marocchino/sticky-pull-request-comment@v2 + if: failure() && env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :x: Test recipe publishing failed + tests: + name: Run ${{ matrix.name }} functional tests + needs: build + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + name: + [ + cli-noncloud, + corerp-noncloud, + daprrp-noncloud, + datastoresrp-noncloud, + kubernetes-noncloud, + msgrp-noncloud, + samples-noncloud, + ucp-noncloud, + ] + runs-on: ${{ matrix.os }} + env: + UNIQUE_ID: ${{ needs.build.outputs.UNIQUE_ID }} + REL_VERSION: ${{ needs.build.outputs.REL_VERSION }} + CHECKOUT_REPO: ${{ needs.build.outputs.CHECKOUT_REPO }} + CHECKOUT_REF: ${{ needs.build.outputs.CHECKOUT_REF }} + PR_NUMBER: ${{ needs.build.outputs.PR_NUMBER }} + RAD_CLI_ARTIFACT_NAME: ${{ needs.build.outputs.RAD_CLI_ARTIFACT_NAME }} + BICEP_RECIPE_TAG_VERSION: ${{ needs.build.outputs.REL_VERSION }} + DE_IMAGE: ${{ needs.build.outputs.DE_IMAGE }} + DE_TAG: ${{ needs.build.outputs.DE_TAG }} + steps: + - name: Get GitHub app token + uses: tibdex/github-app-token@v2 + id: get_installation_token + with: + app_id: ${{ env.FUNCTIONAL_TEST_APP_ID }} + private_key: ${{ secrets.FUNCTIONAL_TEST_APP_PRIVATE_KEY }} + - uses: LouisBrunner/checks-action@v2.0.0 + if: always() + with: + token: ${{ steps.get_installation_token.outputs.token }} + name: "Functional Test Run" + status: in_progress + repo: ${{ github.repository }} + sha: ${{ env.CHECKOUT_REF }} + details_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: Checkout + uses: actions/checkout@v4 + with: + repository: ${{ env.CHECKOUT_REPO }} + ref: ${{ env.CHECKOUT_REF }} + - name: Checkout samples repo + uses: actions/checkout@v4 + if: matrix.name == 'samples' + with: + repository: radius-project/samples + ref: refs/heads/edge + path: samples + - name: Set up Go ${{ env.GOVER }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GOVER }} + - name: Get Go Cache path + id: go-cache-paths + run: | + echo "go-build=$(go env GOCACHE)" >> $GITHUB_OUTPUT + echo "go-mod=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT + - uses: actions/cache@v4 + with: + path: | + ${{ steps.go-cache-paths.outputs.go-build }} + ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Download rad CLI + uses: actions/download-artifact@v4 + with: + name: ${{ env.RAD_CLI_ARTIFACT_NAME }} + path: bin + - uses: marocchino/sticky-pull-request-comment@v2 + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :hourglass: Starting ${{ matrix.name }} functional tests... + - uses: azure/setup-helm@v4 + with: + version: ${{ env.HELM_VER }} + - name: Create KinD cluster + run: | + curl -sSLo "kind" "https://github.com/kubernetes-sigs/kind/releases/download/${{ env.KIND_VER }}/kind-linux-amd64" + chmod +x ./kind + + # Populate the following environment variables for Azure workload identity from secrets. + # AZURE_OIDC_ISSUER_PUBLIC_KEY + # AZURE_OIDC_ISSUER_PRIVATE_KEY + # AZURE_OIDC_ISSUER + eval "export $(echo "${{ secrets.FUNCTEST_AZURE_OIDC_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')" + + AUTHKEY=$(echo -n "${{ github.actor }}:${{ secrets.GH_RAD_CI_BOT_PAT }}" | base64) + echo "{\"auths\":{\"ghcr.io\":{\"auth\":\"${AUTHKEY}\"}}}" > "./ghcr_secret.json" + + # Create KinD cluster with OIDC Issuer keys + echo $AZURE_OIDC_ISSUER_PUBLIC_KEY | base64 -d > sa.pub + echo $AZURE_OIDC_ISSUER_PRIVATE_KEY | base64 -d > sa.key + cat <> $POD_STATE_LOG_FILENAME + kubectl get pods -A >> $POD_STATE_LOG_FILENAME + echo "kubectl describe pods -A" >> $POD_STATE_LOG_FILENAME + kubectl describe pods -A >> $POD_STATE_LOG_FILENAME + - name: Upload container logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.name }}_container_logs + path: ./${{ env.RADIUS_CONTAINER_LOG_BASE }} + - name: Get Terraform recipe publishing logs + if: always() + run: | + # Create pod-logs directory + mkdir -p recipes/pod-logs + # Get pod logs and save to file + namespace="radius-test-tf-module-server" + label="app.kubernetes.io/name=tf-module-server" + pod_names=($(kubectl get pods -l $label -n $namespace -o jsonpath='{.items[*].metadata.name}')) + for pod_name in "${pod_names[@]}"; do + kubectl logs $pod_name -n $namespace > recipes/pod-logs/${pod_name}.txt + done + echo "Pod logs saved to recipes/pod-logs/" + # Get kubernetes events and save to file + kubectl get events -n $namespace > recipes/pod-logs/events.txt + - name: Upload Terraform recipe publishing logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ matrix.name }}_recipes-pod-logs + path: recipes/pod-logs + if-no-files-found: error + - uses: marocchino/sticky-pull-request-comment@v2 + if: success() && env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :white_check_mark: ${{ matrix.name }} functional tests succeeded + - uses: marocchino/sticky-pull-request-comment@v2 + if: failure() && env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :x: ${{ matrix.name }} functional test failed. Please check [the logs](${{ env.ACTION_LINK }}) for more details + - uses: marocchino/sticky-pull-request-comment@v2 + if: cancelled() && env.PR_NUMBER != '' + continue-on-error: true + with: + GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }} + header: teststatus-${{ github.run_id }} + number: ${{ env.PR_NUMBER }} + append: true + message: | + :x: ${{ matrix.name }} functional test cancelled. Please check [the logs](${{ env.ACTION_LINK }}) for more details + report-test-results: + name: Report test results + needs: [build, tests] + runs-on: ubuntu-latest + if: always() + env: + CHECKOUT_REF: ${{ needs.build.outputs.CHECKOUT_REF }} + steps: + - name: Get GitHub app token + uses: tibdex/github-app-token@v2 + id: get_installation_token + with: + app_id: ${{ env.FUNCTIONAL_TEST_APP_ID }} + private_key: ${{ secrets.FUNCTIONAL_TEST_APP_PRIVATE_KEY }} + - name: Get tests job status + id: get_test_status + run: | + # from: https://github.com/orgs/community/discussions/26526#discussioncomment-3252209 + ALL_JOBS_STATUS=$(curl -X GET -s -u "admin:${{ steps.get_installation_token.outputs.token }}" https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs | jq ".jobs[] | {job_status: .conclusion}") + echo "All jobs status: $ALL_JOBS_STATUS" + TEST_STATUS="success" + for job_status in $(echo "$ALL_JOBS_STATUS" | jq -r '.[]'); do + echo "Job Status: $job_status" + if [[ "$job_status" == "failure" ]]; then + echo "Found failed test. Setting test status to failure" + TEST_STATUS="failure" + break + elif [[ "$job_status" == "cancelled" ]]; then + echo "Found cancelled test. Setting test status to cancelled" + TEST_STATUS="cancelled" + fi + done + echo "Test Status: $TEST_STATUS" + echo "test_status=$TEST_STATUS" >> $GITHUB_OUTPUT + - uses: LouisBrunner/checks-action@v2.0.0 + if: always() + with: + token: ${{ steps.get_installation_token.outputs.token }} + name: "Functional Test Run" + repo: ${{ github.repository }} + sha: ${{ env.CHECKOUT_REF }} + status: completed + conclusion: ${{ steps.get_test_status.outputs.test_status }} + output: | + {"summary":"Functional Test run completed. See links for more information.","title":"Functional Test Run"} + details_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + report-failure: + name: Report test failure + needs: [build, tests] + runs-on: ubuntu-latest + if: failure() && github.event_name == 'schedule' && github.repository == 'radius-project/radius' + steps: + - name: Count recently failed tests + id: count_failures + uses: actions/github-script@v7 + with: + script: | + response = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'functional-test.yaml', + event: 'schedule', + per_page: 10 + }); + + failureCount = 1; + for (const run of response.data.workflow_runs) { + if (run.conclusion === 'failure') { + failureCount++; + } else { + break; + } + } + return failureCount; + - name: Create failure issue for failing scheduled run + uses: actions/github-script@v7 + # Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests. + if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD + with: + github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }} + script: | + github.rest.issues.create({ + ...context.repo, + title: `Scheduled functional test (noncloud) failed - Run ID: ${context.runId}`, + labels: ['bug', 'test-failure'], + body: `## Bug information \n\nThis bug is generated automatically if the scheduled functional test fails at least ${process.env.ISSUE_CREATE_THRESHOLD} times in a row. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).` + }) diff --git a/build/test.mk b/build/test.mk index 8c26c71422b..06eca889e16 100644 --- a/build/test.mk +++ b/build/test.mk @@ -58,7 +58,13 @@ test-get-envtools: test-validate-cli: ## Run cli integration tests CGO_ENABLED=1 $(GOTEST_TOOL) -coverpkg= ./pkg/cli/cmd/... ./cmd/rad/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS) -test-functional-all: test-functional-ucp test-functional-kubernetes test-functional-shared test-functional-cli test-functional-msgrp test-functional-daprrp test-functional-datastoresrp test-functional-samples ## Runs all functional tests +test-functional-all: test-functional-ucp test-functional-kubernetes test-functional-corerp test-functional-cli test-functional-msgrp test-functional-daprrp test-functional-datastoresrp test-functional-samples ## Runs all functional tests + +# Run all functional tests that do not require cloud resources +test-functional-all-noncloud: test-functional-ucp-noncloud test-functional-kubernetes-noncloud test-functional-corerp-noncloud test-functional-cli-noncloud test-functional-msgrp-noncloud test-functional-daprrp-noncloud test-functional-datastoresrp-noncloud test-functional-samples-noncloud ## Runs all functional tests that do not require cloud resources + +# Run all functional tests that require cloud resources +test-functional-all-cloud: test-functional-ucp-cloud test-functional-corerp-cloud test-functional-datastoresrp-cloud test-functional-ucp: test-functional-ucp-noncloud test-functional-ucp-cloud ## Runs all UCP functional tests (both cloud and non-cloud) @@ -74,7 +80,7 @@ test-functional-kubernetes: test-functional-kubernetes-noncloud ## Runs all Kube test-functional-kubernetes-noncloud: ## Runs Kubernetes functional tests that do not require cloud resources CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-portable/kubernetes/noncloud/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS) -test-functional-shared: test-functional-corerp-noncloud test-functional-corerp-cloud ## Runs all Core RP functional tests (both cloud and non-cloud) +test-functional-corerp: test-functional-corerp-noncloud test-functional-corerp-cloud ## Runs all Core RP functional tests (both cloud and non-cloud) test-functional-corerp-noncloud: ## Runs corerp functional tests that do not require cloud resources CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-portable/corerp/noncloud/... -timeout ${TEST_TIMEOUT} -v -parallel 10 $(GOTEST_OPTS) @@ -95,7 +101,7 @@ test-functional-cli-noncloud: ## Runs cli functional tests that do not require c test-functional-daprrp: test-functional-daprrp-noncloud ## Runs all Dapr RP functional tests (both cloud and non-cloud) test-functional-daprrp-noncloud: ## Runs Dapr RP functional tests that do not require cloud resources - CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-portable/daprrp/noncloud/... -timeout ${TEST_TIMEOUT} -v -parallel 3 $(GOTEST_OPTS) +CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-portable/daprrp/noncloud/... -timeout ${TEST_TIMEOUT} -v -parallel 3 $(GOTEST_OPTS) test-functional-datastoresrp: test-functional-datastoresrp-noncloud test-functional-datastoresrp-cloud ## Runs all Datastores RP functional tests (non-cloud and cloud) diff --git a/docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md b/docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md index a0f79436095..80143faa6a1 100644 --- a/docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md +++ b/docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md @@ -5,8 +5,7 @@ You can find the functional tests under `./test/functional`. A functional test ( These tests verify whether: - That Radius Environment can be created successfully. -- That Bicep templates of sample applications ca be deployed to the Radius Environment. - +- That Bicep templates of sample applications ca be deployed to the Radius Environment. ## Running via GitHub workflow @@ -14,7 +13,7 @@ These tests automatically run for every PR in the `functional-tests.yml` github We do not run these tests for commits to `main` or tags since they might block the build if they fail. -### How this works +### How this works For each PR we run the following set of steps: @@ -40,45 +39,45 @@ As much as possible, the tests use product functionality such as the Radius CLI 2. Make sure `rad-bicep` is downloaded (`rad bicep download`) 3. Make sure your [local dev environment is setup](../contributing-code-control-plane/running-controlplane-locally.md) 4. Log into your Github account and [Generate PAT](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) -5. Log-in to the container registry of your Github organization. - - `export CR_PAT=` - - `echo $CR_PAT | docker login ghcr.io -u --password-stdin` +5. Log-in to the container registry of your Github organization. + + `export CR_PAT=` + + `echo $CR_PAT | docker login ghcr.io -u --password-stdin` 6. Publish Bicep test recipes by running `BICEP_RECIPE_REGISTRY= make publish-test-bicep-recipes` 7. Publish Terraform test recipes by running `make publish-test-terraform-recipes` 8. Change the visibility of the published packages to 'public' - > ⚠️ The tests assume the Kubernetes namespace in use is `default`. If your environment is set up differently you will see > test failures. > ⚠️ If you set environment variables for functional tests you may need to restart VS Code or other editors for them to take effect. - -### Run + +### Run 1. Run: - ```sh - .{workspace}/radius/test/executeFunctionalTest.sh - ``` + ```sh + .{workspace}/radius/test/executeFunctionalTest.sh + ``` When you're running locally with this configuration, the tests will use your locally selected Radius Environment and your local copy of `rad`. The executeFunctionalTest.sh scripts creates the azure resources and exports the values to be used in the functional test and runs: - ```sh - make test-functional-shared - make test-functional-msgrp - make test-functional-daprrp - make test-functional-datastoresrp - ``` + +```sh + make test-functional-corerp + make test-functional-msgrp + make test-functional-daprrp + make test-functional-datastoresrp +``` You can also run/debug individual tests from VSCode. ### Tips -> 💡 If you make changes to recipes, make sure to re-run the *publish test recipe* step from prerequisites. +> 💡 If you make changes to recipes, make sure to re-run the _publish test recipe_ step from prerequisites. > 💡 Make sure the packages published to your organization have their visibility set to "public" -> 💡 If you make changes to the `rad` CLI make sure to copy it to your path. +> 💡 If you make changes to the `rad` CLI make sure to copy it to your path. ### Seeing log output diff --git a/test/executeFunctionalTest.sh b/test/executeFunctionalTest.sh index c0b72c0ac33..5a389c78354 100755 --- a/test/executeFunctionalTest.sh +++ b/test/executeFunctionalTest.sh @@ -6,7 +6,7 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software @@ -15,12 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------ -usage(){ +usage() { echo -e "$0 requires \n" exit 1 } -if [ $# -lt 1 ];then +if [ $# -lt 1 ]; then usage fi @@ -29,17 +29,17 @@ echo $resourcegroup # set the username and password for msqlDB to be passed as parameters to the bicep template adminUser='coolUser' adminPassword=$(uuidgen) -resp=`az deployment group create --resource-group $resourcegroup --template-file createAzureTestResources.bicep --parameters 'adminUsername=$adminUser' --parameters 'adminPassword=$adminPassword'` +resp=$(az deployment group create --resource-group $resourcegroup --template-file createAzureTestResources.bicep --parameters 'adminUsername=$adminUser' --parameters 'adminPassword=$adminPassword') cat resp -export AZURE_SERVICEBUS_RESOURCE_ID=$( jq -r '.properties.outputs.namespaceId.value' <<< "${resp}" ) -export AZURE_MSSQL_RESOURCE_ID=$( jq -r '.properties.outputs.sqlServerId.value' <<< "${resp}" ) +export AZURE_SERVICEBUS_RESOURCE_ID=$(jq -r '.properties.outputs.namespaceId.value' <<<"${resp}") +export AZURE_MSSQL_RESOURCE_ID=$(jq -r '.properties.outputs.sqlServerId.value' <<<"${resp}") export AZURE_MSSQL_USERNAME=$adminUser export AZURE_MSSQL_PASSWORD=$adminPassword -export AZURE_MONGODB_RESOURCE_ID=$( jq -r '.properties.outputs.mongoDatabaseId.value' <<< "${resp}" ) -export AZURE_REDIS_RESOURCE_ID=$( jq -r '.properties.outputs.redisCacheId.value' <<< "${resp}" ) -export AZURE_TABLESTORAGE_RESOURCE_ID=$( jq -r '.properties.outputs.tableStorageAccId.value' <<< "${resp}" ) -export AZURE_COSMOS_MONGODB_ACCOUNT_ID=$( jq -r '.properties.outputs.cosmosMongoAccountID.value' <<< "${resp}" ) -make test-functional-shared +export AZURE_MONGODB_RESOURCE_ID=$(jq -r '.properties.outputs.mongoDatabaseId.value' <<<"${resp}") +export AZURE_REDIS_RESOURCE_ID=$(jq -r '.properties.outputs.redisCacheId.value' <<<"${resp}") +export AZURE_TABLESTORAGE_RESOURCE_ID=$(jq -r '.properties.outputs.tableStorageAccId.value' <<<"${resp}") +export AZURE_COSMOS_MONGODB_ACCOUNT_ID=$(jq -r '.properties.outputs.cosmosMongoAccountID.value' <<<"${resp}") +make test-functional-corerp make test-functional-msgrp make test-functional-daprrp make test-functional-datastoresrp