retry-functional-test.yaml

# ------------------------------------------------------------
# Copyright 2023 The Radius Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#    
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------------------------------------

name: '[TEST] Functional tests with retry'

permissions:
  id-token: write # Required for requesting the JWT
  contents: read  # Required for actions/checkout
  packages: write # Required for uploading the package

on:
  schedule:
    # Run every 2 hours
    - cron: "0 */2 * * *"

env:
  # Go version
  GOVER: '1.21.7'
  GOPROXY: https://proxy.golang.org

  # gotestsum version - see: https://github.com/gotestyourself/gotestsum
  GOTESTSUM_VER: 1.10.0

  # Helm version
  HELM_VER: 'v3.12.0'
  # KinD cluster version
  KIND_VER: 'v0.20.0'
  # Dapr version
  DAPR_VER: '1.12.0'
  DAPR_DASHBOARD_VER: '0.14.0'
  # Kubectl version
  KUBECTL_VER: 'v1.25.0'
  # Azure Keyvault CSI driver chart version
  AZURE_KEYVAULT_CSI_DRIVER_VER: '1.4.2'
  # Azure workload identity webhook chart version
  AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER: '1.1.0'
  # Container registry for storing container images
  CONTAINER_REGISTRY: ghcr.io/radius-project/dev
  # Container registry for storing Bicep recipe artifacts
  BICEP_RECIPE_REGISTRY: ghcr.io/radius-project/dev
  # The radius functional test timeout
  FUNCTIONALTEST_TIMEOUT: 60m
  # The Azure Location to store test resources
  AZURE_LOCATION: westus3
  # The base directory for storing test logs
  RADIUS_CONTAINER_LOG_BASE: dist/container_logs
  # The Radius helm chart location.
  RADIUS_CHART_LOCATION: deploy/Chart/
  # The region for AWS resources
  AWS_REGION: 'us-west-2'
  # The AWS account ID
  AWS_ACCOUNT_ID: '${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}'
  # The current GitHub action link
  ACTION_LINK: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'
  # Server where terraform test modules are deployed
  TF_RECIPE_MODULE_SERVER_URL: "http://tf-module-server.radius-test-tf-module-server.svc.cluster.local"
  # The functional test GitHub app id
  FUNCTIONAL_TEST_APP_ID: 425843
  # Private Git repository where terraform module for testing is stored.
  TF_RECIPE_PRIVATE_GIT_SOURCE: "git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis"

jobs:
  build:
    name: Build Radius for test
    runs-on: ubuntu-latest
    env:
      DE_IMAGE: 'ghcr.io/radius-project/deployment-engine'
      DE_TAG: 'latest'
    outputs:
      REL_VERSION: ${{ steps.gen-id.outputs.REL_VERSION }}
      UNIQUE_ID: ${{ steps.gen-id.outputs.UNIQUE_ID }}
      PR_NUMBER: ${{ steps.gen-id.outputs.PR_NUMBER }}
      CHECKOUT_REPO: ${{ steps.gen-id.outputs.CHECKOUT_REPO }}
      CHECKOUT_REF: ${{ steps.gen-id.outputs.CHECKOUT_REF }}
      RAD_CLI_ARTIFACT_NAME: ${{ steps.gen-id.outputs.RAD_CLI_ARTIFACT_NAME }}
      DE_IMAGE: ${{ steps.gen-id.outputs.DE_IMAGE }}
      DE_TAG: ${{ steps.gen-id.outputs.DE_TAG }}
    steps:
      - name: Get GitHub app token
        uses: tibdex/github-app-token@v1
        id: get_installation_token
        with: 
          app_id: ${{ env.FUNCTIONAL_TEST_APP_ID }}
          private_key: ${{ secrets.FUNCTIONAL_TEST_APP_PRIVATE_KEY }}
      - name: Set up checkout target
        run: |
          echo "CHECKOUT_REPO=youngbupark/radius" >> $GITHUB_ENV
          echo "CHECKOUT_REF=refs/heads/youngp/enable-retry-test" >> $GITHUB_ENV
      - name: Check out code
        uses: actions/checkout@v3
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Set up Go ${{ env.GOVER }}
        uses: actions/setup-go@v4
        with:
          go-version: ${{ env.GOVER }}
      - name: Generate ID for release
        id: gen-id
        run: |
          BASE_STR="RADIUS|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
          if [ "$GITHUB_EVENT_NAME" == "schedule" ]; then
            # Add run number to randomize unique id for scheduled runs.
            BASE_STR="${GITHUB_RUN_NUMBER}|${BASE_STR}"
          fi
          UNIQUE_ID=retry$(echo $BASE_STR | sha1sum | head -c 10)
          echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_ENV
          
          # Set output variables to be used in the other jobs
          echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_OUTPUT
          echo "UNIQUE_ID=${UNIQUE_ID}" >> $GITHUB_OUTPUT
          echo "CHECKOUT_REPO=${{ env.CHECKOUT_REPO }}" >> $GITHUB_OUTPUT
          echo "CHECKOUT_REF=${{ env.CHECKOUT_REF }}" >> $GITHUB_OUTPUT
          echo "AZURE_TEST_RESOURCE_GROUP=radtest-${UNIQUE_ID}" >> $GITHUB_OUTPUT
          echo "RAD_CLI_ARTIFACT_NAME=rad_cli_linux_amd64" >> $GITHUB_OUTPUT
          echo "PR_NUMBER=${{ env.PR_NUMBER }}" >> $GITHUB_OUTPUT
          echo "DE_IMAGE=${{ env.DE_IMAGE }}" >> $GITHUB_OUTPUT
          echo "DE_TAG=${{ env.DE_TAG }}" >> $GITHUB_OUTPUT
      - uses: marocchino/sticky-pull-request-comment@v2
        if: env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          hide: true
          hide_classify: 'OUTDATED'
          message: |
            ## Radius functional test overview

            :mag: **[Go to test action run](${{ env.ACTION_LINK }})**

            | Name | Value |
            |------|-------|
            |**Repository** | ${{ steps.gen-id.outputs.CHECKOUT_REPO }} |
            |**Commit ref** | ${{ steps.gen-id.outputs.CHECKOUT_REF }} |
            |**Unique ID** | ${{ steps.gen-id.outputs.UNIQUE_ID }} |
            |**Image tag** | ${{ steps.gen-id.outputs.REL_VERSION }} |

            <details>
            <summary> Click here to see the list of tools in the current test run</summary>

            * gotestsum ${{ env.GOTESTSUM_VER }}
            * KinD: ${{ env.KIND_VER }}
            * Dapr: ${{ env.DAPR_VER }}
            * Azure KeyVault CSI driver: ${{ env.AZURE_KEYVAULT_CSI_DRIVER_VER }}
            * Azure Workload identity webhook: ${{ env.AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER }}
            * Bicep recipe location `${{ env.BICEP_RECIPE_REGISTRY }}/test/functional/shared/recipes/<name>:${{ env.REL_VERSION }}`
            * Terraform recipe location `${{ env.TF_RECIPE_MODULE_SERVER_URL }}/<name>.zip` (in cluster)
            * applications-rp test image location: `${{ env.CONTAINER_REGISTRY }}/applications-rp:${{ env.REL_VERSION }}`
            * controller test image location: `${{ env.CONTAINER_REGISTRY }}/controller:${{ env.REL_VERSION }}`
            * ucp test image location: `${{ env.CONTAINER_REGISTRY }}/ucpd:${{ env.REL_VERSION }}`
            * deployment-engine test image location: `${{ env.DE_IMAGE }}:${{ env.DE_TAG }}`

            </details>
            
            ## Test Status
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - uses: marocchino/sticky-pull-request-comment@v2
        if: env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :hourglass: Building Radius and pushing container images for functional tests...
      - name: Build and Push container images
        run: |
          make build && make docker-build && make docker-push
        env:
          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
          DOCKER_TAG_VERSION: ${{ env.REL_VERSION }}
      - name: Upload CLI binary
        uses: actions/upload-artifact@v3
        with:
          name: ${{ steps.gen-id.outputs.RAD_CLI_ARTIFACT_NAME }}
          path: |
            ./dist/linux_amd64/release/rad
      - uses: marocchino/sticky-pull-request-comment@v2
        if: success() && env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :white_check_mark: Container images build succeeded
      - uses: marocchino/sticky-pull-request-comment@v2
        if: failure() && env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :x: Container images build failed
      - uses: marocchino/sticky-pull-request-comment@v2
        if: env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :hourglass: Publishing Bicep Recipes for functional tests...
      - name: Publish Bicep Test Recipes
        run: |
          mkdir ./bin
          cp ./dist/linux_amd64/release/rad ./bin/rad
          chmod +x ./bin/rad
          export PATH=$GITHUB_WORKSPACE/bin:$PATH
          which rad || { echo "cannot find rad"; exit 1; }
          rad bicep download
          rad version
          make publish-test-bicep-recipes
        env:
          BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }}
          BICEP_RECIPE_TAG_VERSION: ${{ env.REL_VERSION }}
      - uses: marocchino/sticky-pull-request-comment@v2
        if: success() && env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :white_check_mark: Recipe publishing succeeded
      - uses: marocchino/sticky-pull-request-comment@v2
        if: failure() && env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :x: Test recipe publishing failed
  tests:
    name: Run ${{ matrix.name }} functional tests
    needs: build
    strategy:
      fail-fast: true
      matrix:
        os: [ubuntu-latest]
        name: [ucp,kubernetes,shared,msgrp,daprrp,samples,cli]
        include:
          # datastorerp functional tests need the larger VM.
          - os: ubuntu-latest-m
            name: datastoresrp
    runs-on: ${{ matrix.os }}
    env:
      UNIQUE_ID: ${{ needs.build.outputs.UNIQUE_ID }}
      REL_VERSION: ${{ needs.build.outputs.REL_VERSION }}
      CHECKOUT_REPO: ${{ needs.build.outputs.CHECKOUT_REPO }}
      CHECKOUT_REF: ${{ needs.build.outputs.CHECKOUT_REF }}
      PR_NUMBER: ${{ needs.build.outputs.PR_NUMBER }}
      AZURE_TEST_RESOURCE_GROUP: radtest-${{ needs.build.outputs.UNIQUE_ID }}-${{ matrix.name }}
      RAD_CLI_ARTIFACT_NAME: ${{ needs.build.outputs.RAD_CLI_ARTIFACT_NAME }}
      BICEP_RECIPE_TAG_VERSION: ${{ needs.build.outputs.REL_VERSION }}
      DE_IMAGE: ${{ needs.build.outputs.DE_IMAGE }}
      DE_TAG: ${{ needs.build.outputs.DE_TAG }}
    steps:
      - name: Get GitHub app token
        uses: tibdex/github-app-token@v1
        id: get_installation_token
        with: 
          app_id: ${{ env.FUNCTIONAL_TEST_APP_ID }}
          private_key: ${{ secrets.FUNCTIONAL_TEST_APP_PRIVATE_KEY }}
      - name: Checkout
        uses: actions/checkout@v3
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Checkout samples repo
        uses: actions/checkout@v3
        if: matrix.name == 'samples'
        with:
          repository: radius-project/samples
          ref: refs/heads/edge
          path: samples
      - name: Set up Go ${{ env.GOVER }}
        uses: actions/setup-go@v3
        with:
          go-version: ${{ env.GOVER }}
      - name: Get Go Cache path
        id: go-cache-paths
        run: |
          echo "go-build=$(go env GOCACHE)" >> $GITHUB_OUTPUT
          echo "go-mod=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
      - uses: actions/cache@v3
        with:
          path: |
            ${{ steps.go-cache-paths.outputs.go-build }}
            ${{ steps.go-cache-paths.outputs.go-mod }}
          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-
      - name: Download rad CLI
        uses: actions/download-artifact@v3
        with:
          name: ${{ env.RAD_CLI_ARTIFACT_NAME }}
          path: bin
      - name: Login to Azure
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_SP_TESTS_APPID }}
          tenant-id: ${{ secrets.AZURE_SP_TESTS_TENANTID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }}
      - uses: marocchino/sticky-pull-request-comment@v2
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :hourglass: Starting ${{ matrix.name }} functional tests...
      - name: Create azure resource group - ${{ env.AZURE_TEST_RESOURCE_GROUP }}
        run: |
          current_time=$(date +%s)
          az group create \
            --location ${{ env.AZURE_LOCATION }} \
            --name $RESOURCE_GROUP \
            --subscription ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} \
            --tags creationTime=$current_time
          while [ $(az group exists --name $RESOURCE_GROUP) = false ]; do sleep 2; done
        env:
          RESOURCE_GROUP: ${{ env.AZURE_TEST_RESOURCE_GROUP }}
      - uses: azure/setup-helm@v3
        with:
          version: ${{ env.HELM_VER }}
      - name: Create KinD cluster
        run: |
          curl -sSLo "kind" "https://github.com/kubernetes-sigs/kind/releases/download/${{ env.KIND_VER }}/kind-linux-amd64"
          chmod +x ./kind

          # Populate the following environment variables for Azure workload identity from secrets.
          # AZURE_OIDC_ISSUER_PUBLIC_KEY
          # AZURE_OIDC_ISSUER_PRIVATE_KEY
          # AZURE_OIDC_ISSUER
          eval "export $(echo "${{ secrets.FUNCTEST_AZURE_OIDC_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')"

          AUTHKEY=$(echo -n "${{ github.actor }}:${{ secrets.GH_RAD_CI_BOT_PAT }}" | base64)
          echo "{\"auths\":{\"ghcr.io\":{\"auth\":\"${AUTHKEY}\"}}}" > "./ghcr_secret.json"

          # Create KinD cluster with OIDC Issuer keys
          echo $AZURE_OIDC_ISSUER_PUBLIC_KEY | base64 -d > sa.pub
          echo $AZURE_OIDC_ISSUER_PRIVATE_KEY | base64 -d > sa.key
          cat <<EOF | ./kind create cluster --name radius --config=-
          kind: Cluster
          apiVersion: kind.x-k8s.io/v1alpha4
          nodes:
          - role: control-plane
            extraMounts:
              - hostPath: ./sa.pub
                containerPath: /etc/kubernetes/pki/sa.pub
              - hostPath: ./sa.key
                containerPath: /etc/kubernetes/pki/sa.key
              - hostPath: ./ghcr_secret.json
                containerPath: /var/lib/kubelet/config.json
            kubeadmConfigPatches:
            - |
              kind: ClusterConfiguration
              apiServer:
                extraArgs:
                  service-account-issuer: $AZURE_OIDC_ISSUER
                  service-account-key-file: /etc/kubernetes/pki/sa.pub
                  service-account-signing-key-file: /etc/kubernetes/pki/sa.key
              controllerManager:
                extraArgs:
                  service-account-private-key-file: /etc/kubernetes/pki/sa.key
          EOF
      - name: Install dapr into cluster
        run: |
          wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash -s ${{ env.DAPR_VER }}
          dapr init -k --wait --timeout 600 --runtime-version ${{ env.DAPR_VER }} --dashboard-version ${{ env.DAPR_DASHBOARD_VER }}
      - name: Install Azure Keyvault CSI driver chart
        run: |
          helm repo add csi-secrets-store-provider-azure https://azure.github.io/secrets-store-csi-driver-provider-azure/charts
          helm install csi csi-secrets-store-provider-azure/csi-secrets-store-provider-azure --version ${{ env.AZURE_KEYVAULT_CSI_DRIVER_VER }}
      - name: Install azure workload identity webhook chart
        run: |
          helm repo add azure-workload-identity https://azure.github.io/azure-workload-identity/charts
          helm install workload-identity-webhook azure-workload-identity/workload-identity-webhook --namespace radius-default --create-namespace --version ${{ env.AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER }} --set azureTenantID=${{ secrets.AZURE_SP_TESTS_TENANTID }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Download Bicep
        run: |
          chmod +x ./bin/rad
          export PATH=$GITHUB_WORKSPACE/bin:$PATH
          which rad || { echo "cannot find rad"; exit 1; }
          rad bicep download
          rad version
      - name: Install gotestsum (test reporting tool)
        run: |
          go install gotest.tools/gotestsum@v${{ env.GOTESTSUM_VER }}
      - uses: marocchino/sticky-pull-request-comment@v2
        if: failure() && env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :x: Test tool installation for ${{ matrix.name }} failed. Please check [the logs](${{ env.ACTION_LINK }}) for more details
      - name: Install Radius
        run: |
          export PATH=$GITHUB_WORKSPACE/bin:$PATH
          which rad || { echo "cannot find rad"; exit 1; }

          echo "*** Installing Radius to Kubernetes ***"
          rad install kubernetes \
            --chart ${{ env.RADIUS_CHART_LOCATION }} \
            --set rp.image=${{ env.CONTAINER_REGISTRY }}/applications-rp,rp.tag=${{ env.REL_VERSION }},controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }},ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }},de.image=${{ env.DE_IMAGE }},de.tag=${{ env.DE_TAG }}

          echo "*** Create workspace, group and environment for test ***"
          rad workspace create kubernetes
          rad group create kind-radius
          rad group switch kind-radius

          # The functional test is designed to use default namespace. So you must create the environment for default namespace.
          rad env create kind-radius --namespace default
          rad env switch kind-radius

          echo "*** Configuring Azure provider ***"
          rad env update kind-radius --azure-subscription-id ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} \
            --azure-resource-group ${{ env.AZURE_TEST_RESOURCE_GROUP }}
          rad credential register azure --client-id ${{ secrets.AZURE_SP_TESTS_APPID }} \
            --client-secret ${{ secrets.INTEGRATION_TEST_SP_PASSWORD }} \
            --tenant-id ${{ secrets.AZURE_SP_TESTS_TENANTID }}

          echo "*** Configuring AWS provider ***"
          rad env update kind-radius --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}
          rad credential register aws \
            --access-key-id ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }} --secret-access-key ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
      - uses: marocchino/sticky-pull-request-comment@v2
        if: failure() && env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :x: Failed to install Radius for ${{ matrix.name }} functional test. Please check [the logs](${{ env.ACTION_LINK }}) for more details
      - name: Publish Terraform test recipes
        run: |
          make publish-test-terraform-recipes
      - name: Run functional tests
        run: |
          # Ensure rad cli is in path before running tests.
          export PATH=$GITHUB_WORKSPACE/bin:$PATH
          cd $GITHUB_WORKSPACE

          which rad || { echo "cannot find rad"; exit 1; }

          # Populate the following test environment variables from JSON secret.
          # AZURE_MONGODB_RESOURCE_ID
          # AZURE_COSMOS_MONGODB_ACCOUNT_ID
          # AZURE_TABLESTORAGE_RESOURCE_ID
          # AZURE_SERVICEBUS_RESOURCE_ID
          # AZURE_REDIS_RESOURCE_ID
          # AZURE_MSSQL_RESOURCE_ID
          # AZURE_MSSQL_USERNAME
          # AZURE_MSSQL_PASSWORD
          eval "export $(echo "${{ secrets.FUNCTEST_PREPROVISIONED_RESOURCE_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')"

          make test-functional-${{ matrix.name }}
        env:
          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
          TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}
          RADIUS_CONTAINER_LOG_PATH: ${{ github.workspace }}/${{ env.RADIUS_CONTAINER_LOG_BASE }}
          AWS_ACCESS_KEY_ID: ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ env.AWS_REGION }}
          RADIUS_SAMPLES_REPO_ROOT: ${{ github.workspace }}/samples
          # Test_MongoDB_Recipe_Parameters is using the following environment variable.
          INTEGRATION_TEST_RESOURCE_GROUP_NAME: ${{ env.AZURE_TEST_RESOURCE_GROUP }}
          BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }}
          BICEP_RECIPE_TAG_VERSION: ${{ env.BICEP_RECIPE_TAG_VERSION }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - uses: azure/setup-kubectl@v3
        if: always()
        with:
          version: ${{ env.KUBECTL_VER }}
      - name: Collect Pod details
        if: always()
        run: |
          POD_STATE_LOG_FILENAME='${{ env.RADIUS_CONTAINER_LOG_BASE }}/${{ matrix.name }}-tests-pod-states.log'
          mkdir -p $(dirname $POD_STATE_LOG_FILENAME)
          echo "kubectl get pods -A" >> $POD_STATE_LOG_FILENAME
          kubectl get pods -A >> $POD_STATE_LOG_FILENAME
          echo "kubectl describe pods -A" >> $POD_STATE_LOG_FILENAME
          kubectl describe pods -A >> $POD_STATE_LOG_FILENAME
      - name: Upload container logs
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.name }}_container_logs
          path: ./${{ env.RADIUS_CONTAINER_LOG_BASE }}
      - name: Get Terraform recipe publishing logs
        if: always()
        run: |
          # Create pod-logs directory
          mkdir -p recipes/pod-logs
          # Get pod logs and save to file
          namespace="radius-test-tf-module-server"
          label="app.kubernetes.io/name=tf-module-server"
          pod_names=($(kubectl get pods -l $label -n $namespace -o jsonpath='{.items[*].metadata.name}'))
          for pod_name in "${pod_names[@]}"; do
            kubectl logs $pod_name -n $namespace > recipes/pod-logs/${pod_name}.txt
          done
          echo "Pod logs saved to recipes/pod-logs/"
          # Get kubernetes events and save to file
          kubectl get events -n $namespace > recipes/pod-logs/events.txt
      - name: Upload Terraform recipe publishing logs 
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: recipes-pod-logs
          path: recipes/pod-logs
          if-no-files-found: error
      - uses: marocchino/sticky-pull-request-comment@v2
        if: success() && env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :white_check_mark: ${{ matrix.name }} functional tests succeeded
      - uses: marocchino/sticky-pull-request-comment@v2
        if: failure() && env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :x: ${{ matrix.name }} functional test failed. Please check [the logs](${{ env.ACTION_LINK }}) for more details
      - uses: marocchino/sticky-pull-request-comment@v2
        if: cancelled() && env.PR_NUMBER != ''
        continue-on-error: true
        with:
          GITHUB_TOKEN: ${{ steps.get_installation_token.outputs.token }}
          header: teststatus-${{ github.run_id }}
          number: ${{ env.PR_NUMBER }}
          append: true
          message: |
            :x: ${{ matrix.name }} functional test cancelled. Please check [the logs](${{ env.ACTION_LINK }}) for more details