Separate cloud/non-cloud tests

.github/workflows/functional-test.yaml

@@ -14,16 +14,13 @@
 # limitations under the License.
 # ------------------------------------------------------------
 
-name: Functional tests
+name: Functional tests with no cloud resources
 on:
-  schedule:
-    # Run every 4 hours on weekdays.
-    - cron: "30 0,4,8,12,16,20 * * 1-5"
-    # Run every 12 hours on weekends.
-    - cron: "30 0,12 * * 0,6"
-  # Dispatch on external events
-  repository_dispatch:
-    types: [functional-tests, de-functional-test]
+  pull_request:
+    branches:
+      - main
+      - features/*
+      - release/*
 
 env:
   # Go version
@@ -51,16 +48,10 @@ env:
   BICEP_RECIPE_REGISTRY: ghcr.io/radius-project/dev
   # The radius functional test timeout
   FUNCTIONALTEST_TIMEOUT: 60m
-  # The Azure Location to store test resources
-  AZURE_LOCATION: westus3
   # The base directory for storing test logs
   RADIUS_CONTAINER_LOG_BASE: dist/container_logs
   # The Radius helm chart location.
   RADIUS_CHART_LOCATION: deploy/Chart/
-  # The region for AWS resources
-  AWS_REGION: 'us-west-2'
-  # The AWS account ID
-  AWS_ACCOUNT_ID: '${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}'
   # The current GitHub action link
   ACTION_LINK: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'
   # Server where terraform test modules are deployed
@@ -148,7 +139,6 @@ jobs:
           echo "UNIQUE_ID=${UNIQUE_ID}" >> $GITHUB_OUTPUT
           echo "CHECKOUT_REPO=${{ env.CHECKOUT_REPO }}" >> $GITHUB_OUTPUT
           echo "CHECKOUT_REF=${{ env.CHECKOUT_REF }}" >> $GITHUB_OUTPUT
-          echo "AZURE_TEST_RESOURCE_GROUP=radtest-${UNIQUE_ID}" >> $GITHUB_OUTPUT
           echo "RAD_CLI_ARTIFACT_NAME=rad_cli_linux_amd64" >> $GITHUB_OUTPUT
           echo "PR_NUMBER=${{ env.PR_NUMBER }}" >> $GITHUB_OUTPUT
           echo "DE_IMAGE=${{ env.DE_IMAGE }}" >> $GITHUB_OUTPUT
@@ -277,7 +267,7 @@ jobs:
           message: |
             :x: Test recipe publishing failed
   tests:
-    name: Run ${{ matrix.name }} functional tests
+    name: Run ${{ matrix.name }} functional tests that do not require cloud resources
     needs: build
     strategy:
       fail-fast: true
@@ -295,7 +285,6 @@ jobs:
       CHECKOUT_REPO: ${{ needs.build.outputs.CHECKOUT_REPO }}
       CHECKOUT_REF: ${{ needs.build.outputs.CHECKOUT_REF }}
       PR_NUMBER: ${{ needs.build.outputs.PR_NUMBER }}
-      AZURE_TEST_RESOURCE_GROUP: radtest-${{ needs.build.outputs.UNIQUE_ID }}-${{ matrix.name }}
       RAD_CLI_ARTIFACT_NAME: ${{ needs.build.outputs.RAD_CLI_ARTIFACT_NAME }}
       BICEP_RECIPE_TAG_VERSION: ${{ needs.build.outputs.REL_VERSION }}
       DE_IMAGE: ${{ needs.build.outputs.DE_IMAGE }}
@@ -336,10 +325,6 @@ jobs:
         with:
           name: ${{ env.RAD_CLI_ARTIFACT_NAME }}
           path: bin
-      - name: Login to Azure
-        uses: azure/login@v1
-        with:
-          creds: '{"clientId":"${{ secrets.INTEGRATION_TEST_SP_APP_ID }}","clientSecret":"${{ secrets.INTEGRATION_TEST_SP_PASSWORD }}","subscriptionId":"${{ secrets.INTEGRATION_TEST_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.INTEGRATION_TEST_TENANT_ID }}"}'
       - uses: marocchino/sticky-pull-request-comment@v2
         continue-on-error: true
         with:
@@ -348,17 +333,6 @@ jobs:
           append: true
           message: |
             :hourglass: Starting ${{ matrix.name }} functional tests...
-      - name: Create azure resource group - ${{ env.AZURE_TEST_RESOURCE_GROUP }}
-        run: |
-          current_time=$(date +%s)
-          az group create \
-            --location ${{ env.AZURE_LOCATION }} \
-            --name $RESOURCE_GROUP \
-            --subscription ${{ secrets.INTEGRATION_TEST_SUBSCRIPTION_ID }} \
-            --tags creationTime=$current_time
-          while [ $(az group exists --name $RESOURCE_GROUP) = false ]; do sleep 2; done
-        env:
-          RESOURCE_GROUP: ${{ env.AZURE_TEST_RESOURCE_GROUP }}
       - uses: azure/setup-helm@v3
         with:
           version: ${{ env.HELM_VER }}
@@ -407,14 +381,6 @@ jobs:
         run: |
           wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash -s ${{ env.DAPR_VER }}
           dapr init -k --wait --timeout 600 --runtime-version ${{ env.DAPR_VER }}
-      - name: Install Azure Keyvault CSI driver chart
-        run: |
-          helm repo add csi-secrets-store-provider-azure https://azure.github.io/secrets-store-csi-driver-provider-azure/charts
-          helm install csi csi-secrets-store-provider-azure/csi-secrets-store-provider-azure --version ${{ env.AZURE_KEYVAULT_CSI_DRIVER_VER }}
-      - name: Install azure workload identity webhook chart
-        run: |
-          helm repo add azure-workload-identity https://azure.github.io/azure-workload-identity/charts
-          helm install workload-identity-webhook azure-workload-identity/workload-identity-webhook --namespace radius-default --create-namespace --version ${{ env.AZURE_WORKLOAD_IDENTITY_WEBHOOK_VER }} --set azureTenantID=${{ secrets.INTEGRATION_TEST_TENANT_ID }}
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -458,18 +424,6 @@ jobs:
           # The functional test is designed to use default namespace. So you must create the environment for default namespace.
           rad env create kind-radius --namespace default
           rad env switch kind-radius
-
-          echo "*** Configuring Azure provider ***"
-          rad env update kind-radius --azure-subscription-id ${{ secrets.INTEGRATION_TEST_SUBSCRIPTION_ID }} \
-            --azure-resource-group ${{ env.AZURE_TEST_RESOURCE_GROUP }}
-          rad credential register azure --client-id ${{ secrets.INTEGRATION_TEST_SP_APP_ID }} \
-            --client-secret ${{ secrets.INTEGRATION_TEST_SP_PASSWORD }} \
-            --tenant-id ${{ secrets.INTEGRATION_TEST_TENANT_ID }}
-
-          echo "*** Configuring AWS provider ***"
-          rad env update kind-radius --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}
-          rad credential register aws \
-            --access-key-id ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }} --secret-access-key ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
       - uses: marocchino/sticky-pull-request-comment@v2
         if: failure() && env.PR_NUMBER != ''
         continue-on-error: true
@@ -482,7 +436,7 @@ jobs:
       - name: Publish Terraform test recipes
         run: |
           make publish-test-terraform-recipes
-      - name: Run functional tests
+      - name: Run functional tests that do not require cloud resources
         run: |
           # Ensure rad cli is in path before running tests.
           export PATH=$GITHUB_WORKSPACE/bin:$PATH
@@ -506,12 +460,8 @@ jobs:
           DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
           TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}
           RADIUS_CONTAINER_LOG_PATH: ${{ github.workspace }}/${{ env.RADIUS_CONTAINER_LOG_BASE }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
-          AWS_REGION: ${{ env.AWS_REGION }}
           RADIUS_SAMPLES_REPO_ROOT: ${{ github.workspace }}/samples
           # Test_MongoDB_Recipe_Parameters is using the following environment variable.
-          INTEGRATION_TEST_RESOURCE_GROUP_NAME: ${{ env.AZURE_TEST_RESOURCE_GROUP }}
           BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }}
           BICEP_RECIPE_TAG_VERSION: ${{ env.BICEP_RECIPE_TAG_VERSION }}
       - uses: azure/setup-kubectl@v3
@@ -551,14 +501,6 @@ jobs:
           append: true
           message: |
             :x: ${{ matrix.name }} functional test failed. Please check [the logs](${{ env.ACTION_LINK }}) for more details
-      - name: Delete azure resource group - ${{ env.AZURE_TEST_RESOURCE_GROUP }}
-        if: always()
-        run: |
-          # if deletion fails, purge workflow will purge the resource group and its resources later.
-          az group delete \
-            --subscription ${{ secrets.INTEGRATION_TEST_SUBSCRIPTION_ID }} \
-            --name ${{ env.AZURE_TEST_RESOURCE_GROUP }} \
-            --yes --verbose
   report-failure:
     name: Report test failure
     needs: [build, tests]

build/test.mk

@@ -66,6 +66,9 @@ test-functional-kubernetes: ## Runs Kubernetes functional tests
 test-functional-shared: ## Runs shared functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/shared/... -timeout ${TEST_TIMEOUT} -v -parallel 10 $(GOTEST_OPTS)
 
+test-functional-shared-cloud: ## Runs shared functional tests that require cloud resources
+	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-cloud/shared/... -timeout ${TEST_TIMEOUT} -v -parallel 10 $(GOTEST_OPTS)
+
 test-functional-msgrp: ## Runs Messaging RP functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/messagingrp/... -timeout ${TEST_TIMEOUT} -v -parallel 2 $(GOTEST_OPTS)
 
@@ -75,12 +78,18 @@ test-functional-daprrp: ## Runs Dapr RP functional tests
 test-functional-datastoresrp: ## Runs Datastores RP functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/datastoresrp/... -timeout ${TEST_TIMEOUT} -v -parallel 3 $(GOTEST_OPTS)
 
+test-functional-datastoresrp-cloud: ## Runs Datastores RP functional tests that require cloud resources
+	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-cloud/datastoresrp/... -timeout ${TEST_TIMEOUT} -v -parallel 3 $(GOTEST_OPTS)
+
 test-functional-samples: ## Runs Samples functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/samples/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
 
 test-functional-ucp: ## Runs UCP functional tests
 	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional/ucp/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
 
+test-functional-ucp-cloud: ## Runs UCP functional tests that require cloud resources
+	CGO_ENABLED=1 $(GOTEST_TOOL) ./test/functional-cloud/ucp/... -timeout ${TEST_TIMEOUT} -v -parallel 5 $(GOTEST_OPTS)
+
 test-validate-bicep: ## Validates that all .bicep files compile cleanly
 	BICEP_PATH="${HOME}/.rad/bin/rad-bicep" ./build/validate-bicep.sh
 

test/functional-cloud/README.md

@@ -0,0 +1,3 @@
+* [Writing functional tests](../../docs/contributing/contributing-code/contributing-code-tests/writing-functional-tests.md)
+
+* [Running functional tests](../../docs/contributing/contributing-code/contributing-code-tests/running-functional-tests.md)

test/functional-cloud/datastoresrp/resources/mongodb_test.go

@@ -0,0 +1,184 @@
+/*
+Copyright 2023 The Radius Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resource_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/radius-project/radius/test/functional"
+	"github.com/radius-project/radius/test/functional/shared"
+	"github.com/radius-project/radius/test/step"
+	"github.com/radius-project/radius/test/validation"
+)
+
+// Test_MongoDB_Recipe validates:
+// the creation of a mongoDB from a recipe that uses an Azure resource
+func Test_MongoDB_Recipe(t *testing.T) {
+	template := "testdata/datastoresrp-resources-mongodb-recipe.bicep"
+	name := "dsrp-resources-mongodb-recipe"
+	appNamespace := "dsrp-resources-mongodb-recipe-app"
+
+	test := shared.NewRPTest(t, name, []shared.TestStep{
+		{
+			Executor: step.NewDeployExecutor(template, functional.GetMagpieImage(), functional.GetBicepRecipeRegistry(), functional.GetBicepRecipeVersion()),
+			RPResources: &validation.RPResourceSet{
+				Resources: []validation.RPResource{
+					{
+						Name: "dsrp-resources-mongodb-recipe-env",
+						Type: validation.EnvironmentsResource,
+					},
+					{
+						Name: "dsrp-resources-mongodb-recipe",
+						Type: validation.ApplicationsResource,
+						App:  name,
+					},
+					{
+						Name: "mongodb-app-ctnr",
+						Type: validation.ContainersResource,
+						App:  name,
+					},
+					{
+						Name: "mongodb-db",
+						Type: validation.MongoDatabasesResource,
+						App:  name,
+					},
+				},
+			},
+			K8sObjects: &validation.K8sObjectSet{
+				Namespaces: map[string][]validation.K8sObject{
+					appNamespace: {
+						validation.NewK8sPodForResource(name, "mongodb-app-ctnr").ValidateLabels(false),
+					},
+				},
+			},
+		},
+	})
+
+	test.Test(t)
+}
+
+// Test_MongoDB_Recipe_Parameters validates the creation of a mongoDB from recipe with parameters passed by operator while linking recipe
+// and developer while creating the mongoDatabase resource.
+// If the same parameters are set by the developer and the operator then the developer parameters are applied in to resolve conflicts.
+// Container uses the mongoDB resource to connect to the mongoDB resource
+func Test_MongoDB_RecipeParameters(t *testing.T) {
+	t.Skip("Skipping test as creating/deleting cosmosdb resource is unreliable - https://github.com/radius-project/radius/issues/5929")
+
+	template := "testdata/datastoresrp-resources-mongodb-recipe-parameters.bicep"
+	name := "dsrp-resources-mongodb-recipe-parameters"
+	appNamespace := "dsrp-resources-mongodb-recipe-param-app"
+	rg := os.Getenv("INTEGRATION_TEST_RESOURCE_GROUP_NAME")
+	// Error the test if INTEGRATION_TEST_RESOURCE_GROUP_NAME is not set
+	// for running locally set the INTEGRATION_TEST_RESOURCE_GROUP_NAME with the test resourceGroup
+	if rg == "" {
+		t.Error("This test needs the env variable INTEGRATION_TEST_RESOURCE_GROUP_NAME to be set")
+	}
+
+	test := shared.NewRPTest(t, name, []shared.TestStep{
+		{
+			Executor: step.NewDeployExecutor(template, functional.GetMagpieImage(), functional.GetBicepRecipeRegistry(), functional.GetBicepRecipeVersion()),
+			RPResources: &validation.RPResourceSet{
+				Resources: []validation.RPResource{
+					{
+						Name: "dsrp-resources-env-recipe-parameters-env",
+						Type: validation.EnvironmentsResource,
+					},
+					{
+						Name: name,
+						Type: validation.ApplicationsResource,
+						App:  name,
+					},
+					{
+						Name: "mdb-param-ctnr",
+						Type: validation.ContainersResource,
+						App:  name,
+					},
+					{
+						Name: "mdb-recipe-param-db",
+						Type: validation.MongoDatabasesResource,
+						App:  name,
+					},
+				},
+			},
+			K8sObjects: &validation.K8sObjectSet{
+				Namespaces: map[string][]validation.K8sObject{
+					appNamespace: {
+						validation.NewK8sPodForResource(name, "mdb-param-ctnr").ValidateLabels(false),
+					},
+				},
+			},
+		},
+	})
+
+	test.Test(t)
+}
+
+// Test_MongoDB_Recipe_ContextParameter validates creation of a mongoDB from
+// a default recipe using the context parameter generated and set by DatastoresRP,
+// and container using the mongoDatabases portable resource to connect to the underlying mongoDB resource.
+func Test_MongoDB_Recipe_ContextParameter(t *testing.T) {
+	t.Skip("Skipping test as creating/deleting cosmosdb resource is unreliable - https://github.com/radius-project/radius/issues/5929")
+
+	template := "testdata/datastoresrp-resources-mongodb-recipe-context.bicep"
+	name := "dsrp-resources-mongodb-recipe-context"
+	appNamespace := "dsrp-resources-mongodb-recipe-context-app"
+	rg := os.Getenv("INTEGRATION_TEST_RESOURCE_GROUP_NAME")
+	// Error the test if INTEGRATION_TEST_RESOURCE_GROUP_NAME is not set
+	// for running locally set the INTEGRATION_TEST_RESOURCE_GROUP_NAME with the test resourceGroup
+	if rg == "" {
+		t.Error("This test needs the env variable INTEGRATION_TEST_RESOURCE_GROUP_NAME to be set")
+	}
+
+	test := shared.NewRPTest(t, name, []shared.TestStep{
+		{
+			Executor: step.NewDeployExecutor(template, functional.GetMagpieImage(), functional.GetBicepRecipeRegistry(), functional.GetBicepRecipeVersion()),
+			RPResources: &validation.RPResourceSet{
+				Resources: []validation.RPResource{
+					{
+						Name: "dsrp-resources-env-recipes-context-env",
+						Type: validation.EnvironmentsResource,
+					},
+					{
+						Name: name,
+						Type: validation.ApplicationsResource,
+						App:  name,
+					},
+					{
+						Name: "mdb-ctx-ctnr",
+						Type: validation.ContainersResource,
+						App:  name,
+					},
+					{
+						Name: "mdb-ctx",
+						Type: validation.MongoDatabasesResource,
+						App:  name,
+					},
+				},
+			},
+			K8sObjects: &validation.K8sObjectSet{
+				Namespaces: map[string][]validation.K8sObject{
+					appNamespace: {
+						validation.NewK8sPodForResource(name, "mdb-ctx-ctnr").ValidateLabels(false),
+					},
+				},
+			},
+		},
+	})
+
+	test.Test(t)
+}