Merge branch 'main' into deploymenttemplatecontroller

Signed-off-by: Will Smith <[email protected]>

.github/workflows/functional-test-cloud.yaml

@@ -85,8 +85,6 @@ env:
   FUNCTIONAL_TEST_APP_ID: 425843
   # Private Git repository where terraform module for testing is stored.
   TF_RECIPE_PRIVATE_GIT_SOURCE: "git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis"
-  # The number of failed tests to report.
-  ISSUE_CREATE_THRESHOLD: 2
   # bicep-types ACR url for uploading Radius Bicep types
   BICEP_TYPES_REGISTRY: 'biceptypes.azurecr.io'
 
@@ -549,7 +547,17 @@ jobs:
       - uses: azure/setup-helm@v4
         with:
           version: ${{ env.HELM_VER }}
+
+      # The role-to-assume is the role that the github action will assume to execute aws commands and 
+      # construct cloud control client in test code.
+      - name: configure aws credentials using assumed role
+        uses: aws-actions/[email protected]
+        with:
+          role-to-assume: ${{ secrets.AWS_GH_ACTIONS_ROLE }}
+          role-session-name: GitHub_to_AWS_via_FederatedOIDC
+          aws-region: ${{ env.AWS_REGION }}
 
+      # create kind cluster with OIDC provider.      
       - name: Create KinD cluster
         run: |
           curl -sSLo "kind" "https://github.com/kubernetes-sigs/kind/releases/download/${{ env.KIND_VER }}/kind-linux-amd64"
@@ -632,7 +640,6 @@ jobs:
           append: true
           message: |
             :x: Test tool installation for ${{ matrix.name }} failed. Please check [the logs](${{ env.ACTION_LINK }}) for more details
-
       - name: Install Radius
         run: |
           export PATH=$GITHUB_WORKSPACE/bin:$PATH
@@ -646,8 +653,9 @@ jobs:
             --set controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }} \
             --set ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }} \
             --set de.image=${{ env.DE_IMAGE }},de.tag=${{ env.DE_TAG }} \
-            --set global.azureWorkloadIdentity.enabled=true
-
+            --set global.azureWorkloadIdentity.enabled=true \
+            --set global.aws.irsa.enabled=true 
+          
           echo "*** Create workspace, group and environment for test ***"
           rad workspace create kubernetes
           rad group create kind-radius
@@ -666,9 +674,10 @@ jobs:
 
           echo "*** Configuring AWS provider ***"
           rad env update kind-radius --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}
-          rad credential register aws access-key \
-            --access-key-id ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }} --secret-access-key ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
-
+          
+          rad credential register aws irsa \
+            --iam-role ${{ secrets.FUNC_TEST_RAD_IRSA_ROLE }}
+          
       - uses: marocchino/sticky-pull-request-comment@v2
         if: failure() && env.PR_NUMBER != ''
         continue-on-error: true
@@ -726,9 +735,8 @@ jobs:
           DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
           TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}
           RADIUS_CONTAINER_LOG_PATH: ${{ github.workspace }}/${{ env.RADIUS_CONTAINER_LOG_BASE }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: ${{ env.AWS_REGION }}
+          AWS_ACCOUNT_ID: ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}
           RADIUS_SAMPLES_REPO_ROOT: ${{ github.workspace }}/samples
           # Test_MongoDB_Recipe_Parameters is using the following environment variable.
           INTEGRATION_TEST_RESOURCE_GROUP_NAME: ${{ env.AZURE_TEST_RESOURCE_GROUP }}
@@ -884,39 +892,14 @@ jobs:
     runs-on: ubuntu-latest
     if: failure() && github.event_name == 'schedule' && github.repository == 'radius-project/radius'
     steps:
-      - name: Count recently failed tests
-        id: count_failures
-        uses: actions/github-script@v7
-        with:
-          script: |
-            response = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'functional-test-cloud.yaml',
-              event: 'schedule',
-              per_page: 10
-            });
-
-            failureCount = 1;
-            for (const run of response.data.workflow_runs) {
-              if (run.conclusion === 'failure') {
-                failureCount++;
-              } else {
-                break;
-              }
-            }
-            return failureCount;
-
       - name: Create failure issue for failing scheduled run
         uses: actions/github-script@v7
-        # Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests.
-        if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD
         with:
           github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
           script: |
             github.rest.issues.create({
               ...context.repo,
               title: `Scheduled functional test failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
-              body: `## Bug information \n\nThis bug is generated automatically if the scheduled functional test fails at least ${process.env.ISSUE_CREATE_THRESHOLD} times in a row. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
+              body: `## Bug information \n\nThis issue is automatically generated if the scheduled functional test fails. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
             })

.github/workflows/functional-test-noncloud.yaml

@@ -66,10 +66,6 @@ env:
   TF_RECIPE_MODULE_SERVER_URL: "http://tf-module-server.radius-test-tf-module-server.svc.cluster.local"
   # Private Git repository where terraform module for testing is stored.
   TF_RECIPE_PRIVATE_GIT_SOURCE: "git::https://github.com/radius-project/terraform-private-modules//kubernetes-redis"
-
-  # The number of failed tests to report.
-  ISSUE_CREATE_THRESHOLD: 2
-
   # Local Docker registry name
   LOCAL_REGISTRY_NAME: "radius-registry"
   # Local Docker registry server
@@ -453,38 +449,14 @@ jobs:
     runs-on: ubuntu-latest
     if: failure() && github.event_name == 'schedule' && github.repository == 'radius-project/radius'
     steps:
-      - name: Count recently failed tests
-        id: count_failures
-        uses: actions/github-script@v7
-        with:
-          script: |
-            response = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'functional-test-noncloud.yaml',
-              event: 'schedule',
-              per_page: 10
-            });
-
-            failureCount = 1;
-            for (const run of response.data.workflow_runs) {
-              if (run.conclusion === 'failure') {
-                failureCount++;
-              } else {
-                break;
-              }
-            }
-            return failureCount;
       - name: Create failure issue for failing scheduled run
         uses: actions/github-script@v7
-        # Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests.
-        if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD
         with:
           github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
           script: |
             github.rest.issues.create({
               ...context.repo,
               title: `Scheduled functional test (noncloud) failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
-              body: `## Bug information \n\nThis bug is generated automatically if the scheduled functional test fails at least ${process.env.ISSUE_CREATE_THRESHOLD} times in a row. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
+              body: `## Bug information \n\nThis issue is automatically generated if the scheduled functional test fails. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
             })

.github/workflows/long-running-azure.yaml

@@ -95,9 +95,6 @@ env:
   # The current GitHub action link
   ACTION_LINK: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
 
-  # The number of failed tests to report.
-  ISSUE_CREATE_THRESHOLD: 2
-
 jobs:
   build:
     name: Build Radius for test
@@ -522,38 +519,14 @@ jobs:
     runs-on: ubuntu-latest
     if: failure() && github.repository == 'radius-project/radius' && github.event_name == 'schedule'
     steps:
-      - name: Count recently failed tests
-        id: count_failures
-        uses: actions/github-script@v7
-        with:
-          script: |
-            response = await github.rest.actions.listWorkflowRuns({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              workflow_id: 'long-running-azure.yaml',
-              event: 'schedule',
-              per_page: 10
-            });
-
-            failureCount = 1;
-            for (const run of response.data.workflow_runs) {
-              if (run.conclusion === 'failure') {
-                failureCount++;
-              } else {
-                break;
-              }
-            }
-            return failureCount;
       - name: Create failure issue for failing long running test run
         uses: actions/github-script@v7
-        # Only create an issue if there are (env.ISSUE_CREATE_THRESHOLD) failures of the recent tests.
-        if: steps.count_failures.outputs.result >= env.ISSUE_CREATE_THRESHOLD
         with:
           github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
           script: |
             github.rest.issues.create({
               ...context.repo,
               title: `Scheduled long running test failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
-              body: `## Bug information \n\nThis bug is generated automatically if the scheduled long running test fails at least ${process.env.ISSUE_CREATE_THRESHOLD} times in a row. The Radius long running test operates on a schedule of every 2 hours everyday. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
+              body: `## Bug information \n\nThis issue is automatically generated if the scheduled long running test fails. The Radius long running test operates on a schedule of every 2 hours everyday. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
             })

.github/workflows/purge-aws-test-resources.yaml

@@ -36,7 +36,7 @@ jobs:
       - name: Checkout the repository
         uses: actions/checkout@v4
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@v4.0.2
         with:
           aws-access-key-id: ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}

.github/workflows/release.yaml

@@ -46,7 +46,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Get supported versions from versions.yaml
         id: get-supported-versions
-        uses: mikefarah/[email protected].5
+        uses: mikefarah/[email protected].6
         with:
           # Get a comma-separated list of supported versions
           cmd: yq '.supported[].version' versions.yaml | tr '\n' ',' | sed 's/,$//'
@@ -165,7 +165,7 @@ jobs:
           git config --global user.email "[email protected]"
       - name: Get supported versions from versions.yaml
         id: get-supported-versions
-        uses: mikefarah/[email protected].5
+        uses: mikefarah/[email protected].6
         with:
           # Get a comma-separated list of supported versions
           cmd: yq '.supported[].version' ./radius/versions.yaml | tr '\n' ',' | sed 's/,$//'

.vscode/launch.json

@@ -58,12 +58,7 @@
       "args": [
         "--config-file",
         "${workspaceFolder}/cmd/ucpd/ucp-dev.yaml"
-      ],
-      "env": {
-        "BASE_PATH": "/apis/api.ucp.dev/v1alpha3",
-        "PORT": "9000",
-        "UCP_CONFIG": "${workspaceFolder}/cmd/ucpd/ucp-self-hosted-dev.yaml"
-      }
+      ]
     },
     {
       "name": "Launch Controller",

build/generate.mk

@@ -62,8 +62,8 @@ generate-controller-gen-installed:
 .PHONY: generate-ucp-crd
 generate-ucp-crd: generate-controller-gen-installed ## Generates the CRDs for UCP APIServer store.
 	@echo "$(ARROW) Generating CRDs for ucp.dev..."
-	controller-gen object:headerFile=./boilerplate.go.txt paths=./pkg/ucp/store/apiserverstore/api/ucp.dev/v1alpha1/...
-	controller-gen crd paths=./pkg/ucp/store/apiserverstore/api/ucp.dev/v1alpha1/... output:crd:dir=./deploy/Chart/crds/ucpd
+	controller-gen object:headerFile=./boilerplate.go.txt paths=./pkg/components/database/apiserverstore/api/ucp.dev/v1alpha1/...
+	controller-gen crd paths=./pkg/components/database/apiserverstore/api/ucp.dev/v1alpha1/... output:crd:dir=./deploy/Chart/crds/ucpd
 
 .PHONY: generate-controller
 generate-controller: generate-controller-gen-installed ## Generates the CRDs for the Radius controller.

cmd/applications-rp/cmd/root.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/spf13/cobra"
-	etcdclient "go.etcd.io/etcd/client/v3"
 	runtimelog "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/radius-project/radius/pkg/armrpc/builder"
@@ -33,8 +32,6 @@ import (
 	"github.com/radius-project/radius/pkg/server"
 	"github.com/radius-project/radius/pkg/trace"
 
-	"github.com/radius-project/radius/pkg/ucp/data"
-	"github.com/radius-project/radius/pkg/ucp/dataprovider"
 	"github.com/radius-project/radius/pkg/ucp/hosting"
 	"github.com/radius-project/radius/pkg/ucp/ucplog"
 
@@ -79,19 +76,6 @@ var rootCmd = &cobra.Command{
 		// Must set the logger before using controller-runtime.
 		runtimelog.SetLogger(logger)
 
-		if options.Config.StorageProvider.Provider == dataprovider.TypeETCD &&
-			options.Config.StorageProvider.ETCD.InMemory {
-			// For in-memory etcd we need to register another service to manage its lifecycle.
-			//
-			// The client will be initialized asynchronously.
-			logger.Info("Enabled in-memory etcd")
-			client := hosting.NewAsyncValue[etcdclient.Client]()
-			options.Config.StorageProvider.ETCD.Client = client
-			options.Config.SecretProvider.ETCD.Client = client
-
-			hostingSvc = append(hostingSvc, data.NewEmbeddedETCDService(data.EmbeddedETCDServiceOptions{ClientConfigSink: client}))
-		}
-
 		builders, err := builders(options)
 		if err != nil {
 			return err
@@ -111,7 +95,12 @@ var rootCmd = &cobra.Command{
 			Services: hostingSvc,
 		}
 
+		// Make the logger available to the services.
 		ctx := logr.NewContext(context.Background(), logger)
+
+		// Make the hosting configuration available to the services.
+		ctx = hostoptions.WithContext(ctx, options.Config)
+
 		return hosting.RunWithInterrupts(ctx, host)
 	},
 }

cmd/applications-rp/radius-dev.yaml

@@ -2,7 +2,7 @@
 environment:
   name: Dev
   roleLocation: "global"
-storageProvider:
+databaseProvider:
   provider: "etcd"
   etcd:
     inmemory: true

cmd/applications-rp/radius-self-hosted.yaml

@@ -8,9 +8,9 @@
 # - Disables metrics and profiler
 #
 environment:
-  name: Dev
+  name: self-hosted
   roleLocation: "global"
-storageProvider:
+databaseProvider:
   provider: "apiserver"
   apiserver:
     context: ''