Refactor and implement shared integration test host

This change updates implements a shared (reusable) integration test host for the Radius control-plane services. The new integration test host enables us to do in-memory testing of UCP and dynamic-rp using a "full stack" approach. This change is a significant refactor because the "glue" code in UCP had many points of divergence with the rest of our codebase. The following major changes are the bulk of the work: - Defining new types for configuration + options in UCP - Updating the UCP configuration file to match the format of other components Signed-off-by: Ryan Nowak <[email protected]>
radius-project · Dec 18, 2024 · 30e655e · 30e655e
1 parent d87e854
commit 30e655e
Show file tree

Hide file tree

Showing 83 changed files with 1,954 additions and 1,590 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -58,12 +58,7 @@
       "args": [
         "--config-file",
         "${workspaceFolder}/cmd/ucpd/ucp-dev.yaml"
-      ],
-      "env": {
-        "BASE_PATH": "/apis/api.ucp.dev/v1alpha3",
-        "PORT": "9000",
-        "UCP_CONFIG": "${workspaceFolder}/cmd/ucpd/ucp-self-hosted-dev.yaml"
-      }
+      ]
     },
     {
       "name": "Launch Controller",

diff --git a/cmd/applications-rp/cmd/root.go b/cmd/applications-rp/cmd/root.go
@@ -111,7 +111,12 @@ var rootCmd = &cobra.Command{
 			Services: hostingSvc,
 		}
 
+		// Make the logger available to the services.
 		ctx := logr.NewContext(context.Background(), logger)
+
+		// Make the hosting configuration available to the services.
+		ctx = hostoptions.WithContext(ctx, options.Config)
+
 		return hosting.RunWithInterrupts(ctx, host)
 	},
 }

diff --git a/cmd/applications-rp/radius-dev.yaml b/cmd/applications-rp/radius-dev.yaml
@@ -2,7 +2,7 @@
 environment:
   name: Dev
   roleLocation: "global"
-storageProvider:
+databaseProvider:
   provider: "etcd"
   etcd:
     inmemory: true

diff --git a/cmd/applications-rp/radius-self-hosted.yaml b/cmd/applications-rp/radius-self-hosted.yaml
@@ -8,9 +8,9 @@
 # - Disables metrics and profiler
 #
 environment:
-  name: Dev
+  name: self-hosted
   roleLocation: "global"
-storageProvider:
+databaseProvider:
   provider: "apiserver"
   apiserver:
     context: ''

diff --git a/cmd/dynamic-rp/dynamicrp-dev.yaml b/cmd/dynamic-rp/dynamicrp-dev.yaml
@@ -2,7 +2,7 @@
 environment:
   name: Dev
   roleLocation: "global"
-storageProvider:
+databaseProvider:
   provider: "apiserver"
   apiserver:
     context: ''

diff --git a/cmd/ucpd/cmd/root.go b/cmd/ucpd/cmd/root.go
@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"os"
 
 	"github.com/go-logr/logr"
 	"github.com/spf13/cobra"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/radius-project/radius/pkg/armrpc/hostoptions"
 	"github.com/radius-project/radius/pkg/components/database/databaseprovider"
+	"github.com/radius-project/radius/pkg/ucp"
 	"github.com/radius-project/radius/pkg/ucp/hosting"
 	"github.com/radius-project/radius/pkg/ucp/server"
 	"github.com/radius-project/radius/pkg/ucp/ucplog"
@@ -38,12 +40,23 @@ var rootCmd = &cobra.Command{
 	Long:  `Server process for the Universal Control Plane (UCP).`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		configFilePath := cmd.Flag("config-file").Value.String()
-		options, err := server.NewServerOptionsFromEnvironment(configFilePath)
+
+		bs, err := os.ReadFile(configFilePath)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to read configuration file: %w", err)
+		}
+
+		config, err := ucp.LoadConfig(bs)
+		if err != nil {
+			return fmt.Errorf("failed to parse configuration file: %w", err)
+		}
+
+		options, err := ucp.NewOptions(cmd.Context(), config)
+		if err != nil {
+			return fmt.Errorf("failed to create server options: %w", err)
 		}
 
-		logger, flush, err := ucplog.NewLogger(ucplog.LoggerName, &options.LoggingOptions)
+		logger, flush, err := ucplog.NewLogger(ucplog.LoggerName, &options.Config.Logging)
 		if err != nil {
 			return err
 		}
@@ -52,17 +65,17 @@ var rootCmd = &cobra.Command{
 		// Must set the logger before using controller-runtime.
 		runtimelog.SetLogger(logger)
 
-		if options.DatabaseProviderOptions.Provider == databaseprovider.TypeETCD &&
-			options.DatabaseProviderOptions.ETCD.InMemory {
+		if options.Config.Database.Provider == databaseprovider.TypeETCD &&
+			options.Config.Database.ETCD.InMemory {
 			// For in-memory etcd we need to register another service to manage its lifecycle.
 			//
 			// The client will be initialized asynchronously.
 			clientconfigSource := hosting.NewAsyncValue[etcdclient.Client]()
-			options.DatabaseProviderOptions.ETCD.Client = clientconfigSource
-			options.SecretProviderOptions.ETCD.Client = clientconfigSource
+			options.Config.Database.ETCD.Client = clientconfigSource
+			options.Config.Secrets.ETCD.Client = clientconfigSource
 		}
 
-		host, err := server.NewServer(&options)
+		host, err := server.NewServer(options)
 		if err != nil {
 			return err
 		}

diff --git a/cmd/ucpd/ucp-dev.yaml b/cmd/ucpd/ucp-dev.yaml
@@ -9,8 +9,14 @@
 # - Talk to Portable Resources' Providers on port 8081
 # - Disables metrics and profiler
 #
-location: 'global'
-storageProvider:
+environment:
+  name: Dev
+  roleLocation: "global"
+server:
+  port: 9000
+  pathBase: /apis/api.ucp.dev/v1alpha3
+
+databaseProvider:
   provider: "apiserver"
   apiserver:
     context: ''
@@ -32,19 +38,20 @@ profilerProvider:
 
 #Default planes configuration with which ucp starts
 # TODO: Remove azure and aws planes once rad provider commands are supported
-planes:
-  - id: "/planes/aws/aws"
-    properties:
-      kind: "AWS"
-  - id: "/planes/radius/local"
-    properties:
-      resourceProviders:
-        Applications.Core: "http://localhost:8080"
-        Applications.Messaging: "http://localhost:8080"
-        Applications.Dapr: "http://localhost:8080"
-        Applications.Datastores: "http://localhost:8080"
-        Microsoft.Resources: "http://localhost:5017"
-      kind: "UCPNative"
+initialization:
+  planes:
+    - id: "/planes/aws/aws"
+      properties:
+        kind: "AWS"
+    - id: "/planes/radius/local"
+      properties:
+        resourceProviders:
+          Applications.Core: "http://localhost:8080"
+          Applications.Messaging: "http://localhost:8080"
+          Applications.Dapr: "http://localhost:8080"
+          Applications.Datastores: "http://localhost:8080"
+          Microsoft.Resources: "http://localhost:5017"
+        kind: "UCPNative"
 
 identity:
   authMethod: default
@@ -76,4 +83,4 @@ logging:
 tracerProvider:
   serviceName: "ucp"
   zipkin:
-    url: "http://localhost:9411/api/v2/spans"
+    url: "http://localhost:9411/api/v2/spans"
diff --git a/deploy/Chart/templates/controller/configmaps.yaml b/deploy/Chart/templates/controller/configmaps.yaml
@@ -12,7 +12,7 @@ data:
       host: "0.0.0.0"
       port: 9443
 
-    storageProvider:
+    databaseProvider:
       provider: "apiserver"
       apiserver:
         context: ""

diff --git a/deploy/Chart/templates/dynamic-rp/configmaps.yaml b/deploy/Chart/templates/dynamic-rp/configmaps.yaml
@@ -13,7 +13,7 @@ data:
     environment:
       name: self-hosted
       roleLocation: "global"
-    storageProvider:
+    databaseProvider:
       provider: "apiserver"
       apiserver:
         context: ""

diff --git a/deploy/Chart/templates/rp/configmaps.yaml b/deploy/Chart/templates/rp/configmaps.yaml
@@ -13,7 +13,7 @@ data:
     environment:
       name: self-hosted
       roleLocation: "global"
-    storageProvider:
+    databaseProvider:
       provider: "apiserver"
       apiserver:
         context: ""

diff --git a/deploy/Chart/templates/ucp/configmaps.yaml b/deploy/Chart/templates/ucp/configmaps.yaml
@@ -10,8 +10,14 @@ data:
   ucp-config.yaml: |-
     # Radius configuration file.
     # See https://github.com/radius-project/radius/blob/main/docs/contributing/contributing-code/contributing-code-control-plane/configSettings.md for more information.
-    location: 'global'
-    storageProvider:
+    environment:
+      name: Dev
+      roleLocation: "global"
+    server:
+      port: 9443
+      pathBase: /apis/api.ucp.dev/v1alpha3
+      tlsCertificateDirectory: /var/tls/cert
+    databaseProvider:
       provider: "apiserver"
       apiserver:
         context: ""
@@ -30,20 +36,20 @@ data:
     profilerProvider:
       enabled: true
       port: 6060
-
-    planes:
-      - id: "/planes/radius/local"
-        properties:
-          resourceProviders:
-            Applications.Core: "http://applications-rp.radius-system:5443"
-            Applications.Dapr: "http://applications-rp.radius-system:5443"
-            Applications.Datastores: "http://applications-rp.radius-system:5443"
-            Applications.Messaging: "http://applications-rp.radius-system:5443"
-            Microsoft.Resources: "http://bicep-de.radius-system:6443"
-          kind: "UCPNative"
-      - id: "/planes/aws/aws"
-        properties:
-          kind: "AWS"
+    initialization:
+      planes:
+        - id: "/planes/radius/local"
+          properties:
+            resourceProviders:
+              Applications.Core: "http://applications-rp.radius-system:5443"
+              Applications.Dapr: "http://applications-rp.radius-system:5443"
+              Applications.Datastores: "http://applications-rp.radius-system:5443"
+              Applications.Messaging: "http://applications-rp.radius-system:5443"
+              Microsoft.Resources: "http://bicep-de.radius-system:6443"
+            kind: "UCPNative"
+        - id: "/planes/aws/aws"
+          properties:
+            kind: "AWS"
 
     identity:
       authMethod: UCPCredential

diff --git a/deploy/Chart/templates/ucp/deployment.yaml b/deploy/Chart/templates/ucp/deployment.yaml
@@ -36,8 +36,6 @@ spec:
         args:
         - --config-file=/etc/config/ucp-config.yaml
         env:
-        - name: BASE_PATH
-          value: '/apis/api.ucp.dev/v1alpha3' # listen for APIService URLs
         - name: TLS_CERT_DIR
           value: '/var/tls/cert'
         - name: PORT

diff --git a/...ontributing/contributing-code/contributing-code-control-plane/configSettings.md b/...ontributing/contributing-code/contributing-code-control-plane/configSettings.md
@@ -20,7 +20,7 @@ The following properties can be specified in configuration for all services:
 |-----|-------------|---------|
 | environment | Environment name and its role location | [**See below**](#environment) |
 | identity | AAD APP authentication for the resource provider | [**See below**](#identity) |
-| storageProvider | Configuration options for the data storage provider | [**See below**](#storageprovider) |
+| databaseProvider | Configuration options for the database provider | [**See below**](#databaseprovider) |
 | queueProvider | Configuration options for the provider to create and manage the queue client | [**See below**](#queueprovider) |
 | secretProvider | Configuration options for the provider to manage credential | [**See below**](#secretprovider) |
 | server | Configuration options for the HTTP server bootstrap | [**See below**](#server) |
@@ -61,18 +61,18 @@ The following are properties that can be specified for UCP:
 | audience | The recipient of the certificate | `https://management.core.windows.net` |
 | pemCertPath | Path to certificate file | `/var/certs/rp-aad-app.pem` |
 
-### storageProvider
+### databaseProvider
 | Key | Description | Example |
 |-----|-------------|---------|
-| provider | The type of storage provider | `apiServer` | 
-| apiServer | Object containing properties for Kubernetes APIServer store | [**See below**](#apiserver) |
-| etcd | Object containing properties for ETCD store | [**See below**](#etcd)|
+| provider | The type of database provider | `apiServer` | 
+| apiServer | Object containing properties for Kubernetes APIServer database | [**See below**](#apiserver) |
+| etcd | Object containing properties for ETCD database | [**See below**](#etcd)|
 
 ### queueProvider
 | Key | Description | Example |
 |-----|-------------|---------|
 | provider | The type of queue provider | `apiServer` | 
-| apiServer |  Object containing properties for Kubernetes APIServer store | [**See below**](#apiserver) |
+| apiServer |  Object containing properties for Kubernetes APIServer queue | [**See below**](#apiserver) |
 | inMemoryQueue | Object containing properties for InMemory Queue client | |
 
 ### secretProvider
@@ -181,7 +181,7 @@ Below are completed examples of possible configurations:
 environment:
   name: self-hosted
   roleLocation: "global"
-storageProvider:
+databaseProvider:
   provider: "apiserver"
   apiserver:
     context: ""
@@ -210,7 +210,7 @@ ucp:
 ### UCP 
 ```yaml
 location: 'global'
-storageProvider:
+databaseProvider:
   provider: "apiserver"
   apiserver:
     context: ""