Initial implementation of user-defined-types (radius-project#7686)

This change implements the skeleton of user-defined types. The changes
here enable the following:

- Users can author a resource of type
`System.Resources/resourceProviders` to create a user-defined-type.
- Users can use the UCP API to register and query `resourceProviders`.
- Users can use the UCP API to execute the full lifecycle of a
user-defined-type.

Right now the user-defined-type RP will use our default operation
(synchronous) controllers to implement the resource lifecycle. There is
no background processing.

The next step will include the ability to execute asynchronous
operations like recipes.

- This pull request fixes a bug in Radius and has an approved issue
(issue link required).
- This pull request adds or changes features of Radius and has an
approved issue (issue link required).

Part of: radius-project#6688

**note: This change is going into a feature-branch where we can iterate
on the user-defined-type design before integrating it with main. The PR
is an FYI 😆.**

---------

Signed-off-by: ytimocin <[email protected]>
Signed-off-by: dependabot[bot] <[email protected]>
Signed-off-by: willdavsmith <[email protected]>
Signed-off-by: Ryan Nowak <[email protected]>
Co-authored-by: Yetkin Timocin <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Will Smith <[email protected]>
Signed-off-by: Ryan Nowak <[email protected]>

.github/scripts/release-verification.sh

@@ -69,10 +69,12 @@ kind create cluster
 ./rad install kubernetes
 
 EXPECTED_APPCORE_RP_IMAGE="ghcr.io/radius-project/applications-rp:${EXPECTED_TAG_VERSION}"
+EXPECTED_DYNAMIC_RP_IMAGE="ghcr.io/radius-project/dynamic-rp:${EXPECTED_TAG_VERSION}"
 EXPECTED_UCP_IMAGE="ghcr.io/radius-project/ucpd:${EXPECTED_TAG_VERSION}"
 EXPECTED_DE_IMAGE="ghcr.io/radius-project/deployment-engine:${EXPECTED_TAG_VERSION}"
 
 APPCORE_RP_IMAGE=$(kubectl describe pods -n radius-system -l control-plane=applications-rp | awk '/^.*Image:/ {print $2}')
+DYNAMIC_RP_IMAGE=$(kubectl describe pods -n radius-system -l control-plane=dynamic-rp | awk '/^.*Image:/ {print $2}')
 UCP_IMAGE=$(kubectl describe pods -n radius-system -l control-plane=ucp | awk '/^.*Image:/ {print $2}')
 DE_IMAGE=$(kubectl describe pods -n radius-system -l control-plane=bicep-de | awk '/^.*Image:/ {print $2}')
 
@@ -81,6 +83,11 @@ if [[ "${APPCORE_RP_IMAGE}" != "${EXPECTED_APPCORE_RP_IMAGE}" ]]; then
     exit 1
 fi
 
+if [[ "${DYNAMIC_RP_IMAGE}" != "${EXPECTED_DYNAMIC_RP_IMAGE}" ]]; then
+    echo "Error: Dynamic RP image: ${DYNAMIC_RP_IMAGE} does not match the desired image: ${EXPECTED_DYNAMIC_RP_IMAGE}."
+    exit 1
+fi
+
 if [[ "${UCP_IMAGE}" != "${EXPECTED_UCP_IMAGE}" ]]; then
     echo "Error: UCP image: ${UCP_IMAGE} does not match the desired image: ${EXPECTED_UCP_IMAGE}."
     exit 1

.github/workflows/functional-test-cloud.yaml

@@ -275,6 +275,7 @@ jobs:
             * Bicep recipe location `${{ env.BICEP_RECIPE_REGISTRY }}/test/testrecipes/test-bicep-recipes/<name>:${{ env.REL_VERSION }}`
             * Terraform recipe location `${{ env.TF_RECIPE_MODULE_SERVER_URL }}/<name>.zip` (in cluster)
             * applications-rp test image location: `${{ env.CONTAINER_REGISTRY }}/applications-rp:${{ env.REL_VERSION }}`
+            * dynamic-rp test image location: `${{ env.CONTAINER_REGISTRY }}/dynamic-rp:${{ env.REL_VERSION }}`
             * controller test image location: `${{ env.CONTAINER_REGISTRY }}/controller:${{ env.REL_VERSION }}`
             * ucp test image location: `${{ env.CONTAINER_REGISTRY }}/ucpd:${{ env.REL_VERSION }}`
             * deployment-engine test image location: `${{ env.DE_IMAGE }}:${{ env.DE_TAG }}`
@@ -647,7 +648,7 @@ jobs:
           echo "*** Installing Radius to Kubernetes ***"
           rad install kubernetes \
             --chart ${{ env.RADIUS_CHART_LOCATION }} \
-            --set rp.image=${{ env.CONTAINER_REGISTRY }}/applications-rp,rp.tag=${{ env.REL_VERSION }},controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }},ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }},de.image=${{ env.DE_IMAGE }},de.tag=${{ env.DE_TAG }},global.azureWorkloadIdentity.enabled=true
+            --set rp.image=${{ env.CONTAINER_REGISTRY }}/applications-rp,rp.tag=${{ env.REL_VERSION }},dynamicrp.image=${{ env.CONTAINER_REGISTRY }}/dynamic-rp,dynamicrp.tag=${{ env.REL_VERSION }}controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }},ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }},de.image=${{ env.DE_IMAGE }},de.tag=${{ env.DE_TAG }},global.azureWorkloadIdentity.enabled=true
 
           echo "*** Create workspace, group and environment for test ***"
           rad workspace create kubernetes

.github/workflows/functional-test-noncloud.yaml

@@ -290,6 +290,7 @@ jobs:
           RAD_COMMAND="rad install kubernetes \
                  --chart ${{ env.RADIUS_CHART_LOCATION }} \
                  --set rp.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/applications-rp,rp.tag=${{ env.REL_VERSION }} \
+                 --set dynamicrp.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/dynamic-rp,dynamicrp.tag=${{ env.REL_VERSION }} \
                  --set controller.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/controller,controller.tag=${{ env.REL_VERSION }} \
                  --set ucp.image=${{ env.LOCAL_REGISTRY_NAME }}:${{ env.LOCAL_REGISTRY_PORT }}/ucpd,ucp.tag=${{ env.REL_VERSION }} \
                  --set de.image=${{ env.DE_IMAGE }},de.tag=${{ env.DE_TAG }}"

.github/workflows/long-running-azure.yaml

@@ -201,6 +201,7 @@ jobs:
           * Bicep recipe location `${{ env.BICEP_RECIPE_REGISTRY }}/test/testrecipes/test-bicep-recipes/<name>:${{ steps.gen-id.outputs.REL_VERSION }}`
           * Terraform recipe location `${{ env.TF_RECIPE_MODULE_SERVER_URL }}/<name>.zip` (in cluster)
           * applications-rp test image location: `${{ env.CONTAINER_REGISTRY }}/applications-rp:${{ steps.gen-id.outputs.REL_VERSION }}`
+          * dynamic-rp test image location: `${{ env.CONTAINER_REGISTRY }}/dynamic-rp:${{ steps.gen-id.outputs.REL_VERSION }}`
           * controller test image location: `${{ env.CONTAINER_REGISTRY }}/controller:${{ steps.gen-id.outputs.REL_VERSION }}`
           * ucp test image location: `${{ env.CONTAINER_REGISTRY }}/ucpd:${{ steps.gen-id.outputs.REL_VERSION }}`
 
@@ -405,7 +406,7 @@ jobs:
           echo "*** Installing Radius to Kubernetes ***"
           rad install kubernetes --reinstall \
             --chart ${{ env.RADIUS_CHART_LOCATION }} \
-            --set rp.image=${{ env.CONTAINER_REGISTRY }}/applications-rp,rp.tag=${{ env.REL_VERSION }},controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }},ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }}
+            --set rp.image=${{ env.CONTAINER_REGISTRY }}/applications-rp,rp.tag=${{ env.REL_VERSION }},dynamicrp.image=${{ env.CONTAINER_REGISTRY }}/dynamic-rp,dynamicrp.tag=${{ env.REL_VERSION }},controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }},ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }}
       - name: Configure Radius test workspace
         run: |
           set -x

.vscode/launch.json

@@ -38,6 +38,13 @@
             "env": {
                 "RADIUS_ENV": "self-hosted"
             }
+        },{
+            "name": "Launch Dynamic RP",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "preLaunchTask": "Build Radius (all)",
+            "program": "${workspaceFolder}/cmd/dynamic-rp/main.go",
         },
         {
             "name": "Launch UCP",

build/build.mk

@@ -119,6 +119,7 @@ GOARCHES := amd64 arm arm64
 BINARIES := docgen:./cmd/docgen \
 	rad:./cmd/rad \
 	applications-rp:./cmd/applications-rp \
+	dynamic-rp:./cmd/dynamic-rp \
 	ucpd:./cmd/ucpd \
 	controller:./cmd/controller \
 	testrp:./test/testrp \

build/docker.mk

@@ -101,6 +101,7 @@ configure-buildx:
 # Define a target for each image with name and Dockerfile location
 APPS_MAP := ucpd:./deploy/images/ucpd \
 	applications-rp:./deploy/images/applications-rp \
+	dynamic-rp:./deploy/images/dynamic-rp \
 	controller:./deploy/images/controller \
 	testrp:./test/testrp \
 	magpiego:./test/magpiego

cmd/dynamic-rp/cmd/root.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2023 The Radius Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/go-logr/logr"
+	"github.com/spf13/cobra"
+	runtimelog "sigs.k8s.io/controller-runtime/pkg/log"
+
+	"github.com/radius-project/radius/pkg/armrpc/hostoptions"
+	"github.com/radius-project/radius/pkg/dynamicrp"
+	"github.com/radius-project/radius/pkg/dynamicrp/server"
+	"github.com/radius-project/radius/pkg/ucp/hosting"
+	"github.com/radius-project/radius/pkg/ucp/ucplog"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "dynamic-rp",
+	Short: "Dyanamic Resource Provider server",
+	Long:  `Server process for the Dynamic Resource Provider (UCP).`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		configFilePath := cmd.Flag("config-file").Value.String()
+		bs, err := os.ReadFile(configFilePath)
+		if err != nil {
+			return fmt.Errorf("failed to read config file %s: %w", configFilePath, err)
+		}
+
+		config, err := dynamicrp.LoadConfig(bs)
+		if err != nil {
+			return fmt.Errorf("failed to parse config file %s: %w", configFilePath, err)
+		}
+
+		options, err := dynamicrp.NewOptions(cmd.Context(), config)
+		if err != nil {
+			return err
+		}
+
+		logger, flush, err := ucplog.NewLogger(ucplog.LoggerName, &options.Config.Logging)
+		if err != nil {
+			log.Fatal(err) //nolint:forbidigo // this is OK inside the main function.
+		}
+		defer flush()
+
+		// Must set the logger before using controller-runtime.
+		runtimelog.SetLogger(logger)
+
+		host, err := server.NewServer(options)
+		if err != nil {
+			return err
+		}
+
+		ctx := logr.NewContext(cmd.Context(), logger)
+
+		return hosting.RunWithInterrupts(ctx, host)
+	},
+}
+
+func Execute() {
+	// Let users override the configuration via `--config-file`.
+	rootCmd.Flags().String("config-file", fmt.Sprintf("dynamicrp-%s.yaml", hostoptions.Environment()), "The service configuration file.")
+
+	cobra.CheckErr(rootCmd.ExecuteContext(context.Background()))
+}

cmd/dynamic-rp/dynamicrp-dev.yaml

@@ -0,0 +1,43 @@
+# This is an example of configuration file.
+environment:
+  name: Dev
+  roleLocation: "global"
+storageProvider:
+  provider: "etcd"
+  etcd:
+    inmemory: true
+queueProvider:
+  provider: inmemory
+  name: radius
+profilerProvider:
+  enabled: true
+  port: 6060
+secretProvider:
+  provider: etcd
+  etcd:
+    inmemory: true
+metricsProvider:
+  prometheus:
+    enabled: true
+    path: "/metrics"
+    port: 9090
+featureFlags:
+  - "PLACEHOLDER"
+server:
+  host: "0.0.0.0"
+  port: 8080
+  enableArmAuth: false
+workerServer:
+  maxOperationConcurrency: 10
+  maxOperationRetryCount: 2
+ucp:
+  kind: kubernetes
+ # Logging configuration   
+logging:
+  level: "info"
+  json: false
+bicep:
+  deleteRetryCount: 20
+  deleteRetryDelaySeconds: 60
+terraform:
+  path: "/terraform"

cmd/dynamic-rp/main.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2023 The Radius Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import "github.com/radius-project/radius/cmd/dynamic-rp/cmd"
+
+func main() {
+	cmd.Execute()
+}

deploy/Chart/values.yaml

@@ -64,6 +64,24 @@ ucp:
     limits:
       memory: "300Mi"
 
+dynamicrp:
+  image: ghcr.io/radius-project/dynamic-rp
+  # Default tag uses Chart AppVersion.
+  # tag: latest
+  resources:
+    requests:
+      # request memory is the average memory usage + 10% buffer.
+      memory: "160Mi"
+    limits:
+      # limit is higher for dynamic-rp because the Terraform execution
+      # can spike memory usage.
+      memory: "500Mi"
+  bicep:
+    deleteRetryCount: 20
+    deleteRetryDelaySeconds: 60
+  terraform:
+    path: "/terraform"
+
 rp:
   image: ghcr.io/radius-project/applications-rp
   # Default tag uses Chart AppVersion.

pkg/armrpc/asyncoperation/worker/registry.go

@@ -25,13 +25,27 @@ import (
 	"github.com/radius-project/radius/pkg/ucp/dataprovider"
 )
 
+const (
+	// ResourceTypeAny is a wildcard for any resource type.
+	ResourceTypeAny = "*"
+
+	// OperationMethodAny is a wildcard for any operation method.
+	OperationMethodAny = "*"
+)
+
+// ControllerFactoryFunc is a factory function to create a controller.
 type ControllerFactoryFunc func(opts ctrl.Options) (ctrl.Controller, error)
 
 // ControllerRegistry is an registry to register async controllers.
 type ControllerRegistry struct {
 	ctrlMap   map[string]ctrl.Controller
 	ctrlMapMu sync.RWMutex
 	sp        dataprovider.DataStorageProvider
+
+	// Fallback allows the registration of a controller that will be used
+	// for operations that don't match any other operation type.
+	fallbackFactory ControllerFactoryFunc
+	fallbackOpts    ctrl.Options
 }
 
 // NewControllerRegistry creates an ControllerRegistry instance.
@@ -48,6 +62,13 @@ func (h *ControllerRegistry) Register(ctx context.Context, resourceType string,
 	defer h.ctrlMapMu.Unlock()
 
 	ot := v1.OperationType{Type: resourceType, Method: method}
+	if resourceType == ResourceTypeAny && method == OperationMethodAny {
+		// This is a fallback controller. Skip registration for now so we can create instances
+		// dynamically when needed.
+		h.fallbackFactory = factoryFn
+		h.fallbackOpts = opts
+		return nil
+	}
 
 	storageClient, err := h.sp.GetStorageClient(ctx, resourceType)
 	if err != nil {
@@ -66,13 +87,29 @@ func (h *ControllerRegistry) Register(ctx context.Context, resourceType string,
 }
 
 // Get gets the registered async controller instance.
-func (h *ControllerRegistry) Get(operationType v1.OperationType) ctrl.Controller {
+func (h *ControllerRegistry) Get(ctx context.Context, operationType v1.OperationType) (ctrl.Controller, error) {
 	h.ctrlMapMu.RLock()
 	defer h.ctrlMapMu.RUnlock()
 
 	if h, ok := h.ctrlMap[operationType.String()]; ok {
-		return h
+		return h, nil
 	}
 
-	return nil
+	// If no controller is found, then look for a default controller.
+	if h.fallbackFactory == nil {
+		return nil, nil
+	}
+
+	storageClient, err := h.sp.GetStorageClient(ctx, operationType.Type)
+	if err != nil {
+		return nil, err
+	}
+
+	// Copy the options so we can update it.
+	opts := h.fallbackOpts
+
+	opts.StorageClient = storageClient
+	opts.ResourceType = operationType.Type
+
+	return h.fallbackFactory(opts)
 }

pkg/armrpc/asyncoperation/worker/registry_test.go

@@ -63,8 +63,11 @@ func TestRegister_Get(t *testing.T) {
 	}, ctrlOpts)
 	require.NoError(t, err)
 
-	ctrl := registry.Get(opGet)
+	ctrl, err := registry.Get(context.Background(), opGet)
+	require.NoError(t, err)
 	require.NotNil(t, ctrl)
-	ctrl = registry.Get(opPut)
+
+	ctrl, err = registry.Get(context.Background(), opPut)
+	require.NoError(t, err)
 	require.NotNil(t, ctrl)
 }

pkg/armrpc/asyncoperation/worker/worker.go

@@ -168,7 +168,15 @@ func (w *AsyncRequestProcessWorker) Start(ctx context.Context) error {
 			}
 			reqCtx = v1.WithARMRequestContext(reqCtx, armReqCtx)
 
-			asyncCtrl := w.registry.Get(armReqCtx.OperationType)
+			asyncCtrl, err := w.registry.Get(reqCtx, armReqCtx.OperationType)
+			if err != nil {
+				opLogger.Error(err, "failed to get async controller.")
+				if err := w.requestQueue.FinishMessage(reqCtx, msgreq); err != nil {
+					opLogger.Error(err, "failed to finish the message")
+				}
+				return
+			}
+
 			if asyncCtrl == nil {
 				opLogger.Error(nil, "cannot process unknown operation: "+armReqCtx.OperationType.String())
 				if err := w.requestQueue.FinishMessage(reqCtx, msgreq); err != nil {

pkg/armrpc/builder/builder_test.go

@@ -261,7 +261,8 @@ func TestApplyAsyncHandler(t *testing.T) {
 	}
 
 	for _, op := range expectedOperations {
-		jobCtrl := registry.Get(op)
+		jobCtrl, err := registry.Get(context.Background(), op)
+		require.NoError(t, err)
 		require.NotNil(t, jobCtrl)
 	}
 }