diff --git a/.vscode/launch.json b/.vscode/launch.json index d98ac8286c..899ef7aa1e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -58,12 +58,7 @@ "args": [ "--config-file", "${workspaceFolder}/cmd/ucpd/ucp-dev.yaml" - ], - "env": { - "BASE_PATH": "/apis/api.ucp.dev/v1alpha3", - "PORT": "9000", - "UCP_CONFIG": "${workspaceFolder}/cmd/ucpd/ucp-self-hosted-dev.yaml" - } + ] }, { "name": "Launch Controller", diff --git a/cmd/applications-rp/cmd/root.go b/cmd/applications-rp/cmd/root.go index 88650ee966..d1c97e1112 100644 --- a/cmd/applications-rp/cmd/root.go +++ b/cmd/applications-rp/cmd/root.go @@ -111,7 +111,12 @@ var rootCmd = &cobra.Command{ Services: hostingSvc, } + // Make the logger available to the services. ctx := logr.NewContext(context.Background(), logger) + + // Make the hosting configuration available to the services. + ctx = hostoptions.WithContext(ctx, options.Config) + return hosting.RunWithInterrupts(ctx, host) }, } diff --git a/cmd/applications-rp/radius-self-hosted.yaml b/cmd/applications-rp/radius-self-hosted.yaml index 2b9dd9368d..e9184d50da 100644 --- a/cmd/applications-rp/radius-self-hosted.yaml +++ b/cmd/applications-rp/radius-self-hosted.yaml @@ -8,7 +8,7 @@ # - Disables metrics and profiler # environment: - name: Dev + name: self-hosted roleLocation: "global" storageProvider: provider: "apiserver" diff --git a/cmd/ucpd/cmd/root.go b/cmd/ucpd/cmd/root.go index c6672ea03f..fd524c5c99 100644 --- a/cmd/ucpd/cmd/root.go +++ b/cmd/ucpd/cmd/root.go @@ -19,6 +19,7 @@ package cmd import ( "context" "fmt" + "os" "github.com/go-logr/logr" "github.com/spf13/cobra" @@ -26,6 +27,7 @@ import ( runtimelog "sigs.k8s.io/controller-runtime/pkg/log" "github.com/radius-project/radius/pkg/armrpc/hostoptions" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/hosting" "github.com/radius-project/radius/pkg/ucp/server" @@ -38,12 +40,23 @@ var rootCmd = &cobra.Command{ Long: `Server process for the Universal Control Plane (UCP).`, RunE: func(cmd *cobra.Command, args []string) error { configFilePath := cmd.Flag("config-file").Value.String() - options, err := server.NewServerOptionsFromEnvironment(configFilePath) + + bs, err := os.ReadFile(configFilePath) if err != nil { - return err + return fmt.Errorf("failed to read configuration file: %w", err) + } + + config, err := ucp.LoadConfig(bs) + if err != nil { + return fmt.Errorf("failed to parse configuration file: %w", err) + } + + options, err := ucp.NewOptions(cmd.Context(), config) + if err != nil { + return fmt.Errorf("failed to create server options: %w", err) } - logger, flush, err := ucplog.NewLogger(ucplog.LoggerName, &options.LoggingOptions) + logger, flush, err := ucplog.NewLogger(ucplog.LoggerName, &options.Config.Logging) if err != nil { return err } @@ -52,17 +65,17 @@ var rootCmd = &cobra.Command{ // Must set the logger before using controller-runtime. runtimelog.SetLogger(logger) - if options.StorageProviderOptions.Provider == dataprovider.TypeETCD && - options.StorageProviderOptions.ETCD.InMemory { + if options.Config.Storage.Provider == dataprovider.TypeETCD && + options.Config.Storage.ETCD.InMemory { // For in-memory etcd we need to register another service to manage its lifecycle. // // The client will be initialized asynchronously. clientconfigSource := hosting.NewAsyncValue[etcdclient.Client]() - options.StorageProviderOptions.ETCD.Client = clientconfigSource - options.SecretProviderOptions.ETCD.Client = clientconfigSource + options.Config.Storage.ETCD.Client = clientconfigSource + options.Config.Storage.ETCD.Client = clientconfigSource } - host, err := server.NewServer(&options) + host, err := server.NewServer(options) if err != nil { return err } diff --git a/cmd/ucpd/ucp-dev.yaml b/cmd/ucpd/ucp-dev.yaml index 32f1a465cf..3138038efa 100644 --- a/cmd/ucpd/ucp-dev.yaml +++ b/cmd/ucpd/ucp-dev.yaml @@ -9,7 +9,13 @@ # - Talk to Portable Resources' Providers on port 8081 # - Disables metrics and profiler # -location: 'global' +environment: + name: Dev + roleLocation: "global" +server: + port: 9000 + pathBase: /apis/api.ucp.dev/v1alpha3 + storageProvider: provider: "apiserver" apiserver: @@ -32,19 +38,20 @@ profilerProvider: #Default planes configuration with which ucp starts # TODO: Remove azure and aws planes once rad provider commands are supported -planes: - - id: "/planes/aws/aws" - properties: - kind: "AWS" - - id: "/planes/radius/local" - properties: - resourceProviders: - Applications.Core: "http://localhost:8080" - Applications.Messaging: "http://localhost:8080" - Applications.Dapr: "http://localhost:8080" - Applications.Datastores: "http://localhost:8080" - Microsoft.Resources: "http://localhost:5017" - kind: "UCPNative" +initialization: + planes: + - id: "/planes/aws/aws" + properties: + kind: "AWS" + - id: "/planes/radius/local" + properties: + resourceProviders: + Applications.Core: "http://localhost:8080" + Applications.Messaging: "http://localhost:8080" + Applications.Dapr: "http://localhost:8080" + Applications.Datastores: "http://localhost:8080" + Microsoft.Resources: "http://localhost:5017" + kind: "UCPNative" identity: authMethod: default @@ -76,4 +83,4 @@ logging: tracerProvider: serviceName: "ucp" zipkin: - url: "http://localhost:9411/api/v2/spans" + url: "http://localhost:9411/api/v2/spans" \ No newline at end of file diff --git a/deploy/Chart/templates/ucp/configmaps.yaml b/deploy/Chart/templates/ucp/configmaps.yaml index d56d0f4f30..aae1a17a15 100644 --- a/deploy/Chart/templates/ucp/configmaps.yaml +++ b/deploy/Chart/templates/ucp/configmaps.yaml @@ -10,7 +10,13 @@ data: ucp-config.yaml: |- # Radius configuration file. # See https://github.com/radius-project/radius/blob/main/docs/contributing/contributing-code/contributing-code-control-plane/configSettings.md for more information. - location: 'global' + environment: + name: Dev + roleLocation: "global" + server: + port: 9443 + pathBase: /apis/api.ucp.dev/v1alpha3 + tlsCertificateDirectory: /var/tls/cert storageProvider: provider: "apiserver" apiserver: @@ -30,20 +36,20 @@ data: profilerProvider: enabled: true port: 6060 - - planes: - - id: "/planes/radius/local" - properties: - resourceProviders: - Applications.Core: "http://applications-rp.radius-system:5443" - Applications.Dapr: "http://applications-rp.radius-system:5443" - Applications.Datastores: "http://applications-rp.radius-system:5443" - Applications.Messaging: "http://applications-rp.radius-system:5443" - Microsoft.Resources: "http://bicep-de.radius-system:6443" - kind: "UCPNative" - - id: "/planes/aws/aws" - properties: - kind: "AWS" + initialization: + planes: + - id: "/planes/radius/local" + properties: + resourceProviders: + Applications.Core: "http://applications-rp.radius-system:5443" + Applications.Dapr: "http://applications-rp.radius-system:5443" + Applications.Datastores: "http://applications-rp.radius-system:5443" + Applications.Messaging: "http://applications-rp.radius-system:5443" + Microsoft.Resources: "http://bicep-de.radius-system:6443" + kind: "UCPNative" + - id: "/planes/aws/aws" + properties: + kind: "AWS" identity: authMethod: UCPCredential diff --git a/deploy/Chart/templates/ucp/deployment.yaml b/deploy/Chart/templates/ucp/deployment.yaml index 776b01ac7b..9c79557899 100644 --- a/deploy/Chart/templates/ucp/deployment.yaml +++ b/deploy/Chart/templates/ucp/deployment.yaml @@ -36,8 +36,6 @@ spec: args: - --config-file=/etc/config/ucp-config.yaml env: - - name: BASE_PATH - value: '/apis/api.ucp.dev/v1alpha3' # listen for APIService URLs - name: TLS_CERT_DIR value: '/var/tls/cert' - name: PORT diff --git a/pkg/armrpc/asyncoperation/worker/service.go b/pkg/armrpc/asyncoperation/worker/service.go index 4a04a19384..52d436a1fc 100644 --- a/pkg/armrpc/asyncoperation/worker/service.go +++ b/pkg/armrpc/asyncoperation/worker/service.go @@ -18,57 +18,64 @@ package worker import ( "context" + "sync" manager "github.com/radius-project/radius/pkg/armrpc/asyncoperation/statusmanager" - "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/ucp/dataprovider" - queue "github.com/radius-project/radius/pkg/ucp/queue/client" qprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" "github.com/radius-project/radius/pkg/ucp/ucplog" ) // Service is the base worker service implementation to initialize and start worker. +// All exported fields should be initialized by the caller. type Service struct { - // ProviderName is the name of provider namespace. - ProviderName string - // Options is the server hosting options. - Options hostoptions.HostOptions - // StorageProvider is the provider of storage client. - StorageProvider dataprovider.DataStorageProvider // OperationStatusManager is the manager of the operation status. OperationStatusManager manager.StatusManager - // Controllers is the registry of the async operation controllers. - Controllers *ControllerRegistry - // RequestQueue is the queue client for async operation request message. - RequestQueue queue.Client + + // Options configures options for the async worker. + Options Options + + // QueueProvider is the queue provider. Will be initialized from config if not provided. + QueueProvider *qprovider.QueueProvider + + // StorageProvider is the provider of storage client. Will be initialized from config if not provided. + StorageProvider dataprovider.DataStorageProvider + + // controllers is the registry of the async operation controllers. + controllers *ControllerRegistry + + // controllersInit is used to ensure single initialization of controllers. + controllersInit sync.Once } -// Init initializes worker service - it initializes the StorageProvider, RequestQueue, OperationStatusManager, Controllers, KubeClient and -// returns an error if any of these operations fail. -func (s *Service) Init(ctx context.Context) error { - s.StorageProvider = dataprovider.NewStorageProvider(s.Options.Config.StorageProvider) - qp := qprovider.New(s.Options.Config.QueueProvider) - var err error - s.RequestQueue, err = qp.GetClient(ctx) - if err != nil { - return err - } - s.OperationStatusManager = manager.New(s.StorageProvider, s.RequestQueue, s.Options.Config.Env.RoleLocation) - s.Controllers = NewControllerRegistry(s.StorageProvider) - return nil +// Controllers returns the controller registry. +func (s *Service) Controllers() *ControllerRegistry { + s.controllersInit.Do(func() { + s.controllers = NewControllerRegistry(s.StorageProvider) + }) + + return s.controllers } -// Start creates and starts a worker, and logs any errors that occur while starting the worker. -func (s *Service) Start(ctx context.Context, opt Options) error { +// Start creates and starts an async worker. +// +// The provided context will be provided to each async controller. +func (s *Service) Start(ctx context.Context) error { logger := ucplog.FromContextOrDiscard(ctx) - ctx = hostoptions.WithContext(ctx, s.Options.Config) + + // Create the queue reader for the worker. + requestQueue, err := s.QueueProvider.GetClient(ctx) + if err != nil { + return err + } // Create and start worker. - worker := New(opt, s.OperationStatusManager, s.RequestQueue, s.Controllers) + worker := New(s.Options, s.OperationStatusManager, requestQueue, s.Controllers()) logger.Info("Start Worker...") if err := worker.Start(ctx); err != nil { logger.Error(err, "failed to start worker...") + return err } logger.Info("Worker stopped...") diff --git a/pkg/armrpc/asyncoperation/worker/worker_runoperation_test.go b/pkg/armrpc/asyncoperation/worker/worker_runoperation_test.go index 3e7d6280ab..b18e55dd16 100644 --- a/pkg/armrpc/asyncoperation/worker/worker_runoperation_test.go +++ b/pkg/armrpc/asyncoperation/worker/worker_runoperation_test.go @@ -114,7 +114,7 @@ func (c *testContext) cancellable(timeout time.Duration) (context.Context, conte func newTestContext(t *testing.T, lockTime time.Duration) (*testContext, *gomock.Controller) { mctrl := gomock.NewController(t) - inmemQ := inmemory.NewInMemQueue(lockTime) + inmemQ := inmemory.NewInMemQueue("test", lockTime) return &testContext{ ctx: context.Background(), mockSC: store.NewMockStorageClient(mctrl), diff --git a/pkg/armrpc/hostoptions/providerconfig.go b/pkg/armrpc/hostoptions/providerconfig.go index 05f4318012..5571756fde 100644 --- a/pkg/armrpc/hostoptions/providerconfig.go +++ b/pkg/armrpc/hostoptions/providerconfig.go @@ -17,12 +17,16 @@ limitations under the License. package hostoptions import ( + "fmt" + metricsprovider "github.com/radius-project/radius/pkg/metrics/provider" profilerprovider "github.com/radius-project/radius/pkg/profiler/provider" "github.com/radius-project/radius/pkg/trace" "github.com/radius-project/radius/pkg/ucp/config" "github.com/radius-project/radius/pkg/ucp/dataprovider" + qprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" + sprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" "github.com/radius-project/radius/pkg/ucp/ucplog" ) @@ -59,6 +63,18 @@ type ServerOptions struct { ArmMetadataEndpoint string `yaml:"armMetadataEndpoint,omitempty"` // EnableAuth when set the arm client authetication will be performed EnableArmAuth bool `yaml:"enableArmAuth,omitempty"` + + // TLSCertificateDirectory is the directory where the TLS certificates are stored. + // + // The server code will expect to find the following files in this directory: + // - tls.crt: The server's certificate. + // - tls.key: The server's private key. + TLSCertificateDirectory string `yaml:"tlsCertificateDirectory,omitempty"` +} + +// Address returns the address of the server in host:port format. +func (s ServerOptions) Address() string { + return s.Host + ":" + fmt.Sprint(s.Port) } // WorkerServerOptions includes the worker server options. diff --git a/pkg/dynamicrp/backend/service.go b/pkg/dynamicrp/backend/service.go index 370563c80b..e9d8339cd5 100644 --- a/pkg/dynamicrp/backend/service.go +++ b/pkg/dynamicrp/backend/service.go @@ -18,56 +18,57 @@ package backend import ( "context" - "fmt" "github.com/radius-project/radius/pkg/armrpc/asyncoperation/worker" - "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/dynamicrp" + "github.com/radius-project/radius/pkg/recipes/controllerconfig" ) // Service runs the backend for the dynamic-rp. type Service struct { worker.Service options *dynamicrp.Options + recipes *controllerconfig.RecipeControllerConfig } // NewService creates a new service to run the dynamic-rp backend. func NewService(options *dynamicrp.Options) *Service { + workerOptions := worker.Options{} + if options.Config.Worker.MaxOperationConcurrency != nil { + workerOptions.MaxOperationConcurrency = *options.Config.Worker.MaxOperationConcurrency + } + if options.Config.Worker.MaxOperationRetryCount != nil { + workerOptions.MaxOperationRetryCount = *options.Config.Worker.MaxOperationRetryCount + } + return &Service{ options: options, Service: worker.Service{ - ProviderName: "dynamic-rp", - Options: hostoptions.HostOptions{ - Config: &hostoptions.ProviderConfig{ - Env: options.Config.Environment, - StorageProvider: options.Config.Storage, - SecretProvider: options.Config.Secrets, - QueueProvider: options.Config.Queue, - }, - }, + OperationStatusManager: options.StatusManager, + Options: workerOptions, + QueueProvider: options.QueueProvider, + StorageProvider: options.StorageProvider, }, + recipes: options.Recipes, } } // Name returns the name of the service used for logging. func (w *Service) Name() string { - return fmt.Sprintf("%s async worker", w.Service.ProviderName) + return "dynamic-rp async worker" } // Run runs the service. func (w *Service) Run(ctx context.Context) error { - err := w.Init(ctx) + err := w.registerControllers(ctx) if err != nil { return err } - workerOptions := worker.Options{} - if w.options.Config.Worker.MaxOperationConcurrency != nil { - workerOptions.MaxOperationConcurrency = *w.options.Config.Worker.MaxOperationConcurrency - } - if w.options.Config.Worker.MaxOperationRetryCount != nil { - workerOptions.MaxOperationRetryCount = *w.options.Config.Worker.MaxOperationRetryCount - } + return w.Start(ctx) +} - return w.Start(ctx, workerOptions) +func (w *Service) registerControllers(ctx context.Context) error { + // No controllers yet. + return nil } diff --git a/pkg/dynamicrp/config.go b/pkg/dynamicrp/config.go index 084c469c21..f40c7ab417 100644 --- a/pkg/dynamicrp/config.go +++ b/pkg/dynamicrp/config.go @@ -32,6 +32,8 @@ import ( ) // Config defines the configuration for the DynamicRP server. +// +// For testability, all fields on this struct MUST be parsable from YAML without any further initialization required. type Config struct { // Bicep configures properties for the Bicep recipe driver. Bicep hostoptions.BicepOptions `yaml:"bicep"` diff --git a/pkg/dynamicrp/integrationtest/dynamic/providers_test.go b/pkg/dynamicrp/integrationtest/dynamic/providers_test.go new file mode 100644 index 0000000000..bcfda112ff --- /dev/null +++ b/pkg/dynamicrp/integrationtest/dynamic/providers_test.go @@ -0,0 +1,174 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dynamic + +import ( + "context" + "net/http" + "testing" + + v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/dynamicrp/testhost" + "github.com/radius-project/radius/pkg/to" + "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" + ucptesthost "github.com/radius-project/radius/pkg/ucp/testhost" + "github.com/stretchr/testify/require" +) + +const ( + radiusPlaneName = "testing" + resourceProviderNamespace = "Applications.Test" + resourceTypeName = "exampleResources" + locationName = v1.LocationGlobal + apiVersion = "2024-01-01" + + resourceGroupName = "test-group" + exampleResourceName = "my-example" + + exampleResourcePlaneID = "/planes/radius/" + radiusPlaneName + exampleResourceGroupID = exampleResourcePlaneID + "/resourceGroups/test-group" + + exampleResourceID = exampleResourceGroupID + "/providers/Applications.Test/exampleResources/" + exampleResourceName + exampleResourceURL = exampleResourceID + "?api-version=" + apiVersion +) + +// This test covers the lifecycle of a dynamic resource. +func Test_Dynamic_Resource_Lifecycle(t *testing.T) { + _, ucp := testhost.Start(t) + + // Setup a resource provider (Applications.Test/exampleResources) + createRadiusPlane(ucp) + createResourceProvider(ucp) + createResourceType(ucp) + createAPIVersion(ucp) + createLocation(ucp) + + // Setup a resource group where we can interact with the new resource type. + createResourceGroup(ucp) + + // We have not yet implemented any functionality for dynamic RP. + // + // This is the hello-worldiest of tests. We're just making sure that all + // of the infrastructure works. + response := ucp.MakeRequest(http.MethodGet, exampleResourceURL, nil) + response.EqualsErrorCode(404, "NotFound") +} + +func createRadiusPlane(server *ucptesthost.TestHost) v20231001preview.RadiusPlanesClientCreateOrUpdateResponse { + ctx := context.Background() + + plane := v20231001preview.RadiusPlaneResource{ + Location: to.Ptr(v1.LocationGlobal), + Properties: &v20231001preview.RadiusPlaneResourceProperties{ + // Note: this is a workaround. Properties is marked as a required field in + // the API. Without passing *something* here the body will be rejected. + ProvisioningState: to.Ptr(v20231001preview.ProvisioningStateSucceeded), + ResourceProviders: map[string]*string{}, + }, + } + + client := server.UCP().NewRadiusPlanesClient() + poller, err := client.BeginCreateOrUpdate(ctx, radiusPlaneName, plane, nil) + require.NoError(server.T(), err) + + response, err := poller.PollUntilDone(ctx, nil) + require.NoError(server.T(), err) + + return response +} + +func createResourceProvider(server *ucptesthost.TestHost) { + ctx := context.Background() + + resourceProvider := v20231001preview.ResourceProviderResource{ + Location: to.Ptr(v1.LocationGlobal), + Properties: &v20231001preview.ResourceProviderProperties{}, + } + + client := server.UCP().NewResourceProvidersClient() + poller, err := client.BeginCreateOrUpdate(ctx, radiusPlaneName, resourceProviderNamespace, resourceProvider, nil) + require.NoError(server.T(), err) + + _, err = poller.PollUntilDone(ctx, nil) + require.NoError(server.T(), err) +} + +func createResourceType(server *ucptesthost.TestHost) { + ctx := context.Background() + + resourceType := v20231001preview.ResourceTypeResource{ + Properties: &v20231001preview.ResourceTypeProperties{}, + } + + client := server.UCP().NewResourceTypesClient() + poller, err := client.BeginCreateOrUpdate(ctx, radiusPlaneName, resourceProviderNamespace, resourceTypeName, resourceType, nil) + require.NoError(server.T(), err) + + _, err = poller.PollUntilDone(ctx, nil) + require.NoError(server.T(), err) +} + +func createAPIVersion(server *ucptesthost.TestHost) { + ctx := context.Background() + + apiVersionResource := v20231001preview.APIVersionResource{ + Properties: &v20231001preview.APIVersionProperties{}, + } + + client := server.UCP().NewAPIVersionsClient() + poller, err := client.BeginCreateOrUpdate(ctx, radiusPlaneName, resourceProviderNamespace, resourceTypeName, apiVersion, apiVersionResource, nil) + require.NoError(server.T(), err) + + _, err = poller.PollUntilDone(ctx, nil) + require.NoError(server.T(), err) +} + +func createLocation(server *ucptesthost.TestHost) { + ctx := context.Background() + + location := v20231001preview.LocationResource{ + Properties: &v20231001preview.LocationProperties{ + ResourceTypes: map[string]*v20231001preview.LocationResourceType{ + resourceTypeName: { + APIVersions: map[string]map[string]any{ + apiVersion: {}, + }, + }, + }, + }, + } + + client := server.UCP().NewLocationsClient() + poller, err := client.BeginCreateOrUpdate(ctx, radiusPlaneName, resourceProviderNamespace, locationName, location, nil) + require.NoError(server.T(), err) + + _, err = poller.PollUntilDone(ctx, nil) + require.NoError(server.T(), err) +} + +func createResourceGroup(server *ucptesthost.TestHost) { + ctx := context.Background() + + resourceGroup := v20231001preview.ResourceGroupResource{ + Location: to.Ptr(v1.LocationGlobal), + Properties: &v20231001preview.ResourceGroupProperties{}, + } + + client := server.UCP().NewResourceGroupsClient() + _, err := client.CreateOrUpdate(ctx, radiusPlaneName, resourceGroupName, resourceGroup, nil) + require.NoError(server.T(), err) +} diff --git a/pkg/dynamicrp/options.go b/pkg/dynamicrp/options.go index ce18389617..c13fb3155f 100644 --- a/pkg/dynamicrp/options.go +++ b/pkg/dynamicrp/options.go @@ -32,7 +32,10 @@ import ( kube_rest "k8s.io/client-go/rest" ) -// Options holds the configuration options and shared services for the server. +// Options holds the configuration options and shared services for the DyanmicRP server. +// +// For testability, all fields on this struct MUST be constructed from the NewOptions function without any +// additional initialization required. type Options struct { // Config is the configuration for the server. Config *Config @@ -75,14 +78,16 @@ func NewOptions(ctx context.Context, config *Config) (*Options, error) { options.StatusManager = statusmanager.New(options.StorageProvider, queueClient, config.Environment.RoleLocation) var cfg *kube_rest.Config - cfg, err = kubeutil.NewClientConfig(&kubeutil.ConfigOptions{ - // TODO: Allow to use custom context via configuration. - https://github.com/radius-project/radius/issues/5433 - ContextName: "", - QPS: kubeutil.DefaultServerQPS, - Burst: kubeutil.DefaultServerBurst, - }) - if err != nil { - return nil, fmt.Errorf("failed to get kubernetes config: %w", err) + if config.UCP.Kind == ucpconfig.UCPConnectionKindKubernetes { + cfg, err = kubeutil.NewClientConfig(&kubeutil.ConfigOptions{ + // TODO: Allow to use custom context via configuration. - https://github.com/radius-project/radius/issues/5433 + ContextName: "", + QPS: kubeutil.DefaultServerQPS, + Burst: kubeutil.DefaultServerBurst, + }) + if err != nil { + return nil, fmt.Errorf("failed to get kubernetes config: %w", err) + } } options.UCP, err = ucpconfig.NewConnectionFromUCPConfig(&config.UCP, cfg) diff --git a/pkg/dynamicrp/testhost/doc.go b/pkg/dynamicrp/testhost/doc.go new file mode 100644 index 0000000000..bdd65e049e --- /dev/null +++ b/pkg/dynamicrp/testhost/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// testhost provides an implementation of a test server for the dynamic RP. +package testhost diff --git a/pkg/dynamicrp/testhost/host.go b/pkg/dynamicrp/testhost/host.go new file mode 100644 index 0000000000..56728c8a4b --- /dev/null +++ b/pkg/dynamicrp/testhost/host.go @@ -0,0 +1,138 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testhost + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/armrpc/hostoptions" + "github.com/radius-project/radius/pkg/dynamicrp" + "github.com/radius-project/radius/pkg/dynamicrp/server" + "github.com/radius-project/radius/pkg/sdk" + "github.com/radius-project/radius/pkg/ucp" + "github.com/radius-project/radius/pkg/ucp/config" + "github.com/radius-project/radius/pkg/ucp/dataprovider" + queueprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" + secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" + ucptesthost "github.com/radius-project/radius/pkg/ucp/testhost" + "github.com/radius-project/radius/test/integrationtest/testhost" + "github.com/stretchr/testify/require" +) + +// TestHostOptions supports configuring the dynamic-rp test host. +type TestHostOption interface { + // Apply applies the option to the dynamic-rp options. + Apply(options *dynamicrp.Options) +} + +// TestHostOptionFunc is a function that implements the TestHostOption interface. +type TestHostOptionFunc func(options *dynamicrp.Options) + +// Apply applies the function to the dynamic-rp options. +func (f TestHostOptionFunc) Apply(options *dynamicrp.Options) { + f(options) +} + +// TestHost provides a test host for the dynamic-rp server. +type TestHost struct { + *testhost.TestHost +} + +func Start(t *testing.T, opts ...TestHostOption) (*TestHost, *ucptesthost.TestHost) { + config := &dynamicrp.Config{ + Environment: hostoptions.EnvironmentOptions{ + Name: "test", + RoleLocation: v1.LocationGlobal, + }, + Queue: queueprovider.QueueProviderOptions{ + Provider: queueprovider.TypeInmemory, + Name: "dynamic-rp", + }, + Secrets: secretprovider.SecretProviderOptions{ + Provider: secretprovider.TypeInMemorySecret, + }, + Server: hostoptions.ServerOptions{ + // Initialized dynamically when the server is started. + }, + Storage: dataprovider.StorageProviderOptions{ + Provider: dataprovider.TypeInMemory, + }, + UCP: config.UCPOptions{ + Kind: config.UCPConnectionKindDirect, + Direct: &config.UCPDirectConnectionOptions{ + Endpoint: "http://localhost:65000", // Initialized dynamically when the server is started. + }, + }, + } + + options, err := dynamicrp.NewOptions(context.Background(), config) + require.NoError(t, err) + + for _, opt := range opts { + opt.Apply(options) + } + + return StartWithOptions(t, options) +} + +// StartWithOptions uses the provided options to start the dynamic-rp test host and an instance of UCP +// configured to route traffic to the dynamic-rp test host. +// +// Manually configuring the server information other than the port is not supported. +func StartWithOptions(t *testing.T, options *dynamicrp.Options) (*TestHost, *ucptesthost.TestHost) { + options.Config.Server.Host = "localhost" + options.Config.Server.PathBase = "/" + uuid.New().String() + if options.Config.Server.Port == 0 { + options.Config.Server.Port = testhost.AllocateFreePort(t) + } + + // Allocate a port for UCP. + ucpPort := testhost.AllocateFreePort(t) + options.Config.UCP.Kind = config.UCPConnectionKindDirect + options.Config.UCP.Direct = &config.UCPDirectConnectionOptions{Endpoint: fmt.Sprintf("http://localhost:%d", ucpPort)} + + var err error + options.UCP, err = sdk.NewDirectConnection(options.Config.UCP.Direct.Endpoint) + require.NoError(t, err) + + baseURL := fmt.Sprintf( + "http://%s%s", + options.Config.Server.Address(), + options.Config.Server.PathBase) + baseURL = strings.TrimSuffix(baseURL, "/") + + host, err := server.NewServer(options) + require.NoError(t, err, "failed to create server") + + th := testhost.StartHost(t, host, baseURL) + return &TestHost{th}, startUCP(t, baseURL, ucpPort) +} + +func startUCP(t *testing.T, dynamicRPURL string, ucpPort int) *ucptesthost.TestHost { + return ucptesthost.Start(t, ucptesthost.TestHostOptionFunc(func(options *ucp.Options) { + // Initialize UCP with its listening port + options.Config.Server.Port = ucpPort + + // Intitialize UCP with the dynamic-rp URL + options.Config.Routing.DefaultDownstreamEndpoint = dynamicRPURL + })) +} diff --git a/pkg/recipes/controllerconfig/config.go b/pkg/recipes/controllerconfig/config.go index 6ddeb1ef19..947d93b38f 100644 --- a/pkg/recipes/controllerconfig/config.go +++ b/pkg/recipes/controllerconfig/config.go @@ -76,6 +76,14 @@ func New(options hostoptions.HostOptions) (*RecipeControllerConfig, error) { return nil, err } + if options.Config.Bicep.DeleteRetryCount == "" { + options.Config.Bicep.DeleteRetryCount = "3" + } + + if options.Config.Bicep.DeleteRetryDelaySeconds == "" { + options.Config.Bicep.DeleteRetryDelaySeconds = "10" + } + bicepDeleteRetryCount, err := strconv.Atoi(options.Config.Bicep.DeleteRetryCount) if err != nil { return nil, err diff --git a/pkg/server/asyncworker.go b/pkg/server/asyncworker.go index e5703885ec..9ef2f630b3 100644 --- a/pkg/server/asyncworker.go +++ b/pkg/server/asyncworker.go @@ -21,29 +21,33 @@ import ( "fmt" ctrl "github.com/radius-project/radius/pkg/armrpc/asyncoperation/controller" + "github.com/radius-project/radius/pkg/armrpc/asyncoperation/statusmanager" "github.com/radius-project/radius/pkg/armrpc/asyncoperation/worker" "github.com/radius-project/radius/pkg/armrpc/builder" "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/corerp/backend/deployment" "github.com/radius-project/radius/pkg/corerp/model" "github.com/radius-project/radius/pkg/kubeutil" + "github.com/radius-project/radius/pkg/ucp/dataprovider" + queueprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" ) // AsyncWorker is a service to run AsyncRequestProcessWorker. type AsyncWorker struct { worker.Service + options hostoptions.HostOptions handlerBuilder []builder.Builder } // NewAsyncWorker creates new service instance to run AsyncRequestProcessWorker. func NewAsyncWorker(options hostoptions.HostOptions, builder []builder.Builder) *AsyncWorker { return &AsyncWorker{ - Service: worker.Service{ - ProviderName: "radius", - Options: options, - }, + options: options, handlerBuilder: builder, + Service: worker.Service{ + // Will be initialized later + }, } } @@ -52,22 +56,54 @@ func (w *AsyncWorker) Name() string { return "radiusasyncworker" } -// Run starts the service and worker. -func (w *AsyncWorker) Run(ctx context.Context) error { - if err := w.Init(ctx); err != nil { +func (w *AsyncWorker) init(ctx context.Context) error { + workerOptions := worker.Options{} + if w.options.Config.WorkerServer != nil { + if w.options.Config.WorkerServer.MaxOperationConcurrency != nil { + workerOptions.MaxOperationConcurrency = *w.options.Config.WorkerServer.MaxOperationConcurrency + } + if w.options.Config.WorkerServer.MaxOperationRetryCount != nil { + workerOptions.MaxOperationRetryCount = *w.options.Config.WorkerServer.MaxOperationRetryCount + } + } + + queueProvider := queueprovider.New(w.options.Config.QueueProvider) + storageProvider := dataprovider.NewStorageProvider(w.options.Config.StorageProvider) + + queueClient, err := queueProvider.GetClient(ctx) + if err != nil { return err } - k8s, err := kubeutil.NewClients(w.Options.K8sConfig) + statusManager := statusmanager.New(storageProvider, queueClient, w.options.Config.Env.RoleLocation) + + w.Service = worker.Service{ + OperationStatusManager: statusManager, + Options: workerOptions, + QueueProvider: queueProvider, + StorageProvider: storageProvider, + } + + return nil +} + +// Run starts the service and worker. +func (w *AsyncWorker) Run(ctx context.Context) error { + k8s, err := kubeutil.NewClients(w.options.K8sConfig) if err != nil { return fmt.Errorf("failed to initialize kubernetes clients: %w", err) } - appModel, err := model.NewApplicationModel(w.Options.Arm, k8s.RuntimeClient, k8s.ClientSet, k8s.DiscoveryClient, k8s.DynamicClient) + appModel, err := model.NewApplicationModel(w.options.Arm, k8s.RuntimeClient, k8s.ClientSet, k8s.DiscoveryClient, k8s.DynamicClient) if err != nil { return fmt.Errorf("failed to initialize application model: %w", err) } + err = w.init(ctx) + if err != nil { + return fmt.Errorf("failed to initialize async worker: %w", err) + } + for _, b := range w.handlerBuilder { opts := ctrl.Options{ DataProvider: w.StorageProvider, @@ -77,21 +113,11 @@ func (w *AsyncWorker) Run(ctx context.Context) error { }, } - err := b.ApplyAsyncHandler(ctx, w.Controllers, opts) + err := b.ApplyAsyncHandler(ctx, w.Controllers(), opts) if err != nil { panic(err) } } - workerOpts := worker.Options{} - if w.Options.Config.WorkerServer != nil { - if w.Options.Config.WorkerServer.MaxOperationConcurrency != nil { - workerOpts.MaxOperationConcurrency = *w.Options.Config.WorkerServer.MaxOperationConcurrency - } - if w.Options.Config.WorkerServer.MaxOperationRetryCount != nil { - workerOpts.MaxOperationRetryCount = *w.Options.Config.WorkerServer.MaxOperationRetryCount - } - } - - return w.Start(ctx, workerOpts) + return w.Start(ctx) } diff --git a/pkg/ucp/backend/controller/resourcegroups/trackedresourceprocess.go b/pkg/ucp/backend/controller/resourcegroups/trackedresourceprocess.go index bb2fa2b3d0..8fc9126c3c 100644 --- a/pkg/ucp/backend/controller/resourcegroups/trackedresourceprocess.go +++ b/pkg/ucp/backend/controller/resourcegroups/trackedresourceprocess.go @@ -25,6 +25,7 @@ import ( v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" ctrl "github.com/radius-project/radius/pkg/armrpc/asyncoperation/controller" + "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" "github.com/radius-project/radius/pkg/ucp/frontend/controller/resourcegroups" "github.com/radius-project/radius/pkg/ucp/resources" @@ -54,14 +55,19 @@ type TrackedResourceProcessController struct { func NewTrackedResourceProcessController(opts ctrl.Options, transport http.RoundTripper, defaultDownstream *url.URL) (ctrl.Controller, error) { return &TrackedResourceProcessController{ BaseController: ctrl.NewBaseAsyncController(opts), - updater: trackedresource.NewUpdater(opts.StorageClient, &http.Client{Transport: transport}), + updater: trackedresource.NewUpdater(opts.DataProvider, &http.Client{Transport: transport}), defaultDownstream: defaultDownstream, }, nil } // Run retrieves a resource from storage, parses the resource ID, and updates our tracked resource entry in the background. func (c *TrackedResourceProcessController) Run(ctx context.Context, request *ctrl.Request) (ctrl.Result, error) { - resource, err := store.GetResource[datamodel.GenericResource](ctx, c.StorageClient(), request.ResourceID) + client, err := c.DataProvider().GetStorageClient(ctx, v20231001preview.ResourceType) + if err != nil { + return ctrl.Result{}, err + } + + resource, err := store.GetResource[datamodel.GenericResource](ctx, client, request.ResourceID) if errors.Is(err, &store.ErrNotFound{}) { return ctrl.NewFailedResult(v1.ErrorDetails{Code: v1.CodeNotFound, Message: fmt.Sprintf("resource %q not found", request.ResourceID), Target: request.ResourceID}), nil } else if err != nil { @@ -73,7 +79,7 @@ func (c *TrackedResourceProcessController) Run(ctx context.Context, request *ctr return ctrl.Result{}, err } - downstreamURL, err := resourcegroups.ValidateDownstream(ctx, c.StorageClient(), originalID, v1.LocationGlobal, resource.Properties.APIVersion) + downstreamURL, err := resourcegroups.ValidateDownstream(ctx, c.DataProvider(), originalID, v1.LocationGlobal, resource.Properties.APIVersion) if errors.Is(err, &resourcegroups.NotFoundError{}) { return ctrl.NewFailedResult(v1.ErrorDetails{Code: v1.CodeNotFound, Message: err.Error(), Target: request.ResourceID}), nil } else if errors.Is(err, &resourcegroups.InvalidError{}) { diff --git a/pkg/ucp/backend/controller/resourcegroups/trackedresourceprocess_test.go b/pkg/ucp/backend/controller/resourcegroups/trackedresourceprocess_test.go index c4db345daf..dc026fbf58 100644 --- a/pkg/ucp/backend/controller/resourcegroups/trackedresourceprocess_test.go +++ b/pkg/ucp/backend/controller/resourcegroups/trackedresourceprocess_test.go @@ -24,6 +24,7 @@ import ( v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" "github.com/radius-project/radius/pkg/armrpc/asyncoperation/controller" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/resources" "github.com/radius-project/radius/pkg/ucp/store" "github.com/radius-project/radius/pkg/ucp/trackedresource" @@ -35,9 +36,12 @@ import ( func Test_Run(t *testing.T) { setup := func(t *testing.T) (*TrackedResourceProcessController, *mockUpdater, *store.MockStorageClient) { ctrl := gomock.NewController(t) + + storageProvider := dataprovider.NewMockDataStorageProvider(ctrl) storageClient := store.NewMockStorageClient(ctrl) + storageProvider.EXPECT().GetStorageClient(gomock.Any(), gomock.Any()).Return(storageClient, nil).AnyTimes() - pc, err := NewTrackedResourceProcessController(controller.Options{StorageClient: storageClient}, nil, nil) + pc, err := NewTrackedResourceProcessController(controller.Options{DataProvider: storageProvider, StorageClient: storageClient}, nil, nil) require.NoError(t, err) updater := mockUpdater{} diff --git a/pkg/ucp/backend/controller/resourceproviders/apiversion_delete.go b/pkg/ucp/backend/controller/resourceproviders/apiversion_delete.go index 4cf969d673..f6f63f61e6 100644 --- a/pkg/ucp/backend/controller/resourceproviders/apiversion_delete.go +++ b/pkg/ucp/backend/controller/resourceproviders/apiversion_delete.go @@ -38,7 +38,7 @@ func (c *APIVersionDeleteController) Run(ctx context.Context, request *ctrl.Requ return ctrl.Result{}, err } - err = updateResourceProviderSummaryWithETag(ctx, c.StorageClient(), summaryID, summaryNotFoundIgnore, c.updateSummary(id)) + err = updateResourceProviderSummaryWithETag(ctx, c.DataProvider(), summaryID, summaryNotFoundIgnore, c.updateSummary(id)) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/ucp/backend/controller/resourceproviders/apiversion_put.go b/pkg/ucp/backend/controller/resourceproviders/apiversion_put.go index ca97bfe308..6f09e60a0d 100644 --- a/pkg/ucp/backend/controller/resourceproviders/apiversion_put.go +++ b/pkg/ucp/backend/controller/resourceproviders/apiversion_put.go @@ -39,7 +39,7 @@ func (c *APIVersionPutController) Run(ctx context.Context, request *ctrl.Request return ctrl.Result{}, err } - err = updateResourceProviderSummaryWithETag(ctx, c.StorageClient(), summaryID, summaryNotFoundFail, c.updateSummary(id)) + err = updateResourceProviderSummaryWithETag(ctx, c.DataProvider(), summaryID, summaryNotFoundFail, c.updateSummary(id)) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/ucp/backend/controller/resourceproviders/location_delete.go b/pkg/ucp/backend/controller/resourceproviders/location_delete.go index 987ab21439..dec42e6429 100644 --- a/pkg/ucp/backend/controller/resourceproviders/location_delete.go +++ b/pkg/ucp/backend/controller/resourceproviders/location_delete.go @@ -38,7 +38,7 @@ func (c *LocationDeleteController) Run(ctx context.Context, request *ctrl.Reques return ctrl.Result{}, err } - err = updateResourceProviderSummaryWithETag(ctx, c.StorageClient(), summaryID, summaryNotFoundIgnore, c.updateSummary(id)) + err = updateResourceProviderSummaryWithETag(ctx, c.DataProvider(), summaryID, summaryNotFoundIgnore, c.updateSummary(id)) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/ucp/backend/controller/resourceproviders/location_put.go b/pkg/ucp/backend/controller/resourceproviders/location_put.go index 7ff0e3a130..f0554f98ab 100644 --- a/pkg/ucp/backend/controller/resourceproviders/location_put.go +++ b/pkg/ucp/backend/controller/resourceproviders/location_put.go @@ -38,7 +38,7 @@ func (c *LocationPutController) Run(ctx context.Context, request *ctrl.Request) return ctrl.Result{}, err } - err = updateResourceProviderSummaryWithETag(ctx, c.StorageClient(), summaryID, summaryNotFoundFail, c.updateSummary(id)) + err = updateResourceProviderSummaryWithETag(ctx, c.DataProvider(), summaryID, summaryNotFoundFail, c.updateSummary(id)) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/ucp/backend/controller/resourceproviders/resourceprovider_delete.go b/pkg/ucp/backend/controller/resourceproviders/resourceprovider_delete.go index 0984c4d2f7..37a1fb14b3 100644 --- a/pkg/ucp/backend/controller/resourceproviders/resourceprovider_delete.go +++ b/pkg/ucp/backend/controller/resourceproviders/resourceprovider_delete.go @@ -25,6 +25,7 @@ import ( aztoken "github.com/radius-project/radius/pkg/azure/tokencredentials" "github.com/radius-project/radius/pkg/sdk" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" + "github.com/radius-project/radius/pkg/ucp/datamodel" "github.com/radius-project/radius/pkg/ucp/resources" resources_radius "github.com/radius-project/radius/pkg/ucp/resources/radius" "github.com/radius-project/radius/pkg/ucp/store" @@ -222,7 +223,12 @@ func (c *ResourceProviderDeleteController) deleteSummary(ctx context.Context, re return err } - err = c.StorageClient().Delete(ctx, summaryID.String()) + client, err := c.DataProvider().GetStorageClient(ctx, datamodel.ResourceProviderSummaryResourceType) + if err != nil { + return err + } + + err = client.Delete(ctx, summaryID.String()) if errors.Is(err, &store.ErrNotFound{}) { // It's OK if the summary was already deleted. return nil diff --git a/pkg/ucp/backend/controller/resourceproviders/resourceprovider_put.go b/pkg/ucp/backend/controller/resourceproviders/resourceprovider_put.go index de416896fa..c3ce69e845 100644 --- a/pkg/ucp/backend/controller/resourceproviders/resourceprovider_put.go +++ b/pkg/ucp/backend/controller/resourceproviders/resourceprovider_put.go @@ -37,7 +37,7 @@ func (c *ResourceProviderPutController) Run(ctx context.Context, request *ctrl.R return ctrl.Result{}, err } - err = updateResourceProviderSummaryWithETag(ctx, c.StorageClient(), summaryID, summaryNotFoundCreate, c.updateSummary()) + err = updateResourceProviderSummaryWithETag(ctx, c.DataProvider(), summaryID, summaryNotFoundCreate, c.updateSummary()) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/ucp/backend/controller/resourceproviders/resourcetype_delete.go b/pkg/ucp/backend/controller/resourceproviders/resourcetype_delete.go index 76758b0b6f..a01f5853d2 100644 --- a/pkg/ucp/backend/controller/resourceproviders/resourcetype_delete.go +++ b/pkg/ucp/backend/controller/resourceproviders/resourcetype_delete.go @@ -53,7 +53,7 @@ func (c *ResourceTypeDeleteController) Run(ctx context.Context, request *ctrl.Re return ctrl.Result{}, err } - err = updateResourceProviderSummaryWithETag(ctx, c.StorageClient(), summaryID, summaryNotFoundIgnore, c.updateSummary(id)) + err = updateResourceProviderSummaryWithETag(ctx, c.DataProvider(), summaryID, summaryNotFoundIgnore, c.updateSummary(id)) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/ucp/backend/controller/resourceproviders/resourcetype_put.go b/pkg/ucp/backend/controller/resourceproviders/resourcetype_put.go index 890be9748e..864a35d5d4 100644 --- a/pkg/ucp/backend/controller/resourceproviders/resourcetype_put.go +++ b/pkg/ucp/backend/controller/resourceproviders/resourcetype_put.go @@ -43,7 +43,7 @@ func (c *ResourceTypePutController) Run(ctx context.Context, request *ctrl.Reque return ctrl.Result{}, err } - err = updateResourceProviderSummaryWithETag(ctx, c.StorageClient(), summaryID, summaryNotFoundFail, c.updateSummary(id, resourceType)) + err = updateResourceProviderSummaryWithETag(ctx, c.DataProvider(), summaryID, summaryNotFoundFail, c.updateSummary(id, resourceType)) if err != nil { return ctrl.Result{}, err } diff --git a/pkg/ucp/backend/controller/resourceproviders/util.go b/pkg/ucp/backend/controller/resourceproviders/util.go index e6c7ccafb5..ddec44ca92 100644 --- a/pkg/ucp/backend/controller/resourceproviders/util.go +++ b/pkg/ucp/backend/controller/resourceproviders/util.go @@ -24,6 +24,7 @@ import ( ctrl "github.com/radius-project/radius/pkg/armrpc/asyncoperation/controller" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/resources" "github.com/radius-project/radius/pkg/ucp/store" "github.com/radius-project/radius/pkg/ucp/ucplog" @@ -64,13 +65,18 @@ func resourceProviderSummaryIDFromRequest(request *ctrl.Request) (resources.ID, } // updateResourceProviderSummaryWithETag updates the summary with the provided function and saves it to the storage client. -func updateResourceProviderSummaryWithETag(ctx context.Context, client store.StorageClient, summaryID resources.ID, policy summaryNotFoundPolicy, update func(summary *datamodel.ResourceProviderSummary) error) error { +func updateResourceProviderSummaryWithETag(ctx context.Context, storageProvider dataprovider.DataStorageProvider, summaryID resources.ID, policy summaryNotFoundPolicy, update func(summary *datamodel.ResourceProviderSummary) error) error { // There are a few cases here: // 1. The summary does not exist and we are allowed to create it (in the resource provider). // 2. The summary does not exist and we are not allowed to create it (in the child-types of resource provider). // 3. Any other error case. summary := &datamodel.ResourceProviderSummary{} + client, err := storageProvider.GetStorageClient(ctx, datamodel.ResourceProviderSummaryResourceType) + if err != nil { + return err + } + obj, err := client.Get(ctx, summaryID.String()) if errors.Is(err, &store.ErrNotFound{}) && policy == summaryNotFoundCreate { // This is fine. We will create a new summary. diff --git a/pkg/ucp/backend/controller/resourceproviders/util_test.go b/pkg/ucp/backend/controller/resourceproviders/util_test.go index 874076ca3e..63c6cc6d37 100644 --- a/pkg/ucp/backend/controller/resourceproviders/util_test.go +++ b/pkg/ucp/backend/controller/resourceproviders/util_test.go @@ -22,6 +22,7 @@ import ( v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" ctrl "github.com/radius-project/radius/pkg/armrpc/asyncoperation/controller" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/resources" "github.com/radius-project/radius/pkg/ucp/store" "github.com/radius-project/radius/pkg/ucp/util/etag" @@ -151,7 +152,9 @@ func Test_UpdateResourceProviderSummaryWithETag(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) + storageProvider := dataprovider.NewMockDataStorageProvider(ctrl) client := store.NewMockStorageClient(ctrl) + storageProvider.EXPECT().GetStorageClient(gomock.Any(), datamodel.ResourceProviderSummaryResourceType).Return(client, nil) expectedETag := "" if tt.existing == nil { @@ -192,7 +195,7 @@ func Test_UpdateResourceProviderSummaryWithETag(t *testing.T) { }) } - err := updateResourceProviderSummaryWithETag(context.Background(), client, tt.summaryID, tt.policy, tt.updateFunc) + err := updateResourceProviderSummaryWithETag(context.Background(), storageProvider, tt.summaryID, tt.policy, tt.updateFunc) if tt.expectedErr { assert.Error(t, err) } else { diff --git a/pkg/ucp/backend/service.go b/pkg/ucp/backend/service.go index fc6ecc1270..57d2a93b81 100644 --- a/pkg/ucp/backend/service.go +++ b/pkg/ucp/backend/service.go @@ -19,83 +19,70 @@ package backend import ( "context" "errors" - "fmt" "net/http" "net/url" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" ctrl "github.com/radius-project/radius/pkg/armrpc/asyncoperation/controller" "github.com/radius-project/radius/pkg/armrpc/asyncoperation/worker" - "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/sdk" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/backend/controller/resourcegroups" "github.com/radius-project/radius/pkg/ucp/backend/controller/resourceproviders" "github.com/radius-project/radius/pkg/ucp/datamodel" - ucpoptions "github.com/radius-project/radius/pkg/ucp/hostoptions" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) -const ( - UCPProviderName = "System.Resources" -) - // Service is a service to run AsyncReqeustProcessWorker. type Service struct { worker.Service - - config ucpoptions.UCPConfig + options *ucp.Options } -// NewService creates new service instance to run AsyncRequestProcessWorker. -func NewService(options hostoptions.HostOptions, config ucpoptions.UCPConfig) *Service { +// NewService creates new backend service instance to run the async worker. +func NewService(options *ucp.Options) *Service { + workerOptions := worker.Options{} + if options.Config.Worker.MaxOperationConcurrency != nil { + workerOptions.MaxOperationConcurrency = *options.Config.Worker.MaxOperationConcurrency + } + if options.Config.Worker.MaxOperationRetryCount != nil { + workerOptions.MaxOperationRetryCount = *options.Config.Worker.MaxOperationRetryCount + } return &Service{ + options: options, Service: worker.Service{ - ProviderName: UCPProviderName, - Options: options, + OperationStatusManager: options.StatusManager, + Options: workerOptions, + QueueProvider: options.QueueProvider, + StorageProvider: options.StorageProvider, }, - config: config, } } -// Name returns a string containing the UCPProviderName and the text "async worker". +// Name returns the service name. func (w *Service) Name() string { - return fmt.Sprintf("%s async worker", UCPProviderName) + return "ucp async worker" } -// Run starts the service and worker. It initializes the service and sets the worker options based on the configuration, -// then starts the service with the given worker options. It returns an error if the initialization fails. +// Run starts the background worker. func (w *Service) Run(ctx context.Context) error { - if err := w.Init(ctx); err != nil { - return err - } - - workerOpts := worker.Options{} - if w.Options.Config.WorkerServer != nil { - if w.Options.Config.WorkerServer.MaxOperationConcurrency != nil { - workerOpts.MaxOperationConcurrency = *w.Options.Config.WorkerServer.MaxOperationConcurrency - } - if w.Options.Config.WorkerServer.MaxOperationRetryCount != nil { - workerOpts.MaxOperationRetryCount = *w.Options.Config.WorkerServer.MaxOperationRetryCount - } - } - opts := ctrl.Options{ DataProvider: w.StorageProvider, } - defaultDownstream, err := url.Parse(w.config.Routing.DefaultDownstreamEndpoint) + defaultDownstream, err := url.Parse(w.options.Config.Routing.DefaultDownstreamEndpoint) if err != nil { return err } transport := otelhttp.NewTransport(http.DefaultTransport) - err = RegisterControllers(ctx, w.Controllers, w.Options.UCPConnection, transport, opts, defaultDownstream) + err = RegisterControllers(ctx, w.Controllers(), w.options.UCP, transport, opts, defaultDownstream) if err != nil { return err } - return w.Start(ctx, workerOpts) + return w.Start(ctx) } // RegisterControllers registers the controllers for the UCP backend. diff --git a/pkg/ucp/config.go b/pkg/ucp/config.go new file mode 100644 index 0000000000..a47b43e225 --- /dev/null +++ b/pkg/ucp/config.go @@ -0,0 +1,144 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ucp + +import ( + "bytes" + + "github.com/radius-project/radius/pkg/armrpc/hostoptions" + metricsprovider "github.com/radius-project/radius/pkg/metrics/provider" + profilerprovider "github.com/radius-project/radius/pkg/profiler/provider" + "github.com/radius-project/radius/pkg/trace" + ucpconfig "github.com/radius-project/radius/pkg/ucp/config" + "github.com/radius-project/radius/pkg/ucp/dataprovider" + queueprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" + secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" + "github.com/radius-project/radius/pkg/ucp/ucplog" + "gopkg.in/yaml.v3" +) + +// Config defines the configuration for the UCP server. +// +// For testability, all fields on this struct MUST be parsable from YAML without any further initialization required. +type Config struct { + // Environment is the configuration for the hosting environment. + Environment hostoptions.EnvironmentOptions `yaml:"environment"` + + // Identity is the configuration for authenticating with external systems like Azure and AWS. + Identity IdentityConfig `yaml:"identity"` + + // Initialization is the configuration for initializing the UCP server. + Initialization InitializationConfig `yaml:"initialization"` + + // Logging is the configuration for the logging system. + Logging ucplog.LoggingOptions `yaml:"logging"` + + // Metrics is the configuration for the metrics endpoint. + Metrics metricsprovider.MetricsProviderOptions `yaml:"metricsProvider"` + + // Profiler is the configuration for the profiler endpoint. + Profiler profilerprovider.ProfilerProviderOptions `yaml:"profilerProvider"` + + // Routing is the configuration for UCP routing. + Routing RoutingConfig `yaml:"routing"` + + // Queue is the configuration for the message queue. + Queue queueprovider.QueueProviderOptions `yaml:"queueProvider"` + + // Secrets is the configuration for the secret storage system. + Secrets secretprovider.SecretProviderOptions `yaml:"secretProvider"` + + // Server is the configuration for the HTTP server. + Server hostoptions.ServerOptions `yaml:"server"` + + // Storage is the configuration for the database used for storage. + Storage dataprovider.StorageProviderOptions `yaml:"storageProvider"` + + // Tracing is the configuration for the tracing system. + Tracing trace.Options `yaml:"tracerProvider"` + + // UCPConfig is the configuration for the connection to UCP. + UCP ucpconfig.UCPOptions `yaml:"ucp"` + + // Worker is the configuration for the backend worker server. + Worker hostoptions.WorkerServerOptions `yaml:"workerServer"` +} + +const ( + // AuthUCPCredential is the authentication method via UCP Credential API. + AuthUCPCredential = "UCPCredential" + + // AuthDefault is the default authentication method, such as environment variables. + AuthDefault = "default" +) + +// Identity represents configuration options for authenticating with external systems like Azure and AWS. +type IdentityConfig struct { + // AuthMethod represents the method of authentication for authenticating with external systems like Azure and AWS. + AuthMethod string `yaml:"authMethod"` +} + +// RoutingConfig provides configuration for UCP routing. +type RoutingConfig struct { + // DefaultDownstreamEndpoint is the default destination when a resource provider does not provide a downstream endpoint. + // In practice, this points to the URL of dynamic-rp. + DefaultDownstreamEndpoint string `yaml:"defaultDownstreamEndpoint"` +} + +// InitializeConfig defines the configuration for initializing the UCP server. +// +// This includes resources that are added to UCP's data on startup. +// +// TODO: this will be generalized as part of the UDT work. Right now it only +// handles planes, and we need to support other kinds of resources. +type InitializationConfig struct { + // Planes is a list of planes to create at startup. + Planes []Plane `yaml:"planes,omitempty"` +} + +const ( + PlaneKindUCPNative = "UCPNative" + PlaneKindAzure = "Azure" + PlaneKindAWS = "AWS" +) + +type Plane struct { + ID string `json:"id" yaml:"id"` + Type string `json:"type" yaml:"type"` + Name string `json:"name" yaml:"name"` + Properties PlaneProperties `json:"properties" yaml:"properties"` +} + +type PlaneProperties struct { + ResourceProviders map[string]string `json:"resourceProviders" yaml:"resourceProviders"` // Used only for UCP native planes + Kind string `json:"kind" yaml:"kind"` + URL string `json:"url" yaml:"url"` // Used only for non UCP native planes and non AWS planes +} + +// LoadConfig loads a Config from bytes. +func LoadConfig(bs []byte) (*Config, error) { + decoder := yaml.NewDecoder(bytes.NewBuffer(bs)) + decoder.KnownFields(true) + + config := Config{} + err := decoder.Decode(&config) + if err != nil { + return nil, err + } + + return &config, nil +} diff --git a/pkg/ucp/config/ucpoptions.go b/pkg/ucp/config/ucpoptions.go index 57583033ae..785a2efcc7 100644 --- a/pkg/ucp/config/ucpoptions.go +++ b/pkg/ucp/config/ucpoptions.go @@ -59,6 +59,9 @@ func NewConnectionFromUCPConfig(option *UCPOptions, k8sConfig *rest.Config) (sdk return nil, errors.New("the property .ucp.direct.endpoint is required when using a direct connection") } return sdk.NewDirectConnection(option.Direct.Endpoint) + } else if option.Kind == UCPConnectionKindKubernetes { + return sdk.NewKubernetesConnectionFromConfig(k8sConfig) } - return sdk.NewKubernetesConnectionFromConfig(k8sConfig) + + return nil, errors.New("invalid connection kind: " + option.Kind) } diff --git a/pkg/ucp/datamodel/resourcegroup.go b/pkg/ucp/datamodel/resourcegroup.go index 0272b954a6..353e2b4a3e 100644 --- a/pkg/ucp/datamodel/resourcegroup.go +++ b/pkg/ucp/datamodel/resourcegroup.go @@ -18,6 +18,11 @@ package datamodel import v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" +const ( + // ResourceGroupResourceType is the type of a resource group. + ResourceGroupResourceType = "System.Resources/resourceGroups" +) + // ResourceGroup represents UCP ResourceGroup. type ResourceGroup struct { v1.BaseResource @@ -25,5 +30,5 @@ type ResourceGroup struct { // ResourceTypeName returns a string representing the resource type name of the ResourceGroup object. func (p ResourceGroup) ResourceTypeName() string { - return "System.Resources/resourceGroups" + return ResourceGroupResourceType } diff --git a/pkg/ucp/dataprovider/factory.go b/pkg/ucp/dataprovider/factory.go index 14696633fc..f318970641 100644 --- a/pkg/ucp/dataprovider/factory.go +++ b/pkg/ucp/dataprovider/factory.go @@ -124,8 +124,8 @@ func InitETCDClient(ctx context.Context, opt StorageProviderOptions, _ string) ( } // initInMemoryClient creates a new in-memory store client. -func initInMemoryClient(ctx context.Context, opt StorageProviderOptions, _ string) (store.StorageClient, error) { - return inmemory.NewClient(), nil +func initInMemoryClient(ctx context.Context, opt StorageProviderOptions, name string) (store.StorageClient, error) { + return inmemory.NewClient(name), nil } // initPostgreSQLClient creates a new PostgreSQL store client. diff --git a/pkg/ucp/doc.go b/pkg/ucp/doc.go new file mode 100644 index 0000000000..44a63b58c4 --- /dev/null +++ b/pkg/ucp/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// ucp holds the configuration and options types for UCP. See the packages nested inside this +// one for the UCP implementation. +package ucp diff --git a/pkg/ucp/frontend/api/routes.go b/pkg/ucp/frontend/api/routes.go index f18a628628..162db281ea 100644 --- a/pkg/ucp/frontend/api/routes.go +++ b/pkg/ucp/frontend/api/routes.go @@ -26,6 +26,7 @@ import ( v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" "github.com/radius-project/radius/pkg/armrpc/frontend/controller" "github.com/radius-project/radius/pkg/armrpc/frontend/server" + "github.com/radius-project/radius/pkg/ucp" kubernetes_ctrl "github.com/radius-project/radius/pkg/ucp/frontend/controller/kubernetes" planes_ctrl "github.com/radius-project/radius/pkg/ucp/frontend/controller/planes" "github.com/radius-project/radius/pkg/ucp/frontend/modules" @@ -74,9 +75,9 @@ func initModules(ctx context.Context, mods []modules.Initializer) (map[string]ht } // Register registers the routes for UCP including modules. -func Register(ctx context.Context, router chi.Router, planeModules []modules.Initializer, options modules.Options) error { +func Register(ctx context.Context, router chi.Router, planeModules []modules.Initializer, options *ucp.Options) error { logger := ucplog.FromContextOrDiscard(ctx) - logger.Info(fmt.Sprintf("Registering routes with path base: %s", options.PathBase)) + logger.Info(fmt.Sprintf("Registering routes with path base: %s", options.Config.Server.PathBase)) router.NotFound(validator.APINotFoundHandler()) router.MethodNotAllowed(validator.APIMethodNotAllowedHandler()) @@ -89,7 +90,7 @@ func Register(ctx context.Context, router chi.Router, planeModules []modules.Ini handlerOptions := []server.HandlerOptions{} // If we're in Kubernetes we have some required routes to implement. - if options.PathBase != "" { + if options.Config.Server.PathBase != "" { // NOTE: the Kubernetes API Server does not include the gvr (base path) in // the URL for swagger routes. handlerOptions = append(handlerOptions, []server.HandlerOptions{ @@ -109,7 +110,7 @@ func Register(ctx context.Context, router chi.Router, planeModules []modules.Ini }, { ParentRouter: router, - Path: options.PathBase, + Path: options.Config.Server.PathBase, OperationType: &v1.OperationType{Type: OperationTypeKubernetesDiscoveryDoc, Method: v1.OperationGet}, Method: v1.OperationGet, ControllerFactory: kubernetes_ctrl.NewDiscoveryDoc, @@ -124,7 +125,7 @@ func Register(ctx context.Context, router chi.Router, planeModules []modules.Ini }) // Configures planes collection and resource routes. - planeCollectionRouter := server.NewSubrouter(router, options.PathBase+planeCollectionPath, apiValidator) + planeCollectionRouter := server.NewSubrouter(router, options.Config.Server.PathBase+planeCollectionPath, apiValidator) // The "list all planes by type" handler is registered here. handlerOptions = append(handlerOptions, []server.HandlerOptions{ @@ -139,9 +140,14 @@ func Register(ctx context.Context, router chi.Router, planeModules []modules.Ini }...) ctrlOptions := controller.Options{ - Address: options.Address, - PathBase: options.PathBase, - DataProvider: options.DataProvider, + Address: options.Config.Server.Address(), + PathBase: options.Config.Server.PathBase, + DataProvider: options.StorageProvider, + StatusManager: options.StatusManager, + + KubeClient: nil, // Unused by UCP + StorageClient: nil, // Set dynamically + ResourceType: "", // Set dynamically } for _, h := range handlerOptions { @@ -151,7 +157,7 @@ func Register(ctx context.Context, router chi.Router, planeModules []modules.Ini } // Register a catch-all route to handle requests that get dispatched to a specific plane. - unknownPlaneRouter := server.NewSubrouter(router, options.PathBase+planeTypeCollectionPath) + unknownPlaneRouter := server.NewSubrouter(router, options.Config.Server.PathBase+planeTypeCollectionPath) unknownPlaneRouter.HandleFunc(server.CatchAllPath, func(w http.ResponseWriter, r *http.Request) { planeType := chi.URLParam(r, "planeType") handler, ok := moduleHandlers[planeType] diff --git a/pkg/ucp/frontend/api/routes_test.go b/pkg/ucp/frontend/api/routes_test.go index d0a7fdc849..84c0d5ca14 100644 --- a/pkg/ucp/frontend/api/routes_test.go +++ b/pkg/ucp/frontend/api/routes_test.go @@ -23,7 +23,9 @@ import ( "github.com/go-chi/chi/v5" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/armrpc/rpctest" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/frontend/modules" "github.com/radius-project/radius/test/testcontext" @@ -82,10 +84,15 @@ func Test_Routes(t *testing.T) { dataProvider := dataprovider.NewMockDataStorageProvider(ctrl) dataProvider.EXPECT().GetStorageClient(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() - options := modules.Options{ - Address: "localhost", - PathBase: pathBase, - DataProvider: dataProvider, + options := &ucp.Options{ + Config: &ucp.Config{ + Server: hostoptions.ServerOptions{ + Host: "localhost", + Port: 8080, + PathBase: pathBase, + }, + }, + StorageProvider: dataProvider, } rpctest.AssertRouters(t, tests, pathBase, "", func(ctx context.Context) (chi.Router, error) { @@ -101,10 +108,15 @@ func Test_Route_ToModule(t *testing.T) { dataProvider := dataprovider.NewMockDataStorageProvider(ctrl) dataProvider.EXPECT().GetStorageClient(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() - options := modules.Options{ - Address: "localhost", - PathBase: pathBase, - DataProvider: dataProvider, + options := &ucp.Options{ + Config: &ucp.Config{ + Server: hostoptions.ServerOptions{ + Host: "localhost", + Port: 8080, + PathBase: pathBase, + }, + }, + StorageProvider: dataProvider, } r := chi.NewRouter() diff --git a/pkg/ucp/frontend/api/server.go b/pkg/ucp/frontend/api/server.go index 8a957c2d89..01d43e2798 100644 --- a/pkg/ucp/frontend/api/server.go +++ b/pkg/ucp/frontend/api/server.go @@ -26,71 +26,34 @@ import ( "strings" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" - "github.com/radius-project/radius/pkg/armrpc/asyncoperation/statusmanager" armrpc_controller "github.com/radius-project/radius/pkg/armrpc/frontend/controller" "github.com/radius-project/radius/pkg/armrpc/frontend/defaultoperation" "github.com/radius-project/radius/pkg/armrpc/servicecontext" "github.com/radius-project/radius/pkg/middleware" - "github.com/radius-project/radius/pkg/sdk" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/datamodel" "github.com/radius-project/radius/pkg/ucp/datamodel/converter" - "github.com/radius-project/radius/pkg/ucp/dataprovider" aws_frontend "github.com/radius-project/radius/pkg/ucp/frontend/aws" azure_frontend "github.com/radius-project/radius/pkg/ucp/frontend/azure" "github.com/radius-project/radius/pkg/ucp/frontend/modules" radius_frontend "github.com/radius-project/radius/pkg/ucp/frontend/radius" "github.com/radius-project/radius/pkg/ucp/frontend/versions" "github.com/radius-project/radius/pkg/ucp/hosting" - "github.com/radius-project/radius/pkg/ucp/hostoptions" - queueprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" "github.com/radius-project/radius/pkg/ucp/resources" - "github.com/radius-project/radius/pkg/ucp/rest" - secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" "github.com/radius-project/radius/pkg/ucp/ucplog" - "github.com/radius-project/radius/pkg/validator" - "github.com/radius-project/radius/swagger" "github.com/go-chi/chi/v5" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel" ) -const ( - DefaultPlanesConfig = "DEFAULT_PLANES_CONFIG" -) - -type ServiceOptions struct { - // Config is the bootstrap configuration loaded from config file. - Config *hostoptions.UCPConfig - - ProviderName string - Address string - PathBase string - Configure func(chi.Router) - TLSCertDir string - DefaultPlanesConfigFile string - StorageProviderOptions dataprovider.StorageProviderOptions - SecretProviderOptions secretprovider.SecretProviderOptions - QueueProviderOptions queueprovider.QueueProviderOptions - InitialPlanes []rest.Plane - Identity hostoptions.Identity - UCPConnection sdk.Connection - Location string - - // Modules is a list of modules that will be registered with the router. - Modules []modules.Initializer -} - // Service implements the hosting.Service interface for the UCP frontend API. type Service struct { - options ServiceOptions - storageProvider dataprovider.DataStorageProvider - queueProvider *queueprovider.QueueProvider - secretProvider *secretprovider.SecretProvider + options *ucp.Options } // DefaultModules returns a list of default modules that will be registered with the router. -func DefaultModules(options modules.Options) []modules.Initializer { +func DefaultModules(options *ucp.Options) []modules.Initializer { return []modules.Initializer{ aws_frontend.NewModule(options), azure_frontend.NewModule(options), @@ -101,7 +64,7 @@ func DefaultModules(options modules.Options) []modules.Initializer { var _ hosting.Service = (*Service)(nil) // NewService creates a server to serve UCP API requests. -func NewService(options ServiceOptions) *Service { +func NewService(options *ucp.Options) *Service { return &Service{ options: options, } @@ -118,52 +81,25 @@ func (s *Service) Name() string { func (s *Service) Initialize(ctx context.Context) (*http.Server, error) { r := chi.NewRouter() - s.storageProvider = dataprovider.NewStorageProvider(s.options.StorageProviderOptions) - s.queueProvider = queueprovider.New(s.options.QueueProviderOptions) - s.secretProvider = secretprovider.NewSecretProvider(s.options.SecretProviderOptions) - - specLoader, err := validator.LoadSpec(ctx, "ucp", swagger.SpecFilesUCP, []string{s.options.PathBase}, "") - if err != nil { - return nil, err + // Allow tests to override the default modules. + modules := s.options.Modules + if modules == nil { + // If unset, use the default modules. + modules = DefaultModules(s.options) } - queueClient, err := s.queueProvider.GetClient(ctx) + err := Register(ctx, r, modules, s.options) if err != nil { return nil, err } - statusManager := statusmanager.New(s.storageProvider, queueClient, s.options.Location) - - moduleOptions := modules.Options{ - Address: s.options.Address, - PathBase: s.options.PathBase, - Config: s.options.Config, - Location: s.options.Location, - DataProvider: s.storageProvider, - QueueProvider: s.queueProvider, - SecretProvider: s.secretProvider, - SpecLoader: specLoader, - StatusManager: statusManager, - UCPConnection: s.options.UCPConnection, - } - - modules := DefaultModules(moduleOptions) - err = Register(ctx, r, modules, moduleOptions) - if err != nil { - return nil, err - } - - if s.options.Configure != nil { - s.options.Configure(r) - } - err = s.configureDefaultPlanes(ctx) if err != nil { return nil, err } app := http.Handler(r) - app = servicecontext.ARMRequestCtx(s.options.PathBase, "global")(app) + app = servicecontext.ARMRequestCtx(s.options.Config.Server.PathBase, s.options.Config.Environment.RoleLocation)(app) app = middleware.WithLogger(app) app = otelhttp.NewHandler( @@ -177,7 +113,7 @@ func (s *Service) Initialize(ctx context.Context) (*http.Server, error) { app = middleware.RemoveRemoteAddr(app) server := &http.Server{ - Addr: s.options.Address, + Addr: s.options.Config.Server.Address(), // Need to be able to respond to requests with planes and resourcegroups segments with any casing e.g.: /Planes, /resourceGroups // AWS SDK is case sensitive. Therefore, cannot use lowercase middleware. Therefore, introducing a new middleware that translates // the path for only these segments and preserves the case for the other parts of the path. @@ -192,7 +128,7 @@ func (s *Service) Initialize(ctx context.Context) (*http.Server, error) { // configureDefaultPlanes reads the configuration file specified by the env var to configure default planes into UCP func (s *Service) configureDefaultPlanes(ctx context.Context) error { - for _, plane := range s.options.InitialPlanes { + for _, plane := range s.options.Config.Initialization.Planes { err := s.createPlane(ctx, plane) if err != nil { return err @@ -202,7 +138,7 @@ func (s *Service) configureDefaultPlanes(ctx context.Context) error { return nil } -func (s *Service) createPlane(ctx context.Context, plane rest.Plane) error { +func (s *Service) createPlane(ctx context.Context, plane ucp.Plane) error { body, err := json.Marshal(plane) if err != nil { return err @@ -217,7 +153,7 @@ func (s *Service) createPlane(ctx context.Context, plane rest.Plane) error { return fmt.Errorf("invalid plane ID: %s", plane.ID) } - db, err := s.storageProvider.GetStorageClient(ctx, "ucp") + db, err := s.options.StorageProvider.GetStorageClient(ctx, "ucp") if err != nil { return err } @@ -266,7 +202,7 @@ func (s *Service) createPlane(ctx context.Context, plane rest.Plane) error { // Wrap the request in an ARM RPC context because this call will bypass the middleware // that normally does this for us. - rpcContext, err := v1.FromARMRequest(request, s.options.PathBase, s.options.Location) + rpcContext, err := v1.FromARMRequest(request, s.options.Config.Server.PathBase, s.options.Config.Environment.RoleLocation) if err != nil { return err } @@ -296,11 +232,11 @@ func (s *Service) Run(ctx context.Context) error { _ = service.Shutdown(ctx) }() - logger.Info(fmt.Sprintf("listening on: '%s'...", s.options.Address)) - if s.options.TLSCertDir == "" { + logger.Info(fmt.Sprintf("listening on: '%s'...", s.options.Config.Server.Address())) + if s.options.Config.Server.TLSCertificateDirectory == "" { err = service.ListenAndServe() } else { - err = service.ListenAndServeTLS(s.options.TLSCertDir+"/tls.crt", s.options.TLSCertDir+"/tls.key") + err = service.ListenAndServeTLS(s.options.Config.Server.TLSCertificateDirectory+"/tls.crt", s.options.Config.Server.TLSCertificateDirectory+"/tls.key") } if err == http.ErrServerClosed { diff --git a/pkg/ucp/frontend/aws/module.go b/pkg/ucp/frontend/aws/module.go index 676b4d0966..776c1df0cd 100644 --- a/pkg/ucp/frontend/aws/module.go +++ b/pkg/ucp/frontend/aws/module.go @@ -18,6 +18,7 @@ package aws import ( "github.com/go-chi/chi/v5" + "github.com/radius-project/radius/pkg/ucp" ucp_aws "github.com/radius-project/radius/pkg/ucp/aws" "github.com/radius-project/radius/pkg/ucp/frontend/modules" "github.com/radius-project/radius/pkg/validator" @@ -32,7 +33,7 @@ const ( ) // NewModule creates a new AWS module. -func NewModule(options modules.Options) *Module { +func NewModule(options *ucp.Options) *Module { m := Module{options: options} m.router = chi.NewRouter() m.router.NotFound(validator.APINotFoundHandler()) @@ -45,7 +46,7 @@ var _ modules.Initializer = &Module{} // Module defines the module for AWS functionality. type Module struct { - options modules.Options + options *ucp.Options router chi.Router // AWSClients provides access to AWS services. This field can be overridden by tests. diff --git a/pkg/ucp/frontend/aws/routes.go b/pkg/ucp/frontend/aws/routes.go index 6a3154f23d..90b10298b2 100644 --- a/pkg/ucp/frontend/aws/routes.go +++ b/pkg/ucp/frontend/aws/routes.go @@ -29,6 +29,7 @@ import ( "github.com/radius-project/radius/pkg/armrpc/frontend/defaultoperation" "github.com/radius-project/radius/pkg/armrpc/frontend/server" aztoken "github.com/radius-project/radius/pkg/azure/tokencredentials" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" ucp_aws "github.com/radius-project/radius/pkg/ucp/aws" sdk_cred "github.com/radius-project/radius/pkg/ucp/credentials" @@ -37,7 +38,6 @@ import ( awsproxy_ctrl "github.com/radius-project/radius/pkg/ucp/frontend/controller/awsproxy" aws_credential_ctrl "github.com/radius-project/radius/pkg/ucp/frontend/controller/credentials/aws" planes_ctrl "github.com/radius-project/radius/pkg/ucp/frontend/controller/planes" - "github.com/radius-project/radius/pkg/ucp/hostoptions" "github.com/radius-project/radius/pkg/ucp/ucplog" "github.com/radius-project/radius/pkg/validator" ) @@ -80,7 +80,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { } } - baseRouter := server.NewSubrouter(m.router, m.options.PathBase) + baseRouter := server.NewSubrouter(m.router, m.options.Config.Server.PathBase+"/") apiValidator := validator.APIValidator(validator.Options{ SpecLoader: m.options.SpecLoader, @@ -101,6 +101,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { // This is a scope query so we can't use the default operation. ParentRouter: planeCollectionRouter, Method: v1.OperationList, + ResourceType: datamodel.AWSPlaneResourceType, OperationType: &v1.OperationType{Type: datamodel.AWSPlaneResourceType, Method: v1.OperationList}, ControllerFactory: func(opts controller.Options) (controller.Controller, error) { return &planes_ctrl.ListPlanesByType[*datamodel.AWSPlane, datamodel.AWSPlane]{ @@ -111,6 +112,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { { ParentRouter: planeResourceRouter, Method: v1.OperationGet, + ResourceType: datamodel.AWSPlaneResourceType, OperationType: &v1.OperationType{Type: datamodel.AWSPlaneResourceType, Method: v1.OperationGet}, ControllerFactory: func(opts controller.Options) (controller.Controller, error) { return defaultoperation.NewGetResource(opts, planeResourceOptions) @@ -119,6 +121,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { { ParentRouter: planeResourceRouter, Method: v1.OperationPut, + ResourceType: datamodel.AWSPlaneResourceType, OperationType: &v1.OperationType{Type: datamodel.AWSPlaneResourceType, Method: v1.OperationPut}, ControllerFactory: func(opts controller.Options) (controller.Controller, error) { return defaultoperation.NewDefaultSyncPut(opts, planeResourceOptions) @@ -127,6 +130,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { { ParentRouter: planeResourceRouter, Method: v1.OperationDelete, + ResourceType: datamodel.AWSPlaneResourceType, OperationType: &v1.OperationType{Type: datamodel.AWSPlaneResourceType, Method: v1.OperationDelete}, ControllerFactory: func(opts controller.Options) (controller.Controller, error) { return defaultoperation.NewDefaultSyncDelete(opts, planeResourceOptions) @@ -280,9 +284,14 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { }...) ctrlOpts := controller.Options{ - Address: m.options.Address, - PathBase: m.options.PathBase, - DataProvider: m.options.DataProvider, + Address: m.options.Config.Server.Address(), + PathBase: m.options.Config.Server.PathBase, + DataProvider: m.options.StorageProvider, + StatusManager: m.options.StatusManager, + + KubeClient: nil, // Unused by AWS module + StorageClient: nil, // Set dynamically + ResourceType: "", // Set dynamically } for _, h := range handlerOptions { @@ -299,8 +308,8 @@ func (m *Module) newAWSConfig(ctx context.Context) (aws.Config, error) { credProviders := []func(*config.LoadOptions) error{} switch m.options.Config.Identity.AuthMethod { - case hostoptions.AuthUCPCredential: - provider, err := sdk_cred.NewAWSCredentialProvider(m.options.SecretProvider, m.options.UCPConnection, &aztoken.AnonymousCredential{}) + case ucp.AuthUCPCredential: + provider, err := sdk_cred.NewAWSCredentialProvider(m.options.SecretProvider, m.options.UCP, &aztoken.AnonymousCredential{}) if err != nil { return aws.Config{}, err } diff --git a/pkg/ucp/frontend/aws/routes_test.go b/pkg/ucp/frontend/aws/routes_test.go index aacd25032a..90d1622f44 100644 --- a/pkg/ucp/frontend/aws/routes_test.go +++ b/pkg/ucp/frontend/aws/routes_test.go @@ -25,12 +25,12 @@ import ( "go.uber.org/mock/gomock" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/armrpc/rpctest" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" "github.com/radius-project/radius/pkg/ucp/dataprovider" - "github.com/radius-project/radius/pkg/ucp/frontend/modules" - "github.com/radius-project/radius/pkg/ucp/hostoptions" "github.com/radius-project/radius/pkg/ucp/secret" secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" ) @@ -118,12 +118,16 @@ func Test_Routes(t *testing.T) { secretProvider := secretprovider.NewSecretProvider(secretprovider.SecretProviderOptions{}) secretProvider.SetClient(secretClient) - options := modules.Options{ - Address: "localhost", - PathBase: pathBase, - Config: &hostoptions.UCPConfig{}, - DataProvider: dataProvider, - SecretProvider: secretProvider, + options := &ucp.Options{ + Config: &ucp.Config{ + Server: hostoptions.ServerOptions{ + Host: "localhost", + Port: 8080, + PathBase: pathBase, + }, + }, + SecretProvider: secretProvider, + StorageProvider: dataProvider, } rpctest.AssertRouters(t, tests, pathBase, "", func(ctx context.Context) (chi.Router, error) { diff --git a/pkg/ucp/frontend/azure/module.go b/pkg/ucp/frontend/azure/module.go index d956d6ce30..24404996f0 100644 --- a/pkg/ucp/frontend/azure/module.go +++ b/pkg/ucp/frontend/azure/module.go @@ -18,25 +18,25 @@ package azure import ( "github.com/go-chi/chi/v5" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/frontend/modules" "github.com/radius-project/radius/pkg/validator" ) // NewModule creates a new Azure module. -func NewModule(options modules.Options) *Module { - m := Module{options: options} - m.router = chi.NewRouter() - m.router.NotFound(validator.APINotFoundHandler()) - m.router.MethodNotAllowed(validator.APIMethodNotAllowedHandler()) +func NewModule(options *ucp.Options) *Module { + router := chi.NewRouter() + router.NotFound(validator.APINotFoundHandler()) + router.MethodNotAllowed(validator.APIMethodNotAllowedHandler()) - return &Module{options: options, router: m.router} + return &Module{options: options, router: router} } var _ modules.Initializer = &Module{} // Module defines the module for Azure functionality. type Module struct { - options modules.Options + options *ucp.Options router chi.Router } diff --git a/pkg/ucp/frontend/azure/routes.go b/pkg/ucp/frontend/azure/routes.go index ca06e44f61..2c4096179a 100644 --- a/pkg/ucp/frontend/azure/routes.go +++ b/pkg/ucp/frontend/azure/routes.go @@ -49,7 +49,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { return nil, err } - baseRouter := server.NewSubrouter(m.router, m.options.PathBase) + baseRouter := server.NewSubrouter(m.router, m.options.Config.Server.PathBase+"/") apiValidator := validator.APIValidator(validator.Options{ SpecLoader: m.options.SpecLoader, @@ -73,6 +73,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { // This is a scope query so we can't use the default operation. ParentRouter: planeCollectionRouter, Method: v1.OperationList, + ResourceType: datamodel.AzurePlaneResourceType, OperationType: &v1.OperationType{Type: datamodel.AzurePlaneResourceType, Method: v1.OperationList}, ControllerFactory: func(opts controller.Options) (controller.Controller, error) { return &planes_ctrl.ListPlanesByType[*datamodel.AzurePlane, datamodel.AzurePlane]{ @@ -83,6 +84,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { { ParentRouter: planeResourceRouter, Method: v1.OperationGet, + ResourceType: datamodel.AzurePlaneResourceType, OperationType: &v1.OperationType{Type: datamodel.AzurePlaneResourceType, Method: v1.OperationGet}, ControllerFactory: func(opts controller.Options) (controller.Controller, error) { return defaultoperation.NewGetResource(opts, planeResourceOptions) @@ -91,6 +93,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { { ParentRouter: planeResourceRouter, Method: v1.OperationPut, + ResourceType: datamodel.AzurePlaneResourceType, OperationType: &v1.OperationType{Type: datamodel.AzurePlaneResourceType, Method: v1.OperationPut}, ControllerFactory: func(opts controller.Options) (controller.Controller, error) { return defaultoperation.NewDefaultSyncPut(opts, planeResourceOptions) @@ -99,6 +102,7 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { { ParentRouter: planeResourceRouter, Method: v1.OperationDelete, + ResourceType: datamodel.AzurePlaneResourceType, OperationType: &v1.OperationType{Type: datamodel.AzurePlaneResourceType, Method: v1.OperationDelete}, ControllerFactory: func(opts controller.Options) (controller.Controller, error) { return defaultoperation.NewDefaultSyncDelete(opts, planeResourceOptions) @@ -161,9 +165,14 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { } ctrlOpts := controller.Options{ - Address: m.options.Address, - PathBase: m.options.PathBase, - DataProvider: m.options.DataProvider, + Address: m.options.Config.Server.Address(), + PathBase: m.options.Config.Server.PathBase, + DataProvider: m.options.StorageProvider, + StatusManager: m.options.StatusManager, + + KubeClient: nil, // Unused by Azure module + StorageClient: nil, // Set dynamically + ResourceType: "", // Set dynamically } for _, h := range handlerOptions { diff --git a/pkg/ucp/frontend/azure/routes_test.go b/pkg/ucp/frontend/azure/routes_test.go index 8690166085..d0c823374e 100644 --- a/pkg/ucp/frontend/azure/routes_test.go +++ b/pkg/ucp/frontend/azure/routes_test.go @@ -25,12 +25,12 @@ import ( "go.uber.org/mock/gomock" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/armrpc/rpctest" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" "github.com/radius-project/radius/pkg/ucp/dataprovider" - "github.com/radius-project/radius/pkg/ucp/frontend/modules" - "github.com/radius-project/radius/pkg/ucp/hostoptions" "github.com/radius-project/radius/pkg/ucp/secret" secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" ) @@ -92,12 +92,16 @@ func Test_Routes(t *testing.T) { secretProvider := secretprovider.NewSecretProvider(secretprovider.SecretProviderOptions{}) secretProvider.SetClient(secretClient) - options := modules.Options{ - Address: "localhost", - PathBase: pathBase, - Config: &hostoptions.UCPConfig{}, - DataProvider: dataProvider, - SecretProvider: secretProvider, + options := &ucp.Options{ + Config: &ucp.Config{ + Server: hostoptions.ServerOptions{ + Host: "localhost", + Port: 8080, + PathBase: pathBase, + }, + }, + SecretProvider: secretProvider, + StorageProvider: dataProvider, } rpctest.AssertRouters(t, tests, pathBase, "", func(ctx context.Context) (chi.Router, error) { diff --git a/pkg/ucp/frontend/controller/planes/listplanes.go b/pkg/ucp/frontend/controller/planes/listplanes.go index 25592d705d..0443443fee 100644 --- a/pkg/ucp/frontend/controller/planes/listplanes.go +++ b/pkg/ucp/frontend/controller/planes/listplanes.go @@ -56,9 +56,9 @@ func (e *ListPlanes) Run(ctx context.Context, w http.ResponseWriter, req *http.R // The plane objects are all stored separately (by plane type). We need to query each type separately. planeTypes := []string{ - "aws", - "azure", - "radius", + "System.AWS/planes", + "System.Azure/planes", + "System.Radius/planes", } objs := []store.Object{} @@ -70,7 +70,12 @@ func (e *ListPlanes) Run(ctx context.Context, w http.ResponseWriter, req *http.R } logger.Info(fmt.Sprintf("Listing planes of type %s in scope %s", query.ResourceType, query.RootScope)) - result, err := e.StorageClient().Query(ctx, query) + client, err := e.DataProvider().GetStorageClient(ctx, planeType) + if err != nil { + return nil, err + } + + result, err := client.Query(ctx, query) if err != nil { return nil, err } diff --git a/pkg/ucp/frontend/controller/planes/listplanes_test.go b/pkg/ucp/frontend/controller/planes/listplanes_test.go index 505a3cd320..af368aadbf 100644 --- a/pkg/ucp/frontend/controller/planes/listplanes_test.go +++ b/pkg/ucp/frontend/controller/planes/listplanes_test.go @@ -26,6 +26,7 @@ import ( "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/store" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -34,9 +35,13 @@ import ( func Test_ListPlanes(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() + mockStorageClient := store.NewMockStorageClient(mockCtrl) - planesCtrl, err := NewListPlanes(armrpc_controller.Options{StorageClient: mockStorageClient}) + mockStorageProvider := dataprovider.NewMockDataStorageProvider(mockCtrl) + mockStorageProvider.EXPECT().GetStorageClient(gomock.Any(), gomock.Any()).Return(mockStorageClient, nil).AnyTimes() + + planesCtrl, err := NewListPlanes(armrpc_controller.Options{DataProvider: mockStorageProvider}) require.NoError(t, err) url := "/planes?api-version=2023-10-01-preview" @@ -59,7 +64,7 @@ func Test_ListPlanes(t *testing.T) { mockStorageClient.EXPECT().Query(gomock.Any(), store.Query{ RootScope: "/planes", - ResourceType: "aws", + ResourceType: "System.AWS/planes", IsScopeQuery: true, }).Return(&store.ObjectQueryResult{ Items: []store.Object{ @@ -72,13 +77,13 @@ func Test_ListPlanes(t *testing.T) { mockStorageClient.EXPECT().Query(gomock.Any(), store.Query{ RootScope: "/planes", - ResourceType: "azure", + ResourceType: "System.Azure/planes", IsScopeQuery: true, }).Return(&store.ObjectQueryResult{}, nil) mockStorageClient.EXPECT().Query(gomock.Any(), store.Query{ RootScope: "/planes", - ResourceType: "radius", + ResourceType: "System.Radius/planes", IsScopeQuery: true, }).Return(&store.ObjectQueryResult{}, nil) diff --git a/pkg/ucp/frontend/controller/planes/listplanesbytype.go b/pkg/ucp/frontend/controller/planes/listplanesbytype.go index 69ca21f49d..e0682185fb 100644 --- a/pkg/ucp/frontend/controller/planes/listplanesbytype.go +++ b/pkg/ucp/frontend/controller/planes/listplanesbytype.go @@ -46,8 +46,22 @@ type ListPlanesByType[P interface { // an error occurs, it returns an error. func (e *ListPlanesByType[P, T]) Run(ctx context.Context, w http.ResponseWriter, req *http.Request) (armrpc_rest.Response, error) { path := middleware.GetRelativePath(e.Options().PathBase, req.URL.Path) - // The path is /planes/{planeType} - planeType := strings.Split(path, resources.SegmentSeparator)[2] + // The path is /planes/{planeShortType} + planeShortType := strings.Split(path, resources.SegmentSeparator)[2] + + // Map that onto the known plane fully-qualified types, so we can do the database + // lookup. + knownPlaneTypes := map[string]string{ + "aws": "System.AWS/planes", + "azure": "System.Azure/planes", + "radius": "System.Radius/planes", + } + + planeType, ok := knownPlaneTypes[planeShortType] + if !ok { + return armrpc_rest.NewBadRequestResponse(fmt.Sprintf("Unknown plane type %s", planeShortType)), nil + } + query := store.Query{ RootScope: resources.SegmentSeparator + resources.PlanesSegment, IsScopeQuery: true, @@ -55,16 +69,23 @@ func (e *ListPlanesByType[P, T]) Run(ctx context.Context, w http.ResponseWriter, } logger := ucplog.FromContextOrDiscard(ctx) logger.Info(fmt.Sprintf("Listing planes in scope %s/%s", query.RootScope, planeType)) - result, err := e.StorageClient().Query(ctx, query) + + client, err := e.DataProvider().GetStorageClient(ctx, planeType) if err != nil { return nil, err } + + result, err := client.Query(ctx, query) + if err != nil { + return nil, err + } + listOfPlanes, err := e.createResponse(ctx, result) if err != nil { return nil, err } - var ok = armrpc_rest.NewOKResponse(listOfPlanes) - return ok, nil + + return armrpc_rest.NewOKResponse(listOfPlanes), nil } func (p *ListPlanesByType[P, T]) createResponse(ctx context.Context, result *store.ObjectQueryResult) (*v1.PaginatedList, error) { diff --git a/pkg/ucp/frontend/controller/planes/listplanesbytype_test.go b/pkg/ucp/frontend/controller/planes/listplanesbytype_test.go index 58bf997eed..f6bd7a31db 100644 --- a/pkg/ucp/frontend/controller/planes/listplanesbytype_test.go +++ b/pkg/ucp/frontend/controller/planes/listplanesbytype_test.go @@ -27,6 +27,7 @@ import ( "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" "github.com/radius-project/radius/pkg/ucp/datamodel/converter" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/store" "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" @@ -35,11 +36,15 @@ import ( func Test_ListPlanesByType(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() + mockStorageClient := store.NewMockStorageClient(mockCtrl) + mockStorageProvider := dataprovider.NewMockDataStorageProvider(mockCtrl) + mockStorageProvider.EXPECT().GetStorageClient(gomock.Any(), gomock.Any()).Return(mockStorageClient, nil).AnyTimes() + ctrl := &ListPlanesByType[*datamodel.RadiusPlane, datamodel.RadiusPlane]{ Operation: armrpc_controller.NewOperation[*datamodel.RadiusPlane, datamodel.RadiusPlane]( - armrpc_controller.Options{StorageClient: mockStorageClient}, + armrpc_controller.Options{DataProvider: mockStorageProvider}, armrpc_controller.ResourceOptions[datamodel.RadiusPlane]{ ResponseConverter: converter.RadiusPlaneDataModelToVersioned, }), @@ -50,7 +55,7 @@ func Test_ListPlanesByType(t *testing.T) { query := store.Query{ RootScope: "/planes", IsScopeQuery: true, - ResourceType: "radius", + ResourceType: "System.Radius/planes", } testPlaneId := "/planes/radius/local" diff --git a/pkg/ucp/frontend/controller/planes/proxycontroller.go b/pkg/ucp/frontend/controller/planes/proxycontroller.go index 07a05e5be8..1a4c53b86f 100644 --- a/pkg/ucp/frontend/controller/planes/proxycontroller.go +++ b/pkg/ucp/frontend/controller/planes/proxycontroller.go @@ -17,6 +17,7 @@ package planes import ( "context" + "errors" "fmt" http "net/http" "net/url" @@ -28,6 +29,7 @@ import ( "github.com/radius-project/radius/pkg/ucp/datamodel" "github.com/radius-project/radius/pkg/ucp/proxy" "github.com/radius-project/radius/pkg/ucp/resources" + "github.com/radius-project/radius/pkg/ucp/store" "github.com/radius-project/radius/pkg/ucp/ucplog" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) @@ -76,14 +78,35 @@ func (p *ProxyController) Run(ctx context.Context, w http.ResponseWriter, req *h return nil, err } + // The only supported proxy-plane type is Azure. + knownPlaneResourceTypes := map[string]string{ + "azure": "System.Azure/planes", + } + + resourceType, ok := knownPlaneResourceTypes[planeType] + if !ok { + return armrpc_rest.NewBadRequestResponse(fmt.Sprintf("unknown plane type %q", planeType)), nil + } + serviceCtx := v1.ARMRequestContextFromContext(ctx) - plane, _, err := p.GetResource(ctx, planeID) + + client, err := p.DataProvider().GetStorageClient(ctx, resourceType) if err != nil { return nil, err } - if plane == nil { + + obj, err := client.Get(ctx, planeID.String()) + if errors.Is(err, &store.ErrNotFound{}) { restResponse := armrpc_rest.NewNotFoundResponse(serviceCtx.ResourceID) return restResponse, nil + } else if err != nil { + return nil, err + } + + plane := &datamodel.AzurePlane{} + err = obj.As(plane) + if err != nil { + return nil, err } // Get the resource provider diff --git a/pkg/ucp/frontend/controller/radius/proxy.go b/pkg/ucp/frontend/controller/radius/proxy.go index 84cc7360a2..08033c4a01 100644 --- a/pkg/ucp/frontend/controller/radius/proxy.go +++ b/pkg/ucp/frontend/controller/radius/proxy.go @@ -31,6 +31,7 @@ import ( armrpc_controller "github.com/radius-project/radius/pkg/armrpc/frontend/controller" armrpc_rest "github.com/radius-project/radius/pkg/armrpc/rest" "github.com/radius-project/radius/pkg/middleware" + "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" "github.com/radius-project/radius/pkg/ucp/frontend/controller/resourcegroups" "github.com/radius-project/radius/pkg/ucp/proxy" @@ -82,7 +83,7 @@ func NewProxyController(opts armrpc_controller.Options, transport http.RoundTrip return nil, fmt.Errorf("failed to parse default downstream URL: %w", err) } - updater := trackedresource.NewUpdater(opts.StorageClient, &http.Client{Transport: transport}) + updater := trackedresource.NewUpdater(opts.DataProvider, &http.Client{Transport: transport}) return &ProxyController{ Operation: armrpc_controller.NewOperation(opts, armrpc_controller.ResourceOptions[datamodel.RadiusPlane]{}), transport: transport, @@ -113,7 +114,7 @@ func (p *ProxyController) Run(ctx context.Context, w http.ResponseWriter, req *h return armrpc_rest.NewBadRequestARMResponse(response), nil } - downstreamURL, err := resourcegroups.ValidateDownstream(ctx, p.StorageClient(), id, v1.LocationGlobal, apiVersion) + downstreamURL, err := resourcegroups.ValidateDownstream(ctx, p.DataProvider(), id, v1.LocationGlobal, apiVersion) if errors.Is(err, &resourcegroups.NotFoundError{}) { return armrpc_rest.NewNotFoundResponseWithCause(id, err.Error()), nil } else if errors.Is(err, &resourcegroups.InvalidError{}) { @@ -279,7 +280,12 @@ func (p *ProxyController) EnqueueTrackedResourceUpdate(ctx context.Context, id r queueOperation := false retry: for retryCount := 1; retryCount <= EnqueueOperationRetryCount; retryCount++ { - obj, err := p.StorageClient().Get(ctx, trackingID.String()) + client, err := p.DataProvider().GetStorageClient(ctx, v20231001preview.ResourceType) + if err != nil { + return err + } + + obj, err := client.Get(ctx, trackingID.String()) if errors.Is(err, &store.ErrNotFound{}) { // Safe to ignore. This means that the resource has not been tracked yet. } else if err != nil { @@ -302,7 +308,7 @@ retry: } logger.V(ucplog.LevelDebug).Info("enqueuing tracked resource update") - err = p.StorageClient().Save(ctx, &store.Object{Metadata: store.Metadata{ID: trackingID.String()}, Data: entry}, store.WithETag(etag)) + err = client.Save(ctx, &store.Object{Metadata: store.Metadata{ID: trackingID.String()}, Data: entry}, store.WithETag(etag)) if errors.Is(err, &store.ErrConcurrency{}) { // This means we hit a concurrency error saving the tracked resource entry. This means that the resource // was updated in the background. We should retry. diff --git a/pkg/ucp/frontend/controller/radius/proxy_test.go b/pkg/ucp/frontend/controller/radius/proxy_test.go index 8e51472637..e52e8ce7a0 100644 --- a/pkg/ucp/frontend/controller/radius/proxy_test.go +++ b/pkg/ucp/frontend/controller/radius/proxy_test.go @@ -29,6 +29,7 @@ import ( "github.com/radius-project/radius/pkg/armrpc/frontend/controller" "github.com/radius-project/radius/pkg/armrpc/rest" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/resources" "github.com/radius-project/radius/pkg/ucp/store" "github.com/radius-project/radius/pkg/ucp/trackedresource" @@ -46,13 +47,16 @@ const ( func createController(t *testing.T) (*ProxyController, *store.MockStorageClient, *mockUpdater, *mockRoundTripper, *statusmanager.MockStatusManager) { ctrl := gomock.NewController(t) + storageProvider := dataprovider.NewMockDataStorageProvider(ctrl) storageClient := store.NewMockStorageClient(ctrl) + storageProvider.EXPECT().GetStorageClient(gomock.Any(), gomock.Any()).Return(storageClient, nil).AnyTimes() + statusManager := statusmanager.NewMockStatusManager(ctrl) roundTripper := mockRoundTripper{} p, err := NewProxyController( - controller.Options{StorageClient: storageClient, StatusManager: statusManager}, + controller.Options{DataProvider: storageProvider, StorageClient: storageClient, StatusManager: statusManager}, &roundTripper, "http://localhost:1234") require.NoError(t, err) diff --git a/pkg/ucp/frontend/controller/resourcegroups/listresources.go b/pkg/ucp/frontend/controller/resourcegroups/listresources.go index 4cfc82a53c..ff7a35abd3 100644 --- a/pkg/ucp/frontend/controller/resourcegroups/listresources.go +++ b/pkg/ucp/frontend/controller/resourcegroups/listresources.go @@ -62,7 +62,12 @@ func (r *ListResources) Run(ctx context.Context, w http.ResponseWriter, req *htt resourceGroupID := id.Truncate() // First check if the resource group exists. - _, err = r.StorageClient().Get(ctx, resourceGroupID.String()) + client, err := r.DataProvider().GetStorageClient(ctx, v20231001preview.ResourceGroupType) + if err != nil { + return nil, err + } + + _, err = client.Get(ctx, resourceGroupID.String()) if errors.Is(err, &store.ErrNotFound{}) { return armrpc_rest.NewNotFoundResponse(id), nil } else if err != nil { diff --git a/pkg/ucp/frontend/controller/resourcegroups/listresources_test.go b/pkg/ucp/frontend/controller/resourcegroups/listresources_test.go index c7f77c9934..823ecdf6fc 100644 --- a/pkg/ucp/frontend/controller/resourcegroups/listresources_test.go +++ b/pkg/ucp/frontend/controller/resourcegroups/listresources_test.go @@ -30,6 +30,7 @@ import ( "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/resources" "github.com/radius-project/radius/pkg/ucp/store" ) @@ -136,8 +137,10 @@ func Test_ListResources(t *testing.T) { func setupListResources(t *testing.T) (*store.MockStorageClient, *ListResources) { ctrl := gomock.NewController(t) storage := store.NewMockStorageClient(ctrl) + storageProvider := dataprovider.NewMockDataStorageProvider(ctrl) + storageProvider.EXPECT().GetStorageClient(gomock.Any(), gomock.Any()).Return(storage, nil).AnyTimes() - c, err := NewListResources(armrpc_controller.Options{StorageClient: storage, PathBase: "/" + uuid.New().String()}) + c, err := NewListResources(armrpc_controller.Options{DataProvider: storageProvider, StorageClient: storage, PathBase: "/" + uuid.New().String()}) require.NoError(t, err) return storage, c.(*ListResources) diff --git a/pkg/ucp/frontend/controller/resourcegroups/util.go b/pkg/ucp/frontend/controller/resourcegroups/util.go index a1d5f8c507..cc0ae3f274 100644 --- a/pkg/ucp/frontend/controller/resourcegroups/util.go +++ b/pkg/ucp/frontend/controller/resourcegroups/util.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/resources" resources_radius "github.com/radius-project/radius/pkg/ucp/resources/radius" "github.com/radius-project/radius/pkg/ucp/store" @@ -62,26 +63,27 @@ func (e *InvalidError) Is(err error) bool { } // ValidateRadiusPlane validates that the plane specified in the id exists. Returns NotFoundError if the plane does not exist. -func ValidateRadiusPlane(ctx context.Context, client store.StorageClient, id resources.ID) (*datamodel.RadiusPlane, error) { +func ValidateRadiusPlane(ctx context.Context, storageProvider dataprovider.DataStorageProvider, id resources.ID) (*datamodel.RadiusPlane, error) { planeID, err := resources.ParseScope(id.PlaneScope()) if err != nil { // Not expected to happen. return nil, err } - plane, err := store.GetResource[datamodel.RadiusPlane](ctx, client, planeID.String()) + plane := datamodel.RadiusPlane{} + err = fetchResource(ctx, storageProvider, planeID, datamodel.RadiusPlaneResourceType, &plane) if errors.Is(err, &store.ErrNotFound{}) { return nil, &NotFoundError{Message: fmt.Sprintf("plane %q not found", planeID.String())} } else if err != nil { return nil, fmt.Errorf("failed to fetch plane %q: %w", planeID.String(), err) } - return plane, nil + return &plane, nil } // ValidateResourceGroup validates that the resource group specified in the id exists (if applicable). // Returns NotFoundError if the resource group does not exist. -func ValidateResourceGroup(ctx context.Context, client store.StorageClient, id resources.ID) error { +func ValidateResourceGroup(ctx context.Context, storageProvider dataprovider.DataStorageProvider, id resources.ID) error { // If the ID contains a resource group, validate it now. if id.FindScope(resources_radius.ScopeResourceGroups) == "" { return nil @@ -93,7 +95,8 @@ func ValidateResourceGroup(ctx context.Context, client store.StorageClient, id r return err } - _, err = store.GetResource[datamodel.ResourceGroup](ctx, client, resourceGroupID.String()) + resourceGroup := datamodel.ResourceGroup{} + err = fetchResource(ctx, storageProvider, resourceGroupID, datamodel.ResourceGroupResourceType, &resourceGroup) if errors.Is(err, &store.ErrNotFound{}) { return &NotFoundError{Message: fmt.Sprintf("resource group %q not found", resourceGroupID.String())} } else if err != nil { @@ -108,7 +111,7 @@ func ValidateResourceGroup(ctx context.Context, client store.StorageClient, id r // // Returns NotFoundError if the resource type does not exist. // Returns InvalidError if the request cannot be routed due to an invalid configuration. -func ValidateResourceType(ctx context.Context, client store.StorageClient, id resources.ID, locationName string, apiVersion string) (*url.URL, error) { +func ValidateResourceType(ctx context.Context, storageProvider dataprovider.DataStorageProvider, id resources.ID, locationName string, apiVersion string) (*url.URL, error) { // The strategy is to: // - Look up the resource type and validate that it exists .. then // - Look up the location resource, and validate that it supports the requested resource type and API version. @@ -121,7 +124,8 @@ func ValidateResourceType(ctx context.Context, client store.StorageClient, id re return nil, err } - _, err = store.GetResource[datamodel.ResourceType](ctx, client, resourceTypeID.String()) + resourceType := datamodel.ResourceType{} + err = fetchResource(ctx, storageProvider, resourceTypeID, datamodel.ResourceTypeResourceType, &resourceType) if errors.Is(err, &store.ErrNotFound{}) { // Return the error as-is to fallback to the legacy routing behavior. @@ -139,7 +143,8 @@ func ValidateResourceType(ctx context.Context, client store.StorageClient, id re return nil, err } - location, err := store.GetResource[datamodel.Location](ctx, client, locationID.String()) + location := datamodel.Location{} + err = fetchResource(ctx, storageProvider, locationID, datamodel.LocationResourceType, &location) if errors.Is(err, &store.ErrNotFound{}) { // Return the error as-is to fallback to the legacy routing behavior. @@ -155,7 +160,10 @@ func ValidateResourceType(ctx context.Context, client store.StorageClient, id re // Resource types are case-insensitive so we have to iterate. var locationResourceType *datamodel.LocationResourceTypeConfiguration - // We special-case two pseudo-resource types: "locations/operationstatuses" and "locations/operationresults". + // We special-case two pseudo-resource types: "operationstatuses" and "operationresults". + // + // These are implemented by all resource providers, and don't require the resource provider to register them. + // // If the resource type is one of these, we can return the downstream URL directly. if isOperationResourceType(id) { locationResourceType = &datamodel.LocationResourceTypeConfiguration{ @@ -215,10 +223,11 @@ func isOperationResourceType(id resources.ID) bool { return true } - // An older pattern is to use a child resource + // An older pattern is to use a child resource, it might also use the name "operations" typeSegments := id.TypeSegments() if len(typeSegments) >= 2 && (strings.EqualFold(typeSegments[len(typeSegments)-1].Type, "operationstatuses") || - strings.EqualFold(typeSegments[len(typeSegments)-1].Type, "operationresults")) { + strings.EqualFold(typeSegments[len(typeSegments)-1].Type, "operationresults") || + strings.EqualFold(typeSegments[len(typeSegments)-1].Type, "operations")) { return true } @@ -228,7 +237,7 @@ func isOperationResourceType(id resources.ID) bool { // ValidateLegacyResourceProvider validates that the resource provider specified in the id exists. Returns InvalidError if the plane // contains invalid data. -func ValidateLegacyResourceProvider(ctx context.Context, client store.StorageClient, id resources.ID, plane *datamodel.RadiusPlane) (*url.URL, error) { +func ValidateLegacyResourceProvider(ctx context.Context, id resources.ID, plane *datamodel.RadiusPlane) (*url.URL, error) { downstream := plane.LookupResourceProvider(id.ProviderNamespace()) if downstream == "" { return nil, &InvalidError{Message: fmt.Sprintf("resource provider %s not configured", id.ProviderNamespace())} @@ -245,7 +254,7 @@ func ValidateLegacyResourceProvider(ctx context.Context, client store.StorageCli // ValidateDownstream can be used to find and validate the downstream URL for a resource. // Returns NotFoundError for the case where the plane or resource group does not exist. // Returns InvalidError for cases where the data is invalid, like when the resource provider is not configured. -func ValidateDownstream(ctx context.Context, client store.StorageClient, id resources.ID, location string, apiVersion string) (*url.URL, error) { +func ValidateDownstream(ctx context.Context, storageProvider dataprovider.DataStorageProvider, id resources.ID, location string, apiVersion string) (*url.URL, error) { // There are a few steps to validation: // // - The plane exists @@ -256,25 +265,50 @@ func ValidateDownstream(ctx context.Context, client store.StorageClient, id reso // // The plane exists. - plane, err := ValidateRadiusPlane(ctx, client, id) + plane, err := ValidateRadiusPlane(ctx, storageProvider, id) if err != nil { return nil, err } // The resource group exists (if applicable). - err = ValidateResourceGroup(ctx, client, id) + err = ValidateResourceGroup(ctx, storageProvider, id) if err != nil { return nil, err } // If this returns success, it means the resource type is configured using new/UDT routing. - downstreamURL, err := ValidateResourceType(ctx, client, id, location, apiVersion) + downstreamURL, err := ValidateResourceType(ctx, storageProvider, id, location, apiVersion) if errors.Is(err, &store.ErrNotFound{}) { // If the resource provider is not found, treat it like a legacy provider. - return ValidateLegacyResourceProvider(ctx, client, id, plane) + return ValidateLegacyResourceProvider(ctx, id, plane) } else if err != nil { return nil, err } return downstreamURL, nil } + +// fetchResource is a helper that fetches a resource from UCP storage. +// +// We need this helper, because we need to use the storage client that corresponds to each resource type. +// +// Our storage client layer is designed to have a "client" for each resource type, but not all of the storage +// implementations enforce this. We use the in-memory client for testing, which is strict. +func fetchResource(ctx context.Context, storageProvider dataprovider.DataStorageProvider, id resources.ID, resourceType string, resource any) error { + client, err := storageProvider.GetStorageClient(ctx, resourceType) + if err != nil { + return fmt.Errorf("failed to get storage client for resource type %q: %w", resourceType, err) + } + + obj, err := client.Get(ctx, id.String()) + if err != nil { + return err + } + + err = obj.As(resource) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/ucp/frontend/controller/resourcegroups/util_test.go b/pkg/ucp/frontend/controller/resourcegroups/util_test.go index f266458196..d5a93f23ab 100644 --- a/pkg/ucp/frontend/controller/resourcegroups/util_test.go +++ b/pkg/ucp/frontend/controller/resourcegroups/util_test.go @@ -24,6 +24,7 @@ import ( v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/resources" "github.com/radius-project/radius/pkg/ucp/store" "github.com/radius-project/radius/test/testcontext" @@ -93,9 +94,15 @@ func Test_ValidateDownstream(t *testing.T) { }, } - setup := func(t *testing.T) *store.MockStorageClient { + setup := func(t *testing.T) (*dataprovider.MockDataStorageProvider, *store.MockStorageClient) { ctrl := gomock.NewController(t) - return store.NewMockStorageClient(ctrl) + + storageProvider := dataprovider.NewMockDataStorageProvider(ctrl) + client := store.NewMockStorageClient(ctrl) + + storageProvider.EXPECT().GetStorageClient(gomock.Any(), gomock.Any()).Return(client, nil).AnyTimes() + + return storageProvider, client } t.Run("success (resource group)", func(t *testing.T) { @@ -107,138 +114,138 @@ func Test_ValidateDownstream(t *testing.T) { }, } - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeResource}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeResource}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) expectedURL, err := url.Parse(downstream) require.NoError(t, err) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.NoError(t, err) require.Equal(t, expectedURL, downstreamURL) }) t.Run("success (non resource group)", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeResource}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeResource}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) expectedURL, err := url.Parse(downstream) require.NoError(t, err) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, idWithoutResourceGroup, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, idWithoutResourceGroup, location, apiVersion) require.NoError(t, err) require.Equal(t, expectedURL, downstreamURL) }) // The deployment engine models its operation status resources as child resources of the deployment resource. t.Run("success (operationstatuses as child resource)", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) operationStatusID := resources.MustParse("/planes/radius/local/providers/System.TestRP/deployments/xzy/operationStatuses/abcd") expectedURL, err := url.Parse(downstream) require.NoError(t, err) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, operationStatusID, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, operationStatusID, location, apiVersion) require.NoError(t, err) require.Equal(t, expectedURL, downstreamURL) }) // All of the Radius RPs include a location in the operation status child resource. t.Run("success (operationstatuses with location)", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) operationStatusID := resources.MustParse("/planes/radius/local/providers/System.TestRP/locations/east/operationStatuses/abcd") expectedURL, err := url.Parse(downstream) require.NoError(t, err) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, operationStatusID, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, operationStatusID, location, apiVersion) require.NoError(t, err) require.Equal(t, expectedURL, downstreamURL) }) // The deployment engine models its operation result resources as child resources of the deployment resource. t.Run("success (operationresults as child resource)", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) operationResultID := resources.MustParse("/planes/radius/local/providers/System.TestRP/deployments/xzy/operationResults/abcd") expectedURL, err := url.Parse(downstream) require.NoError(t, err) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, operationResultID, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, operationResultID, location, apiVersion) require.NoError(t, err) require.Equal(t, expectedURL, downstreamURL) }) // All of the Radius RPs include a location in the operation result child resource. t.Run("success (operationresults with location)", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) operationResultID := resources.MustParse("/planes/radius/local/providers/System.TestRP/locations/east/operationResults/abcd") expectedURL, err := url.Parse(downstream) require.NoError(t, err) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, operationResultID, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, operationResultID, location, apiVersion) require.NoError(t, err) require.Equal(t, expectedURL, downstreamURL) }) t.Run("plane not found", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(nil, &store.ErrNotFound{}).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(nil, &store.ErrNotFound{}).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &NotFoundError{Message: "plane \"/planes/radius/local\" not found"}, err) require.Nil(t, downstreamURL) }) t.Run("plane retreival failure", func(t *testing.T) { - mock := setup(t) + provider, client := setup(t) expected := fmt.Errorf("failed to fetch plane \"/planes/radius/local\": %w", errors.New("test error")) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(nil, errors.New("test error")).Times(1) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(nil, errors.New("test error")).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, expected, err) require.Nil(t, downstreamURL) }) t.Run("resource group not found", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(nil, &store.ErrNotFound{}).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(nil, &store.ErrNotFound{}).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &NotFoundError{Message: "resource group \"/planes/radius/local/resourceGroups/test-group\" not found"}, err) require.Nil(t, downstreamURL) }) t.Run("resource group err", func(t *testing.T) { - mock := setup(t) + provider, client := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(nil, errors.New("test error")).Times(1) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(nil, errors.New("test error")).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, "failed to fetch resource group \"/planes/radius/local/resourceGroups/test-group\": test error", err.Error()) require.Nil(t, downstreamURL) @@ -255,12 +262,12 @@ func Test_ValidateDownstream(t *testing.T) { expected := fmt.Errorf("failed to fetch resource type %q: %w", "System.TestRP/testResources", errors.New("test error")) - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(nil, errors.New("test error")).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(nil, errors.New("test error")).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, expected, err) require.Nil(t, downstreamURL) @@ -277,13 +284,13 @@ func Test_ValidateDownstream(t *testing.T) { expected := fmt.Errorf("failed to fetch location %q: %w", locationResource.ID, errors.New("test error")) - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeID}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(nil, errors.New("test error")).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeID}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(nil, errors.New("test error")).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, expected, err) require.Nil(t, downstreamURL) @@ -317,13 +324,13 @@ func Test_ValidateDownstream(t *testing.T) { }, } - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeID}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeID}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &InvalidError{Message: "resource type \"System.TestRP/testResources\" not supported by location \"east\""}, err) require.Nil(t, downstreamURL) @@ -357,13 +364,13 @@ func Test_ValidateDownstream(t *testing.T) { }, } - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeID}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeID}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &InvalidError{Message: "api version \"2025-01-01\" is not supported for resource type \"System.TestRP/testResources\" by location \"east\""}, err) require.Nil(t, downstreamURL) @@ -397,13 +404,13 @@ func Test_ValidateDownstream(t *testing.T) { }, } - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeID}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeResource.ID).Return(&store.Object{Data: resourceTypeID}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), locationResource.ID).Return(&store.Object{Data: locationResource}, nil).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &InvalidError{Message: "failed to parse location address: parse \"\\ninvalid\": net/url: invalid control character in URL"}, err) require.Nil(t, downstreamURL) @@ -437,9 +444,15 @@ func Test_ValidateDownstream_Legacy(t *testing.T) { }, } - setup := func(t *testing.T) *store.MockStorageClient { + setup := func(t *testing.T) (*dataprovider.MockDataStorageProvider, *store.MockStorageClient) { ctrl := gomock.NewController(t) - return store.NewMockStorageClient(ctrl) + + storageProvider := dataprovider.NewMockDataStorageProvider(ctrl) + client := store.NewMockStorageClient(ctrl) + + storageProvider.EXPECT().GetStorageClient(gomock.Any(), gomock.Any()).Return(client, nil).AnyTimes() + + return storageProvider, client } t.Run("success (resource group)", func(t *testing.T) { @@ -451,72 +464,72 @@ func Test_ValidateDownstream_Legacy(t *testing.T) { }, } - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) expectedURL, err := url.Parse(downstream) require.NoError(t, err) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.NoError(t, err) require.Equal(t, expectedURL, downstreamURL) }) t.Run("success (non resource group)", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), idWithoutResourceGroup.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) expectedURL, err := url.Parse(downstream) require.NoError(t, err) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, idWithoutResourceGroup, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, idWithoutResourceGroup, location, apiVersion) require.NoError(t, err) require.Equal(t, expectedURL, downstreamURL) }) t.Run("plane not found", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(nil, &store.ErrNotFound{}).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(nil, &store.ErrNotFound{}).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &NotFoundError{Message: "plane \"/planes/radius/local\" not found"}, err) require.Nil(t, downstreamURL) }) t.Run("plane retrieval failure", func(t *testing.T) { - mock := setup(t) + provider, client := setup(t) expected := fmt.Errorf("failed to fetch plane \"/planes/radius/local\": %w", errors.New("test error")) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(nil, errors.New("test error")).Times(1) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(nil, errors.New("test error")).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, expected, err) require.Nil(t, downstreamURL) }) t.Run("resource group not found", func(t *testing.T) { - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(nil, &store.ErrNotFound{}).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(nil, &store.ErrNotFound{}).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &NotFoundError{Message: "resource group \"/planes/radius/local/resourceGroups/test-group\" not found"}, err) require.Nil(t, downstreamURL) }) t.Run("resource group err", func(t *testing.T) { - mock := setup(t) + provider, client := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(nil, errors.New("test error")).Times(1) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(nil, errors.New("test error")).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, "failed to fetch resource group \"/planes/radius/local/resourceGroups/test-group\": test error", err.Error()) require.Nil(t, downstreamURL) @@ -542,12 +555,12 @@ func Test_ValidateDownstream_Legacy(t *testing.T) { }, } - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &InvalidError{Message: "resource provider System.TestRP not configured"}, err) require.Nil(t, downstreamURL) @@ -573,12 +586,12 @@ func Test_ValidateDownstream_Legacy(t *testing.T) { }, } - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &InvalidError{Message: "resource provider System.TestRP not configured"}, err) require.Nil(t, downstreamURL) @@ -606,12 +619,12 @@ func Test_ValidateDownstream_Legacy(t *testing.T) { }, } - mock := setup(t) - mock.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) - mock.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) + provider, client := setup(t) + client.EXPECT().Get(gomock.Any(), id.PlaneScope()).Return(&store.Object{Data: plane}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), id.RootScope()).Return(&store.Object{Data: resourceGroup}, nil).Times(1) + client.EXPECT().Get(gomock.Any(), resourceTypeID.String()).Return(nil, &store.ErrNotFound{}).Times(1) - downstreamURL, err := ValidateDownstream(testcontext.New(t), mock, id, location, apiVersion) + downstreamURL, err := ValidateDownstream(testcontext.New(t), provider, id, location, apiVersion) require.Error(t, err) require.Equal(t, &InvalidError{Message: "failed to parse downstream URL: parse \"\\ninvalid\": net/url: invalid control character in URL"}, err) require.Nil(t, downstreamURL) diff --git a/pkg/ucp/frontend/controller/resourceproviders/getresourceprovidersummary.go b/pkg/ucp/frontend/controller/resourceproviders/getresourceprovidersummary.go index ac0b92acd4..6839a31948 100644 --- a/pkg/ucp/frontend/controller/resourceproviders/getresourceprovidersummary.go +++ b/pkg/ucp/frontend/controller/resourceproviders/getresourceprovidersummary.go @@ -61,7 +61,15 @@ func (r *GetResourceProviderSummary) Run(ctx context.Context, w http.ResponseWri } // First check if the plane exists. - _, err = r.StorageClient().Get(ctx, scope.String()) + // + // The only supported plane type is Radius. + planeResourceType := datamodel.RadiusPlaneResourceType + client, err := r.DataProvider().GetStorageClient(ctx, planeResourceType) + if err != nil { + return nil, err + } + + _, err = client.Get(ctx, scope.String()) if errors.Is(err, &store.ErrNotFound{}) { return armrpc_rest.NewNotFoundResponse(scope), nil } else if err != nil { diff --git a/pkg/ucp/frontend/controller/resourceproviders/listresourceprovidersummaries.go b/pkg/ucp/frontend/controller/resourceproviders/listresourceprovidersummaries.go index edd72c4575..0512d0c4a7 100644 --- a/pkg/ucp/frontend/controller/resourceproviders/listresourceprovidersummaries.go +++ b/pkg/ucp/frontend/controller/resourceproviders/listresourceprovidersummaries.go @@ -68,7 +68,13 @@ func (r *ListResourceProviderSummaries) Run(ctx context.Context, w http.Response } // First check if the plane exists. - _, err = r.StorageClient().Get(ctx, scope.String()) + planeResourceType := datamodel.RadiusPlaneResourceType + client, err := r.DataProvider().GetStorageClient(ctx, planeResourceType) + if err != nil { + return nil, err + } + + _, err = client.Get(ctx, scope.String()) if errors.Is(err, &store.ErrNotFound{}) { return armrpc_rest.NewNotFoundResponse(scope), nil } else if err != nil { diff --git a/pkg/ucp/frontend/modules/types.go b/pkg/ucp/frontend/modules/types.go index 6b0a0d6e9a..6b93a359d6 100644 --- a/pkg/ucp/frontend/modules/types.go +++ b/pkg/ucp/frontend/modules/types.go @@ -19,14 +19,6 @@ package modules import ( "context" "net/http" - - "github.com/radius-project/radius/pkg/armrpc/asyncoperation/statusmanager" - "github.com/radius-project/radius/pkg/sdk" - "github.com/radius-project/radius/pkg/ucp/dataprovider" - "github.com/radius-project/radius/pkg/ucp/hostoptions" - queueprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" - secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" - "github.com/radius-project/radius/pkg/validator" ) // Initializer is an interface that can be implemented by modules that want to provide functionality for a plane. @@ -45,36 +37,3 @@ type Initializer interface { // - radius PlaneType() string } - -// Options defines the options for a module. -type Options struct { - // Config is the bootstrap configuration loaded from config file. - Config *hostoptions.UCPConfig - - // Address is the hostname + port of the server hosting UCP. - Address string - - // PathBase is the base path of the server as it appears in the URL. eg: '/apis/api.ucp.dev/v1alpha3'. - PathBase string - - // Location is the location of the server hosting UCP. - Location string - - // DataProvider is the data storage provider. - DataProvider dataprovider.DataStorageProvider - - // QeueueProvider provides access to the queue for async operations. - QueueProvider *queueprovider.QueueProvider - - // SecretProvider is the secret store provider used for managing credentials. - SecretProvider *secretprovider.SecretProvider - - // SpecLoader is the OpenAPI spec loader containing specs for the UCP APIs. - SpecLoader *validator.Loader - - // StatusManager is the async operation status manager. - StatusManager statusmanager.StatusManager - - // UCPConnection is the connection used to communicate with UCP APIs. - UCPConnection sdk.Connection -} diff --git a/pkg/ucp/frontend/radius/module.go b/pkg/ucp/frontend/radius/module.go index 811e5692d4..6b2a867429 100644 --- a/pkg/ucp/frontend/radius/module.go +++ b/pkg/ucp/frontend/radius/module.go @@ -18,12 +18,13 @@ package radius import ( "github.com/go-chi/chi/v5" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/frontend/modules" "github.com/radius-project/radius/pkg/validator" ) // NewModule creates a new Radius module. -func NewModule(options modules.Options) *Module { +func NewModule(options *ucp.Options) *Module { router := chi.NewRouter() router.NotFound(validator.APINotFoundHandler()) router.MethodNotAllowed(validator.APIMethodNotAllowedHandler()) @@ -35,7 +36,7 @@ var _ modules.Initializer = &Module{} // Module defines the module for Radius functionality. type Module struct { - options modules.Options + options *ucp.Options router chi.Router defaultDownstream string } diff --git a/pkg/ucp/frontend/radius/routes.go b/pkg/ucp/frontend/radius/routes.go index 2631bd9afd..624ffc76e5 100644 --- a/pkg/ucp/frontend/radius/routes.go +++ b/pkg/ucp/frontend/radius/routes.go @@ -65,14 +65,18 @@ func (m *Module) Initialize(ctx context.Context) (http.Handler, error) { } ctrlOptions := controller.Options{ - Address: m.options.Address, - PathBase: m.options.PathBase, - DataProvider: m.options.DataProvider, + Address: m.options.Config.Server.Address(), + PathBase: m.options.Config.Server.PathBase, + DataProvider: m.options.StorageProvider, StatusManager: m.options.StatusManager, + + KubeClient: nil, // Unused by Radius module + StorageClient: nil, // Set dynamically + ResourceType: "", // Set dynamically } // NOTE: we're careful where we use the `apiValidator` middleware. It's not used for the proxy routes. - m.router.Route(m.options.PathBase+"/planes/radius", func(r chi.Router) { + m.router.Route(m.options.Config.Server.PathBase+"/planes/radius", func(r chi.Router) { r.With(apiValidator).Get("/", capture(radiusPlaneListHandler(ctx, ctrlOptions))) r.Route("/{planeName}", func(r chi.Router) { r.With(apiValidator).Get("/", capture(radiusPlaneGetHandler(ctx, ctrlOptions))) diff --git a/pkg/ucp/frontend/radius/routes_test.go b/pkg/ucp/frontend/radius/routes_test.go index c47bb9ab1a..b0d763197f 100644 --- a/pkg/ucp/frontend/radius/routes_test.go +++ b/pkg/ucp/frontend/radius/routes_test.go @@ -23,12 +23,12 @@ import ( "github.com/go-chi/chi/v5" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/armrpc/rpctest" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" "github.com/radius-project/radius/pkg/ucp/dataprovider" - "github.com/radius-project/radius/pkg/ucp/frontend/modules" - "github.com/radius-project/radius/pkg/ucp/hostoptions" "github.com/radius-project/radius/pkg/ucp/secret" secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" "go.uber.org/mock/gomock" @@ -192,12 +192,16 @@ func Test_Routes(t *testing.T) { secretProvider := secretprovider.NewSecretProvider(secretprovider.SecretProviderOptions{}) secretProvider.SetClient(secretClient) - options := modules.Options{ - Address: "localhost", - PathBase: pathBase, - Config: &hostoptions.UCPConfig{}, - DataProvider: dataProvider, - SecretProvider: secretProvider, + options := &ucp.Options{ + Config: &ucp.Config{ + Server: hostoptions.ServerOptions{ + Host: "localhost", + Port: 8080, + PathBase: pathBase, + }, + }, + SecretProvider: secretProvider, + StorageProvider: dataProvider, } rpctest.AssertRouters(t, tests, pathBase, "", func(ctx context.Context) (chi.Router, error) { diff --git a/pkg/ucp/hostoptions/hostoptions.go b/pkg/ucp/hostoptions/hostoptions.go deleted file mode 100644 index db708f73b5..0000000000 --- a/pkg/ucp/hostoptions/hostoptions.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2023 The Radius Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// hostoptions defines and reads options for the RP's execution environment. - -package hostoptions - -import ( - "bytes" - "fmt" - "os" - - "gopkg.in/yaml.v3" -) - -// HostOptions defines all of the settings that our RP's execution environment provides. -type HostOptions struct { - // Config is the bootstrap configuration loaded from config file. - Config *UCPConfig -} - -// NewHostOptionsFromEnvironment reads the configuration from the given path and returns a HostOptions object, or an -// error if the configuration could not be loaded. -func NewHostOptionsFromEnvironment(configPath string) (HostOptions, error) { - conf, err := loadConfig(configPath) - if err != nil { - return HostOptions{}, err - } - - return HostOptions{ - Config: conf, - }, nil -} - -func loadConfig(configPath string) (*UCPConfig, error) { - buf, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } - - conf := &UCPConfig{} - decoder := yaml.NewDecoder(bytes.NewBuffer(buf)) - decoder.KnownFields(true) - - err = decoder.Decode(conf) - if err != nil { - return nil, fmt.Errorf("failed to load yaml: %w", err) - } - - return conf, nil -} diff --git a/pkg/ucp/hostoptions/providerconfig.go b/pkg/ucp/hostoptions/providerconfig.go deleted file mode 100644 index 4b37e6a8cf..0000000000 --- a/pkg/ucp/hostoptions/providerconfig.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2023 The Radius Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package hostoptions - -import ( - metricsprovider "github.com/radius-project/radius/pkg/metrics/provider" - profilerprovider "github.com/radius-project/radius/pkg/profiler/provider" - "github.com/radius-project/radius/pkg/trace" - "github.com/radius-project/radius/pkg/ucp/config" - "github.com/radius-project/radius/pkg/ucp/dataprovider" - qprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" - "github.com/radius-project/radius/pkg/ucp/rest" - "github.com/radius-project/radius/pkg/ucp/secret/provider" - "github.com/radius-project/radius/pkg/ucp/ucplog" -) - -// UCPConfig includes the resource provider configuration. -type UCPConfig struct { - StorageProvider dataprovider.StorageProviderOptions `yaml:"storageProvider"` - Planes []rest.Plane `yaml:"planes"` - SecretProvider provider.SecretProviderOptions `yaml:"secretProvider"` - MetricsProvider metricsprovider.MetricsProviderOptions `yaml:"metricsProvider"` - ProfilerProvider profilerprovider.ProfilerProviderOptions `yaml:"profilerProvider"` - QueueProvider qprovider.QueueProviderOptions `yaml:"queueProvider"` - TracerProvider trace.Options `yaml:"tracerProvider"` - Logging ucplog.LoggingOptions `yaml:"logging"` - Identity Identity `yaml:"identity,omitempty"` - UCP config.UCPOptions `yaml:"ucp"` - Location string `yaml:"location"` - Routing RoutingConfig `yaml:"routing"` -} - -const ( - // AuthUCPCredential is the authentication method via UCP Credential API. - AuthUCPCredential = "UCPCredential" - // AuthDefault is the default authentication method, such as environment variables. - AuthDefault = "default" -) - -// Identity represents configuration options for authenticating with external systems like Azure and AWS. -type Identity struct { - // AuthMethod represents the method of authentication for authenticating with external systems like Azure and AWS. - AuthMethod string `yaml:"authMethod"` -} - -// RoutingConfig provides configuration for UCP routing. -type RoutingConfig struct { - // DefaultDownstreamEndpoint is the default destination when a resource provider does not provide a downstream endpoint. - // In practice, this points to the URL of dynamic-rp. - DefaultDownstreamEndpoint string `yaml:"defaultDownstreamEndpoint"` -} diff --git a/pkg/ucp/integrationtests/aws/awstest.go b/pkg/ucp/integrationtests/aws/awstest.go index 52aa311d44..59e77df5cf 100644 --- a/pkg/ucp/integrationtests/aws/awstest.go +++ b/pkg/ucp/integrationtests/aws/awstest.go @@ -21,12 +21,11 @@ package aws import ( "testing" + "github.com/radius-project/radius/pkg/ucp" ucp_aws "github.com/radius-project/radius/pkg/ucp/aws" ucp_aws_frontend "github.com/radius-project/radius/pkg/ucp/frontend/aws" "github.com/radius-project/radius/pkg/ucp/frontend/modules" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" - "github.com/radius-project/radius/pkg/ucp/secret" - "github.com/radius-project/radius/pkg/ucp/store" + "github.com/radius-project/radius/pkg/ucp/testhost" "go.uber.org/mock/gomock" ) @@ -38,17 +37,17 @@ const ( testAWSRequestToken = "79B9F0DA-4882-4DC8-A367-6FD3BC122DED" // Random UUID ) -func initializeAWSTest(t *testing.T) (*testserver.TestServer, *store.MockStorageClient, *secret.MockClient, *ucp_aws.MockAWSCloudControlClient, *ucp_aws.MockAWSCloudFormationClient) { +func initializeAWSTest(t *testing.T) (*testhost.TestHost, *ucp_aws.MockAWSCloudControlClient, *ucp_aws.MockAWSCloudFormationClient) { ctrl := gomock.NewController(t) cloudControlClient := ucp_aws.NewMockAWSCloudControlClient(ctrl) cloudFormationClient := ucp_aws.NewMockAWSCloudFormationClient(ctrl) - ucp := testserver.StartWithMocks(t, func(options modules.Options) []modules.Initializer { + ucp := testhost.Start(t, testhost.TestHostOptionFunc(func(options *ucp.Options) { module := ucp_aws_frontend.NewModule(options) module.AWSClients.CloudControl = cloudControlClient module.AWSClients.CloudFormation = cloudFormationClient - return []modules.Initializer{module} - }) + options.Modules = []modules.Initializer{module} + })) - return ucp, ucp.Mocks.Storage, ucp.Mocks.Secrets, cloudControlClient, cloudFormationClient + return ucp, cloudControlClient, cloudFormationClient } diff --git a/pkg/ucp/integrationtests/aws/createresource_test.go b/pkg/ucp/integrationtests/aws/createresource_test.go index 78475c40a8..055f753b52 100644 --- a/pkg/ucp/integrationtests/aws/createresource_test.go +++ b/pkg/ucp/integrationtests/aws/createresource_test.go @@ -35,7 +35,7 @@ import ( ) func Test_CreateAWSResource(t *testing.T) { - ucp, _, _, cloudcontrolClient, _ := initializeAWSTest(t) + ucp, cloudcontrolClient, _ := initializeAWSTest(t) cloudcontrolClient.EXPECT().GetResource(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, params *cloudcontrol.GetResourceInput, optFns ...func(*cloudcontrol.Options)) (*cloudcontrol.GetResourceOutput, error) { notfound := types.ResourceNotFoundException{ @@ -63,7 +63,7 @@ func Test_CreateAWSResource(t *testing.T) { body, err := json.Marshal(requestBody) require.NoError(t, err) - createRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPut, ucp.BaseURL+testProxyRequestAWSPath, body) + createRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPut, ucp.BaseURL()+testProxyRequestAWSPath, body) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(createRequest) diff --git a/pkg/ucp/integrationtests/aws/createresourcewithpost_test.go b/pkg/ucp/integrationtests/aws/createresourcewithpost_test.go index 4909caa092..19d96f852a 100644 --- a/pkg/ucp/integrationtests/aws/createresourcewithpost_test.go +++ b/pkg/ucp/integrationtests/aws/createresourcewithpost_test.go @@ -37,7 +37,7 @@ import ( ) func Test_CreateAWSResourceWithPost(t *testing.T) { - ucp, _, _, cloudcontrolClient, cloudformationClient := initializeAWSTest(t) + ucp, cloudcontrolClient, cloudformationClient := initializeAWSTest(t) primaryIdentifiers := map[string]any{ "primaryIdentifier": []any{ @@ -81,7 +81,7 @@ func Test_CreateAWSResourceWithPost(t *testing.T) { body, err := json.Marshal(requestBody) require.NoError(t, err) - createRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPost, ucp.BaseURL+testProxyRequestAWSCollectionPath+"/:put", body) + createRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPost, ucp.BaseURL()+testProxyRequestAWSCollectionPath+"/:put", body) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(createRequest) diff --git a/pkg/ucp/integrationtests/aws/deleteresource_test.go b/pkg/ucp/integrationtests/aws/deleteresource_test.go index f497772f76..5edca23c39 100644 --- a/pkg/ucp/integrationtests/aws/deleteresource_test.go +++ b/pkg/ucp/integrationtests/aws/deleteresource_test.go @@ -34,7 +34,7 @@ import ( ) func Test_DeleteAWSResource(t *testing.T) { - ucp, _, _, cloudcontrolClient, _ := initializeAWSTest(t) + ucp, cloudcontrolClient, _ := initializeAWSTest(t) cloudcontrolClient.EXPECT().DeleteResource(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, params *cloudcontrol.DeleteResourceInput, optFns ...func(*cloudcontrol.Options)) (*cloudcontrol.DeleteResourceOutput, error) { output := cloudcontrol.DeleteResourceOutput{ @@ -46,7 +46,7 @@ func Test_DeleteAWSResource(t *testing.T) { return &output, nil }) - deleteRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodDelete, ucp.BaseURL+testProxyRequestAWSPath, nil) + deleteRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodDelete, ucp.BaseURL()+testProxyRequestAWSPath, nil) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(deleteRequest) diff --git a/pkg/ucp/integrationtests/aws/deleteresourcewithpost_test.go b/pkg/ucp/integrationtests/aws/deleteresourcewithpost_test.go index fc78fb54a4..f9f5078d05 100644 --- a/pkg/ucp/integrationtests/aws/deleteresourcewithpost_test.go +++ b/pkg/ucp/integrationtests/aws/deleteresourcewithpost_test.go @@ -37,7 +37,7 @@ import ( ) func Test_DeleteAWSResourceWithPost(t *testing.T) { - ucp, _, _, cloudcontrolClient, cloudformationClient := initializeAWSTest(t) + ucp, cloudcontrolClient, cloudformationClient := initializeAWSTest(t) primaryIdentifiers := map[string]any{ "primaryIdentifier": []any{ @@ -73,7 +73,7 @@ func Test_DeleteAWSResourceWithPost(t *testing.T) { body, err := json.Marshal(requestBody) require.NoError(t, err) - deleteRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPost, ucp.BaseURL+testProxyRequestAWSCollectionPath+"/:delete", body) + deleteRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPost, ucp.BaseURL()+testProxyRequestAWSCollectionPath+"/:delete", body) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(deleteRequest) diff --git a/pkg/ucp/integrationtests/aws/getresource_test.go b/pkg/ucp/integrationtests/aws/getresource_test.go index c9e17c38d9..501be7565f 100644 --- a/pkg/ucp/integrationtests/aws/getresource_test.go +++ b/pkg/ucp/integrationtests/aws/getresource_test.go @@ -35,7 +35,7 @@ import ( ) func Test_GetAWSResource(t *testing.T) { - ucp, _, _, cloudcontrolClient, _ := initializeAWSTest(t) + ucp, cloudcontrolClient, _ := initializeAWSTest(t) getResponseBody := map[string]any{ "RetentionPeriodHours": 178, @@ -54,7 +54,7 @@ func Test_GetAWSResource(t *testing.T) { return &output, nil }) - getRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodGet, ucp.BaseURL+testProxyRequestAWSPath, nil) + getRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodGet, ucp.BaseURL()+testProxyRequestAWSPath, nil) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(getRequest) diff --git a/pkg/ucp/integrationtests/aws/getresourcewithpost_test.go b/pkg/ucp/integrationtests/aws/getresourcewithpost_test.go index 95d72ee5c6..e33a972e15 100644 --- a/pkg/ucp/integrationtests/aws/getresourcewithpost_test.go +++ b/pkg/ucp/integrationtests/aws/getresourcewithpost_test.go @@ -37,7 +37,7 @@ import ( ) func Test_GetAWSResourceWithPost(t *testing.T) { - ucp, _, _, cloudcontrolClient, cloudformationClient := initializeAWSTest(t) + ucp, cloudcontrolClient, cloudformationClient := initializeAWSTest(t) primaryIdentifiers := map[string]any{ "primaryIdentifier": []any{ @@ -80,7 +80,7 @@ func Test_GetAWSResourceWithPost(t *testing.T) { body, err := json.Marshal(requestBody) require.NoError(t, err) - getRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPost, ucp.BaseURL+testProxyRequestAWSCollectionPath+"/:get", body) + getRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPost, ucp.BaseURL()+testProxyRequestAWSCollectionPath+"/:get", body) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(getRequest) diff --git a/pkg/ucp/integrationtests/aws/listresources_test.go b/pkg/ucp/integrationtests/aws/listresources_test.go index ca97d8ba9a..c068073a4f 100644 --- a/pkg/ucp/integrationtests/aws/listresources_test.go +++ b/pkg/ucp/integrationtests/aws/listresources_test.go @@ -37,7 +37,7 @@ import ( const testProxyRequestAWSListPath = "/planes/aws/aws/accounts/1234567/regions/us-east-1/providers/AWS.Kinesis/Stream" func Test_ListAWSResources(t *testing.T) { - ucp, _, _, cloudcontrolClient, _ := initializeAWSTest(t) + ucp, cloudcontrolClient, _ := initializeAWSTest(t) getResponseBody := map[string]any{ "RetentionPeriodHours": 178, @@ -58,7 +58,7 @@ func Test_ListAWSResources(t *testing.T) { return &output, nil }) - listRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodGet, ucp.BaseURL+testProxyRequestAWSListPath, nil) + listRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodGet, ucp.BaseURL()+testProxyRequestAWSListPath, nil) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(listRequest) diff --git a/pkg/ucp/integrationtests/aws/operationresults_test.go b/pkg/ucp/integrationtests/aws/operationresults_test.go index 354c79c44b..730480bf71 100644 --- a/pkg/ucp/integrationtests/aws/operationresults_test.go +++ b/pkg/ucp/integrationtests/aws/operationresults_test.go @@ -35,7 +35,7 @@ import ( ) func Test_GetOperationResults(t *testing.T) { - ucp, _, _, cloudcontrolClient, _ := initializeAWSTest(t) + ucp, cloudcontrolClient, _ := initializeAWSTest(t) cloudcontrolClient.EXPECT().GetResourceRequestStatus(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, params *cloudcontrol.GetResourceRequestStatusInput, optFns ...func(*cloudcontrol.Options)) (*cloudcontrol.GetResourceRequestStatusOutput, error) { output := cloudcontrol.GetResourceRequestStatusOutput{ @@ -46,7 +46,7 @@ func Test_GetOperationResults(t *testing.T) { return &output, nil }) - operationResultsRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodGet, ucp.BaseURL+testProxyRequestAWSAsyncPath+"/operationResults/"+strings.ToLower(testAWSRequestToken), nil) + operationResultsRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodGet, ucp.BaseURL()+testProxyRequestAWSAsyncPath+"/operationResults/"+strings.ToLower(testAWSRequestToken), nil) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(operationResultsRequest) diff --git a/pkg/ucp/integrationtests/aws/operationstatuses_test.go b/pkg/ucp/integrationtests/aws/operationstatuses_test.go index 1de5766732..ae78037c6b 100644 --- a/pkg/ucp/integrationtests/aws/operationstatuses_test.go +++ b/pkg/ucp/integrationtests/aws/operationstatuses_test.go @@ -36,7 +36,7 @@ import ( ) func Test_GetOperationStatuses(t *testing.T) { - ucp, _, _, cloudcontrolClient, _ := initializeAWSTest(t) + ucp, cloudcontrolClient, _ := initializeAWSTest(t) cloudcontrolClient.EXPECT().GetResourceRequestStatus(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, params *cloudcontrol.GetResourceRequestStatusInput, optFns ...func(*cloudcontrol.Options)) (*cloudcontrol.GetResourceRequestStatusOutput, error) { output := cloudcontrol.GetResourceRequestStatusOutput{ @@ -48,7 +48,7 @@ func Test_GetOperationStatuses(t *testing.T) { return &output, nil }) - operationResultsRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodGet, ucp.BaseURL+testProxyRequestAWSAsyncPath+"/operationStatuses/"+strings.ToLower(testAWSRequestToken), nil) + operationResultsRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodGet, ucp.BaseURL()+testProxyRequestAWSAsyncPath+"/operationStatuses/"+strings.ToLower(testAWSRequestToken), nil) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(operationResultsRequest) diff --git a/pkg/ucp/integrationtests/aws/updateresource_test.go b/pkg/ucp/integrationtests/aws/updateresource_test.go index 6afdcc7138..3f5b9d1a48 100644 --- a/pkg/ucp/integrationtests/aws/updateresource_test.go +++ b/pkg/ucp/integrationtests/aws/updateresource_test.go @@ -39,7 +39,7 @@ import ( const ZeroAWSRequestToken = "00000000-0000-0000-0000-000000000000" func Test_UpdateAWSResource(t *testing.T) { - ucp, _, _, cloudcontrolClient, cloudFormationClient := initializeAWSTest(t) + ucp, cloudcontrolClient, cloudFormationClient := initializeAWSTest(t) getResponseBody := map[string]any{ "RetentionPeriodHours": 178, @@ -94,7 +94,7 @@ func Test_UpdateAWSResource(t *testing.T) { body, err := json.Marshal(requestBody) require.NoError(t, err) - updateRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPut, ucp.BaseURL+testProxyRequestAWSPath, body) + updateRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPut, ucp.BaseURL()+testProxyRequestAWSPath, body) require.NoError(t, err, "creating request failed") ctx := rpctest.NewARMRequestContext(updateRequest) diff --git a/pkg/ucp/integrationtests/aws/updateresourcewithpost_test.go b/pkg/ucp/integrationtests/aws/updateresourcewithpost_test.go index 015074783d..8c23b7d1a0 100644 --- a/pkg/ucp/integrationtests/aws/updateresourcewithpost_test.go +++ b/pkg/ucp/integrationtests/aws/updateresourcewithpost_test.go @@ -37,7 +37,7 @@ import ( ) func Test_UpdateAWSResourceWithPost(t *testing.T) { - ucp, _, _, cloudcontrolClient, cloudformationClient := initializeAWSTest(t) + ucp, cloudcontrolClient, cloudformationClient := initializeAWSTest(t) primaryIdentifiers := map[string]any{ "primaryIdentifier": []any{ @@ -90,7 +90,7 @@ func Test_UpdateAWSResourceWithPost(t *testing.T) { body, err := json.Marshal(requestBody) require.NoError(t, err) - updateRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPost, ucp.BaseURL+testProxyRequestAWSCollectionPath+"/:put", body) + updateRequest, err := rpctest.NewHTTPRequestWithContent(context.Background(), http.MethodPost, ucp.BaseURL()+testProxyRequestAWSCollectionPath+"/:put", body) require.NoError(t, err, "update request failed") ctx := rpctest.NewARMRequestContext(updateRequest) diff --git a/pkg/ucp/integrationtests/azure/proxy_test.go b/pkg/ucp/integrationtests/azure/proxy_test.go index 74261d45f4..d672c83168 100644 --- a/pkg/ucp/integrationtests/azure/proxy_test.go +++ b/pkg/ucp/integrationtests/azure/proxy_test.go @@ -25,9 +25,8 @@ import ( v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" - "github.com/radius-project/radius/pkg/ucp/frontend/api" "github.com/radius-project/radius/pkg/ucp/integrationtests/testrp" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/stretchr/testify/require" ) @@ -38,7 +37,7 @@ const ( ) func Test_AzurePlane_ProxyRequest(t *testing.T) { - ucp := testserver.StartWithETCD(t, api.DefaultModules) + ucp := testhost.Start(t) data := map[string]string{ "message": "here is some test data", @@ -63,7 +62,7 @@ func Test_AzurePlane_ProxyRequest(t *testing.T) { require.Equal(t, "SomeValue", response.Raw.Header.Get("SomeHeader")) } -func createAzurePlane(ucp *testserver.TestServer, rp *testrp.Server) { +func createAzurePlane(ucp *testhost.TestHost, rp *testrp.Server) { body := v20231001preview.AzurePlaneResource{ Location: to.Ptr(v1.LocationGlobal), Properties: &v20231001preview.AzurePlaneResourceProperties{ diff --git a/pkg/ucp/integrationtests/handler_test.go b/pkg/ucp/integrationtests/handler_test.go index c131536c73..b8c0ee2a40 100644 --- a/pkg/ucp/integrationtests/handler_test.go +++ b/pkg/ucp/integrationtests/handler_test.go @@ -20,20 +20,32 @@ import ( "net/http" "testing" + "github.com/google/uuid" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/stretchr/testify/require" ) func Test_Handler_MethodNotAllowed(t *testing.T) { - ucp := testserver.StartWithMocks(t, testserver.NoModules) + ucp := testhost.Start(t, testhost.NoModules()) response := ucp.MakeRequest(http.MethodDelete, "/planes?api-version=2023-10-01-preview", nil) require.Equal(t, "failed to parse route: undefined route path", response.Error.Error.Details[0].Message) } func Test_Handler_NotFound(t *testing.T) { - ucp := testserver.StartWithMocks(t, testserver.NoModules) + ucp := testhost.Start(t, testhost.NoModules()) + + response := ucp.MakeRequest(http.MethodGet, "/abc", nil) + response.EqualsErrorCode(http.StatusNotFound, v1.CodeNotFound) + require.Regexp(t, "The request 'GET /abc' is invalid.", response.Error.Error.Message) +} + +func Test_Handler_NotFound_PathBase(t *testing.T) { + ucp := testhost.Start(t, testhost.NoModules(), testhost.TestHostOptionFunc(func(options *ucp.Options) { + options.Config.Server.PathBase = "/" + uuid.New().String() + })) response := ucp.MakeRequest(http.MethodGet, "/abc", nil) response.EqualsErrorCode(http.StatusNotFound, v1.CodeNotFound) diff --git a/pkg/ucp/integrationtests/planes/aws_test.go b/pkg/ucp/integrationtests/planes/aws_test.go index fa707bf3c8..b810b648cf 100644 --- a/pkg/ucp/integrationtests/planes/aws_test.go +++ b/pkg/ucp/integrationtests/planes/aws_test.go @@ -20,8 +20,7 @@ import ( "testing" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" ) const ( @@ -35,7 +34,7 @@ const ( ) func Test_AWSPlane_PUT_Create(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", awsPlaneResourceURL, awsPlaneRequestFixture) @@ -43,7 +42,7 @@ func Test_AWSPlane_PUT_Create(t *testing.T) { } func Test_AWSPlane_PUT_Update(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", awsPlaneResourceURL, awsPlaneRequestFixture) @@ -54,7 +53,7 @@ func Test_AWSPlane_PUT_Update(t *testing.T) { } func Test_AWSPlane_GET_Empty(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeRequest("GET", awsPlaneResourceURL, nil) @@ -62,7 +61,7 @@ func Test_AWSPlane_GET_Empty(t *testing.T) { } func Test_AWSPlane_GET_Found(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", awsPlaneResourceURL, awsPlaneRequestFixture) @@ -73,8 +72,7 @@ func Test_AWSPlane_GET_Found(t *testing.T) { } func Test_AWSPlane_LIST(t *testing.T) { - - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", awsPlaneResourceURL, awsPlaneRequestFixture) @@ -85,7 +83,7 @@ func Test_AWSPlane_LIST(t *testing.T) { } func Test_AWSPlane_DELETE_DoesNotExist(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeRequest("DELETE", awsPlaneResourceURL, nil) @@ -93,7 +91,7 @@ func Test_AWSPlane_DELETE_DoesNotExist(t *testing.T) { } func Test_AWSPlane_DELETE_Found(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", awsPlaneResourceURL, awsPlaneRequestFixture) diff --git a/pkg/ucp/integrationtests/planes/azure_test.go b/pkg/ucp/integrationtests/planes/azure_test.go index 5ce199c9f0..49e9cf4b1f 100644 --- a/pkg/ucp/integrationtests/planes/azure_test.go +++ b/pkg/ucp/integrationtests/planes/azure_test.go @@ -20,8 +20,7 @@ import ( "testing" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" ) const ( @@ -35,7 +34,7 @@ const ( ) func Test_AzurePlane_PUT_Create(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", azurePlaneResourceURL, azurePlaneRequestFixture) @@ -43,7 +42,7 @@ func Test_AzurePlane_PUT_Create(t *testing.T) { } func Test_AzurePlane_PUT_Update(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", azurePlaneResourceURL, azurePlaneRequestFixture) @@ -54,7 +53,7 @@ func Test_AzurePlane_PUT_Update(t *testing.T) { } func Test_AzurePlane_GET_Empty(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeRequest("GET", azurePlaneResourceURL, nil) @@ -62,7 +61,7 @@ func Test_AzurePlane_GET_Empty(t *testing.T) { } func Test_AzurePlane_GET_Found(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", azurePlaneResourceURL, azurePlaneRequestFixture) @@ -73,7 +72,7 @@ func Test_AzurePlane_GET_Found(t *testing.T) { } func Test_AzurePlane_LIST(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() // Add a azure plane @@ -86,7 +85,7 @@ func Test_AzurePlane_LIST(t *testing.T) { } func Test_AzurePlane_DELETE_DoesNotExist(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeRequest("DELETE", azurePlaneResourceURL, nil) @@ -94,7 +93,7 @@ func Test_AzurePlane_DELETE_DoesNotExist(t *testing.T) { } func Test_AzurePlane_DELETE_Found(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", azurePlaneResourceURL, azurePlaneRequestFixture) diff --git a/pkg/ucp/integrationtests/planes/planes_test.go b/pkg/ucp/integrationtests/planes/planes_test.go index 2a6eb9c35e..03dbd1baad 100644 --- a/pkg/ucp/integrationtests/planes/planes_test.go +++ b/pkg/ucp/integrationtests/planes/planes_test.go @@ -19,8 +19,7 @@ package planes import ( "testing" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" ) const ( @@ -30,7 +29,7 @@ const ( ) func Test_AllPlanes_LIST(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", radiusPlaneResourceURL, radiusPlaneRequestFixture) diff --git a/pkg/ucp/integrationtests/planes/radius_test.go b/pkg/ucp/integrationtests/planes/radius_test.go index d2e8fbfa69..667e4a621c 100644 --- a/pkg/ucp/integrationtests/planes/radius_test.go +++ b/pkg/ucp/integrationtests/planes/radius_test.go @@ -20,8 +20,7 @@ import ( "testing" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" ) const ( @@ -35,7 +34,7 @@ const ( ) func Test_RadiusPlane_PUT_Create(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", radiusPlaneResourceURL, radiusPlaneRequestFixture) @@ -43,7 +42,7 @@ func Test_RadiusPlane_PUT_Create(t *testing.T) { } func Test_RadiusPlane_PUT_Update(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", radiusPlaneResourceURL, radiusPlaneRequestFixture) @@ -54,7 +53,7 @@ func Test_RadiusPlane_PUT_Update(t *testing.T) { } func Test_RadiusPlane_GET_Empty(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeRequest("GET", radiusPlaneResourceURL, nil) @@ -62,7 +61,7 @@ func Test_RadiusPlane_GET_Empty(t *testing.T) { } func Test_RadiusPlane_GET_Found(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", radiusPlaneResourceURL, radiusPlaneRequestFixture) @@ -73,7 +72,7 @@ func Test_RadiusPlane_GET_Found(t *testing.T) { } func Test_RadiusPlane_LIST(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() // Add a radius plane @@ -86,7 +85,7 @@ func Test_RadiusPlane_LIST(t *testing.T) { } func Test_RadiusPlane_DELETE_DoesNotExist(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeRequest("DELETE", radiusPlaneResourceURL, nil) @@ -94,7 +93,7 @@ func Test_RadiusPlane_DELETE_DoesNotExist(t *testing.T) { } func Test_RadiusPlane_DELETE_Found(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() response := server.MakeFixtureRequest("PUT", radiusPlaneResourceURL, radiusPlaneRequestFixture) diff --git a/pkg/ucp/integrationtests/planes/validation_test.go b/pkg/ucp/integrationtests/planes/validation_test.go index 015f49b12f..e6607c45c0 100644 --- a/pkg/ucp/integrationtests/planes/validation_test.go +++ b/pkg/ucp/integrationtests/planes/validation_test.go @@ -23,8 +23,7 @@ import ( v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/stretchr/testify/require" ) @@ -33,7 +32,7 @@ const ( ) func Test_Planes_GET_BadAPIVersion(t *testing.T) { - ucp := testserver.StartWithMocks(t, api.DefaultModules) + ucp := testhost.Start(t) response := ucp.MakeRequest(http.MethodGet, "/planes?api-version=unsupported-version", nil) response.EqualsErrorCode(http.StatusBadRequest, v1.CodeInvalidApiVersionParameter) @@ -41,7 +40,7 @@ func Test_Planes_GET_BadAPIVersion(t *testing.T) { } func Test_Planes_PUT_BadAPIVersion(t *testing.T) { - ucp := testserver.StartWithMocks(t, api.DefaultModules) + ucp := testhost.Start(t) requestBody := v20231001preview.RadiusPlaneResource{ Location: to.Ptr(v1.LocationGlobal), diff --git a/pkg/ucp/integrationtests/radius/proxy_test.go b/pkg/ucp/integrationtests/radius/proxy_test.go index 371a79af0a..c9e84c4b5c 100644 --- a/pkg/ucp/integrationtests/radius/proxy_test.go +++ b/pkg/ucp/integrationtests/radius/proxy_test.go @@ -28,9 +28,8 @@ import ( backend_ctrl "github.com/radius-project/radius/pkg/armrpc/asyncoperation/controller" "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" - "github.com/radius-project/radius/pkg/ucp/frontend/api" "github.com/radius-project/radius/pkg/ucp/integrationtests/testrp" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -48,7 +47,7 @@ const ( ) func Test_RadiusPlane_Proxy_ResourceGroupDoesNotExist(t *testing.T) { - ucp := testserver.StartWithETCD(t, api.DefaultModules) + ucp := testhost.Start(t) rp := testrp.Start(t) rps := map[string]*string{ @@ -62,7 +61,7 @@ func Test_RadiusPlane_Proxy_ResourceGroupDoesNotExist(t *testing.T) { } func Test_RadiusPlane_ResourceSync(t *testing.T) { - ucp := testserver.StartWithETCD(t, api.DefaultModules) + ucp := testhost.Start(t) rp := testrp.Start(t) rp.Handler = testrp.SyncResource(t, ucp, testResourceGroupID) @@ -158,7 +157,7 @@ func Test_RadiusPlane_ResourceSync(t *testing.T) { } func Test_RadiusPlane_ResourceAsync(t *testing.T) { - ucp := testserver.StartWithETCD(t, api.DefaultModules) + ucp := testhost.Start(t) rp := testrp.Start(t) // Block background work item completion until we're ready. @@ -176,7 +175,7 @@ func Test_RadiusPlane_ResourceAsync(t *testing.T) { return result, nil } - client, err := ucp.Clients.StorageProvider.GetStorageClient(ctx, "System.Test/testResources") + client, err := ucp.Options().StorageProvider.GetStorageClient(ctx, "System.Test/testResources") require.NoError(t, err) err = client.Delete(ctx, testResourceID) require.NoError(t, err) @@ -222,8 +221,8 @@ func Test_RadiusPlane_ResourceAsync(t *testing.T) { location := response.Raw.Header.Get("Location") azureAsyncOperation := response.Raw.Header.Get("Azure-AsyncOperation") - require.True(t, strings.HasPrefix(location, ucp.BaseURL), "Location starts with UCP URL") - require.True(t, strings.HasPrefix(azureAsyncOperation, ucp.BaseURL), "Azure-AsyncOperation starts with UCP URL") + require.True(t, strings.HasPrefix(location, ucp.BaseURL()), "Location starts with UCP URL") + require.True(t, strings.HasPrefix(azureAsyncOperation, ucp.BaseURL()), "Azure-AsyncOperation starts with UCP URL") }) t.Run("LIST (during PUT)", func(t *testing.T) { @@ -382,7 +381,7 @@ func Test_RadiusPlane_ResourceAsync(t *testing.T) { }) } -func createRadiusPlane(ucp *testserver.TestServer, resourceProviders map[string]*string) { +func createRadiusPlane(ucp *testhost.TestHost, resourceProviders map[string]*string) { body := v20231001preview.RadiusPlaneResource{ Location: to.Ptr(v1.LocationGlobal), Properties: &v20231001preview.RadiusPlaneResourceProperties{ @@ -393,7 +392,7 @@ func createRadiusPlane(ucp *testserver.TestServer, resourceProviders map[string] response.EqualsStatusCode(http.StatusOK) } -func createResourceGroup(ucp *testserver.TestServer, id string) { +func createResourceGroup(ucp *testhost.TestHost, id string) { body := v20231001preview.ResourceGroupResource{ Location: to.Ptr(v1.LocationGlobal), Properties: &v20231001preview.ResourceGroupProperties{}, diff --git a/pkg/ucp/integrationtests/resourcegroups/resourcegroups_test.go b/pkg/ucp/integrationtests/resourcegroups/resourcegroups_test.go index 5106fcf314..a047c3bde2 100644 --- a/pkg/ucp/integrationtests/resourcegroups/resourcegroups_test.go +++ b/pkg/ucp/integrationtests/resourcegroups/resourcegroups_test.go @@ -20,8 +20,7 @@ import ( "testing" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" ) const ( @@ -40,13 +39,13 @@ const ( resourceGroupInvalidResponseFixture = "testdata/resourcegroup_invalid_v20231001preview_responsebody.json" ) -func createRadiusPlane(server *testserver.TestServer) { +func createRadiusPlane(server *testhost.TestHost) { response := server.MakeFixtureRequest("PUT", radiusPlaneResourceURL, radiusPlaneRequestFixture) response.EqualsFixture(200, radiusPlaneResponseFixture) } func Test_ResourceGroup_PUT_Create(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) @@ -56,7 +55,7 @@ func Test_ResourceGroup_PUT_Create(t *testing.T) { } func Test_ResourceGroup_PUT_Update(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) @@ -69,7 +68,7 @@ func Test_ResourceGroup_PUT_Update(t *testing.T) { } func Test_ResourceGroup_PUT_APIValidation(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) @@ -79,7 +78,7 @@ func Test_ResourceGroup_PUT_APIValidation(t *testing.T) { } func Test_ResourceGroup_GET_Empty(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) @@ -89,7 +88,7 @@ func Test_ResourceGroup_GET_Empty(t *testing.T) { } func Test_ResourceGroup_GET_Found(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) @@ -102,7 +101,7 @@ func Test_ResourceGroup_GET_Found(t *testing.T) { } func Test_ResourceGroup_LIST(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) diff --git a/pkg/ucp/integrationtests/resourceproviders/apiversions_test.go b/pkg/ucp/integrationtests/resourceproviders/apiversions_test.go index 8ba1d22fce..71619894fd 100644 --- a/pkg/ucp/integrationtests/resourceproviders/apiversions_test.go +++ b/pkg/ucp/integrationtests/resourceproviders/apiversions_test.go @@ -20,8 +20,7 @@ import ( "net/http" "testing" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/stretchr/testify/require" ) @@ -31,7 +30,7 @@ const ( ) func Test_APIVersion_Lifecycle(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) diff --git a/pkg/ucp/integrationtests/resourceproviders/locations_test.go b/pkg/ucp/integrationtests/resourceproviders/locations_test.go index 64923fbc9a..7fdd82045a 100644 --- a/pkg/ucp/integrationtests/resourceproviders/locations_test.go +++ b/pkg/ucp/integrationtests/resourceproviders/locations_test.go @@ -20,8 +20,7 @@ import ( "net/http" "testing" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/stretchr/testify/require" ) @@ -31,7 +30,7 @@ const ( ) func Test_Location_Lifecycle(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) diff --git a/pkg/ucp/integrationtests/resourceproviders/resourceproviders_test.go b/pkg/ucp/integrationtests/resourceproviders/resourceproviders_test.go index a67000cf8b..7ad93f4096 100644 --- a/pkg/ucp/integrationtests/resourceproviders/resourceproviders_test.go +++ b/pkg/ucp/integrationtests/resourceproviders/resourceproviders_test.go @@ -20,8 +20,7 @@ import ( "net/http" "testing" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/stretchr/testify/require" ) @@ -31,7 +30,7 @@ const ( ) func Test_ResourceProvider_Lifecycle(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) @@ -61,7 +60,7 @@ func Test_ResourceProvider_Lifecycle(t *testing.T) { } func Test_ResourceProvider_CascadingDelete(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) diff --git a/pkg/ucp/integrationtests/resourceproviders/resourcetypes_test.go b/pkg/ucp/integrationtests/resourceproviders/resourcetypes_test.go index b8ae588548..6db34e64ce 100644 --- a/pkg/ucp/integrationtests/resourceproviders/resourcetypes_test.go +++ b/pkg/ucp/integrationtests/resourceproviders/resourcetypes_test.go @@ -20,8 +20,7 @@ import ( "net/http" "testing" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/stretchr/testify/require" ) @@ -31,7 +30,7 @@ const ( ) func Test_ResourceType_Lifecycle(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) @@ -64,7 +63,7 @@ func Test_ResourceType_Lifecycle(t *testing.T) { } func Test_ResourceType_CascadingDelete(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) diff --git a/pkg/ucp/integrationtests/resourceproviders/summary_test.go b/pkg/ucp/integrationtests/resourceproviders/summary_test.go index 9fbfb19b35..096f3165ec 100644 --- a/pkg/ucp/integrationtests/resourceproviders/summary_test.go +++ b/pkg/ucp/integrationtests/resourceproviders/summary_test.go @@ -23,13 +23,12 @@ import ( "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/stretchr/testify/require" ) func Test_ResourceProviderSummary_Lifecycle(t *testing.T) { - server := testserver.StartWithETCD(t, api.DefaultModules) + server := testhost.Start(t) defer server.Close() createRadiusPlane(server) diff --git a/pkg/ucp/integrationtests/resourceproviders/util_test.go b/pkg/ucp/integrationtests/resourceproviders/util_test.go index efca2af228..3f56206e02 100644 --- a/pkg/ucp/integrationtests/resourceproviders/util_test.go +++ b/pkg/ucp/integrationtests/resourceproviders/util_test.go @@ -16,7 +16,9 @@ limitations under the License. package resourceproviders -import "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" +import ( + "github.com/radius-project/radius/pkg/ucp/testhost" +) const ( radiusAPIVersion = "?api-version=2023-10-01-preview" @@ -56,7 +58,7 @@ const ( resourceProviderSummaryURL = "/planes/radius/local/providers/" + resourceProviderNamespace + radiusAPIVersion ) -func createRadiusPlane(server *testserver.TestServer) { +func createRadiusPlane(server *testhost.TestHost) { response := server.MakeFixtureRequest("PUT", radiusPlaneResourceURL, radiusPlaneRequestFixture) response.WaitForOperationComplete(nil) @@ -64,7 +66,7 @@ func createRadiusPlane(server *testserver.TestServer) { response.EqualsFixture(200, radiusPlaneResponseFixture) } -func createResourceProvider(server *testserver.TestServer) { +func createResourceProvider(server *testhost.TestHost) { response := server.MakeFixtureRequest("PUT", resourceProviderURL, resourceProviderRequestFixture) response.WaitForOperationComplete(nil) @@ -72,7 +74,7 @@ func createResourceProvider(server *testserver.TestServer) { response.EqualsFixture(200, resourceProviderResponseFixture) } -func deleteResourceProvider(server *testserver.TestServer) { +func deleteResourceProvider(server *testhost.TestHost) { response := server.MakeRequest("DELETE", resourceProviderURL, nil) response.WaitForOperationComplete(nil) @@ -80,7 +82,7 @@ func deleteResourceProvider(server *testserver.TestServer) { response.EqualsStatusCode(404) } -func createResourceType(server *testserver.TestServer) { +func createResourceType(server *testhost.TestHost) { response := server.MakeFixtureRequest("PUT", resourceTypeURL, resourceTypeRequestFixture) response.WaitForOperationComplete(nil) @@ -88,7 +90,7 @@ func createResourceType(server *testserver.TestServer) { response.EqualsFixture(200, resourceTypeResponseFixture) } -func deleteResourceType(server *testserver.TestServer) { +func deleteResourceType(server *testhost.TestHost) { response := server.MakeRequest("DELETE", resourceTypeURL, nil) response.WaitForOperationComplete(nil) @@ -96,7 +98,7 @@ func deleteResourceType(server *testserver.TestServer) { response.EqualsStatusCode(404) } -func createAPIVersion(server *testserver.TestServer) { +func createAPIVersion(server *testhost.TestHost) { response := server.MakeFixtureRequest("PUT", apiVersionURL, apiVersionRequestFixture) response.WaitForOperationComplete(nil) @@ -104,7 +106,7 @@ func createAPIVersion(server *testserver.TestServer) { response.EqualsFixture(200, apiVersionResponseFixture) } -func deleteAPIVersion(server *testserver.TestServer) { +func deleteAPIVersion(server *testhost.TestHost) { response := server.MakeRequest("DELETE", apiVersionURL, nil) response.WaitForOperationComplete(nil) @@ -112,7 +114,7 @@ func deleteAPIVersion(server *testserver.TestServer) { response.EqualsStatusCode(404) } -func createLocation(server *testserver.TestServer) { +func createLocation(server *testhost.TestHost) { response := server.MakeFixtureRequest("PUT", locationURL, locationRequestFixture) response.WaitForOperationComplete(nil) @@ -120,7 +122,7 @@ func createLocation(server *testserver.TestServer) { response.EqualsFixture(200, locationResponseFixture) } -func deleteLocation(server *testserver.TestServer) { +func deleteLocation(server *testhost.TestHost) { response := server.MakeRequest("DELETE", locationURL, nil) response.WaitForOperationComplete(nil) diff --git a/pkg/ucp/integrationtests/testrp/async.go b/pkg/ucp/integrationtests/testrp/async.go index 144b598c8c..c4d89fe34c 100644 --- a/pkg/ucp/integrationtests/testrp/async.go +++ b/pkg/ucp/integrationtests/testrp/async.go @@ -32,8 +32,8 @@ import ( "github.com/radius-project/radius/pkg/armrpc/frontend/server" "github.com/radius-project/radius/pkg/armrpc/servicecontext" "github.com/radius-project/radius/pkg/middleware" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" queueprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/radius-project/radius/test/testcontext" "github.com/stretchr/testify/require" ) @@ -50,7 +50,7 @@ func (b *BackendFuncController) Run(ctx context.Context, request *backend_ctrl.R } // AsyncResource creates an HTTP handler that can be used to test asynchronous resource lifecycle operations. -func AsyncResource(t *testing.T, ts *testserver.TestServer, rootScope string, put BackendFunc, delete BackendFunc) func(w http.ResponseWriter, r *http.Request) { +func AsyncResource(t *testing.T, ts *testhost.TestHost, rootScope string, put BackendFunc, delete BackendFunc) func(w http.ResponseWriter, r *http.Request) { rootScope = strings.ToLower(rootScope) ctx := testcontext.New(t) @@ -60,7 +60,7 @@ func AsyncResource(t *testing.T, ts *testserver.TestServer, rootScope string, pu resourceType := "System.Test/testResources" // We can share the storage provider with the test server. - _, err := ts.Clients.StorageProvider.GetStorageClient(ctx, "System.Test/operationStatuses") + _, err := ts.Options().StorageProvider.GetStorageClient(ctx, "System.Test/operationStatuses") require.NoError(t, err) // Do not share the queue. @@ -73,13 +73,13 @@ func AsyncResource(t *testing.T, ts *testserver.TestServer, rootScope string, pu queueClient, err := queueProvider.GetClient(ctx) require.NoError(t, err) - statusManager := statusmanager.New(ts.Clients.StorageProvider, queueClient, v1.LocationGlobal) + statusManager := statusmanager.New(ts.Options().StorageProvider, queueClient, v1.LocationGlobal) backendOpts := backend_ctrl.Options{ - DataProvider: ts.Clients.StorageProvider, + DataProvider: ts.Options().StorageProvider, } - registry := worker.NewControllerRegistry(ts.Clients.StorageProvider) + registry := worker.NewControllerRegistry(ts.Options().StorageProvider) err = registry.Register(ctx, resourceType, v1.OperationPut, func(opts backend_ctrl.Options) (backend_ctrl.Controller, error) { return &BackendFuncController{BaseController: backend_ctrl.NewBaseAsyncController(opts), Func: put}, nil }, backendOpts) @@ -100,7 +100,7 @@ func AsyncResource(t *testing.T, ts *testserver.TestServer, rootScope string, pu }() frontendOpts := frontend_ctrl.Options{ - DataProvider: ts.Clients.StorageProvider, + DataProvider: ts.Options().StorageProvider, StatusManager: statusManager, } diff --git a/pkg/ucp/integrationtests/testrp/sync.go b/pkg/ucp/integrationtests/testrp/sync.go index 53ca8b4438..e027409dd1 100644 --- a/pkg/ucp/integrationtests/testrp/sync.go +++ b/pkg/ucp/integrationtests/testrp/sync.go @@ -28,13 +28,13 @@ import ( "github.com/radius-project/radius/pkg/armrpc/frontend/server" "github.com/radius-project/radius/pkg/armrpc/servicecontext" "github.com/radius-project/radius/pkg/middleware" - "github.com/radius-project/radius/pkg/ucp/integrationtests/testserver" + "github.com/radius-project/radius/pkg/ucp/testhost" "github.com/radius-project/radius/test/testcontext" "github.com/stretchr/testify/require" ) // SyncResource creates an HTTP handler that can be used to test synchronous resource lifecycle operations. -func SyncResource(t *testing.T, ts *testserver.TestServer, rootScope string) func(w http.ResponseWriter, r *http.Request) { +func SyncResource(t *testing.T, ts *testhost.TestHost, rootScope string) func(w http.ResponseWriter, r *http.Request) { rootScope = strings.ToLower(rootScope) ctx := testcontext.New(t) @@ -42,7 +42,7 @@ func SyncResource(t *testing.T, ts *testserver.TestServer, rootScope string) fun r.Use(servicecontext.ARMRequestCtx("", v1.LocationGlobal), middleware.LowercaseURLPath) ctrlOpts := frontend_ctrl.Options{ - DataProvider: ts.Clients.StorageProvider, + DataProvider: ts.Options().StorageProvider, } err := server.ConfigureDefaultHandlers(ctx, r, rootScope, false, "System.Test", nil, ctrlOpts) diff --git a/pkg/ucp/integrationtests/testserver/testserver.go b/pkg/ucp/integrationtests/testserver/testserver.go deleted file mode 100644 index f1eba83023..0000000000 --- a/pkg/ucp/integrationtests/testserver/testserver.go +++ /dev/null @@ -1,605 +0,0 @@ -/* -Copyright 2023 The Radius Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package testserver - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net" - "net/http" - "net/http/httptest" - "net/url" - "os" - "sync" - "testing" - "time" - - "github.com/go-chi/chi/v5" - "github.com/go-logr/logr" - "github.com/google/uuid" - "github.com/stretchr/testify/require" - etcdclient "go.etcd.io/etcd/client/v3" - "go.uber.org/mock/gomock" - - v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" - backend_ctrl "github.com/radius-project/radius/pkg/armrpc/asyncoperation/controller" - "github.com/radius-project/radius/pkg/armrpc/asyncoperation/statusmanager" - "github.com/radius-project/radius/pkg/armrpc/asyncoperation/worker" - "github.com/radius-project/radius/pkg/armrpc/rpctest" - "github.com/radius-project/radius/pkg/armrpc/servicecontext" - "github.com/radius-project/radius/pkg/middleware" - "github.com/radius-project/radius/pkg/sdk" - "github.com/radius-project/radius/pkg/ucp/backend" - "github.com/radius-project/radius/pkg/ucp/data" - "github.com/radius-project/radius/pkg/ucp/dataprovider" - "github.com/radius-project/radius/pkg/ucp/frontend/api" - "github.com/radius-project/radius/pkg/ucp/frontend/modules" - "github.com/radius-project/radius/pkg/ucp/hosting" - "github.com/radius-project/radius/pkg/ucp/hostoptions" - queue "github.com/radius-project/radius/pkg/ucp/queue/client" - queueprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" - "github.com/radius-project/radius/pkg/ucp/secret" - secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" - "github.com/radius-project/radius/pkg/ucp/server" - "github.com/radius-project/radius/pkg/ucp/store" - "github.com/radius-project/radius/pkg/validator" - "github.com/radius-project/radius/swagger" - "github.com/radius-project/radius/test/testcontext" -) - -// NoModules can be used to start a test server without any modules. This is useful for testing the server itself and core functionality -// like planes. -func NoModules(options modules.Options) []modules.Initializer { - return nil -} - -// TestServer can run a UCP server using the Go httptest package. It provides access to an isolated ETCD instances for storage -// of resources and secrets. Alteratively, it can also be used with gomock. -// -// Do not create a TestServer directly, use StartWithETCD or StartWithMocks instead. -type TestServer struct { - // BaseURL is the base URL of the server, including the path base. - BaseURL string - - // Clients gets access to the clients created by TestServer regardless of whether - // they are mocks. - Clients *TestServerClients - - // Mocks gets access to the mock clients. Will be nil if StartWithETCD is used. - Mocks *TestServerMocks - - // Server provides access to the test HTTP server. - Server *httptest.Server - - cancel context.CancelFunc - etcdService *data.EmbeddedETCDService - etcdClient *etcdclient.Client - t *testing.T - stoppedChan <-chan struct{} - shutdown sync.Once -} - -// TestServerClients provides access to the clients created by the TestServer. -type TestServerClients struct { - // QueueProvider is the queue client provider. - QueueProvider *queueprovider.QueueProvider - - // SecretProvider is the secret client provider. - SecretProvider *secretprovider.SecretProvider - - // StorageProvider is the storage client provider. - StorageProvider dataprovider.DataStorageProvider -} - -// TestServerMocks provides access to mock instances created by the TestServer. -type TestServerMocks struct { - // Secrets is the mock secret client. - Secrets *secret.MockClient - - // Storage is the mock storage client. - Storage *store.MockStorageClient -} - -// Client provides access to an http.Client that can be used to send requests. Most tests should use the functionality -// like MakeRequest instead of testing the client directly. -func (ts *TestServer) Client() *http.Client { - return ts.Server.Client() -} - -// Close shuts down the server and will block until shutdown completes. -func (ts *TestServer) Close() { - // We're being picking about resource cleanup here, because unless we are picky we hit scalability - // problems in tests pretty quickly. - ts.shutdown.Do(func() { - ts.cancel() // Start ETCD shutdown - ts.Server.Close() // Stop HTTP server - - if ts.etcdClient != nil { - ts.etcdClient.Close() // Stop ETCD Client - } - - if ts.stoppedChan != nil { - <-ts.stoppedChan // ETCD stopped - } - }) -} - -// StartWithMocks creates and starts a new TestServer that used an mocks for storage. -func StartWithMocks(t *testing.T, configureModules func(options modules.Options) []modules.Initializer) *TestServer { - ctx, cancel := testcontext.NewWithCancel(t) - - // Generate a random base path to ensure we're handling it correctly. - pathBase := "/" + uuid.New().String() - - ctrl := gomock.NewController(t) - dataClient := store.NewMockStorageClient(ctrl) - dataProvider := dataprovider.NewMockDataStorageProvider(ctrl) - dataProvider.EXPECT(). - GetStorageClient(gomock.Any(), gomock.Any()). - Return(dataClient, nil). - AnyTimes() - - queueClient := queue.NewMockClient(ctrl) - queueProvider := queueprovider.New(queueprovider.QueueProviderOptions{Name: "System.Resources"}) - queueProvider.SetClient(queueClient) - - secretClient := secret.NewMockClient(ctrl) - secretProvider := secretprovider.NewSecretProvider(secretprovider.SecretProviderOptions{}) - secretProvider.SetClient(secretClient) - - statusManager := statusmanager.NewMockStatusManager(ctrl) - - router := chi.NewRouter() - router.Use(servicecontext.ARMRequestCtx(pathBase, "global")) - - app := http.Handler(router) - app = middleware.NormalizePath(app) - server := httptest.NewUnstartedServer(app) - server.Config.BaseContext = func(l net.Listener) context.Context { - return ctx - } - - specLoader, err := validator.LoadSpec(ctx, "ucp", swagger.SpecFilesUCP, []string{pathBase}, "") - require.NoError(t, err, "failed to load OpenAPI spec") - - options := modules.Options{ - Address: server.URL, - PathBase: pathBase, - Config: &hostoptions.UCPConfig{}, - DataProvider: dataProvider, - SecretProvider: secretProvider, - SpecLoader: specLoader, - StatusManager: statusManager, - } - - if configureModules == nil { - configureModules = api.DefaultModules - } - - modules := configureModules(options) - - err = api.Register(ctx, router, modules, options) - require.NoError(t, err) - - logger := logr.FromContextOrDiscard(ctx) - logger.Info("Starting HTTP server...") - server.Start() - logger.Info(fmt.Sprintf("Started HTTP server on %s...", server.URL)) - - ucp := &TestServer{ - BaseURL: server.URL + pathBase, - Clients: &TestServerClients{ - QueueProvider: queueProvider, - SecretProvider: secretProvider, - StorageProvider: dataProvider, - }, - Mocks: &TestServerMocks{ - Secrets: secretClient, - Storage: dataClient, - }, - Server: server, - cancel: cancel, - t: t, - } - - t.Cleanup(ucp.Close) - return ucp -} - -// StartWithETCD creates and starts a new TestServer that used an embedded ETCD instance for storage. -func StartWithETCD(t *testing.T, configureModules func(options modules.Options) []modules.Initializer) *TestServer { - config := hosting.NewAsyncValue[etcdclient.Client]() - etcd := data.NewEmbeddedETCDService(data.EmbeddedETCDServiceOptions{ - ClientConfigSink: config, - AssignRandomPorts: true, - Quiet: false, - }) - - ctx, cancel := testcontext.NewWithCancel(t) - t.Cleanup(cancel) - - stoppedChan := make(chan struct{}) - defer close(stoppedChan) - go func() { - // We can't pass the test logger into the etcd service because it is forbidden to log - // using the test logger after the test finishes. - // - // https://github.com/golang/go/issues/40343 - // - // If you need to see the logging output while you are testing, then comment out the next line - // and you'll be able to see the spam from etcd. - // - // This is caught by the race checker and will fail your pr if you do it. - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - err := etcd.Run(ctx) - if err != nil { - t.Logf("error from etcd: %v", err) - } - }() - - storageOptions := dataprovider.StorageProviderOptions{ - Provider: dataprovider.TypeETCD, - ETCD: dataprovider.ETCDOptions{ - InMemory: true, - Client: config, - }, - } - secretOptions := secretprovider.SecretProviderOptions{ - Provider: secretprovider.TypeETCDSecret, - ETCD: storageOptions.ETCD, - } - queueOptions := queueprovider.QueueProviderOptions{ - Name: server.UCPProviderName, - Provider: queueprovider.TypeInmemory, - InMemory: &queueprovider.InMemoryQueueOptions{}, - } - - // Generate a random base path to ensure we're handling it correctly. - pathBase := "/" + uuid.New().String() - dataProvider := dataprovider.NewStorageProvider(storageOptions) - secretProvider := secretprovider.NewSecretProvider(secretOptions) - queueProvider := queueprovider.New(queueOptions) - - queueClient, err := queueProvider.GetClient(ctx) - require.NoError(t, err) - - router := chi.NewRouter() - router.Use(servicecontext.ARMRequestCtx(pathBase, "global")) - - app := middleware.NormalizePath(router) - server := httptest.NewUnstartedServer(app) - server.Config.BaseContext = func(l net.Listener) context.Context { - return ctx - } - - // Unfortunately the server doesn't populate 'server.URL' until it is started, so we have to build it ourselves. - address := "http://" + server.Listener.Addr().String() - connection, err := sdk.NewDirectConnection(address + pathBase) - require.NoError(t, err) - - statusManager := statusmanager.New(dataProvider, queueClient, v1.LocationGlobal) - - specLoader, err := validator.LoadSpec(ctx, "ucp", swagger.SpecFilesUCP, []string{pathBase}, "") - require.NoError(t, err, "failed to load OpenAPI spec") - - options := modules.Options{ - Address: address, - PathBase: pathBase, - Config: &hostoptions.UCPConfig{}, - DataProvider: dataProvider, - SecretProvider: secretProvider, - SpecLoader: specLoader, - QueueProvider: queueProvider, - StatusManager: statusManager, - } - - if configureModules == nil { - configureModules = api.DefaultModules - } - - modules := configureModules(options) - - // The URL for the dynamic-rp needs to be specified in configuration, however not all of our tests - // need to use the dynamic-rp. We can just use a placeholder value here. - if options.Config.Routing.DefaultDownstreamEndpoint == "" { - options.Config.Routing.DefaultDownstreamEndpoint = "http://localhost:65535" - } - - defaultDownstream, err := url.Parse(options.Config.Routing.DefaultDownstreamEndpoint) - require.NoError(t, err) - - registry := worker.NewControllerRegistry(dataProvider) - err = backend.RegisterControllers(ctx, registry, connection, http.DefaultTransport, backend_ctrl.Options{DataProvider: dataProvider}, defaultDownstream) - require.NoError(t, err) - - w := worker.New(worker.Options{}, statusManager, queueClient, registry) - go func() { - err = w.Start(ctx) - require.NoError(t, err) - }() - - err = api.Register(ctx, router, modules, options) - require.NoError(t, err) - - logger := logr.FromContextOrDiscard(ctx) - logger.Info("Starting HTTP server...") - server.Start() - logger.Info(fmt.Sprintf("Started HTTP server on %s...", server.URL)) - - logger.Info("Connecting to data store...") - client, err := config.Get(ctx) - require.NoError(t, err, "failed to access etcd client") - _, err = client.Cluster.MemberList(ctx) - require.NoError(t, err, "failed to query etcd") - logger.Info("Connected to data store") - - // TODO: start worker - - ucp := &TestServer{ - BaseURL: server.URL + pathBase, - Clients: &TestServerClients{ - QueueProvider: queueProvider, - SecretProvider: secretProvider, - StorageProvider: dataProvider, - }, - Server: server, - cancel: cancel, - etcdService: etcd, - etcdClient: client, - t: t, - stoppedChan: stoppedChan, - } - t.Cleanup(ucp.Close) - return ucp -} - -// TestResponse is return from requests made against a TestServer. Tests should use the functions defined -// on TestResponse for valiation. -type TestResponse struct { - Raw *http.Response - Body *bytes.Buffer - Error *v1.ErrorResponse - t *testing.T - server *TestServer -} - -// MakeFixtureRequest sends a request to the server using a file on disk as the payload (body). Use the fixture -// parameter to specify the path to a file. -func (ts *TestServer) MakeFixtureRequest(method string, pathAndQuery string, fixture string) *TestResponse { - body, err := os.ReadFile(fixture) - require.NoError(ts.t, err, "reading fixture failed") - return ts.MakeRequest(method, pathAndQuery, body) -} - -// MakeTypedRequest sends a request to the server by marshalling the provided object to JSON. -func (ts *TestServer) MakeTypedRequest(method string, pathAndQuery string, body any) *TestResponse { - if body == nil { - return ts.MakeRequest(method, pathAndQuery, nil) - } - - b, err := json.Marshal(body) - require.NoError(ts.t, err, "marshalling body failed") - return ts.MakeRequest(method, pathAndQuery, b) -} - -// MakeRequest sends a request to the server. -func (ts *TestServer) MakeRequest(method string, pathAndQuery string, body []byte) *TestResponse { - // Prepend the base path if this is a relative URL. - requestUrl := pathAndQuery - parsed, err := url.Parse(pathAndQuery) - require.NoError(ts.t, err, "parsing URL failed") - if !parsed.IsAbs() { - requestUrl = ts.BaseURL + pathAndQuery - } - - client := ts.Server.Client() - request, err := rpctest.NewHTTPRequestWithContent(context.Background(), method, requestUrl, body) - require.NoError(ts.t, err, "creating request failed") - - ctx := rpctest.NewARMRequestContext(request) - request = request.WithContext(ctx) - - response, err := client.Do(request) - require.NoError(ts.t, err, "sending request failed") - - // Buffer the response so we can read multiple times. - responseBuffer := &bytes.Buffer{} - _, err = io.Copy(responseBuffer, response.Body) - response.Body.Close() - require.NoError(ts.t, err, "copying response failed") - - response.Body = io.NopCloser(responseBuffer) - - // Pretty-print response for logs. - if len(responseBuffer.Bytes()) > 0 { - var data any - err = json.Unmarshal(responseBuffer.Bytes(), &data) - require.NoError(ts.t, err, "unmarshalling response failed") - - text, err := json.MarshalIndent(&data, "", " ") - require.NoError(ts.t, err, "marshalling response failed") - ts.t.Log("Response Body: \n" + string(text)) - } - - var errorResponse *v1.ErrorResponse - if response.StatusCode >= 400 { - // The response MUST be an arm error for a non-success status code. - errorResponse = &v1.ErrorResponse{} - err := json.Unmarshal(responseBuffer.Bytes(), &errorResponse) - require.NoError(ts.t, err, "unmarshalling error response failed - THIS IS A SERIOUS BUG. ALL ERROR RESPONSES MUST USE THE STANDARD FORMAT") - } - - return &TestResponse{Raw: response, Body: responseBuffer, Error: errorResponse, server: ts, t: ts.t} -} - -// EqualsErrorCode compares a TestResponse against an expected status code and error code. EqualsErrorCode assumes the response -// uses the ARM error format (required for our APIs). -func (tr *TestResponse) EqualsErrorCode(statusCode int, code string) { - require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") - require.NotNil(tr.t, tr.Error, "expected an error but actual response did not contain one") - require.Equal(tr.t, code, tr.Error.Error.Code, "actual error code was different from expected") -} - -// EqualsFixture compares a TestResponse against an expected status code and body payload. Use the fixture parameter to specify -// the path to a file. -func (tr *TestResponse) EqualsFixture(statusCode int, fixture string) { - body, err := os.ReadFile(fixture) - require.NoError(tr.t, err, "reading fixture failed") - tr.EqualsResponse(statusCode, body) -} - -// EqualsStatusCode compares a TestResponse against an expected status code (ingnores the body payload). -func (tr *TestResponse) EqualsStatusCode(statusCode int) { - require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") -} - -// EqualsFixture compares a TestResponse against an expected status code and body payload. -func (tr *TestResponse) EqualsResponse(statusCode int, body []byte) { - if len(body) == 0 { - require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") - require.Empty(tr.t, tr.Body.Bytes(), "expected an empty response but actual response had a body") - return - } - - var expected map[string]any - err := json.Unmarshal(body, &expected) - require.NoError(tr.t, err, "unmarshalling expected response failed") - - var actual map[string]any - err = json.Unmarshal(tr.Body.Bytes(), &actual) - - tr.removeSystemData(actual) - - require.NoError(tr.t, err, "unmarshalling actual response failed") - require.EqualValues(tr.t, expected, actual, "response body did not match expected") - require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") -} - -// EqualsValue compares a TestResponse against an expected status code and an response body. -// -// If the systemData propert is present in the response, it will be removed. -func (tr *TestResponse) EqualsValue(statusCode int, expected any) { - var actual map[string]any - err := json.Unmarshal(tr.Body.Bytes(), &actual) - require.NoError(tr.t, err, "unmarshalling actual response failed") - - // Convert expected input to map[string]any to compare with actual response. - expectedBytes, err := json.Marshal(expected) - require.NoError(tr.t, err, "marshalling expected response failed") - - var expectedMap map[string]any - err = json.Unmarshal(expectedBytes, &expectedMap) - require.NoError(tr.t, err, "unmarshalling expected response failed") - - tr.removeSystemData(expectedMap) - tr.removeSystemData(actual) - - require.EqualValues(tr.t, expectedMap, actual, "response body did not match expected") - require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") -} - -// EqualsEmptyList compares a TestResponse against an expected status code and an empty resource list. -func (tr *TestResponse) EqualsEmptyList() { - expected := map[string]any{ - "value": []any{}, - } - - var actual map[string]any - err := json.Unmarshal(tr.Body.Bytes(), &actual) - - tr.removeSystemData(actual) - - require.NoError(tr.t, err, "unmarshalling actual response failed") - require.EqualValues(tr.t, expected, actual, "response body did not match expected") - require.Equal(tr.t, http.StatusOK, tr.Raw.StatusCode, "status code did not match expected") -} - -func (tr *TestResponse) ReadAs(obj any) { - tr.t.Helper() - - decoder := json.NewDecoder(tr.Body) - decoder.DisallowUnknownFields() - - err := decoder.Decode(obj) - require.NoError(tr.t, err, "unmarshalling expected response failed") -} - -func (tr *TestResponse) WaitForOperationComplete(timeout *time.Duration) *TestResponse { - if tr.Raw.StatusCode != http.StatusCreated && tr.Raw.StatusCode != http.StatusAccepted { - // Response is already terminal. - return tr - } - - if timeout == nil { - x := 30 * time.Second - timeout = &x - } - - timer := time.After(*timeout) - poller := time.NewTicker(1 * time.Second) - defer poller.Stop() - for { - select { - case <-timer: - tr.t.Fatalf("timed out waiting for operation to complete") - return nil // unreachable - case <-poller.C: - // The Location header should give us the operation status URL. - response := tr.server.MakeRequest(http.MethodGet, tr.Raw.Header.Get("Azure-AsyncOperation"), nil) - // To determine if the response is terminal we need to read the provisioning state field. - - operationStatus := v1.AsyncOperationStatus{} - response.ReadAs(&operationStatus) - if operationStatus.Status.IsTerminal() { - // Response is terminal. - return response - } - - continue - } - } -} - -func (tr *TestResponse) removeSystemData(responseBody map[string]any) { - // Delete systemData property if found, it's not stable so we don't include it in baselines. - _, ok := responseBody["systemData"] - if ok { - delete(responseBody, "systemData") - return - } - - value, ok := responseBody["value"] - if !ok { - return - } - - valueSlice, ok := value.([]any) - if !ok { - return - } - - for _, v := range valueSlice { - if vMap, ok := v.(map[string]any); ok { - tr.removeSystemData(vMap) - } - } -} diff --git a/pkg/ucp/options.go b/pkg/ucp/options.go new file mode 100644 index 0000000000..2b334188ba --- /dev/null +++ b/pkg/ucp/options.go @@ -0,0 +1,111 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ucp + +import ( + "context" + "fmt" + + "github.com/radius-project/radius/pkg/armrpc/asyncoperation/statusmanager" + "github.com/radius-project/radius/pkg/kubeutil" + "github.com/radius-project/radius/pkg/sdk" + ucpconfig "github.com/radius-project/radius/pkg/ucp/config" + "github.com/radius-project/radius/pkg/ucp/dataprovider" + "github.com/radius-project/radius/pkg/ucp/frontend/modules" + queueprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" + secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" + "github.com/radius-project/radius/pkg/validator" + "github.com/radius-project/radius/swagger" + kube_rest "k8s.io/client-go/rest" +) + +// Options holds the configuration options and shared services for the DyanmicRP server. +// +// For testability, all fields on this struct MUST be constructed from the NewOptions function without any +// additional initialization required. +type Options struct { + // Config is the configuration for the server. + Config *Config + + // Modules is the list of modules to initialize. This will default to nil (implying the default set), and + // can be overridden by tests. + Modules []modules.Initializer + + // QueueProvider provides access to the message queue client. + QueueProvider *queueprovider.QueueProvider + + // SecretProvider provides access to the secret storage system. + SecretProvider *secretprovider.SecretProvider + + // SpecLoader is the loader for the OpenAPI spec. + SpecLoader *validator.Loader + + // StatusManager implements operations on async operation statuses. + StatusManager statusmanager.StatusManager + + // StorageProvider provides access to the data storage system. + StorageProvider dataprovider.DataStorageProvider + + // UCP is the connection to UCP + UCP sdk.Connection +} + +// NewOptions creates a new Options instance from the given configuration. +func NewOptions(ctx context.Context, config *Config) (*Options, error) { + var err error + options := Options{ + Config: config, + + Modules: nil, // Default to nil, which implies the default set of modules. + } + + options.QueueProvider = queueprovider.New(config.Queue) + options.SecretProvider = secretprovider.NewSecretProvider(config.Secrets) + options.StorageProvider = dataprovider.NewStorageProvider(config.Storage) + + queueClient, err := options.QueueProvider.GetClient(ctx) + if err != nil { + return nil, err + } + + options.StatusManager = statusmanager.New(options.StorageProvider, queueClient, config.Environment.RoleLocation) + + options.SpecLoader, err = validator.LoadSpec(ctx, "ucp", swagger.SpecFilesUCP, []string{config.Server.PathBase}, "") + if err != nil { + return nil, err + } + + var cfg *kube_rest.Config + if config.UCP.Kind == ucpconfig.UCPConnectionKindKubernetes { + cfg, err = kubeutil.NewClientConfig(&kubeutil.ConfigOptions{ + // TODO: Allow to use custom context via configuration. - https://github.com/radius-project/radius/issues/5433 + ContextName: "", + QPS: kubeutil.DefaultServerQPS, + Burst: kubeutil.DefaultServerBurst, + }) + if err != nil { + return nil, fmt.Errorf("failed to get kubernetes config: %w", err) + } + } + + options.UCP, err = ucpconfig.NewConnectionFromUCPConfig(&config.UCP, cfg) + if err != nil { + return nil, err + } + + return &options, nil +} diff --git a/pkg/ucp/queue/inmemory/client.go b/pkg/ucp/queue/inmemory/client.go index 70347ca322..1abd302999 100644 --- a/pkg/ucp/queue/inmemory/client.go +++ b/pkg/ucp/queue/inmemory/client.go @@ -44,12 +44,17 @@ func New(queue *InmemQueue) *Client { // New creates the named in-memory queue Client instance. func NewNamedQueue(name string) *Client { - inmemq, _ := namedQueue.LoadOrStore(name, NewInMemQueue(messageLockDuration)) + inmemq, _ := namedQueue.LoadOrStore(name, NewInMemQueue(name, messageLockDuration)) return &Client{ queue: inmemq.(*InmemQueue), } } +// String returns the name of the queue for diagnostics purposes. +func (c *Client) String() string { + return c.queue.name +} + // Enqueue enqueues message to the in-memory queue. func (c *Client) Enqueue(ctx context.Context, msg *client.Message, options ...client.EnqueueOptions) error { if msg == nil || msg.Data == nil || len(msg.Data) == 0 { diff --git a/pkg/ucp/queue/inmemory/client_test.go b/pkg/ucp/queue/inmemory/client_test.go index e3b35b9281..d410e777ff 100644 --- a/pkg/ucp/queue/inmemory/client_test.go +++ b/pkg/ucp/queue/inmemory/client_test.go @@ -44,7 +44,7 @@ func TestNamedQueue(t *testing.T) { } func TestClient(t *testing.T) { - inmem := NewInMemQueue(sharedtest.TestMessageLockTime) + inmem := NewInMemQueue("test", sharedtest.TestMessageLockTime) cli := New(inmem) clean := func(t *testing.T) { diff --git a/pkg/ucp/queue/inmemory/queue.go b/pkg/ucp/queue/inmemory/queue.go index 5a428db340..9014870ca1 100644 --- a/pkg/ucp/queue/inmemory/queue.go +++ b/pkg/ucp/queue/inmemory/queue.go @@ -29,7 +29,7 @@ var ( messageLockDuration = 5 * time.Minute messageExpireDuration = 24 * time.Hour - defaultQueue = NewInMemQueue(messageLockDuration) + defaultQueue = NewInMemQueue("shared", messageLockDuration) ) type element struct { @@ -40,14 +40,16 @@ type element struct { // InmemQueue implements in-memory queue for dev/test type InmemQueue struct { - v *list.List - vMu sync.Mutex + v *list.List + vMu sync.Mutex + name string lockDuration time.Duration } -func NewInMemQueue(lockDuration time.Duration) *InmemQueue { +func NewInMemQueue(name string, lockDuration time.Duration) *InmemQueue { return &InmemQueue{ + name: name, v: &list.List{}, lockDuration: lockDuration, } diff --git a/pkg/ucp/queue/inmemory/queue_test.go b/pkg/ucp/queue/inmemory/queue_test.go index 1a13e8d769..611de3111b 100644 --- a/pkg/ucp/queue/inmemory/queue_test.go +++ b/pkg/ucp/queue/inmemory/queue_test.go @@ -27,7 +27,7 @@ import ( ) func TestEnqueueDequeueMulti(t *testing.T) { - q := NewInMemQueue(messageLockDuration) + q := NewInMemQueue("test", messageLockDuration) for i := 0; i < 10; i++ { q.Enqueue(&client.Message{ Data: []byte(fmt.Sprintf("test%d", i)), @@ -46,7 +46,7 @@ func TestEnqueueDequeueMulti(t *testing.T) { } func TestMessageLock(t *testing.T) { - q := NewInMemQueue(2 * time.Millisecond) + q := NewInMemQueue("test", 2*time.Millisecond) q.Enqueue(&client.Message{ Data: []byte("test"), @@ -65,7 +65,7 @@ func TestMessageLock(t *testing.T) { } func TestExpiry(t *testing.T) { - q := NewInMemQueue(messageLockDuration) + q := NewInMemQueue("test", messageLockDuration) q.Enqueue(&client.Message{ Data: []byte("test"), @@ -84,7 +84,7 @@ func TestExpiry(t *testing.T) { } func TestComplete(t *testing.T) { - q := NewInMemQueue(messageLockDuration) + q := NewInMemQueue("test", messageLockDuration) q.Enqueue(&client.Message{ Data: []byte("test"), diff --git a/pkg/ucp/rest/objects.go b/pkg/ucp/rest/objects.go deleted file mode 100644 index 0558fc7c58..0000000000 --- a/pkg/ucp/rest/objects.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2023 The Radius Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package rest - -import "strings" - -type PlaneProperties struct { - ResourceProviders map[string]string `json:"resourceProviders" yaml:"resourceProviders"` // Used only for UCP native planes - Kind string `json:"kind" yaml:"kind"` - URL string `json:"url" yaml:"url"` // Used only for non UCP native planes and non AWS planes -} - -// Plane kinds -const ( - PlaneKindUCPNative = "UCPNative" - PlaneKindAzure = "Azure" - PlaneKindAWS = "AWS" -) - -type Plane struct { - ID string `json:"id" yaml:"id"` - Type string `json:"type" yaml:"type"` - Name string `json:"name" yaml:"name"` - Properties PlaneProperties `json:"properties" yaml:"properties"` -} - -// PlaneList represents a list of UCP planes in the ARM wire-format -type PlaneList struct { - Value []Plane `json:"value" yaml:"value"` -} - -// Resource represents a resource within a UCP resource group -type Resource struct { - ID string `json:"id" yaml:"id"` - Name string `json:"name" yaml:"name"` - ProvisioningState string `json:"provisioningState" yaml:"provisioningState"` - Type string `json:"type" yaml:"type"` -} - -// ResourceList represents a list of resources -type ResourceList struct { - Value []Resource `json:"value" yaml:"value"` -} - -// LookupResourceProvider searches through the ResourceProviders configured in UCP. -func (plane *Plane) LookupResourceProvider(key string) string { - var value string - for k, v := range plane.Properties.ResourceProviders { - if strings.EqualFold(k, key) { - value = v - break - } - } - return value -} diff --git a/pkg/ucp/server/server.go b/pkg/ucp/server/server.go index 83a1e9cd8b..7737374e48 100644 --- a/pkg/ucp/server/server.go +++ b/pkg/ucp/server/server.go @@ -17,33 +17,17 @@ limitations under the License. package server import ( - "errors" - "fmt" - "os" - "strings" "time" - hostopts "github.com/radius-project/radius/pkg/armrpc/hostoptions" - "github.com/radius-project/radius/pkg/kubeutil" - metricsprovider "github.com/radius-project/radius/pkg/metrics/provider" metricsservice "github.com/radius-project/radius/pkg/metrics/service" - profilerprovider "github.com/radius-project/radius/pkg/profiler/provider" profilerservice "github.com/radius-project/radius/pkg/profiler/service" - "github.com/radius-project/radius/pkg/sdk" "github.com/radius-project/radius/pkg/trace" + "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/backend" - "github.com/radius-project/radius/pkg/ucp/config" "github.com/radius-project/radius/pkg/ucp/data" "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/frontend/api" "github.com/radius-project/radius/pkg/ucp/hosting" - "github.com/radius-project/radius/pkg/ucp/hostoptions" - qprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" - "github.com/radius-project/radius/pkg/ucp/rest" - "github.com/radius-project/radius/pkg/ucp/secret/provider" - "github.com/radius-project/radius/pkg/ucp/ucplog" - - kube_rest "k8s.io/client-go/rest" ) const ( @@ -51,158 +35,47 @@ const ( ServiceName = "ucp" ) -type Options struct { - Config *hostoptions.UCPConfig - Port string - StorageProviderOptions dataprovider.StorageProviderOptions - LoggingOptions ucplog.LoggingOptions - SecretProviderOptions provider.SecretProviderOptions - QueueProviderOptions qprovider.QueueProviderOptions - MetricsProviderOptions metricsprovider.MetricsProviderOptions - ProfilerProviderOptions profilerprovider.ProfilerProviderOptions - TracerProviderOptions trace.Options - TLSCertDir string - PathBase string - InitialPlanes []rest.Plane - Identity hostoptions.Identity - UCPConnection sdk.Connection - Location string -} - const UCPProviderName = "System.Resources" -// NewServerOptionsFromEnvironment creates a new Options struct from environment variables and returns it along with any errors. -func NewServerOptionsFromEnvironment(configFilePath string) (Options, error) { - basePath, ok := os.LookupEnv("BASE_PATH") - if ok && len(basePath) > 0 && (!strings.HasPrefix(basePath, "/") || strings.HasSuffix(basePath, "/")) { - return Options{}, errors.New("env: BASE_PATH must begin with '/' and must not end with '/'") - } - - tlsCertDir := os.Getenv("TLS_CERT_DIR") - port := os.Getenv("PORT") - if port == "" { - return Options{}, errors.New("UCP Port number must be set") - } - - opts, err := hostoptions.NewHostOptionsFromEnvironment(configFilePath) - if err != nil { - return Options{}, err - } - - storeOpts := opts.Config.StorageProvider - planes := opts.Config.Planes - secretOpts := opts.Config.SecretProvider - qproviderOpts := opts.Config.QueueProvider - metricsOpts := opts.Config.MetricsProvider - traceOpts := opts.Config.TracerProvider - profilerOpts := opts.Config.ProfilerProvider - loggingOpts := opts.Config.Logging - identity := opts.Config.Identity - // Set the default authentication method if AuthMethod is not set. - if identity.AuthMethod == "" { - identity.AuthMethod = hostoptions.AuthDefault - } - - location := opts.Config.Location - if location == "" { - location = "global" - } - - var cfg *kube_rest.Config - if opts.Config.UCP.Kind == config.UCPConnectionKindKubernetes { - cfg, err = kubeutil.NewClientConfig(&kubeutil.ConfigOptions{ - // TODO: Allow to use custom context via configuration. - https://github.com/radius-project/radius/issues/5433 - ContextName: "", - QPS: kubeutil.DefaultServerQPS, - Burst: kubeutil.DefaultServerBurst, - }) - if err != nil { - return Options{}, fmt.Errorf("failed to get kubernetes config: %w", err) - } - } - - ucpConn, err := config.NewConnectionFromUCPConfig(&opts.Config.UCP, cfg) - if err != nil { - return Options{}, err - } - - return Options{ - Config: opts.Config, - Port: port, - TLSCertDir: tlsCertDir, - PathBase: basePath, - StorageProviderOptions: storeOpts, - SecretProviderOptions: secretOpts, - QueueProviderOptions: qproviderOpts, - MetricsProviderOptions: metricsOpts, - TracerProviderOptions: traceOpts, - ProfilerProviderOptions: profilerOpts, - LoggingOptions: loggingOpts, - InitialPlanes: planes, - Identity: identity, - UCPConnection: ucpConn, - Location: location, - }, nil -} - -// NewServer creates a new hosting.Host instance with services for API, EmbeddedETCD, Metrics, Profiler and Backend (if -// enabled) based on the given Options. -func NewServer(options *Options) (*hosting.Host, error) { +// NewServer creates a new hosting.Host instance for UCP. +// +// This includes services for UCP's core functionality: +// - frontend API +// - backend worker +// +// This also includes services for diagnostics (when enabled): +// - Metrics +// - Tracing +// - Profiler +// +// Lastly, when embedded etcd is enabled, it includes the embedded etcd service. +func NewServer(options *ucp.Options) (*hosting.Host, error) { hostingServices := []hosting.Service{ - api.NewService(api.ServiceOptions{ - ProviderName: UCPProviderName, - Address: ":" + options.Port, - PathBase: options.PathBase, - Config: options.Config, - Location: options.Location, - TLSCertDir: options.TLSCertDir, - StorageProviderOptions: options.StorageProviderOptions, - SecretProviderOptions: options.SecretProviderOptions, - QueueProviderOptions: options.QueueProviderOptions, - InitialPlanes: options.InitialPlanes, - Identity: options.Identity, - UCPConnection: options.UCPConnection, - }), + api.NewService(options), + backend.NewService(options), } - if options.StorageProviderOptions.Provider == dataprovider.TypeETCD && - options.StorageProviderOptions.ETCD.InMemory { - hostingServices = append(hostingServices, data.NewEmbeddedETCDService(data.EmbeddedETCDServiceOptions{ClientConfigSink: options.StorageProviderOptions.ETCD.Client})) - } - - options.MetricsProviderOptions.ServiceName = ServiceName - if options.MetricsProviderOptions.Prometheus.Enabled { + options.Config.Metrics.ServiceName = ServiceName + if options.Config.Metrics.Prometheus.Enabled { metricOptions := metricsservice.HostOptions{ - Config: &options.MetricsProviderOptions, + Config: &options.Config.Metrics, } hostingServices = append(hostingServices, metricsservice.NewService(metricOptions)) } - if options.ProfilerProviderOptions.Enabled { + if options.Config.Profiler.Enabled { profilerOptions := profilerservice.HostOptions{ - Config: &options.ProfilerProviderOptions, + Config: &options.Config.Profiler, } hostingServices = append(hostingServices, profilerservice.NewService(profilerOptions)) } - backendServiceOptions := hostopts.HostOptions{ + hostingServices = append(hostingServices, &trace.Service{Options: options.Config.Tracing}) - Config: &hostopts.ProviderConfig{ - Env: hostopts.EnvironmentOptions{ - RoleLocation: options.Config.Location, - }, - StorageProvider: options.StorageProviderOptions, - SecretProvider: options.SecretProviderOptions, - QueueProvider: options.QueueProviderOptions, - MetricsProvider: options.MetricsProviderOptions, - TracerProvider: options.TracerProviderOptions, - ProfilerProvider: options.ProfilerProviderOptions, - }, + if options.Config.Storage.Provider == dataprovider.TypeETCD && + options.Config.Storage.ETCD.InMemory { + hostingServices = append(hostingServices, data.NewEmbeddedETCDService(data.EmbeddedETCDServiceOptions{ClientConfigSink: options.Config.Storage.ETCD.Client})) } - hostingServices = append(hostingServices, backend.NewService(backendServiceOptions, *options.Config)) - - options.TracerProviderOptions.ServiceName = "ucp" - hostingServices = append(hostingServices, &trace.Service{Options: options.TracerProviderOptions}) return &hosting.Host{ Services: hostingServices, diff --git a/pkg/ucp/store/inmemory/client.go b/pkg/ucp/store/inmemory/client.go index 97adc07337..91de64dc0b 100644 --- a/pkg/ucp/store/inmemory/client.go +++ b/pkg/ucp/store/inmemory/client.go @@ -34,6 +34,8 @@ var _ store.StorageClient = (*Client)(nil) // Client is an in-memory implementation of store.StorageClient. type Client struct { + name string + // mutex is used to synchronize access to the resources map. mutex sync.Mutex @@ -80,13 +82,19 @@ type entry struct { } // NewClient creates a new in-memory store client. -func NewClient() *Client { +func NewClient(name string) *Client { return &Client{ + name: name, mutex: sync.Mutex{}, resources: map[string]entry{}, } } +// String returns the name of the client. +func (c *Client) String() string { + return c.name +} + // Get implements store.StorageClient. func (c *Client) Get(ctx context.Context, id string, options ...store.GetOptions) (*store.Object, error) { if ctx == nil { diff --git a/pkg/ucp/store/inmemory/client_test.go b/pkg/ucp/store/inmemory/client_test.go index 1581accfa4..02e389994b 100644 --- a/pkg/ucp/store/inmemory/client_test.go +++ b/pkg/ucp/store/inmemory/client_test.go @@ -23,7 +23,7 @@ import ( ) func Test_InMemoryClient(t *testing.T) { - client := NewClient() + client := NewClient("test") clear := func(t *testing.T) { client.Clear() diff --git a/pkg/ucp/testhost/doc.go b/pkg/ucp/testhost/doc.go new file mode 100644 index 0000000000..7b000bd986 --- /dev/null +++ b/pkg/ucp/testhost/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// testhost provides an implementation of a test server for UCP. +package testhost diff --git a/pkg/ucp/testhost/host.go b/pkg/ucp/testhost/host.go new file mode 100644 index 0000000000..5ece0a3e29 --- /dev/null +++ b/pkg/ucp/testhost/host.go @@ -0,0 +1,218 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testhost + +import ( + "context" + "fmt" + "strings" + "testing" + + v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/armrpc/asyncoperation/statusmanager" + "github.com/radius-project/radius/pkg/armrpc/hostoptions" + aztoken "github.com/radius-project/radius/pkg/azure/tokencredentials" + "github.com/radius-project/radius/pkg/sdk" + "github.com/radius-project/radius/pkg/ucp" + "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" + "github.com/radius-project/radius/pkg/ucp/config" + "github.com/radius-project/radius/pkg/ucp/dataprovider" + "github.com/radius-project/radius/pkg/ucp/frontend/modules" + queue "github.com/radius-project/radius/pkg/ucp/queue/client" + queueprovider "github.com/radius-project/radius/pkg/ucp/queue/provider" + "github.com/radius-project/radius/pkg/ucp/secret" + secretprovider "github.com/radius-project/radius/pkg/ucp/secret/provider" + "github.com/radius-project/radius/pkg/ucp/server" + "github.com/radius-project/radius/pkg/ucp/store" + "github.com/radius-project/radius/test/integrationtest/testhost" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +// TestHostOption can be used to configure the UCP options before the server is started. +type TestHostOption interface { + // Apply applies the configuration to the UCP options. + Apply(options *ucp.Options) +} + +// TestHostOptionFunc is a function that implements the TestHostOption interface. +type TestHostOptionFunc func(options *ucp.Options) + +// Apply applies the function to the UCP options. +func (f TestHostOptionFunc) Apply(options *ucp.Options) { + f(options) +} + +// NoModules is a TestHostOption that disables all UCP modules. +func NoModules() TestHostOptionFunc { + return func(options *ucp.Options) { + options.Modules = []modules.Initializer{} + } +} + +// TestServerMocks provides access to mock instances created by the TestServer. +type TestServerMocks struct { + // DataProvider is the mock data provider. + DataProvider *dataprovider.MockDataStorageProvider + + // QueueProvider is the mock queue provider. + QueueProvider *queueprovider.QueueProvider + + // SecretProvider is the mock secret provider. + SecretProvider *secretprovider.SecretProvider + + // Secrets is the mock secret client. + Secrets *secret.MockClient + + // StatusManager is the mock status manager. + StatusManager *statusmanager.MockStatusManager + + // Storage is the mock storage client. + Storage *store.MockStorageClient +} + +// NewMocks creates a new set of mocks for the test server. +func NewMocks(t *testing.T) *TestServerMocks { + ctrl := gomock.NewController(t) + dataClient := store.NewMockStorageClient(ctrl) + dataProvider := dataprovider.NewMockDataStorageProvider(ctrl) + dataProvider.EXPECT(). + GetStorageClient(gomock.Any(), gomock.Any()). + Return(dataClient, nil). + AnyTimes() + + queueClient := queue.NewMockClient(ctrl) + queueProvider := queueprovider.New(queueprovider.QueueProviderOptions{Name: "System.Resources"}) + queueProvider.SetClient(queueClient) + + secretClient := secret.NewMockClient(ctrl) + secretProvider := secretprovider.NewSecretProvider(secretprovider.SecretProviderOptions{}) + secretProvider.SetClient(secretClient) + + statusManager := statusmanager.NewMockStatusManager(ctrl) + return &TestServerMocks{ + DataProvider: dataProvider, + QueueProvider: queueProvider, + SecretProvider: secretProvider, + Secrets: secretClient, + StatusManager: statusManager, + Storage: dataClient, + } +} + +// Apply updates the UCP options to use the mocks. +func (m *TestServerMocks) Apply(options *ucp.Options) { + options.SecretProvider = m.SecretProvider + options.StorageProvider = m.DataProvider + options.QueueProvider = m.QueueProvider + options.StatusManager = m.StatusManager +} + +// TestHost provides a test host for the UCP server. +type TestHost struct { + *testhost.TestHost + options *ucp.Options + + clientFactoryUCP *v20231001preview.ClientFactory +} + +// Internals provides access to the internal options of the server. This allows tests +// to access the data stores and manipulate the server state. +func (th *TestHost) Options() *ucp.Options { + return th.options +} + +// UCP provides access to the generated clients for the UCP API. +func (ts *TestHost) UCP() *v20231001preview.ClientFactory { + if ts.clientFactoryUCP == nil { + connection, err := sdk.NewDirectConnection(ts.BaseURL()) + require.NoError(ts.T(), err) + + ts.clientFactoryUCP, err = v20231001preview.NewClientFactory(&aztoken.AnonymousCredential{}, sdk.NewClientOptions(connection)) + require.NoError(ts.T(), err) + } + + return ts.clientFactoryUCP +} + +// Start creates and starts a new TestServer. +func Start(t *testing.T, opts ...TestHostOption) *TestHost { + config := &ucp.Config{ + Environment: hostoptions.EnvironmentOptions{ + Name: "test", + RoleLocation: v1.LocationGlobal, + }, + Queue: queueprovider.QueueProviderOptions{ + Provider: queueprovider.TypeInmemory, + Name: "ucp", + }, + Secrets: secretprovider.SecretProviderOptions{ + Provider: secretprovider.TypeInMemorySecret, + }, + Server: hostoptions.ServerOptions{ + // Initialized dynamically when the server is started. + }, + Storage: dataprovider.StorageProviderOptions{ + Provider: dataprovider.TypeInMemory, + }, + UCP: config.UCPOptions{ + Kind: config.UCPConnectionKindDirect, + Direct: &config.UCPDirectConnectionOptions{ + Endpoint: "http://localhost:65000", // Initialized dynamically when the server is started. + }, + }, + } + + options, err := ucp.NewOptions(context.Background(), config) + require.NoError(t, err) + + for _, opt := range opts { + opt.Apply(options) + } + + return StartWithOptions(t, options) + +} + +func StartWithOptions(t *testing.T, options *ucp.Options) *TestHost { + options.Config.Server.Host = "localhost" + if options.Config.Server.Port == 0 { + options.Config.Server.Port = testhost.AllocateFreePort(t) + } + + baseURL := fmt.Sprintf( + "http://%s%s", + options.Config.Server.Address(), + options.Config.Server.PathBase) + baseURL = strings.TrimSuffix(baseURL, "/") + + options.Config.UCP.Kind = config.UCPConnectionKindDirect + options.Config.UCP.Direct = &config.UCPDirectConnectionOptions{Endpoint: baseURL} + + // Instantiate the UCP client now that we know the URL. + var err error + options.UCP, err = sdk.NewDirectConnection(baseURL) + require.NoError(t, err) + + host, err := server.NewServer(options) + require.NoError(t, err, "failed to create server") + + return &TestHost{ + TestHost: testhost.StartHost(t, host, baseURL), + options: options, + } +} diff --git a/pkg/ucp/trackedresource/update.go b/pkg/ucp/trackedresource/update.go index 49f9de3c8c..386576f547 100644 --- a/pkg/ucp/trackedresource/update.go +++ b/pkg/ucp/trackedresource/update.go @@ -29,7 +29,9 @@ import ( "github.com/go-logr/logr" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/resources" "github.com/radius-project/radius/pkg/ucp/store" "github.com/radius-project/radius/pkg/ucp/ucplog" @@ -42,20 +44,20 @@ const ( ) // NewUpdater creates a new Updater. -func NewUpdater(storeClient store.StorageClient, httpClient *http.Client) *Updater { +func NewUpdater(storageProvider dataprovider.DataStorageProvider, httpClient *http.Client) *Updater { return &Updater{ - Store: storeClient, - Client: httpClient, - AttemptCount: retryCount, - RetryDelay: retryDelay, - RequestTimeout: requestTimeout, + StorageProvider: storageProvider, + Client: httpClient, + AttemptCount: retryCount, + RetryDelay: retryDelay, + RequestTimeout: requestTimeout, } } // Updater is a utility struct that can perform updates on tracked resources. type Updater struct { - // Store is the storage client used to access the database. - Store store.StorageClient + // StorageProvider is the storage provider used to access the database. + StorageProvider dataprovider.DataStorageProvider // Client is the HTTP client used to make requests to the downstream API. Client *http.Client @@ -153,7 +155,14 @@ func (u *Updater) Update(ctx context.Context, downstream string, id resources.ID func (u *Updater) run(ctx context.Context, id resources.ID, trackingID resources.ID, destination *url.URL, apiVersion string) error { logger := ucplog.FromContextOrDiscard(ctx) - obj, err := u.Store.Get(ctx, trackingID.String()) + + // Note: All tracked resources use the same resource type as they're stored in the same table. + client, err := u.StorageProvider.GetStorageClient(ctx, v20231001preview.ResourceType) + if err != nil { + return err + } + + obj, err := client.Get(ctx, trackingID.String()) if errors.Is(err, &store.ErrNotFound{}) { // This is fine. It might be a new resource. } else if err != nil { @@ -179,7 +188,7 @@ func (u *Updater) run(ctx context.Context, id resources.ID, trackingID resources if data == nil { // Resource was not found. We can delete the tracked resource entry. logger.V(ucplog.LevelDebug).Info("deleting tracked resource entry") - err = u.Store.Delete(ctx, trackingID.String(), store.WithETag(etag)) + err = client.Delete(ctx, trackingID.String(), store.WithETag(etag)) if errors.Is(err, &store.ErrNotFound{}) { return nil } else if err != nil { @@ -208,7 +217,7 @@ func (u *Updater) run(ctx context.Context, id resources.ID, trackingID resources Data: entry, } logger.V(ucplog.LevelDebug).Info("updating tracked resource entry") - err = u.Store.Save(ctx, obj, store.WithETag(etag)) + err = client.Save(ctx, obj, store.WithETag(etag)) if errors.Is(err, &store.ErrConcurrency{}) { logger.V(ucplog.LevelDebug).Info("tracked resource was updated concurrently") return &InProgressErr{} diff --git a/pkg/ucp/trackedresource/update_test.go b/pkg/ucp/trackedresource/update_test.go index 98d3ba173b..8763faffae 100644 --- a/pkg/ucp/trackedresource/update_test.go +++ b/pkg/ucp/trackedresource/update_test.go @@ -28,7 +28,9 @@ import ( v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" "github.com/radius-project/radius/pkg/to" + "github.com/radius-project/radius/pkg/ucp/api/v20231001preview" "github.com/radius-project/radius/pkg/ucp/datamodel" + "github.com/radius-project/radius/pkg/ucp/dataprovider" "github.com/radius-project/radius/pkg/ucp/store" "github.com/radius-project/radius/test/testcontext" "github.com/stretchr/testify/require" @@ -49,8 +51,11 @@ func setupUpdater(t *testing.T) (*Updater, *store.MockStorageClient, *mockRoundT ctrl := gomock.NewController(t) storeClient := store.NewMockStorageClient(ctrl) + storageProvider := dataprovider.NewMockDataStorageProvider(ctrl) + storageProvider.EXPECT().GetStorageClient(gomock.Any(), v20231001preview.ResourceType).Return(storeClient, nil).AnyTimes() + roundTripper := &mockRoundTripper{} - updater := NewUpdater(storeClient, &http.Client{Transport: roundTripper}) + updater := NewUpdater(storageProvider, &http.Client{Transport: roundTripper}) // Optimize these values for testability. We don't want to wait for retries or timeouts unless // the test is specifically testing that behavior. diff --git a/test/integrationtest/testhost/clients.go b/test/integrationtest/testhost/clients.go new file mode 100644 index 0000000000..b430937816 --- /dev/null +++ b/test/integrationtest/testhost/clients.go @@ -0,0 +1,274 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testhost + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "os" + "testing" + "time" + + v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/armrpc/rpctest" + "github.com/stretchr/testify/require" +) + +// TestResponse is return from requests made against a TestHost. Tests should use the functions defined +// on TestResponse for valiation. +type TestResponse struct { + // Raw is the raw HTTP response. + Raw *http.Response + + // Body is the response body. + Body *bytes.Buffer + + // Error is the ARM error response if the response status code is >= 400. + Error *v1.ErrorResponse + + // t is the test object. + t *testing.T + + // host is the TestHost that served this response. + host *TestHost +} + +// MakeFixtureRequest sends a request to the server using a file on disk as the payload (body). Use the fixture +// parameter to specify the path to a file. +func (th *TestHost) MakeFixtureRequest(method string, pathAndQuery string, fixture string) *TestResponse { + body, err := os.ReadFile(fixture) + require.NoError(th.t, err, "reading fixture failed") + return th.MakeRequest(method, pathAndQuery, body) +} + +// MakeTypedRequest sends a request to the server by marshalling the provided object to JSON. +func (th *TestHost) MakeTypedRequest(method string, pathAndQuery string, body any) *TestResponse { + if body == nil { + return th.MakeRequest(method, pathAndQuery, nil) + } + + b, err := json.Marshal(body) + require.NoError(th.t, err, "marshalling body failed") + return th.MakeRequest(method, pathAndQuery, b) +} + +// MakeRequest sends a request to the server. +func (th *TestHost) MakeRequest(method string, pathAndQuery string, body []byte) *TestResponse { + // Prepend the base path if this is a relative URL. + requestUrl := pathAndQuery + parsed, err := url.Parse(pathAndQuery) + require.NoError(th.t, err, "parsing URL failed") + if !parsed.IsAbs() { + requestUrl = th.BaseURL() + pathAndQuery + } + + client := th.Client() + request, err := rpctest.NewHTTPRequestWithContent(context.Background(), method, requestUrl, body) + require.NoError(th.t, err, "creating request failed") + + ctx := rpctest.NewARMRequestContext(request) + request = request.WithContext(ctx) + + response, err := client.Do(request) + require.NoError(th.t, err, "sending request failed") + + // Buffer the response so we can read multiple times. + responseBuffer := &bytes.Buffer{} + _, err = io.Copy(responseBuffer, response.Body) + response.Body.Close() + require.NoError(th.t, err, "copying response failed") + + response.Body = io.NopCloser(responseBuffer) + + // Pretty-print response for logs. + if len(responseBuffer.Bytes()) > 0 { + var data any + err = json.Unmarshal(responseBuffer.Bytes(), &data) + require.NoError(th.t, err, "unmarshalling response failed") + + text, err := json.MarshalIndent(&data, "", " ") + require.NoError(th.t, err, "marshalling response failed") + th.t.Log("Response Body: \n" + string(text)) + } + + var errorResponse *v1.ErrorResponse + if response.StatusCode >= 400 { + // The response MUST be an arm error for a non-success status code. + errorResponse = &v1.ErrorResponse{} + err := json.Unmarshal(responseBuffer.Bytes(), &errorResponse) + require.NoError(th.t, err, "unmarshalling error response failed - THIS IS A SERIOUS BUG. ALL ERROR RESPONSES MUST USE THE STANDARD FORMAT") + } + + return &TestResponse{Raw: response, Body: responseBuffer, Error: errorResponse, host: th, t: th.t} +} + +// EqualsErrorCode compares a TestResponse against an expected status code and error code. EqualsErrorCode assumes the response +// uses the ARM error format (required for our APIs). +func (tr *TestResponse) EqualsErrorCode(statusCode int, code string) { + require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") + require.NotNil(tr.t, tr.Error, "expected an error but actual response did not contain one") + require.Equal(tr.t, code, tr.Error.Error.Code, "actual error code was different from expected") +} + +// EqualsFixture compares a TestResponse against an expected status code and body payload. Use the fixture parameter to specify +// the path to a file. +func (tr *TestResponse) EqualsFixture(statusCode int, fixture string) { + body, err := os.ReadFile(fixture) + require.NoError(tr.t, err, "reading fixture failed") + tr.EqualsResponse(statusCode, body) +} + +// EqualsStatusCode compares a TestResponse against an expected status code (ingnores the body payload). +func (tr *TestResponse) EqualsStatusCode(statusCode int) { + require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") +} + +// EqualsFixture compares a TestResponse against an expected status code and body payload. +func (tr *TestResponse) EqualsResponse(statusCode int, body []byte) { + if len(body) == 0 { + require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") + require.Empty(tr.t, tr.Body.Bytes(), "expected an empty response but actual response had a body") + return + } + + var expected map[string]any + err := json.Unmarshal(body, &expected) + require.NoError(tr.t, err, "unmarshalling expected response failed") + + var actual map[string]any + err = json.Unmarshal(tr.Body.Bytes(), &actual) + + tr.removeSystemData(actual) + + require.NoError(tr.t, err, "unmarshalling actual response failed. Got '%v'", tr.Body.String()) + require.EqualValues(tr.t, expected, actual, "response body did not match expected") + require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") +} + +// EqualsValue compares a TestResponse against an expected status code and an response body. +// +// If the systemData propert is present in the response, it will be removed. +func (tr *TestResponse) EqualsValue(statusCode int, expected any) { + var actual map[string]any + err := json.Unmarshal(tr.Body.Bytes(), &actual) + require.NoError(tr.t, err, "unmarshalling actual response failed") + + // Convert expected input to map[string]any to compare with actual response. + expectedBytes, err := json.Marshal(expected) + require.NoError(tr.t, err, "marshalling expected response failed") + + var expectedMap map[string]any + err = json.Unmarshal(expectedBytes, &expectedMap) + require.NoError(tr.t, err, "unmarshalling expected response failed") + + tr.removeSystemData(expectedMap) + tr.removeSystemData(actual) + + require.EqualValues(tr.t, expectedMap, actual, "response body did not match expected") + require.Equal(tr.t, statusCode, tr.Raw.StatusCode, "status code did not match expected") +} + +// EqualsEmptyList compares a TestResponse against an expected status code and an empty resource list. +func (tr *TestResponse) EqualsEmptyList() { + expected := map[string]any{ + "value": []any{}, + } + + var actual map[string]any + err := json.Unmarshal(tr.Body.Bytes(), &actual) + + tr.removeSystemData(actual) + + require.NoError(tr.t, err, "unmarshalling actual response failed") + require.EqualValues(tr.t, expected, actual, "response body did not match expected") + require.Equal(tr.t, http.StatusOK, tr.Raw.StatusCode, "status code did not match expected") +} + +func (tr *TestResponse) ReadAs(obj any) { + tr.t.Helper() + + decoder := json.NewDecoder(tr.Body) + decoder.DisallowUnknownFields() + + err := decoder.Decode(obj) + require.NoError(tr.t, err, "unmarshalling expected response failed") +} + +func (tr *TestResponse) WaitForOperationComplete(timeout *time.Duration) *TestResponse { + if tr.Raw.StatusCode != http.StatusCreated && tr.Raw.StatusCode != http.StatusAccepted { + // Response is already terminal. + return tr + } + + if timeout == nil { + x := 30 * time.Second + timeout = &x + } + + timer := time.After(*timeout) + poller := time.NewTicker(1 * time.Second) + defer poller.Stop() + for { + select { + case <-timer: + tr.t.Fatalf("timed out waiting for operation to complete") + return nil // unreachable + case <-poller.C: + // The Location header should give us the operation status URL. + response := tr.host.MakeRequest(http.MethodGet, tr.Raw.Header.Get("Azure-AsyncOperation"), nil) + + // To determine if the response is terminal we need to read the provisioning state field. + operationStatus := v1.AsyncOperationStatus{} + response.ReadAs(&operationStatus) + if operationStatus.Status.IsTerminal() { + // Response is terminal. + return response + } + + continue + } + } +} + +func (tr *TestResponse) removeSystemData(responseBody map[string]any) { + // Delete systemData property if found, it's not stable so we don't include it in baselines. + _, ok := responseBody["systemData"] + if ok { + delete(responseBody, "systemData") + return + } + + value, ok := responseBody["value"] + if !ok { + return + } + + valueSlice, ok := value.([]any) + if !ok { + return + } + + for _, v := range valueSlice { + if vMap, ok := v.(map[string]any); ok { + tr.removeSystemData(vMap) + } + } +} diff --git a/test/integrationtest/testhost/doc.go b/test/integrationtest/testhost/doc.go new file mode 100644 index 0000000000..0d187588b9 --- /dev/null +++ b/test/integrationtest/testhost/doc.go @@ -0,0 +1,23 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package testhost provides a host for running any Radius control-plane component +// as an in-memory server for testing purposes. +// +// This package should be wrapped in a test package specific to the component under test. +// The wrapping design allows for component-specific depenendendencies to be defined without +// polluting the shared code. +package testhost diff --git a/test/integrationtest/testhost/host.go b/test/integrationtest/testhost/host.go new file mode 100644 index 0000000000..83f5f19cc9 --- /dev/null +++ b/test/integrationtest/testhost/host.go @@ -0,0 +1,148 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package testhost provides a host for running any Radius control-plane component +// as an in-memory server for testing purposes. +// +// This package should be wrapped in a test package specific to the component under test. +// The wrapping design allows for component-specific depenendendencies to be defined without +// polluting the shared code. +package testhost + +import ( + "context" + "net" + "net/http" + "net/url" + "sync" + "testing" + "time" + + "github.com/radius-project/radius/pkg/ucp/hosting" + "github.com/radius-project/radius/test/testcontext" + "github.com/stretchr/testify/require" +) + +// StartHost starts a new test host for the given hosting.Host and returns a TestHost instance. +// The TestHost will have its lifecycle managed by the test context, and will be shut down when the test +// completes. +func StartHost(t *testing.T, host *hosting.Host, baseURL string) *TestHost { + ctx, cancel := context.WithCancel(testcontext.New(t)) + errs, messages := host.RunAsync(ctx) + + go func() { + for msg := range messages { + t.Logf("Message: %s", msg) + } + }() + + th := &TestHost{ + baseURL: baseURL, + host: host, + messages: messages, + cancel: cancel, + stoppedChan: errs, + t: t, + } + t.Cleanup(th.Close) + + // Wait for the server to start listening on the port. + require.Eventuallyf(t, func() bool { + u, err := url.Parse(baseURL) + if err != nil { + panic("Invalid URL: " + baseURL) + } + + conn, err := net.Dial("tcp", net.JoinHostPort(u.Hostname(), u.Port())) + if err != nil { + t.Logf("Waiting for server to start listening on port: %v", err) + return false + } + defer conn.Close() + + return true + }, time.Second*5, time.Millisecond*20, "server did not start listening on port") + + return th +} + +// TestHost is a test server for any Radius control-plane component. Do not construct this type directly, use the Start function. +type TestHost struct { + // baseURL is the base URL of the server, including the path base. + baseURL string + + // host is the hosting process running the component. + host *hosting.Host + + // messages is the channel that will receive lifecycle messages from the host. + messages <-chan hosting.LifecycleMessage + + // cancel is the function to call to stop the server. + cancel context.CancelFunc + + // stoppedChan is the channel that will be closed when the server has stopped. + stoppedChan <-chan error + + // shutdown is used to ensure that Close is only called once. + shutdown sync.Once + + // t is the testing.T instance to use for assertions. + t *testing.T +} + +// Close shuts down the server and will block until shutdown completes. +func (th *TestHost) Close() { + // We're being picking about resource cleanup here, because unless we are picky we hit scalability + // problems in tests pretty quickly. + th.shutdown.Do(func() { + // Shut down the host. + th.cancel() + + if th.stoppedChan != nil { + <-th.stoppedChan // host stopped + } + }) +} + +// BaseURL returns the base URL of the server, including the path base. +// +// This should be used as a URL prefix for all requests to the server. +func (th *TestHost) BaseURL() string { + return th.baseURL +} + +// Client returns the HTTP client to use to make requests to the server. +func (th *TestHost) Client() *http.Client { + return http.DefaultClient +} + +// T returns the testing.T instance associated with the test host. +func (th *TestHost) T() *testing.T { + return th.t +} + +// AllocateFreePort chooses a random port for use in tests. +func AllocateFreePort(t *testing.T) int { + listener, err := net.Listen("tcp", ":0") + require.NoError(t, err, "failed to allocate port") + + port := listener.Addr().(*net.TCPAddr).Port + + err = listener.Close() + require.NoError(t, err, "failed to close listener") + + return port +}