From c8296aecb51298afa7de8c423549134e5e4ec7fe Mon Sep 17 00:00:00 2001 From: Ryan Nowak Date: Thu, 2 Jan 2025 11:41:48 -0800 Subject: [PATCH] Add recipe engine Signed-off-by: Ryan Nowak --- cmd/dynamic-rp/dynamicrp-dev.yaml | 2 + .../templates/dynamic-rp/configmaps.yaml | 2 + .../kubernetesclientprovider/types.go | 194 ++++++++++++++++++ pkg/dynamicrp/backend/service.go | 13 +- pkg/dynamicrp/config.go | 4 + pkg/dynamicrp/options.go | 162 +++++++++++---- pkg/dynamicrp/testhost/host.go | 8 + .../processors/resourceclient.go | 31 +-- .../processors/resourceclient_test.go | 41 ++-- pkg/recipes/controllerconfig/config.go | 16 +- pkg/recipes/driver/terraform.go | 6 +- pkg/recipes/terraform/execute.go | 37 +++- test/k8sutil/fake.go | 30 +++ 13 files changed, 456 insertions(+), 90 deletions(-) create mode 100644 pkg/components/kubernetesclient/kubernetesclientprovider/types.go diff --git a/cmd/dynamic-rp/dynamicrp-dev.yaml b/cmd/dynamic-rp/dynamicrp-dev.yaml index 0df8054d64..31ab67d13b 100644 --- a/cmd/dynamic-rp/dynamicrp-dev.yaml +++ b/cmd/dynamic-rp/dynamicrp-dev.yaml @@ -29,6 +29,8 @@ tracerProvider: serviceName: "dynamic-rp" zipkin: url: "http://localhost:9411/api/v2/spans" +kubernetes: + kind: default server: host: "0.0.0.0" port: 8082 diff --git a/deploy/Chart/templates/dynamic-rp/configmaps.yaml b/deploy/Chart/templates/dynamic-rp/configmaps.yaml index 231e81d2f4..7f4c9e0d74 100644 --- a/deploy/Chart/templates/dynamic-rp/configmaps.yaml +++ b/deploy/Chart/templates/dynamic-rp/configmaps.yaml @@ -35,6 +35,8 @@ data: port: 6062 secretProvider: provider: kubernetes + kubernetes: + kind: default server: host: "0.0.0.0" port: 8082 diff --git a/pkg/components/kubernetesclient/kubernetesclientprovider/types.go b/pkg/components/kubernetesclient/kubernetesclientprovider/types.go new file mode 100644 index 0000000000..592978459f --- /dev/null +++ b/pkg/components/kubernetesclient/kubernetesclientprovider/types.go @@ -0,0 +1,194 @@ +/* +Copyright 2023 The Radius Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetesclientprovider + +import ( + "fmt" + + contourv1 "github.com/projectcontour/contour/apis/projectcontour/v1" + "github.com/radius-project/radius/pkg/kubeutil" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + csidriver "sigs.k8s.io/secrets-store-csi-driver/apis/v1alpha1" + + // Import kubernetes auth plugins + _ "k8s.io/client-go/plugin/pkg/client/auth" +) + +const ( + errUnsetMockText = "the Kubernetes config is nil. This is likely a test with improperly set up mocks. Use %s to set the client for testing" + + // Kind + KindDefault ConnectionKind = "default" + KindNone ConnectionKind = "none" +) + +// ConnectionKind is the kind of connection to use for accessing Kubernetes. +type ConnectionKind string + +// Options holds the configuration options for the Kubernetes client provider. +type Options struct { + // Kind is the kind of connection to use. + Kind ConnectionKind `yaml:"kind"` +} + +// FromOptions creates a new Kubernetes client provider from the given options. +func FromOptions(options Options) (*KubernetesClientProvider, error) { + if options.Kind == KindNone { + return FromEmpty(), nil + } else if options.Kind == KindDefault { + config, err := kubeutil.NewClientConfig(&kubeutil.ConfigOptions{ + QPS: kubeutil.DefaultServerQPS, + Burst: kubeutil.DefaultServerBurst, + }) + if err != nil { + return nil, err + } + + return FromConfig(config), nil + } + + return nil, fmt.Errorf("unknown connection kind: %s", options.Kind) +} + +// FromConfig creates a new Kubernetes client provider from the given config. +func FromConfig(config *rest.Config) *KubernetesClientProvider { + return &KubernetesClientProvider{ + config: config, + } +} + +// FromEmpty creates a new Kubernetes client provider with an empty config. +// +// This is useful for testing. Use the Set* methods to set the clients. +func FromEmpty() *KubernetesClientProvider { + return &KubernetesClientProvider{} +} + +// KubernetesClientProvider provides access to Kubernetes clients. +type KubernetesClientProvider struct { + config *rest.Config + + // These clients are all settable for testing purposes. + clientGoClient kubernetes.Interface + discoveryClient discovery.DiscoveryInterface + dynamicClient dynamic.Interface + runtimeClient runtimeclient.Client +} + +// Config returns the Kubernetes client provider's config. +func (k *KubernetesClientProvider) Config() *rest.Config { + return k.config +} + +// ClientGoClient returns a Kubernetes client-go client. +func (k *KubernetesClientProvider) ClientGoClient() (kubernetes.Interface, error) { + if k.clientGoClient != nil { + return k.clientGoClient, nil + } + + config := k.Config() + if config == nil { + return nil, fmt.Errorf(errUnsetMockText, "SetClientGoClient") + } + + return kubernetes.NewForConfig(config) +} + +// SetClientGoClient sets the Kubernetes client-go client. This is useful for testing. +func (k *KubernetesClientProvider) SetClientGoClient(client kubernetes.Interface) { + k.clientGoClient = client +} + +// DiscoveryClient returns a Kubernetes discovery client. +func (k *KubernetesClientProvider) DiscoveryClient() (discovery.DiscoveryInterface, error) { + if k.discoveryClient != nil { + return k.discoveryClient, nil + } + + config := k.Config() + if config == nil { + return nil, fmt.Errorf(errUnsetMockText, "SetDiscoveryClient") + } + + client, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, err + } + + // Use legacy discovery client to avoid the issue of the staled GroupVersion discovery(api.ucp.dev/v1alpha3). + // TODO: Disable UseLegacyDiscovery once https://github.com/radius-project/radius/issues/5974 is resolved. + client.DiscoveryClient.UseLegacyDiscovery = true + return client, nil +} + +// SetDiscoveryClient sets the Kubernetes discovery client. This is useful for testing. +func (k *KubernetesClientProvider) SetDiscoveryClient(client discovery.DiscoveryInterface) { + k.discoveryClient = client +} + +// DynamicClient returns a Kubernetes dynamic client. +func (k *KubernetesClientProvider) DynamicClient() (dynamic.Interface, error) { + if k.dynamicClient != nil { + return k.dynamicClient, nil + } + + config := k.Config() + if config == nil { + return nil, fmt.Errorf(errUnsetMockText, "SetDiscoveryClient") + } + + return dynamic.NewForConfig(config) +} + +// SetDynamicClient sets the Kubernetes dynamic client. This is useful for testing. +func (k *KubernetesClientProvider) SetDynamicClient(client dynamic.Interface) { + k.dynamicClient = client +} + +// RuntimeClient returns a Kubernetes controller runtime client. +func (k *KubernetesClientProvider) RuntimeClient() (runtimeclient.Client, error) { + if k.runtimeClient != nil { + return k.runtimeClient, nil + } + + config := k.Config() + if config == nil { + return nil, fmt.Errorf(errUnsetMockText, "SetRuntimeClient") + } + + scheme := runtime.NewScheme() + + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(csidriver.AddToScheme(scheme)) + utilruntime.Must(apiextv1.AddToScheme(scheme)) + utilruntime.Must(contourv1.AddToScheme(scheme)) + + return runtimeclient.New(k.Config(), runtimeclient.Options{Scheme: scheme}) +} + +// SetRuntimeClient sets the Kubernetes controller runtime client. This is useful for testing. +func (k *KubernetesClientProvider) SetRuntimeClient(client runtimeclient.Client) { + k.runtimeClient = client +} diff --git a/pkg/dynamicrp/backend/service.go b/pkg/dynamicrp/backend/service.go index 834f1aad7c..ce3569b1dc 100644 --- a/pkg/dynamicrp/backend/service.go +++ b/pkg/dynamicrp/backend/service.go @@ -23,14 +23,14 @@ import ( "github.com/radius-project/radius/pkg/armrpc/asyncoperation/worker" "github.com/radius-project/radius/pkg/dynamicrp" - "github.com/radius-project/radius/pkg/recipes/controllerconfig" + "github.com/radius-project/radius/pkg/recipes/engine" ) // Service runs the backend for the dynamic-rp. type Service struct { worker.Service options *dynamicrp.Options - recipes *controllerconfig.RecipeControllerConfig + recipes engine.Engine } // NewService creates a new service to run the dynamic-rp backend. @@ -40,7 +40,7 @@ func NewService(options *dynamicrp.Options) *Service { Service: worker.Service{ // Will be initialized later }, - recipes: options.Recipes, + recipes: nil, // Will be initialized later } } @@ -58,6 +58,13 @@ func (w *Service) Run(ctx context.Context) error { w.Service.Options.MaxOperationRetryCount = *w.options.Config.Worker.MaxOperationRetryCount } + e, err := w.options.RecipeEngine() + if err != nil { + return err + } + + w.recipes = e + databaseClient, err := w.options.DatabaseProvider.GetClient(ctx) if err != nil { return err diff --git a/pkg/dynamicrp/config.go b/pkg/dynamicrp/config.go index 8a658ae039..64669670e4 100644 --- a/pkg/dynamicrp/config.go +++ b/pkg/dynamicrp/config.go @@ -21,6 +21,7 @@ import ( "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/components/database/databaseprovider" + "github.com/radius-project/radius/pkg/components/kubernetesclient/kubernetesclientprovider" "github.com/radius-project/radius/pkg/components/metrics/metricsservice" "github.com/radius-project/radius/pkg/components/profiler/profilerservice" "github.com/radius-project/radius/pkg/components/queue/queueprovider" @@ -44,6 +45,9 @@ type Config struct { // Environment is the configuration for the hosting environment. Environment hostoptions.EnvironmentOptions `yaml:"environment"` + // Kubernetes is the configuration for the Kubernetes client. + Kubernetes kubernetesclientprovider.Options `yaml:"kubernetes"` + // Logging is the configuration for the logging system. Logging ucplog.LoggingOptions `yaml:"logging"` diff --git a/pkg/dynamicrp/options.go b/pkg/dynamicrp/options.go index ac32ce8f62..00ca8121cb 100644 --- a/pkg/dynamicrp/options.go +++ b/pkg/dynamicrp/options.go @@ -18,17 +18,26 @@ package dynamicrp import ( "context" + "errors" "fmt" + "strconv" "github.com/radius-project/radius/pkg/armrpc/asyncoperation/statusmanager" + "github.com/radius-project/radius/pkg/azure/armauth" + aztoken "github.com/radius-project/radius/pkg/azure/tokencredentials" "github.com/radius-project/radius/pkg/components/database/databaseprovider" + "github.com/radius-project/radius/pkg/components/kubernetesclient/kubernetesclientprovider" "github.com/radius-project/radius/pkg/components/queue/queueprovider" "github.com/radius-project/radius/pkg/components/secret/secretprovider" - "github.com/radius-project/radius/pkg/kubeutil" - "github.com/radius-project/radius/pkg/recipes/controllerconfig" + "github.com/radius-project/radius/pkg/portableresources/processors" + "github.com/radius-project/radius/pkg/recipes" + "github.com/radius-project/radius/pkg/recipes/configloader" + "github.com/radius-project/radius/pkg/recipes/driver" + "github.com/radius-project/radius/pkg/recipes/engine" "github.com/radius-project/radius/pkg/sdk" + "github.com/radius-project/radius/pkg/sdk/clients" ucpconfig "github.com/radius-project/radius/pkg/ucp/config" - kube_rest "k8s.io/client-go/rest" + sdk_cred "github.com/radius-project/radius/pkg/ucp/credentials" ) // Options holds the configuration options and shared services for the DyanmicRP server. @@ -42,11 +51,14 @@ type Options struct { // DatabaseProvider provides access to the database. DatabaseProvider *databaseprovider.DatabaseProvider + // KubernetesProvider provides access to the Kubernetes clients. + KubernetesProvider *kubernetesclientprovider.KubernetesClientProvider + // QueueProvider provides access to the message queue client. QueueProvider *queueprovider.QueueProvider - // Recipes is the configuration for the recipe subsystem. - Recipes *controllerconfig.RecipeControllerConfig + // Recipes is the configuration for the recipe engine subsystem. + Recipes RecipeOptions // SecretProvider provides access to the secret storage system. SecretProvider *secretprovider.SecretProvider @@ -58,6 +70,19 @@ type Options struct { UCP sdk.Connection } +// RecipeOptions holds the configuration options for the recipe engine subsystem. +type RecipeOptions struct { + // ConfigurationLoader is the loader for recipe configurations. + ConfigurationLoader configloader.ConfigurationLoader + + // Drivers is a map of recipe driver names to driver constructors. If nil, the default drivers are used (Bicep, Terraform) will + // be used. + Drivers map[string]func(options *Options) (driver.Driver, error) + + // SecretsLoader provides access to secrets for recipes. + SecretsLoader configloader.SecretsLoader +} + // NewOptions creates a new Options instance from the given configuration. func NewOptions(ctx context.Context, config *Config) (*Options, error) { var err error @@ -81,46 +106,107 @@ func NewOptions(ctx context.Context, config *Config) (*Options, error) { options.StatusManager = statusmanager.New(databaseClient, queueClient, config.Environment.RoleLocation) - var cfg *kube_rest.Config - if config.UCP.Kind == ucpconfig.UCPConnectionKindKubernetes { - cfg, err = kubeutil.NewClientConfig(&kubeutil.ConfigOptions{ - // TODO: Allow to use custom context via configuration. - https://github.com/radius-project/radius/issues/5433 - ContextName: "", - QPS: kubeutil.DefaultServerQPS, - Burst: kubeutil.DefaultServerBurst, - }) - if err != nil { - return nil, fmt.Errorf("failed to get kubernetes config: %w", err) - } + options.KubernetesProvider, err = kubernetesclientprovider.FromOptions(config.Kubernetes) + if err != nil { + return nil, err } - options.UCP, err = ucpconfig.NewConnectionFromUCPConfig(&config.UCP, cfg) + options.UCP, err = ucpconfig.NewConnectionFromUCPConfig(&config.UCP, options.KubernetesProvider.Config()) if err != nil { return nil, err } - // TODO: This is the right place to initialize the recipe infrastructure. Unfortunately this - // has a dependency on Kubernetes right now, which isn't available for integration tests. - // - // We have a future work item to untangle this dependency and then this code can be uncommented. - // For now this is a placeholder/reminder of the code we need, and where to put it. - // - // The recipe infrastructure is tied to corerp's dependencies, so we need to create it here. - // recipes, err := controllerconfig.New(hostoptions.HostOptions{ - // Config: &hostoptions.ProviderConfig{ - // Bicep: config.Bicep, - // Env: config.Environment, - // Terraform: config.Terraform, - // UCP: config.UCP, - // }, - // K8sConfig: cfg, - // UCPConnection: options.UCP, - // }) - // if err != nil { - // return nil, err - // } + options.Recipes.ConfigurationLoader = configloader.NewEnvironmentLoader(sdk.NewClientOptions(options.UCP)) + options.Recipes.SecretsLoader = configloader.NewSecretStoreLoader(sdk.NewClientOptions(options.UCP)) + + // If this is set to nil, then the service will use the default recipe drivers. // - // options.Recipes = recipes + // This pattern allows us to override the drivers for testing. + options.Recipes.Drivers = nil return &options, nil } + +// RecipeEngine creates a new recipe engine from the options. +func (o *Options) RecipeEngine() (engine.Engine, error) { + var errs error + drivers := map[string]driver.Driver{} + + // Use the default drivers if not otherwise specified. + if o.Recipes.Drivers == nil { + o.Recipes.Drivers = map[string]func(options *Options) (driver.Driver, error){ + recipes.TemplateKindBicep: bicepDriver, + recipes.TemplateKindTerraform: terraformDriver, + } + } + + for name, driverConstructor := range o.Recipes.Drivers { + driver, err := driverConstructor(o) + if err != nil { + errs = errors.Join(errs, err) + continue + } + + drivers[name] = driver + } + + if errs != nil { + return nil, fmt.Errorf("failed to create recipe drivers: %w", errs) + } + + return engine.NewEngine(engine.Options{ + ConfigurationLoader: o.Recipes.ConfigurationLoader, + SecretsLoader: o.Recipes.SecretsLoader, + Drivers: drivers}), nil +} + +func bicepDriver(options *Options) (driver.Driver, error) { + deploymentEngineClient, err := clients.NewResourceDeploymentsClient(&clients.Options{ + Cred: &aztoken.AnonymousCredential{}, + BaseURI: options.UCP.Endpoint(), + ARMClientOptions: sdk.NewClientOptions(options.UCP), + }) + if err != nil { + return nil, err + } + + provider, err := sdk_cred.NewAzureCredentialProvider(options.SecretProvider, options.UCP, &aztoken.AnonymousCredential{}) + if err != nil { + return nil, err + } + + armConfig, err := armauth.NewArmConfig(&armauth.Options{CredentialProvider: provider}) + if err != nil { + return nil, err + } + + resourceClient := processors.NewResourceClient(armConfig, options.UCP, options.KubernetesProvider) + + bicepDeleteRetryCount, err := strconv.Atoi(options.Config.Bicep.DeleteRetryCount) + if err != nil { + return nil, err + } + + bicepDeleteRetryDeleteSeconds, err := strconv.Atoi(options.Config.Bicep.DeleteRetryDelaySeconds) + if err != nil { + return nil, err + } + + return driver.NewBicepDriver( + sdk.NewClientOptions(options.UCP), + deploymentEngineClient, + resourceClient, + driver.BicepOptions{ + DeleteRetryCount: bicepDeleteRetryCount, + DeleteRetryDelaySeconds: bicepDeleteRetryDeleteSeconds, + }), nil +} + +func terraformDriver(options *Options) (driver.Driver, error) { + return driver.NewTerraformDriver( + options.UCP, + options.SecretProvider, + driver.TerraformOptions{ + Path: options.Config.Terraform.Path, + }, *options.KubernetesProvider), nil +} diff --git a/pkg/dynamicrp/testhost/host.go b/pkg/dynamicrp/testhost/host.go index ab01efdc01..6387b37d99 100644 --- a/pkg/dynamicrp/testhost/host.go +++ b/pkg/dynamicrp/testhost/host.go @@ -26,11 +26,13 @@ import ( v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" "github.com/radius-project/radius/pkg/armrpc/hostoptions" "github.com/radius-project/radius/pkg/components/database/databaseprovider" + "github.com/radius-project/radius/pkg/components/kubernetesclient/kubernetesclientprovider" "github.com/radius-project/radius/pkg/components/queue/queueprovider" "github.com/radius-project/radius/pkg/components/secret/secretprovider" "github.com/radius-project/radius/pkg/components/testhost" "github.com/radius-project/radius/pkg/dynamicrp" "github.com/radius-project/radius/pkg/dynamicrp/server" + "github.com/radius-project/radius/pkg/recipes/driver" "github.com/radius-project/radius/pkg/sdk" "github.com/radius-project/radius/pkg/ucp" "github.com/radius-project/radius/pkg/ucp/config" @@ -71,6 +73,9 @@ func Start(t *testing.T, opts ...TestHostOption) (*TestHost, *ucptesthost.TestHo Name: "test", RoleLocation: v1.LocationGlobal, }, + Kubernetes: kubernetesclientprovider.Options{ + Kind: kubernetesclientprovider.KindNone, + }, Queue: queueprovider.QueueProviderOptions{ Provider: queueprovider.TypeInmemory, Name: "dynamic-rp", @@ -92,6 +97,9 @@ func Start(t *testing.T, opts ...TestHostOption) (*TestHost, *ucptesthost.TestHo options, err := dynamicrp.NewOptions(context.Background(), config) require.NoError(t, err) + // Prevent the default recipe drivers from being registered. + options.Recipes.Drivers = map[string]func(options *dynamicrp.Options) (driver.Driver, error){} + for _, opt := range opts { opt.Apply(options) } diff --git a/pkg/portableresources/processors/resourceclient.go b/pkg/portableresources/processors/resourceclient.go index f53140d5a0..6827a52a18 100644 --- a/pkg/portableresources/processors/resourceclient.go +++ b/pkg/portableresources/processors/resourceclient.go @@ -28,6 +28,7 @@ import ( aztoken "github.com/radius-project/radius/pkg/azure/tokencredentials" "github.com/radius-project/radius/pkg/cli/clients" "github.com/radius-project/radius/pkg/cli/clients_new/generated" + "github.com/radius-project/radius/pkg/components/kubernetesclient/kubernetesclientprovider" "github.com/radius-project/radius/pkg/components/trace" "github.com/radius-project/radius/pkg/sdk" "github.com/radius-project/radius/pkg/ucp/resources" @@ -38,7 +39,6 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/discovery" runtime_client "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -51,16 +51,13 @@ type resourceClient struct { // connection is the connection to use for UCP resources. Override this for testing. connection sdk.Connection - // k8sClient is the Kubernetes client used to delete Kubernetes resources. Override this for testing. - k8sClient runtime_client.Client - - // k8sDiscoveryClient is the Kubernetes client to used for API version lookups on Kubernetes resources. Override this for testing. - k8sDiscoveryClient discovery.ServerResourcesInterface + // kubernetesClient is the Kubernetes client provider used to create Kubernetes clients. Override this for testing. + kubernetesClient *kubernetesclientprovider.KubernetesClientProvider } // NewResourceClient creates a new resourceClient instance with the given parameters. -func NewResourceClient(arm *armauth.ArmConfig, connection sdk.Connection, k8sClient runtime_client.Client, k8sDiscoveryClient discovery.ServerResourcesInterface) *resourceClient { - return &resourceClient{arm: arm, connection: connection, k8sClient: k8sClient, k8sDiscoveryClient: k8sDiscoveryClient} +func NewResourceClient(arm *armauth.ArmConfig, connection sdk.Connection, kubernetesClient *kubernetesclientprovider.KubernetesClientProvider) *resourceClient { + return &resourceClient{arm: arm, connection: connection, kubernetesClient: kubernetesClient} } // Delete attempts to delete a resource, either through UCP, Azure, or Kubernetes, depending on the resource type. @@ -234,7 +231,12 @@ func (c *resourceClient) deleteKubernetesResource(ctx context.Context, id resour }, } - err = runtime_client.IgnoreNotFound(c.k8sClient.Delete(ctx, &obj)) + runtimeClient, err := c.kubernetesClient.RuntimeClient() + if err != nil { + return err + } + + err = runtime_client.IgnoreNotFound(runtimeClient.Delete(ctx, &obj)) if err != nil { return err } @@ -245,14 +247,19 @@ func (c *resourceClient) deleteKubernetesResource(ctx context.Context, id resour func (c *resourceClient) lookupKubernetesAPIVersion(id resources.ID) (string, error) { group, kind, namespace, _ := resources_kubernetes.ToParts(id) var resourceLists []*v1.APIResourceList - var err error + + discoveryClient, err := c.kubernetesClient.DiscoveryClient() + if err != nil { + return "", err + } + if namespace == "" { - resourceLists, err = c.k8sDiscoveryClient.ServerPreferredResources() + resourceLists, err = discoveryClient.ServerPreferredResources() if err != nil { return "", fmt.Errorf("could not find API version for type %q: %w", id.Type(), err) } } else { - resourceLists, err = c.k8sDiscoveryClient.ServerPreferredNamespacedResources() + resourceLists, err = discoveryClient.ServerPreferredNamespacedResources() if err != nil { return "", fmt.Errorf("could not find API version for type %q: %w", id.Type(), err) } diff --git a/pkg/portableresources/processors/resourceclient_test.go b/pkg/portableresources/processors/resourceclient_test.go index 447400e511..3cb9cb1269 100644 --- a/pkg/portableresources/processors/resourceclient_test.go +++ b/pkg/portableresources/processors/resourceclient_test.go @@ -31,6 +31,7 @@ import ( "github.com/radius-project/radius/pkg/azure/armauth" "github.com/radius-project/radius/pkg/azure/clientv2" aztoken "github.com/radius-project/radius/pkg/azure/tokencredentials" + "github.com/radius-project/radius/pkg/components/kubernetesclient/kubernetesclientprovider" "github.com/radius-project/radius/pkg/sdk" "github.com/radius-project/radius/pkg/to" "github.com/radius-project/radius/test/k8sutil" @@ -51,7 +52,7 @@ const ( ) func Test_Delete_InvalidResourceID(t *testing.T) { - c := NewResourceClient(nil, nil, nil, nil) + c := NewResourceClient(nil, nil, nil) err := c.Delete(context.Background(), "invalid") require.Error(t, err) } @@ -68,7 +69,7 @@ func Test_Delete_ARM(t *testing.T) { server := httptest.NewServer(mux) defer server.Close() - c := NewResourceClient(newArmOptions(server.URL), nil, nil, nil) + c := NewResourceClient(newArmOptions(server.URL), nil, nil) c.armClientOptions = newClientOptions(server.Client(), server.URL) err := c.Delete(context.Background(), ARMResourceID) @@ -96,7 +97,7 @@ func Test_Delete_ARM(t *testing.T) { server := httptest.NewServer(mux) defer server.Close() - c := NewResourceClient(newArmOptions(server.URL), nil, nil, nil) + c := NewResourceClient(newArmOptions(server.URL), nil, nil) c.armClientOptions = newClientOptions(server.Client(), server.URL) err := c.Delete(context.Background(), ARMResourceID) @@ -119,7 +120,7 @@ func Test_Delete_ARM(t *testing.T) { server := httptest.NewServer(mux) defer server.Close() - c := NewResourceClient(newArmOptions(server.URL), nil, nil, nil) + c := NewResourceClient(newArmOptions(server.URL), nil, nil) c.armClientOptions = newClientOptions(server.Client(), server.URL) err := c.Delete(context.Background(), ARMResourceID) @@ -146,7 +147,7 @@ func Test_Delete_ARM(t *testing.T) { server := httptest.NewServer(mux) defer server.Close() - c := NewResourceClient(newArmOptions(server.URL), nil, nil, nil) + c := NewResourceClient(newArmOptions(server.URL), nil, nil) c.armClientOptions = newClientOptions(server.Client(), server.URL) err := c.Delete(context.Background(), ARMResourceID) @@ -160,7 +161,7 @@ func Test_Delete_ARM(t *testing.T) { server := httptest.NewServer(mux) defer server.Close() - c := NewResourceClient(newArmOptions(server.URL), nil, nil, nil) + c := NewResourceClient(newArmOptions(server.URL), nil, nil) c.armClientOptions = newClientOptions(server.Client(), server.URL) err := c.Delete(context.Background(), ARMResourceID) @@ -178,7 +179,7 @@ func Test_Delete_ARM(t *testing.T) { server := httptest.NewServer(mux) defer server.Close() - c := NewResourceClient(newArmOptions(server.URL), nil, nil, nil) + c := NewResourceClient(newArmOptions(server.URL), nil, nil) c.armClientOptions = newClientOptions(server.Client(), server.URL) err := c.Delete(context.Background(), ARMResourceID) @@ -202,7 +203,7 @@ func Test_Delete_ARM(t *testing.T) { server := httptest.NewServer(mux) defer server.Close() - c := NewResourceClient(newArmOptions(server.URL), nil, nil, nil) + c := NewResourceClient(newArmOptions(server.URL), nil, nil) c.armClientOptions = newClientOptions(server.Client(), server.URL) err := c.Delete(context.Background(), ARMResourceID) @@ -238,7 +239,11 @@ func Test_Delete_Kubernetes(t *testing.T) { }, } - c := NewResourceClient(nil, nil, client, dc) + kcp := kubernetesclientprovider.FromEmpty() + kcp.SetRuntimeClient(client) + kcp.SetDiscoveryClient(dc) + + c := NewResourceClient(nil, nil, kcp) err := c.Delete(context.Background(), KubernetesCoreGroupResourceID) require.NoError(t, err) @@ -266,7 +271,11 @@ func Test_Delete_Kubernetes(t *testing.T) { }, } - c := NewResourceClient(nil, nil, client, dc) + kcp := kubernetesclientprovider.FromEmpty() + kcp.SetRuntimeClient(client) + kcp.SetDiscoveryClient(dc) + + c := NewResourceClient(nil, nil, kcp) err := c.Delete(context.Background(), KubernetesCoreGroupResourceID) require.NoError(t, err) @@ -284,7 +293,11 @@ func Test_Delete_Kubernetes(t *testing.T) { Resources: []*metav1.APIResourceList{}, } - c := NewResourceClient(nil, nil, client, dc) + kcp := kubernetesclientprovider.FromEmpty() + kcp.SetRuntimeClient(client) + kcp.SetDiscoveryClient(dc) + + c := NewResourceClient(nil, nil, kcp) err := c.Delete(context.Background(), KubernetesCoreGroupResourceID) require.Error(t, err) @@ -303,7 +316,7 @@ func Test_Delete_UCP(t *testing.T) { connection, err := sdk.NewDirectConnection(server.URL) require.NoError(t, err) - c := NewResourceClient(nil, connection, nil, nil) + c := NewResourceClient(nil, connection, nil) err = c.Delete(context.Background(), AWSResourceID) require.NoError(t, err) @@ -319,7 +332,7 @@ func Test_Delete_UCP(t *testing.T) { connection, err := sdk.NewDirectConnection(server.URL) require.NoError(t, err) - c := NewResourceClient(nil, connection, nil, nil) + c := NewResourceClient(nil, connection, nil) err = c.Delete(context.Background(), AWSResourceID) require.NoError(t, err) @@ -339,7 +352,7 @@ func Test_Delete_UCP(t *testing.T) { connection, err := sdk.NewDirectConnection(server.URL) require.NoError(t, err) - c := NewResourceClient(nil, connection, nil, nil) + c := NewResourceClient(nil, connection, nil) err = c.Delete(context.Background(), AWSResourceID) require.Error(t, err) diff --git a/pkg/recipes/controllerconfig/config.go b/pkg/recipes/controllerconfig/config.go index 40fbd98a2d..6ae1e3a82d 100644 --- a/pkg/recipes/controllerconfig/config.go +++ b/pkg/recipes/controllerconfig/config.go @@ -21,8 +21,8 @@ import ( "github.com/radius-project/radius/pkg/armrpc/hostoptions" aztoken "github.com/radius-project/radius/pkg/azure/tokencredentials" + "github.com/radius-project/radius/pkg/components/kubernetesclient/kubernetesclientprovider" "github.com/radius-project/radius/pkg/components/secret/secretprovider" - "github.com/radius-project/radius/pkg/kubeutil" "github.com/radius-project/radius/pkg/portableresources/processors" "github.com/radius-project/radius/pkg/recipes" "github.com/radius-project/radius/pkg/recipes/configloader" @@ -34,8 +34,8 @@ import ( // RecipeControllerConfig is the configuration for the controllers which uses recipe. type RecipeControllerConfig struct { - // K8sClients is the collections of Kubernetes clients. - K8sClients *kubeutil.Clients + // Kubernetes provides access to the Kubernetes clients. + Kubernetes *kubernetesclientprovider.KubernetesClientProvider // ResourceClient is a client used by resource processors for interacting with UCP resources. ResourceClient processors.ResourceClient @@ -57,14 +57,12 @@ type RecipeControllerConfig struct { func New(options hostoptions.HostOptions) (*RecipeControllerConfig, error) { cfg := &RecipeControllerConfig{} var err error - cfg.K8sClients, err = kubeutil.NewClients(options.K8sConfig) - if err != nil { - return nil, err - } + + cfg.Kubernetes = kubernetesclientprovider.FromConfig(options.K8sConfig) cfg.UCPConnection = &options.UCPConnection - cfg.ResourceClient = processors.NewResourceClient(options.Arm, options.UCPConnection, cfg.K8sClients.RuntimeClient, cfg.K8sClients.DiscoveryClient) + cfg.ResourceClient = processors.NewResourceClient(options.Arm, options.UCPConnection, cfg.Kubernetes) clientOptions := sdk.NewClientOptions(options.UCPConnection) cfg.DeploymentEngineClient, err = clients.NewResourceDeploymentsClient(&clients.Options{ @@ -111,7 +109,7 @@ func New(options hostoptions.HostOptions) (*RecipeControllerConfig, error) { recipes.TemplateKindTerraform: driver.NewTerraformDriver(options.UCPConnection, secretprovider.NewSecretProvider(options.Config.SecretProvider), driver.TerraformOptions{ Path: options.Config.Terraform.Path, - }, cfg.K8sClients.ClientSet), + }, *cfg.Kubernetes), }, }) diff --git a/pkg/recipes/driver/terraform.go b/pkg/recipes/driver/terraform.go index 72387a4b8f..6f2aec5778 100644 --- a/pkg/recipes/driver/terraform.go +++ b/pkg/recipes/driver/terraform.go @@ -27,10 +27,10 @@ import ( "github.com/google/uuid" v1 "github.com/radius-project/radius/pkg/armrpc/api/v1" + "github.com/radius-project/radius/pkg/components/kubernetesclient/kubernetesclientprovider" "github.com/radius-project/radius/pkg/components/secret/secretprovider" rpv1 "github.com/radius-project/radius/pkg/rp/v1" "golang.org/x/exp/slices" - "k8s.io/client-go/kubernetes" "github.com/radius-project/radius/pkg/recipes" "github.com/radius-project/radius/pkg/recipes/terraform" @@ -48,9 +48,9 @@ import ( var _ Driver = (*terraformDriver)(nil) // NewTerraformDriver creates a new instance of driver to execute a Terraform recipe. -func NewTerraformDriver(ucpConn sdk.Connection, secretProvider *secretprovider.SecretProvider, options TerraformOptions, k8sClientSet kubernetes.Interface) Driver { +func NewTerraformDriver(ucpConn sdk.Connection, secretProvider *secretprovider.SecretProvider, options TerraformOptions, kubernetesClients kubernetesclientprovider.KubernetesClientProvider) Driver { return &terraformDriver{ - terraformExecutor: terraform.NewExecutor(ucpConn, secretProvider, k8sClientSet), + terraformExecutor: terraform.NewExecutor(ucpConn, secretProvider, kubernetesClients), options: options, } } diff --git a/pkg/recipes/terraform/execute.go b/pkg/recipes/terraform/execute.go index 623cd827af..5b85170a0a 100644 --- a/pkg/recipes/terraform/execute.go +++ b/pkg/recipes/terraform/execute.go @@ -27,6 +27,7 @@ import ( install "github.com/hashicorp/hc-install" "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" + "github.com/radius-project/radius/pkg/components/kubernetesclient/kubernetesclientprovider" "github.com/radius-project/radius/pkg/components/metrics" "github.com/radius-project/radius/pkg/components/secret/secretprovider" "github.com/radius-project/radius/pkg/recipes/recipecontext" @@ -37,7 +38,6 @@ import ( "github.com/radius-project/radius/pkg/ucp/ucplog" "go.opentelemetry.io/otel/attribute" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" ) var ( @@ -48,8 +48,8 @@ var ( var _ TerraformExecutor = (*executor)(nil) // NewExecutor creates a new Executor with the given UCP connection and secret provider, to execute a Terraform recipe. -func NewExecutor(ucpConn sdk.Connection, secretProvider *secretprovider.SecretProvider, k8sClientSet kubernetes.Interface) *executor { - return &executor{ucpConn: ucpConn, secretProvider: secretProvider, k8sClientSet: k8sClientSet} +func NewExecutor(ucpConn sdk.Connection, secretProvider *secretprovider.SecretProvider, kubernetesClients kubernetesclientprovider.KubernetesClientProvider) *executor { + return &executor{ucpConn: ucpConn, secretProvider: secretProvider, kubernetesClients: kubernetesClients} } type executor struct { @@ -59,8 +59,8 @@ type executor struct { // secretProvider is the secret store provider used for managing credentials in UCP. secretProvider *secretprovider.SecretProvider - // k8sClientSet is the Kubernetes client. - k8sClientSet kubernetes.Interface + // kubernetesClients provides access to the Kubernetes clients. + kubernetesClients kubernetesclientprovider.KubernetesClientProvider } // Deploy installs Terraform, creates a working directory, generates a config, and runs Terraform init and @@ -104,7 +104,12 @@ func (e *executor) Deploy(ctx context.Context, options Options) (*tfjson.State, // Validate that the terraform state file backend source exists. // Currently only Kubernetes secret backend is supported, which is created by Terraform as a part of Terraform apply. - backendExists, err := backends.NewKubernetesBackend(e.k8sClientSet).ValidateBackendExists(ctx, backends.KubernetesBackendNamePrefix+kubernetesBackendSuffix) + kubernetesClient, err := e.kubernetesClients.ClientGoClient() + if err != nil { + return nil, fmt.Errorf("error getting kubernetes client: %w", err) + } + + backendExists, err := backends.NewKubernetesBackend(kubernetesClient).ValidateBackendExists(ctx, backends.KubernetesBackendNamePrefix+kubernetesBackendSuffix) if err != nil { return nil, fmt.Errorf("error retrieving kubernetes secret for terraform state: %w", err) } else if !backendExists { @@ -142,7 +147,12 @@ func (e *executor) Delete(ctx context.Context, options Options) error { // Before running terraform init and destroy, ensure that the Terraform state file storage source exists. // If the state file source has been deleted or wasn't created due to a failure during apply then // terraform initialization will fail due to missing backend source. - backendExists, err := backends.NewKubernetesBackend(e.k8sClientSet).ValidateBackendExists(ctx, backends.KubernetesBackendNamePrefix+kubernetesBackendSuffix) + kubernetesClient, err := e.kubernetesClients.ClientGoClient() + if err != nil { + return fmt.Errorf("error getting kubernetes client: %w", err) + } + + backendExists, err := backends.NewKubernetesBackend(kubernetesClient).ValidateBackendExists(ctx, backends.KubernetesBackendNamePrefix+kubernetesBackendSuffix) if err != nil { // Continue with the delete flow for all errors other than backend not found. // If it is an intermittent error then the delete flow will fail and should be retried from the client. @@ -160,7 +170,7 @@ func (e *executor) Delete(ctx context.Context, options Options) error { } // Delete the kubernetes secret created for terraform state file. - err = e.k8sClientSet.CoreV1(). + err = kubernetesClient.CoreV1(). Secrets(backends.RadiusNamespace). Delete(ctx, backends.KubernetesBackendNamePrefix+kubernetesBackendSuffix, metav1.DeleteOptions{}) if err != nil { @@ -214,14 +224,14 @@ func (e executor) setEnvironmentVariables(tf *tfexec.Terraform, options Options) recipeConfig := &options.EnvConfig.RecipeConfig var envVarUpdate bool - if recipeConfig != nil && recipeConfig.Env.AdditionalProperties != nil && len(recipeConfig.Env.AdditionalProperties) > 0 { + if len(recipeConfig.Env.AdditionalProperties) > 0 { envVarUpdate = true for key, value := range recipeConfig.Env.AdditionalProperties { envVars[key] = value } } - if recipeConfig != nil && recipeConfig.EnvSecrets != nil && len(recipeConfig.EnvSecrets) > 0 { + if len(recipeConfig.EnvSecrets) > 0 { for secretName, secretReference := range recipeConfig.EnvSecrets { // Extract secret value from the secrets input if secretData, ok := options.Secrets[secretReference.Source]; ok { @@ -282,7 +292,12 @@ func (e *executor) generateConfig(ctx context.Context, tf *tfexec.Terraform, opt return "", err } - backendConfig, err := tfConfig.AddTerraformBackend(options.ResourceRecipe, backends.NewKubernetesBackend(e.k8sClientSet)) + kubernetesClient, err := e.kubernetesClients.ClientGoClient() + if err != nil { + return "", fmt.Errorf("error getting kubernetes client: %w", err) + } + + backendConfig, err := tfConfig.AddTerraformBackend(options.ResourceRecipe, backends.NewKubernetesBackend(kubernetesClient)) if err != nil { return "", err } diff --git a/test/k8sutil/fake.go b/test/k8sutil/fake.go index ea45baba19..9faaa0d1bf 100644 --- a/test/k8sutil/fake.go +++ b/test/k8sutil/fake.go @@ -19,15 +19,20 @@ package k8sutil import ( "context" + openapi_v2 "github.com/google/gnostic-models/openapiv2" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/version" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/openapi" + "k8s.io/client-go/rest" clienttesting "k8s.io/client-go/testing" ) @@ -105,6 +110,31 @@ type DiscoveryClient struct { APIGroup []*metav1.APIGroup } +// OpenAPISchema implements discovery.DiscoveryInterface. +func (d *DiscoveryClient) OpenAPISchema() (*openapi_v2.Document, error) { + panic("unimplemented") +} + +// OpenAPIV3 implements discovery.DiscoveryInterface. +func (d *DiscoveryClient) OpenAPIV3() openapi.Client { + panic("unimplemented") +} + +// RESTClient implements discovery.DiscoveryInterface. +func (d *DiscoveryClient) RESTClient() rest.Interface { + panic("unimplemented") +} + +// ServerVersion implements discovery.DiscoveryInterface. +func (d *DiscoveryClient) ServerVersion() (*version.Info, error) { + panic("unimplemented") +} + +// WithLegacy implements discovery.DiscoveryInterface. +func (d *DiscoveryClient) WithLegacy() discovery.DiscoveryInterface { + panic("unimplemented") +} + // ServerGroups returns a list of API groups supported by the server. func (d *DiscoveryClient) ServerGroups() (*metav1.APIGroupList, error) { return d.Groups, nil