From f5693f6bc1bba4d770e0fea9d9e2b64617f2140b Mon Sep 17 00:00:00 2001 From: sh2 Date: Wed, 11 Sep 2024 10:22:35 +0800 Subject: [PATCH] feat: add file resource provider for EG standalone mode (#3159) * add validations for envoy-gateway file resource type Signed-off-by: shawnh2 * improve eg validation and add resource provider interface for various provider Signed-off-by: shawnh2 * extract common gatewayapi layer translate logic in egctl translate Signed-off-by: shawnh2 * add notifier support Signed-off-by: shawnh2 * fix lint and move read yaml bytes function back to translate Signed-off-by: shawnh2 * add resources store support Signed-off-by: shawnh2 * fix lint Signed-off-by: shawnh2 * fix ci Signed-off-by: shawnh2 * update infra provider api and address comments Signed-off-by: shawnh2 * update custom provider comments and validate method test Signed-off-by: shawnh2 * restore extension manager and add health probe server for file provider Signed-off-by: shawnh2 * update envoy gateway helper functions Signed-off-by: shawnh2 * add some unit tests Signed-off-by: shawnh2 * properly handle the remove event for the file provider Signed-off-by: shawnh2 * fix lint Signed-off-by: shawnh2 * no default to k8s for infra provider Signed-off-by: shawnh2 * fix runner Signed-off-by: shawnh2 --------- Signed-off-by: shawnh2 Co-authored-by: Xunzhuo --- api/v1alpha1/envoygateway_helpers.go | 1 + api/v1alpha1/envoygateway_types.go | 12 +- api/v1alpha1/shared_types.go | 7 +- .../validation/envoygateway_validate.go | 52 +++ .../validation/envoygateway_validate_test.go | 168 ++++++++- .../validation/envoyproxy_validate_test.go | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 6 +- .../gateway.envoyproxy.io_envoyproxies.yaml | 1 + go.mod | 2 +- internal/cmd/server.go | 35 +- internal/infrastructure/manager.go | 16 +- internal/infrastructure/runner/runner.go | 9 +- internal/provider/file/file.go | 131 +++++++ internal/provider/file/notifier.go | 316 +++++++++++++++++ internal/provider/file/path.go | 47 +++ internal/provider/file/path_test.go | 54 +++ internal/provider/file/resources.go | 335 ++++++++++++++++++ internal/provider/file/store.go | 78 ++++ internal/provider/file/testdata/paths/dir/bar | 1 + internal/provider/file/testdata/paths/foo | 1 + internal/provider/kubernetes/kubernetes.go | 5 + internal/provider/resource_provider.go | 20 ++ internal/provider/runner/runner.go | 69 +++- internal/provider/runner/runner_test.go | 67 ---- site/content/en/latest/api/extension_types.md | 10 +- site/content/zh/latest/api/extension_types.md | 10 +- 26 files changed, 1330 insertions(+), 125 deletions(-) create mode 100644 internal/provider/file/file.go create mode 100644 internal/provider/file/notifier.go create mode 100644 internal/provider/file/path.go create mode 100644 internal/provider/file/path_test.go create mode 100644 internal/provider/file/resources.go create mode 100644 internal/provider/file/store.go create mode 100644 internal/provider/file/testdata/paths/dir/bar create mode 100644 internal/provider/file/testdata/paths/foo create mode 100644 internal/provider/resource_provider.go delete mode 100644 internal/provider/runner/runner_test.go diff --git a/api/v1alpha1/envoygateway_helpers.go b/api/v1alpha1/envoygateway_helpers.go index 0b1faf7e66a..fed2f6fa075 100644 --- a/api/v1alpha1/envoygateway_helpers.go +++ b/api/v1alpha1/envoygateway_helpers.go @@ -237,6 +237,7 @@ func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubern if r.Kubernetes.ShutdownManager == nil { r.Kubernetes.ShutdownManager = &ShutdownManager{Image: ptr.To(DefaultShutdownManagerImage)} } + return r.Kubernetes } diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 6e84d07ba7e..6cf8e334182 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -174,7 +174,7 @@ type ExtensionAPISettings struct { // EnvoyGatewayProvider defines the desired configuration of a provider. // +union type EnvoyGatewayProvider struct { - // Type is the type of provider to use. Supported types are "Kubernetes". + // Type is the type of provider to use. Supported types are "Kubernetes", "Custom". // // +unionDiscriminator Type ProviderType `json:"type"` @@ -186,7 +186,7 @@ type EnvoyGatewayProvider struct { Kubernetes *EnvoyGatewayKubernetesProvider `json:"kubernetes,omitempty"` // Custom defines the configuration for the Custom provider. This provider - // allows you to define a specific resource provider and a infrastructure + // allows you to define a specific resource provider and an infrastructure // provider. // // +optional @@ -271,7 +271,11 @@ type EnvoyGatewayCustomProvider struct { // This provider is used to specify the provider to be used // to provide an environment to deploy the out resources like // the Envoy Proxy data plane. - Infrastructure EnvoyGatewayInfrastructureProvider `json:"infrastructure"` + // + // Infrastructure is optional, if provider is not specified, + // No infrastructure provider is available. + // +optional + Infrastructure *EnvoyGatewayInfrastructureProvider `json:"infrastructure,omitempty"` } // ResourceProviderType defines the types of custom resource providers supported by Envoy Gateway. @@ -300,7 +304,7 @@ type EnvoyGatewayResourceProvider struct { // EnvoyGatewayFileResourceProvider defines configuration for the File Resource provider. type EnvoyGatewayFileResourceProvider struct { // Paths are the paths to a directory or file containing the resource configuration. - // Recursive sub directories are not currently supported. + // Recursive subdirectories are not currently supported. Paths []string `json:"paths"` } diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index c74504a6138..56440f096ca 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -47,16 +47,15 @@ type GroupVersionKind struct { // ProviderType defines the types of providers supported by Envoy Gateway. // -// +kubebuilder:validation:Enum=Kubernetes +// +kubebuilder:validation:Enum=Kubernetes;Custom type ProviderType string const ( // ProviderTypeKubernetes defines the "Kubernetes" provider. ProviderTypeKubernetes ProviderType = "Kubernetes" - // ProviderTypeFile defines the "File" provider. This type is not implemented - // until https://github.com/envoyproxy/gateway/issues/1001 is fixed. - ProviderTypeFile ProviderType = "File" + // ProviderTypeCustom defines the "Custom" provider. + ProviderTypeCustom ProviderType = "Custom" ) // KubernetesDeploymentSpec defines the desired state of the Kubernetes deployment resource. diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go index 8d7c97dc94a..d27e2e1e416 100644 --- a/api/v1alpha1/validation/envoygateway_validate.go +++ b/api/v1alpha1/validation/envoygateway_validate.go @@ -35,6 +35,10 @@ func ValidateEnvoyGateway(eg *egv1a1.EnvoyGateway) error { if err := validateEnvoyGatewayKubernetesProvider(eg.Provider.Kubernetes); err != nil { return err } + case egv1a1.ProviderTypeCustom: + if err := validateEnvoyGatewayCustomProvider(eg.Provider.Custom); err != nil { + return err + } default: return fmt.Errorf("unsupported provider type") } @@ -79,6 +83,54 @@ func validateEnvoyGatewayKubernetesProvider(provider *egv1a1.EnvoyGatewayKuberne return nil } +func validateEnvoyGatewayCustomProvider(provider *egv1a1.EnvoyGatewayCustomProvider) error { + if provider == nil { + return fmt.Errorf("empty custom provider settings") + } + + if err := validateEnvoyGatewayCustomResourceProvider(provider.Resource); err != nil { + return err + } + + if err := validateEnvoyGatewayCustomInfrastructureProvider(provider.Infrastructure); err != nil { + return err + } + + return nil +} + +func validateEnvoyGatewayCustomResourceProvider(resource egv1a1.EnvoyGatewayResourceProvider) error { + switch resource.Type { + case egv1a1.ResourceProviderTypeFile: + if resource.File == nil { + return fmt.Errorf("field 'file' should be specified when resource type is 'File'") + } + + if len(resource.File.Paths) == 0 { + return fmt.Errorf("no paths were assigned for file resource provider to watch") + } + default: + return fmt.Errorf("unsupported resource provider: %s", resource.Type) + } + return nil +} + +func validateEnvoyGatewayCustomInfrastructureProvider(infra *egv1a1.EnvoyGatewayInfrastructureProvider) error { + if infra == nil { + return nil + } + + switch infra.Type { + case egv1a1.InfrastructureProviderTypeHost: + if infra.Host == nil { + return fmt.Errorf("field 'host' should be specified when infrastructure type is 'Host'") + } + default: + return fmt.Errorf("unsupported infrastructure provdier: %s", infra.Type) + } + return nil +} + func validateEnvoyGatewayLogging(logging *egv1a1.EnvoyGatewayLogging) error { if logging == nil || len(logging.Level) == 0 { return nil diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index 7bc6bbf3b23..a0cbc7b059e 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -68,11 +68,173 @@ func TestValidateEnvoyGateway(t *testing.T) { expect: false, }, { - name: "unsupported provider", + name: "nil custom provider", eg: &egv1a1.EnvoyGateway{ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ - Gateway: egv1a1.DefaultGateway(), - Provider: &egv1a1.EnvoyGatewayProvider{Type: egv1a1.ProviderTypeFile}, + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: nil, + }, + }, + }, + expect: false, + }, + { + name: "empty custom provider", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{}, + }, + }, + }, + expect: false, + }, + { + name: "custom provider with file resource provider and host infra provider", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: egv1a1.ResourceProviderTypeFile, + File: &egv1a1.EnvoyGatewayFileResourceProvider{ + Paths: []string{"foo", "bar"}, + }, + }, + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ + Type: egv1a1.InfrastructureProviderTypeHost, + Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{}, + }, + }, + }, + }, + }, + expect: true, + }, + { + name: "custom provider with file provider and k8s infra provider", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: egv1a1.ResourceProviderTypeFile, + File: &egv1a1.EnvoyGatewayFileResourceProvider{ + Paths: []string{"foo", "bar"}, + }, + }, + }, + }, + }, + }, + expect: true, + }, + { + name: "custom provider with unsupported resource provider", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: "foobar", + }, + }, + }, + }, + }, + expect: false, + }, + { + name: "custom provider with file provider but no file struct", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: egv1a1.ResourceProviderTypeFile, + }, + }, + }, + }, + }, + expect: false, + }, + { + name: "custom provider with file provider and host infra provider but no host struct", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: egv1a1.ResourceProviderTypeFile, + File: &egv1a1.EnvoyGatewayFileResourceProvider{ + Paths: []string{"a", "b"}, + }, + }, + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ + Type: egv1a1.InfrastructureProviderTypeHost, + }, + }, + }, + }, + }, + expect: false, + }, + { + name: "custom provider with file provider and unsupported infra provider", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: egv1a1.ResourceProviderTypeFile, + File: &egv1a1.EnvoyGatewayFileResourceProvider{ + Paths: []string{"a", "b"}, + }, + }, + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ + Type: "foobar", + }, + }, + }, + }, + }, + expect: false, + }, + { + name: "custom provider with file provider and host infra provider but no paths assign in resource", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: egv1a1.ResourceProviderTypeFile, + File: &egv1a1.EnvoyGatewayFileResourceProvider{}, + }, + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ + Type: egv1a1.InfrastructureProviderTypeHost, + Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{}, + }, + }, + }, }, }, expect: false, diff --git a/api/v1alpha1/validation/envoyproxy_validate_test.go b/api/v1alpha1/validation/envoyproxy_validate_test.go index bd7e4bc18e0..e4b400b34dd 100644 --- a/api/v1alpha1/validation/envoyproxy_validate_test.go +++ b/api/v1alpha1/validation/envoyproxy_validate_test.go @@ -52,7 +52,7 @@ func TestValidateEnvoyProxy(t *testing.T) { }, Spec: egv1a1.EnvoyProxySpec{ Provider: &egv1a1.EnvoyProxyProvider{ - Type: egv1a1.ProviderTypeFile, + Type: egv1a1.ProviderTypeCustom, }, }, }, diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 56b3332f2db..f2c88ae0c29 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1352,7 +1352,11 @@ func (in *EnvoyGatewayAdminAddress) DeepCopy() *EnvoyGatewayAdminAddress { func (in *EnvoyGatewayCustomProvider) DeepCopyInto(out *EnvoyGatewayCustomProvider) { *out = *in in.Resource.DeepCopyInto(&out.Resource) - in.Infrastructure.DeepCopyInto(&out.Infrastructure) + if in.Infrastructure != nil { + in, out := &in.Infrastructure, &out.Infrastructure + *out = new(EnvoyGatewayInfrastructureProvider) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyGatewayCustomProvider. diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index c227b546cb0..4211599adf8 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -10227,6 +10227,7 @@ spec: optional auxiliary control planes. Supported types are "Kubernetes". enum: - Kubernetes + - Custom type: string required: - type diff --git a/go.mod b/go.mod index cc1e8bb745c..45f69e52768 100644 --- a/go.mod +++ b/go.mod @@ -214,7 +214,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect github.com/evanphx/json-patch v5.9.0+incompatible github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/internal/cmd/server.go b/internal/cmd/server.go index e4bf36a5ae5..25add4c8541 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -9,9 +9,11 @@ import ( "github.com/spf13/cobra" ctrl "sigs.k8s.io/controller-runtime" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/admin" "github.com/envoyproxy/gateway/internal/envoygateway/config" extensionregistry "github.com/envoyproxy/gateway/internal/extension/registry" + "github.com/envoyproxy/gateway/internal/extension/types" gatewayapirunner "github.com/envoyproxy/gateway/internal/gatewayapi/runner" ratelimitrunner "github.com/envoyproxy/gateway/internal/globalratelimit/runner" infrarunner "github.com/envoyproxy/gateway/internal/infrastructure/runner" @@ -108,28 +110,31 @@ func getConfigByPath(cfgPath string) (*config.Server, error) { // setupRunners starts all the runners required for the Envoy Gateway to // fulfill its tasks. -func setupRunners(cfg *config.Server) error { +func setupRunners(cfg *config.Server) (err error) { // TODO - Setup a Config Manager // https://github.com/envoyproxy/gateway/issues/43 ctx := ctrl.SetupSignalHandler() // Setup the Extension Manager - extMgr, err := extensionregistry.NewManager(cfg) - if err != nil { - return err + var extMgr types.Manager + if cfg.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes { + extMgr, err = extensionregistry.NewManager(cfg) + if err != nil { + return err + } } pResources := new(message.ProviderResources) // Start the Provider Service // It fetches the resources from the configured provider type - // and publishes it + // and publishes it. // It also subscribes to status resources and once it receives // a status resource back, it writes it out. providerRunner := providerrunner.New(&providerrunner.Config{ Server: *cfg, ProviderResources: pResources, }) - if err := providerRunner.Start(ctx); err != nil { + if err = providerRunner.Start(ctx); err != nil { return err } @@ -145,7 +150,7 @@ func setupRunners(cfg *config.Server) error { InfraIR: infraIR, ExtensionManager: extMgr, }) - if err := gwRunner.Start(ctx); err != nil { + if err = gwRunner.Start(ctx); err != nil { return err } @@ -160,7 +165,7 @@ func setupRunners(cfg *config.Server) error { ExtensionManager: extMgr, ProviderResources: pResources, }) - if err := xdsTranslatorRunner.Start(ctx); err != nil { + if err = xdsTranslatorRunner.Start(ctx); err != nil { return err } @@ -171,7 +176,7 @@ func setupRunners(cfg *config.Server) error { Server: *cfg, InfraIR: infraIR, }) - if err := infraRunner.Start(ctx); err != nil { + if err = infraRunner.Start(ctx); err != nil { return err } @@ -182,7 +187,7 @@ func setupRunners(cfg *config.Server) error { Server: *cfg, Xds: xds, }) - if err := xdsServerRunner.Start(ctx); err != nil { + if err = xdsServerRunner.Start(ctx); err != nil { return err } @@ -194,7 +199,7 @@ func setupRunners(cfg *config.Server) error { Server: *cfg, XdsIR: xdsIR, }) - if err := rateLimitRunner.Start(ctx); err != nil { + if err = rateLimitRunner.Start(ctx); err != nil { return err } } @@ -209,9 +214,11 @@ func setupRunners(cfg *config.Server) error { cfg.Logger.Info("shutting down") - // Close connections to extension services - if mgr, ok := extMgr.(*extensionregistry.Manager); ok { - mgr.CleanupHookConns() + if extMgr != nil { + // Close connections to extension services + if mgr, ok := extMgr.(*extensionregistry.Manager); ok { + mgr.CleanupHookConns() + } } return nil diff --git a/internal/infrastructure/manager.go b/internal/infrastructure/manager.go index 34fc1b24b12..198acef8708 100644 --- a/internal/infrastructure/manager.go +++ b/internal/infrastructure/manager.go @@ -36,15 +36,23 @@ type Manager interface { // NewManager returns a new infrastructure Manager. func NewManager(cfg *config.Server) (Manager, error) { var mgr Manager - if cfg.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes { + + switch cfg.EnvoyGateway.Provider.Type { + case egv1a1.ProviderTypeKubernetes: cli, err := client.New(clicfg.GetConfigOrDie(), client.Options{Scheme: envoygateway.GetScheme()}) if err != nil { return nil, err } mgr = kubernetes.NewInfra(cli, cfg) - } else { - // Kube is the only supported provider type for now. - return nil, fmt.Errorf("unsupported provider type %v", cfg.EnvoyGateway.Provider.Type) + case egv1a1.ProviderTypeCustom: + infra := cfg.EnvoyGateway.Provider.Custom.Infrastructure + switch infra.Type { + case egv1a1.InfrastructureProviderTypeHost: + // TODO(sh2): implement host provider + return nil, fmt.Errorf("host provider is not available yet") + default: + return nil, fmt.Errorf("unsupported provider type: %s", infra.Type) + } } return mgr, nil diff --git a/internal/infrastructure/runner/runner.go b/internal/infrastructure/runner/runner.go index 7574c493090..6c261aff3f3 100644 --- a/internal/infrastructure/runner/runner.go +++ b/internal/infrastructure/runner/runner.go @@ -38,6 +38,12 @@ func New(cfg *Config) *Runner { // Start starts the infrastructure runner func (r *Runner) Start(ctx context.Context) (err error) { r.Logger = r.Logger.WithName(r.Name()).WithValues("runner", r.Name()) + if r.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeCustom && + r.EnvoyGateway.Provider.Custom.Infrastructure == nil { + r.Logger.Info("provider is not specified, no provider is available") + return nil + } + r.mgr, err = infrastructure.NewManager(&r.Config.Server) if err != nil { r.Logger.Error(err, "failed to create new manager") @@ -56,7 +62,8 @@ func (r *Runner) Start(ctx context.Context) (err error) { // When leader election is active, infrastructure initialization occurs only upon acquiring leadership // to avoid multiple EG instances processing envoy proxy infra resources. - if !ptr.Deref(r.EnvoyGateway.Provider.Kubernetes.LeaderElection.Disable, false) { + if r.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes && + !ptr.Deref(r.EnvoyGateway.Provider.Kubernetes.LeaderElection.Disable, false) { go func() { select { case <-ctx.Done(): diff --git a/internal/provider/file/file.go b/internal/provider/file/file.go new file mode 100644 index 00000000000..79ccd04e763 --- /dev/null +++ b/internal/provider/file/file.go @@ -0,0 +1,131 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/message" +) + +type Provider struct { + paths []string + logger logr.Logger + notifier *Notifier + resourcesStore *resourcesStore +} + +func New(svr *config.Server, resources *message.ProviderResources) (*Provider, error) { + logger := svr.Logger.Logger + + notifier, err := NewNotifier(logger) + if err != nil { + return nil, err + } + + return &Provider{ + paths: svr.EnvoyGateway.Provider.Custom.Resource.File.Paths, + logger: logger, + notifier: notifier, + resourcesStore: newResourcesStore(svr.EnvoyGateway.Gateway.ControllerName, resources, logger), + }, nil +} + +func (p *Provider) Type() egv1a1.ProviderType { + return egv1a1.ProviderTypeCustom +} + +func (p *Provider) Start(ctx context.Context) error { + dirs, files, err := getDirsAndFilesForWatcher(p.paths) + if err != nil { + return fmt.Errorf("failed to get directories and files for the watcher: %w", err) + } + + // Start runnable servers. + go p.startHealthProbeServer(ctx) + + // Initially load resources from paths on host. + if err = p.resourcesStore.LoadAndStore(files.UnsortedList(), dirs.UnsortedList()); err != nil { + return fmt.Errorf("failed to load resources into store: %w", err) + } + + // Start watchers in notifier. + p.notifier.Watch(ctx, dirs, files) + defer p.notifier.Close() + + for { + select { + case <-ctx.Done(): + return nil + case event := <-p.notifier.Events: + switch event.Op { + case fsnotify.Create: + dirs.Insert(event.Name) + files.Insert(event.Name) + case fsnotify.Remove: + dirs.Delete(event.Name) + files.Delete(event.Name) + } + + p.resourcesStore.HandleEvent(event, files.UnsortedList(), dirs.UnsortedList()) + } + } +} + +func (p *Provider) startHealthProbeServer(ctx context.Context) { + const ( + readyzEndpoint = "/readyz" + healthzEndpoint = "/healthz" + ) + + mux := http.NewServeMux() + srv := &http.Server{ + Addr: ":8081", + Handler: mux, + MaxHeaderBytes: 1 << 20, + IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout + ReadHeaderTimeout: 32 * time.Second, + } + + readyzHandler := &healthz.Handler{ + Checks: map[string]healthz.Checker{ + readyzEndpoint: healthz.Ping, + }, + } + mux.Handle(readyzEndpoint, http.StripPrefix(readyzEndpoint, readyzHandler)) + // Append '/' suffix to handle subpaths. + mux.Handle(readyzEndpoint+"/", http.StripPrefix(readyzEndpoint, readyzHandler)) + + healthzHandler := &healthz.Handler{ + Checks: map[string]healthz.Checker{ + healthzEndpoint: healthz.Ping, + }, + } + mux.Handle(healthzEndpoint, http.StripPrefix(healthzEndpoint, healthzHandler)) + // Append '/' suffix to handle subpaths. + mux.Handle(healthzEndpoint+"/", http.StripPrefix(healthzEndpoint, readyzHandler)) + + go func() { + <-ctx.Done() + if err := srv.Close(); err != nil { + p.logger.Error(err, "failed to close health probe server") + } + }() + + p.logger.Info("starting health probe server", "address", srv.Addr) + if err := srv.ListenAndServe(); err != nil { + p.logger.Error(err, "failed to start health probe server") + } +} diff --git a/internal/provider/file/notifier.go b/internal/provider/file/notifier.go new file mode 100644 index 00000000000..fca8465e3af --- /dev/null +++ b/internal/provider/file/notifier.go @@ -0,0 +1,316 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "context" + "os" + "path/filepath" + "strings" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/sets" +) + +const ( + defaultCleanUpRemoveEventsPeriod = 300 * time.Millisecond +) + +type Notifier struct { + // Events record events used to update ResourcesStore, + // which only include two types of events: Write/Remove. + Events chan fsnotify.Event + + filesWatcher *fsnotify.Watcher + dirsWatcher *fsnotify.Watcher + cleanUpRemoveEventsPeriod time.Duration + + logger logr.Logger +} + +func NewNotifier(logger logr.Logger) (*Notifier, error) { + fw, err := fsnotify.NewBufferedWatcher(10) + if err != nil { + return nil, err + } + + dw, err := fsnotify.NewBufferedWatcher(10) + if err != nil { + return nil, err + } + + return &Notifier{ + Events: make(chan fsnotify.Event), + filesWatcher: fw, + dirsWatcher: dw, + cleanUpRemoveEventsPeriod: defaultCleanUpRemoveEventsPeriod, + logger: logger, + }, nil +} + +func (n *Notifier) Watch(ctx context.Context, dirs, files sets.Set[string]) { + n.watchDirs(ctx, dirs) + n.watchFiles(ctx, files) +} + +func (n *Notifier) Close() error { + if err := n.filesWatcher.Close(); err != nil { + return err + } + if err := n.dirsWatcher.Close(); err != nil { + return err + } + return nil +} + +// watchFiles watches one or more files, but instead of watching the file directly, +// it watches its parent directory. This solves various issues where files are +// frequently renamed. +func (n *Notifier) watchFiles(ctx context.Context, files sets.Set[string]) { + if len(files) < 1 { + return + } + + go n.runFilesWatcher(ctx, files) + + for p := range files { + if err := n.filesWatcher.Add(filepath.Dir(p)); err != nil { + n.logger.Error(err, "error adding file to notifier", "path", p) + + continue + } + } +} + +func (n *Notifier) runFilesWatcher(ctx context.Context, files sets.Set[string]) { + var ( + cleanUpTicker = time.NewTicker(n.cleanUpRemoveEventsPeriod) + + // This map records the exact previous Op of one event. + preEventOp = make(map[string]fsnotify.Op) + // This set records the name of event that related to Remove Op. + curRemoveEvents = sets.NewString() + ) + + for { + select { + case <-ctx.Done(): + return + + case err, ok := <-n.filesWatcher.Errors: + if !ok { + return + } + n.logger.Error(err, "error from files watcher in notifier") + + case event, ok := <-n.filesWatcher.Events: + if !ok { + return + } + + // Ignore file and operation the watcher not interested in. + if !files.Has(event.Name) || event.Has(fsnotify.Chmod) { + continue + } + + // This logic is trying to avoid files be removed and then created + // frequently by considering Remove/Rename and the follow Create + // Op as one Write Notifier.Event. + // + // Actually, this approach is also suitable for commands like vi/vim. + // It creates a temporary file, removes the existing one and replace + // it with the temporary file when file is saved. So instead of Write + // Op, the watcher will receive Rename and Create Op. + + var writeEvent bool + switch event.Op { + case fsnotify.Create: + if op, ok := preEventOp[event.Name]; ok && + op.Has(fsnotify.Rename) || op.Has(fsnotify.Remove) { + writeEvent = true + // If the exact previous Op of Create is Rename/Remove, + // then consider them as a Write Notifier.Event instead of Remove. + curRemoveEvents.Delete(event.Name) + } + case fsnotify.Write: + writeEvent = true + case fsnotify.Remove, fsnotify.Rename: + curRemoveEvents.Insert(event.Name) + } + + if writeEvent { + n.logger.Info("sending write event", + "name", event.Name, "watcher", "files") + + n.Events <- fsnotify.Event{ + Name: event.Name, + Op: fsnotify.Write, + } + } + preEventOp[event.Name] = event.Op + + case <-cleanUpTicker.C: + // As for collected Remove Notifier.Event, clean them up + // in a period of time to avoid neglect of dealing with + // Remove/Rename Op. + for e := range curRemoveEvents { + n.logger.Info("sending remove event", + "name", e, "watcher", "files") + + n.Events <- fsnotify.Event{ + Name: e, + Op: fsnotify.Remove, + } + } + curRemoveEvents = sets.NewString() + } + } +} + +// watchDirs watches one or more directories. +func (n *Notifier) watchDirs(ctx context.Context, dirs sets.Set[string]) { + if len(dirs) < 1 { + return + } + + // This map maintains the subdirectories ignored by each directory. + ignoredSubDirs := make(map[string]sets.Set[string]) + + for p := range dirs { + if err := n.dirsWatcher.Add(p); err != nil { + n.logger.Error(err, "error adding dir to notifier", "path", p) + + continue + } + + // Find current exist subdirectories to init ignored subdirectories set. + entries, err := os.ReadDir(p) + if err != nil { + n.logger.Error(err, "error reading dir in notifier", "path", p) + + if err = n.dirsWatcher.Remove(p); err != nil { + n.logger.Error(err, "error removing dir from notifier", "path", p) + } + + continue + } + + ignoredSubDirs[p] = sets.New[string]() + for _, entry := range entries { + if entry.IsDir() { + // The entry name is dir name, not dir path. + ignoredSubDirs[p].Insert(entry.Name()) + } + } + } + + go n.runDirsWatcher(ctx, ignoredSubDirs) +} + +func (n *Notifier) runDirsWatcher(ctx context.Context, ignoredSubDirs map[string]sets.Set[string]) { + var ( + cleanUpTicker = time.NewTicker(n.cleanUpRemoveEventsPeriod) + + // This map records the exact previous Op of one event. + preEventOp = make(map[string]fsnotify.Op) + // This set records the name of event that related to Remove Op. + curRemoveEvents = sets.NewString() + ) + + for { + select { + case <-ctx.Done(): + return + + case err, ok := <-n.dirsWatcher.Errors: + if !ok { + return + } + n.logger.Error(err, "error from dirs watcher in notifier") + + case event, ok := <-n.dirsWatcher.Events: + if !ok { + return + } + + // Ignore the hidden or temporary file related event. + _, name := filepath.Split(event.Name) + if event.Has(fsnotify.Chmod) || + strings.HasPrefix(name, ".") || + strings.HasSuffix(name, "~") { + continue + } + + // Ignore any subdirectory related event. + switch event.Op { + case fsnotify.Create: + if fi, err := os.Lstat(event.Name); err == nil && fi.IsDir() { + parentDir := filepath.Dir(event.Name) + if _, ok := ignoredSubDirs[parentDir]; ok { + ignoredSubDirs[parentDir].Insert(name) + continue + } + } + case fsnotify.Remove, fsnotify.Rename: + parentDir := filepath.Dir(event.Name) + if sub, ok := ignoredSubDirs[parentDir]; ok && sub.Has(name) { + ignoredSubDirs[parentDir].Delete(name) + continue + } + } + + // Share the similar logic as in files watcher. + var writeEvent bool + switch event.Op { + case fsnotify.Create: + if op, ok := preEventOp[event.Name]; ok && + op.Has(fsnotify.Rename) || op.Has(fsnotify.Remove) { + curRemoveEvents.Delete(event.Name) + } + // Since the watcher watches the whole dir, the creation of file + // should also be able to trigger the Write event. + writeEvent = true + case fsnotify.Write: + writeEvent = true + case fsnotify.Remove, fsnotify.Rename: + curRemoveEvents.Insert(event.Name) + } + + if writeEvent { + n.logger.Info("sending write event", + "name", event.Name, "watcher", "dirs") + + n.Events <- fsnotify.Event{ + Name: event.Name, + Op: fsnotify.Write, + } + } + preEventOp[event.Name] = event.Op + + case <-cleanUpTicker.C: + // Merge files to be removed in the same parent directory + // to suppress events, because the file has already been + // removed and is unnecessary to send event for each of them. + parentDirs := sets.NewString() + for e := range curRemoveEvents { + parentDirs.Insert(filepath.Dir(e)) + } + + for parentDir := range parentDirs { + n.logger.Info("sending remove event", + "name", parentDir, "watcher", "dirs") + + n.Events <- fsnotify.Event{ + Name: parentDir, + Op: fsnotify.Remove, + } + } + curRemoveEvents = sets.NewString() + } + } +} diff --git a/internal/provider/file/path.go b/internal/provider/file/path.go new file mode 100644 index 00000000000..fe3ad7539f6 --- /dev/null +++ b/internal/provider/file/path.go @@ -0,0 +1,47 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "os" + "path/filepath" + + "k8s.io/apimachinery/pkg/util/sets" +) + +// getDirsAndFilesForWatcher prepares dirs and files for the watcher in notifier. +func getDirsAndFilesForWatcher(paths []string) ( + dirs sets.Set[string], files sets.Set[string], err error, +) { + dirs, files = sets.New[string](), sets.New[string]() + + // Separate paths by whether is a directory or not. + paths = sets.NewString(paths...).List() + for _, path := range paths { + var p os.FileInfo + p, err = os.Lstat(path) + if err != nil { + return + } + + if p.IsDir() { + dirs.Insert(path) + } else { + files.Insert(path) + } + } + + // Ignore filepath if its parent directory is also be watched. + var ignoreFiles []string + for fp := range files { + if dirs.Has(filepath.Dir(fp)) { + ignoreFiles = append(ignoreFiles, fp) + } + } + files.Delete(ignoreFiles...) + + return +} diff --git a/internal/provider/file/path_test.go b/internal/provider/file/path_test.go new file mode 100644 index 00000000000..183c24efa97 --- /dev/null +++ b/internal/provider/file/path_test.go @@ -0,0 +1,54 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "path" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetDirsAndFilesForWatcher(t *testing.T) { + testPath := path.Join("testdata", "paths") + testCases := []struct { + name string + paths []string + expectDirs []string + expectFiles []string + }{ + { + name: "get file and dir path", + paths: []string{ + path.Join(testPath, "dir"), path.Join(testPath, "foo"), + }, + expectDirs: []string{ + path.Join(testPath, "dir"), + }, + expectFiles: []string{ + path.Join(testPath, "foo"), + }, + }, + { + name: "overlap file path will be ignored", + paths: []string{ + path.Join(testPath, "dir"), path.Join(testPath, "dir", "bar"), + }, + expectDirs: []string{ + path.Join(testPath, "dir"), + }, + expectFiles: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dirs, paths, _ := getDirsAndFilesForWatcher(tc.paths) + require.ElementsMatch(t, dirs.UnsortedList(), tc.expectDirs) + require.ElementsMatch(t, paths.UnsortedList(), tc.expectFiles) + }) + } +} diff --git a/internal/provider/file/resources.go b/internal/provider/file/resources.go new file mode 100644 index 00000000000..8dcd60ac78a --- /dev/null +++ b/internal/provider/file/resources.go @@ -0,0 +1,335 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + "sigs.k8s.io/yaml" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/gatewayapi" +) + +// loadFromFilesAndDirs loads resources from specific files and directories. +func loadFromFilesAndDirs(files, dirs []string) ([]*gatewayapi.Resources, error) { + var rs []*gatewayapi.Resources + + for _, file := range files { + r, err := loadFromFile(file) + if err != nil { + return nil, err + } + rs = append(rs, r) + } + + for _, dir := range dirs { + r, err := loadFromDir(dir) + if err != nil { + return nil, err + } + rs = append(rs, r...) + } + + return rs, nil +} + +// loadFromFile loads resources from a specific file. +func loadFromFile(path string) (*gatewayapi.Resources, error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("file %s is not exist", path) + } + return nil, err + } + + bytes, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + return convertKubernetesYAMLToResources(string(bytes)) +} + +// loadFromDir loads resources from all the files under a specific directory excluding subdirectories. +func loadFromDir(path string) ([]*gatewayapi.Resources, error) { + entries, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + var rs []*gatewayapi.Resources + for _, entry := range entries { + // Ignoring subdirectories and all hidden files and directories. + if entry.IsDir() || strings.HasPrefix(entry.Name(), ".") { + continue + } + + r, err := loadFromFile(filepath.Join(path, entry.Name())) + if err != nil { + return nil, err + } + + rs = append(rs, r) + } + + return rs, nil +} + +// TODO(sh2): This function is copied and updated from internal/cmd/egctl/translate.go. +// This function should be able to process arbitrary number of resources, so we +// need to come up with a way to extend the GatewayClass and EnvoyProxy field to array +// instead of single variable in gatewayapi.Resources structure. +// +// - This issue is tracked by https://github.com/envoyproxy/gateway/issues/3207 +// +// convertKubernetesYAMLToResources converts a Kubernetes YAML string into GatewayAPI Resources. +func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) { + resources := gatewayapi.NewResources() + var useDefaultNamespace bool + providedNamespaceMap := sets.New[string]() + requiredNamespaceMap := sets.New[string]() + yamls := strings.Split(str, "\n---") + combinedScheme := envoygateway.GetScheme() + for _, y := range yamls { + if strings.TrimSpace(y) == "" { + continue + } + var obj map[string]interface{} + err := yaml.Unmarshal([]byte(y), &obj) + if err != nil { + return nil, err + } + un := unstructured.Unstructured{Object: obj} + gvk := un.GroupVersionKind() + name, namespace := un.GetName(), un.GetNamespace() + if namespace == "" { + // When kubectl applies a resource in yaml which doesn't have a namespace, + // the current namespace is applied. Here we do the same thing before translating + // the GatewayAPI resource. Otherwise, the resource can't pass the namespace validation + useDefaultNamespace = true + namespace = config.DefaultNamespace + } + requiredNamespaceMap.Insert(namespace) + kobj, err := combinedScheme.New(gvk) + if err != nil { + return nil, err + } + err = combinedScheme.Convert(&un, kobj, nil) + if err != nil { + return nil, err + } + + objType := reflect.TypeOf(kobj) + if objType.Kind() != reflect.Ptr { + return nil, fmt.Errorf("expected pointer type, but got %s", objType.Kind().String()) + } + kobjVal := reflect.ValueOf(kobj).Elem() + spec := kobjVal.FieldByName("Spec") + + switch gvk.Kind { + case gatewayapi.KindEnvoyProxy: + typedSpec := spec.Interface() + envoyProxy := &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(egv1a1.EnvoyProxySpec), + } + resources.EnvoyProxyForGatewayClass = envoyProxy + case gatewayapi.KindGatewayClass: + typedSpec := spec.Interface() + gatewayClass := &gwapiv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.GatewayClassSpec), + } + // fill controller name by default controller name when gatewayclass controller name empty. + if gatewayClass.Spec.ControllerName == "" { + gatewayClass.Spec.ControllerName = egv1a1.GatewayControllerName + } + resources.GatewayClass = gatewayClass + case gatewayapi.KindGateway: + typedSpec := spec.Interface() + gateway := &gwapiv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.GatewaySpec), + } + resources.Gateways = append(resources.Gateways, gateway) + case gatewayapi.KindTCPRoute: + typedSpec := spec.Interface() + tcpRoute := &gwapiv1a2.TCPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindTCPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.TCPRouteSpec), + } + resources.TCPRoutes = append(resources.TCPRoutes, tcpRoute) + case gatewayapi.KindUDPRoute: + typedSpec := spec.Interface() + udpRoute := &gwapiv1a2.UDPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindUDPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.UDPRouteSpec), + } + resources.UDPRoutes = append(resources.UDPRoutes, udpRoute) + case gatewayapi.KindTLSRoute: + typedSpec := spec.Interface() + tlsRoute := &gwapiv1a2.TLSRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindTLSRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.TLSRouteSpec), + } + resources.TLSRoutes = append(resources.TLSRoutes, tlsRoute) + case gatewayapi.KindHTTPRoute: + typedSpec := spec.Interface() + httpRoute := &gwapiv1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindHTTPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.HTTPRouteSpec), + } + resources.HTTPRoutes = append(resources.HTTPRoutes, httpRoute) + case gatewayapi.KindGRPCRoute: + typedSpec := spec.Interface() + grpcRoute := &gwapiv1.GRPCRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindGRPCRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.GRPCRouteSpec), + } + resources.GRPCRoutes = append(resources.GRPCRoutes, grpcRoute) + case gatewayapi.KindNamespace: + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + resources.Namespaces = append(resources.Namespaces, namespace) + providedNamespaceMap.Insert(name) + case gatewayapi.KindService: + typedSpec := spec.Interface() + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(corev1.ServiceSpec), + } + resources.Services = append(resources.Services, service) + case egv1a1.KindEnvoyPatchPolicy: + typedSpec := spec.Interface() + envoyPatchPolicy := &egv1a1.EnvoyPatchPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindEnvoyPatchPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.EnvoyPatchPolicySpec), + } + resources.EnvoyPatchPolicies = append(resources.EnvoyPatchPolicies, envoyPatchPolicy) + case egv1a1.KindClientTrafficPolicy: + typedSpec := spec.Interface() + clientTrafficPolicy := &egv1a1.ClientTrafficPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindClientTrafficPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.ClientTrafficPolicySpec), + } + resources.ClientTrafficPolicies = append(resources.ClientTrafficPolicies, clientTrafficPolicy) + case egv1a1.KindBackendTrafficPolicy: + typedSpec := spec.Interface() + backendTrafficPolicy := &egv1a1.BackendTrafficPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindBackendTrafficPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.BackendTrafficPolicySpec), + } + resources.BackendTrafficPolicies = append(resources.BackendTrafficPolicies, backendTrafficPolicy) + case egv1a1.KindSecurityPolicy: + typedSpec := spec.Interface() + securityPolicy := &egv1a1.SecurityPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindSecurityPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.SecurityPolicySpec), + } + resources.SecurityPolicies = append(resources.SecurityPolicies, securityPolicy) + } + } + + if useDefaultNamespace { + if !providedNamespaceMap.Has(config.DefaultNamespace) { + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.DefaultNamespace, + }, + } + resources.Namespaces = append(resources.Namespaces, namespace) + providedNamespaceMap.Insert(config.DefaultNamespace) + } + } + + return resources, nil +} diff --git a/internal/provider/file/store.go b/internal/provider/file/store.go new file mode 100644 index 00000000000..5a45f1fd638 --- /dev/null +++ b/internal/provider/file/store.go @@ -0,0 +1,78 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" + + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/message" +) + +type resourcesStore struct { + name string + resources *message.ProviderResources + + logger logr.Logger +} + +func newResourcesStore(name string, resources *message.ProviderResources, logger logr.Logger) *resourcesStore { + return &resourcesStore{ + name: name, + resources: resources, + logger: logger, + } +} + +func (r *resourcesStore) HandleEvent(event fsnotify.Event, files, dirs []string) { + r.logger.Info("receive an event", "name", event.Name, "op", event.Op.String()) + + // TODO(sh2): Support multiple GatewayClass. + switch event.Op { + case fsnotify.Write: + if err := r.LoadAndStore(files, dirs); err != nil { + r.logger.Error(err, "failed to load and store resources") + } + case fsnotify.Remove: + // Under our current assumption, one file only contains one GatewayClass and + // all its other related resources, so we can remove them safely. + r.resources.GatewayAPIResources.Delete(r.name) + } +} + +// LoadAndStore loads and stores all resources from files and directories. +func (r *resourcesStore) LoadAndStore(files, dirs []string) error { + rs, err := loadFromFilesAndDirs(files, dirs) + if err != nil { + return err + } + + // TODO(sh2): For now, we assume that one file only contains one GatewayClass and all its other + // related resources, like Gateway, HTTPRoute, etc. If we managed to extend Resources structure, + // we also need to process all the resources and its relationship, like what is done in + // Kubernetes provider. However, this will cause us to maintain two places of the same logic + // in each provider. The ideal case is two different providers share the same resources process logic. + // + // - This issue is tracked by https://github.com/envoyproxy/gateway/issues/3213 + + // We cannot make sure by the time the Write event was triggered, whether the GatewayClass exist, + // so here we just simply Store the first gatewayapi.Resources that has GatewayClass. + gwcResources := make(gatewayapi.ControllerResources, 0, 1) + for _, resource := range rs { + if resource.GatewayClass != nil { + gwcResources = append(gwcResources, resource) + } + } + if len(gwcResources) == 0 { + return nil + } + + r.resources.GatewayAPIResources.Store(r.name, &gwcResources) + r.logger.Info("loaded and stored resources successfully") + + return nil +} diff --git a/internal/provider/file/testdata/paths/dir/bar b/internal/provider/file/testdata/paths/dir/bar new file mode 100644 index 00000000000..e1878797a7c --- /dev/null +++ b/internal/provider/file/testdata/paths/dir/bar @@ -0,0 +1 @@ +THIS FILE IS FOR TEST ONLY \ No newline at end of file diff --git a/internal/provider/file/testdata/paths/foo b/internal/provider/file/testdata/paths/foo new file mode 100644 index 00000000000..e1878797a7c --- /dev/null +++ b/internal/provider/file/testdata/paths/foo @@ -0,0 +1 @@ +THIS FILE IS FOR TEST ONLY \ No newline at end of file diff --git a/internal/provider/kubernetes/kubernetes.go b/internal/provider/kubernetes/kubernetes.go index bd805fd3048..b909eced608 100644 --- a/internal/provider/kubernetes/kubernetes.go +++ b/internal/provider/kubernetes/kubernetes.go @@ -20,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" ec "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/message" @@ -118,6 +119,10 @@ func New(cfg *rest.Config, svr *ec.Server, resources *message.ProviderResources) }, nil } +func (p *Provider) Type() egv1a1.ProviderType { + return egv1a1.ProviderTypeKubernetes +} + // Start starts the Provider synchronously until a message is received from ctx. func (p *Provider) Start(ctx context.Context) error { errChan := make(chan error) diff --git a/internal/provider/resource_provider.go b/internal/provider/resource_provider.go new file mode 100644 index 00000000000..d14f95d158d --- /dev/null +++ b/internal/provider/resource_provider.go @@ -0,0 +1,20 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package provider + +import ( + "context" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" +) + +type Provider interface { + // Start starts the resource provider. + Start(ctx context.Context) error + + // Type returns the type of resource provider. + Type() egv1a1.ProviderType +} diff --git a/internal/provider/runner/runner.go b/internal/provider/runner/runner.go index 32f5a30fe8c..94488489376 100644 --- a/internal/provider/runner/runner.go +++ b/internal/provider/runner/runner.go @@ -14,6 +14,8 @@ import ( egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/message" + "github.com/envoyproxy/gateway/internal/provider" + "github.com/envoyproxy/gateway/internal/provider/file" "github.com/envoyproxy/gateway/internal/provider/kubernetes" ) @@ -37,24 +39,61 @@ func (r *Runner) Name() string { // Start the provider runner func (r *Runner) Start(ctx context.Context) (err error) { r.Logger = r.Logger.WithName(r.Name()).WithValues("runner", r.Name()) - if r.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes { - r.Logger.Info("Using provider", "type", egv1a1.ProviderTypeKubernetes) - cfg, err := ctrl.GetConfig() + + var p provider.Provider + switch r.EnvoyGateway.Provider.Type { + case egv1a1.ProviderTypeKubernetes: + p, err = r.createKubernetesProvider() if err != nil { - return fmt.Errorf("failed to get kubeconfig: %w", err) + return fmt.Errorf("failed to create kubernetes provider: %w", err) } - p, err := kubernetes.New(cfg, &r.Config.Server, r.ProviderResources) + + case egv1a1.ProviderTypeCustom: + p, err = r.createCustomResourceProvider() if err != nil { - return fmt.Errorf("failed to create provider %s: %w", egv1a1.ProviderTypeKubernetes, err) + return fmt.Errorf("failed to create custom provider: %w", err) + } + + default: + // Unsupported provider. + return fmt.Errorf("unsupported provider type %v", r.EnvoyGateway.Provider.Type) + } + + r.Logger.Info("Running provider", "type", p.Type()) + go func() { + if err = p.Start(ctx); err != nil { + r.Logger.Error(err, "unable to start provider") } - go func() { - err := p.Start(ctx) - if err != nil { - r.Logger.Error(err, "unable to start provider") - } - }() - return nil + }() + + return nil +} + +func (r *Runner) createKubernetesProvider() (*kubernetes.Provider, error) { + cfg, err := ctrl.GetConfig() + if err != nil { + return nil, fmt.Errorf("failed to get kubeconfig: %w", err) } - // Unsupported provider. - return fmt.Errorf("unsupported provider type %v", r.EnvoyGateway.Provider.Type) + + p, err := kubernetes.New(cfg, &r.Config.Server, r.ProviderResources) + if err != nil { + return nil, fmt.Errorf("failed to create provider %s: %w", egv1a1.ProviderTypeKubernetes, err) + } + + return p, err +} + +func (r *Runner) createCustomResourceProvider() (p provider.Provider, err error) { + switch r.EnvoyGateway.Provider.Custom.Resource.Type { + case egv1a1.ResourceProviderTypeFile: + p, err = file.New(&r.Config.Server, r.ProviderResources) + if err != nil { + return nil, fmt.Errorf("failed to create provider %s: %w", egv1a1.ProviderTypeCustom, err) + } + + default: + return nil, fmt.Errorf("unsupported resource provider type") + } + + return } diff --git a/internal/provider/runner/runner_test.go b/internal/provider/runner/runner_test.go deleted file mode 100644 index 8252bb407ca..00000000000 --- a/internal/provider/runner/runner_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package runner - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" - "github.com/envoyproxy/gateway/internal/envoygateway/config" - "github.com/envoyproxy/gateway/internal/logging" - "github.com/envoyproxy/gateway/internal/message" -) - -func TestStart(t *testing.T) { - logger := logging.DefaultLogger(egv1a1.LogLevelInfo) - - testCases := []struct { - name string - cfg *config.Server - expect bool - }{ - { - name: "file provider", - cfg: &config.Server{ - EnvoyGateway: &egv1a1.EnvoyGateway{ - TypeMeta: metav1.TypeMeta{ - APIVersion: egv1a1.GroupVersion.String(), - Kind: egv1a1.KindEnvoyGateway, - }, - EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ - Provider: &egv1a1.EnvoyGatewayProvider{ - Type: egv1a1.ProviderTypeFile, - }, - }, - }, - Logger: logger, - }, - expect: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - runner := &Runner{ - Config: Config{ - Server: *tc.cfg, - ProviderResources: new(message.ProviderResources), - }, - } - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - err := runner.Start(ctx) - if tc.expect { - require.NoError(t, err) - } else { - require.Error(t, err, "An error was expected") - } - }) - } -} diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 4dc8b30f31b..fa46214800b 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -1045,7 +1045,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `resource` | _[EnvoyGatewayResourceProvider](#envoygatewayresourceprovider)_ | true | Resource defines the desired resource provider.
This provider is used to specify the provider to be used
to retrieve the resource configurations such as Gateway API
resources | -| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | true | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane. | +| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | false | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane.

Infrastructure is optional, if provider is not specified,
No infrastructure provider is available. | #### EnvoyGatewayFileResourceProvider @@ -1059,7 +1059,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `paths` | _string array_ | true | Paths are the paths to a directory or file containing the resource configuration.
Recursive sub directories are not currently supported. | +| `paths` | _string array_ | true | Paths are the paths to a directory or file containing the resource configuration.
Recursive subdirectories are not currently supported. | #### EnvoyGatewayHostInfrastructureProvider @@ -1217,9 +1217,9 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes". | +| `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes", "Custom". | | `kubernetes` | _[EnvoyGatewayKubernetesProvider](#envoygatewaykubernetesprovider)_ | false | Kubernetes defines the configuration of the Kubernetes provider. Kubernetes
provides runtime configuration via the Kubernetes API. | -| `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and a infrastructure
provider. | +| `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and an infrastructure
provider. | #### EnvoyGatewayResourceProvider @@ -2754,7 +2754,7 @@ _Appears in:_ | Value | Description | | ----- | ----------- | | `Kubernetes` | ProviderTypeKubernetes defines the "Kubernetes" provider.
| -| `File` | ProviderTypeFile defines the "File" provider. This type is not implemented
until https://github.com/envoyproxy/gateway/issues/1001 is fixed.
| +| `Custom` | ProviderTypeCustom defines the "Custom" provider.
| #### ProxyAccessLog diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md index 4dc8b30f31b..fa46214800b 100644 --- a/site/content/zh/latest/api/extension_types.md +++ b/site/content/zh/latest/api/extension_types.md @@ -1045,7 +1045,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `resource` | _[EnvoyGatewayResourceProvider](#envoygatewayresourceprovider)_ | true | Resource defines the desired resource provider.
This provider is used to specify the provider to be used
to retrieve the resource configurations such as Gateway API
resources | -| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | true | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane. | +| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | false | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane.

Infrastructure is optional, if provider is not specified,
No infrastructure provider is available. | #### EnvoyGatewayFileResourceProvider @@ -1059,7 +1059,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `paths` | _string array_ | true | Paths are the paths to a directory or file containing the resource configuration.
Recursive sub directories are not currently supported. | +| `paths` | _string array_ | true | Paths are the paths to a directory or file containing the resource configuration.
Recursive subdirectories are not currently supported. | #### EnvoyGatewayHostInfrastructureProvider @@ -1217,9 +1217,9 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes". | +| `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes", "Custom". | | `kubernetes` | _[EnvoyGatewayKubernetesProvider](#envoygatewaykubernetesprovider)_ | false | Kubernetes defines the configuration of the Kubernetes provider. Kubernetes
provides runtime configuration via the Kubernetes API. | -| `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and a infrastructure
provider. | +| `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and an infrastructure
provider. | #### EnvoyGatewayResourceProvider @@ -2754,7 +2754,7 @@ _Appears in:_ | Value | Description | | ----- | ----------- | | `Kubernetes` | ProviderTypeKubernetes defines the "Kubernetes" provider.
| -| `File` | ProviderTypeFile defines the "File" provider. This type is not implemented
until https://github.com/envoyproxy/gateway/issues/1001 is fixed.
| +| `Custom` | ProviderTypeCustom defines the "Custom" provider.
| #### ProxyAccessLog