From 93e4dd3d74ae920a5ba6dc0f5d6e263ba186f35c Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Mon, 1 Apr 2024 21:27:39 +0800 Subject: [PATCH 01/17] add validations for envoy-gateway file resource type Signed-off-by: shawnh2 --- api/v1alpha1/envoygateway_types.go | 4 +- .../validation/envoygateway_validate.go | 246 ++++++++++++------ .../validation/envoygateway_validate_test.go | 60 ++++- internal/cmd/server.go | 2 +- internal/provider/runner/runner_test.go | 2 +- site/content/en/latest/api/extension_types.md | 4 +- 6 files changed, 231 insertions(+), 87 deletions(-) diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 777b6c50950..e30009f4d5e 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -172,7 +172,7 @@ type EnvoyGatewayProvider struct { Kubernetes *EnvoyGatewayKubernetesProvider `json:"kubernetes,omitempty"` // Custom defines the configuration for the Custom provider. This provider - // allows you to define a specific resource provider and a infrastructure + // allows you to define a specific resource provider and an infrastructure // provider. // // +optional @@ -278,7 +278,7 @@ type EnvoyGatewayResourceProvider struct { // EnvoyGatewayFileResourceProvider defines configuration for the File Resource provider. type EnvoyGatewayFileResourceProvider struct { // Paths are the paths to a directory or file containing the resource configuration. - // Recursive sub directories are not currently supported. + // Recursive subdirectories are not currently supported. Paths []string `json:"paths"` } diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go index 629aae4ae6e..8ae0378c4f8 100644 --- a/api/v1alpha1/validation/envoygateway_validate.go +++ b/api/v1alpha1/validation/envoygateway_validate.go @@ -6,103 +6,193 @@ package validation import ( - "errors" "fmt" "net/url" - gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" - "github.com/envoyproxy/gateway/api/v1alpha1" ) // ValidateEnvoyGateway validates the provided EnvoyGateway. func ValidateEnvoyGateway(eg *v1alpha1.EnvoyGateway) error { - switch { - case eg == nil: - return errors.New("envoy gateway config is unspecified") - case eg.Gateway == nil: - return errors.New("gateway is unspecified") - case len(eg.Gateway.ControllerName) == 0: - return errors.New("gateway controllerName is unspecified") - case eg.Provider == nil: - return errors.New("provider is unspecified") - case eg.Provider.Type != v1alpha1.ProviderTypeKubernetes: - return fmt.Errorf("unsupported provider %v", eg.Provider.Type) - case eg.Provider.Kubernetes != nil && eg.Provider.Kubernetes.Watch != nil: - watch := eg.Provider.Kubernetes.Watch - switch watch.Type { - case v1alpha1.KubernetesWatchModeTypeNamespaces: - if len(watch.Namespaces) == 0 { - return errors.New("namespaces should be specified when envoy gateway watch mode is 'Namespaces'") - } - case v1alpha1.KubernetesWatchModeTypeNamespaceSelector: - if watch.NamespaceSelector == nil { - return errors.New("namespaceSelector should be specified when envoy gateway watch mode is 'NamespaceSelector'") - } - default: - return errors.New("envoy gateway watch mode invalid, should be 'Namespaces' or 'NamespaceSelector'") - } - case eg.Logging != nil && len(eg.Logging.Level) != 0: - level := eg.Logging.Level - for component, logLevel := range level { - switch component { - case v1alpha1.LogComponentGatewayDefault, - v1alpha1.LogComponentProviderRunner, - v1alpha1.LogComponentGatewayAPIRunner, - v1alpha1.LogComponentXdsTranslatorRunner, - v1alpha1.LogComponentXdsServerRunner, - v1alpha1.LogComponentInfrastructureRunner, - v1alpha1.LogComponentGlobalRateLimitRunner: - switch logLevel { - case v1alpha1.LogLevelDebug, v1alpha1.LogLevelError, v1alpha1.LogLevelWarn, v1alpha1.LogLevelInfo: - default: - return errors.New("envoy gateway logging level invalid. valid options: info/debug/warn/error") - } - default: - return errors.New("envoy gateway logging components invalid. valid options: system/provider/gateway-api/xds-translator/xds-server/infrastructure") - } - } - case eg.RateLimit != nil: - if eg.RateLimit.Backend.Type != v1alpha1.RedisBackendType { - return fmt.Errorf("unsupported ratelimit backend %v", eg.RateLimit.Backend.Type) + if eg == nil { + return fmt.Errorf("envoy gateway config is unspecified") + } + + if eg.Gateway == nil { + return fmt.Errorf("gateway is unspecified") + } + + if len(eg.Gateway.ControllerName) == 0 { + return fmt.Errorf("gateway controllerName is unspecified") + } + + if eg.Provider == nil { + return fmt.Errorf("provider is unspecified") + } + + switch eg.Provider.Type { + case v1alpha1.ProviderTypeKubernetes: + if err := validateEnvoyGatewayKubernetesProvider(eg.Provider.Kubernetes); err != nil { + return err } - if eg.RateLimit.Backend.Redis == nil || eg.RateLimit.Backend.Redis.URL == "" { - return fmt.Errorf("empty ratelimit redis settings") + case v1alpha1.ProviderTypeFile: + if err := validateEnvoyGatewayFileProvider(eg.Provider.Custom); err != nil { + return err } - if _, err := url.Parse(eg.RateLimit.Backend.Redis.URL); err != nil { - return fmt.Errorf("unknown ratelimit redis url format: %w", err) + default: + return fmt.Errorf("unsupported provider type") + } + + if err := validateEnvoyGatewayLogging(eg.Logging); err != nil { + return err + } + + if err := validateEnvoyGatewayRateLimit(eg.RateLimit); err != nil { + return err + } + + if err := validateEnvoyGatewayExtensionManager(eg.ExtensionManager); err != nil { + return err + } + + if err := validateEnvoyGatewayTelemetry(eg.Telemetry); err != nil { + return err + } + + return nil +} + +func validateEnvoyGatewayKubernetesProvider(provider *v1alpha1.EnvoyGatewayKubernetesProvider) error { + if provider == nil { + return nil + } + + watch := provider.Watch + if watch == nil { + return nil + } + + switch watch.Type { + case v1alpha1.KubernetesWatchModeTypeNamespaces: + if len(watch.Namespaces) == 0 { + return fmt.Errorf("namespaces should be specified when envoy gateway watch mode is 'Namespaces'") } - case eg.ExtensionManager != nil: - if eg.ExtensionManager.Hooks == nil || eg.ExtensionManager.Hooks.XDSTranslator == nil { - return fmt.Errorf("registered extension has no hooks specified") + case v1alpha1.KubernetesWatchModeTypeNamespaceSelector: + if watch.NamespaceSelector == nil { + return fmt.Errorf("namespaceSelector should be specified when envoy gateway watch mode is 'NamespaceSelector'") } + default: + return fmt.Errorf("envoy gateway watch mode invalid, should be 'Namespaces' or 'NamespaceSelector'") + } + return nil +} - if len(eg.ExtensionManager.Hooks.XDSTranslator.Pre) == 0 && len(eg.ExtensionManager.Hooks.XDSTranslator.Post) == 0 { - return fmt.Errorf("registered extension has no hooks specified") - } +func validateEnvoyGatewayFileProvider(provider *v1alpha1.EnvoyGatewayCustomProvider) error { + if provider == nil { + return nil + } - if eg.ExtensionManager.Service == nil { - return fmt.Errorf("extension service config is empty") - } + rType, iType := provider.Resource.Type, provider.Infrastructure.Type + if rType != v1alpha1.ResourceProviderTypeFile || iType != v1alpha1.InfrastructureProviderTypeHost { + return fmt.Errorf("file provider only supports 'File' resource type and 'Host' infra type") + } - if eg.ExtensionManager.Service.TLS != nil { - certificateRefKind := eg.ExtensionManager.Service.TLS.CertificateRef.Kind + if provider.Resource.File == nil { + return fmt.Errorf("field 'file' should be specified when resource type is 'File'") + } - if certificateRefKind == nil { - return fmt.Errorf("certificateRef empty in extension service server TLS settings") - } + if provider.Infrastructure.Host == nil { + return fmt.Errorf("field 'host' should be specified when infrastructure type is 'Host'") + } + + // TODO(sh2): add more validations for infra.host + + return nil +} - if *certificateRefKind != gwapiv1.Kind("Secret") { - return fmt.Errorf("unsupported extension server TLS certificateRef %v", certificateRefKind) +func validateEnvoyGatewayLogging(logging *v1alpha1.EnvoyGatewayLogging) error { + if logging == nil || len(logging.Level) == 0 { + return nil + } + + for component, logLevel := range logging.Level { + switch component { + case v1alpha1.LogComponentGatewayDefault, + v1alpha1.LogComponentProviderRunner, + v1alpha1.LogComponentGatewayAPIRunner, + v1alpha1.LogComponentXdsTranslatorRunner, + v1alpha1.LogComponentXdsServerRunner, + v1alpha1.LogComponentInfrastructureRunner, + v1alpha1.LogComponentGlobalRateLimitRunner: + switch logLevel { + case v1alpha1.LogLevelDebug, v1alpha1.LogLevelError, v1alpha1.LogLevelWarn, v1alpha1.LogLevelInfo: + default: + return fmt.Errorf("envoy gateway logging level invalid. valid options: info/debug/warn/error") } + default: + return fmt.Errorf("envoy gateway logging components invalid. valid options: system/provider/gateway-api/xds-translator/xds-server/infrastructure") + } + } + return nil +} + +func validateEnvoyGatewayRateLimit(rateLimit *v1alpha1.RateLimit) error { + if rateLimit == nil { + return nil + } + if rateLimit.Backend.Type != v1alpha1.RedisBackendType { + return fmt.Errorf("unsupported ratelimit backend %v", rateLimit.Backend.Type) + } + if rateLimit.Backend.Redis == nil || rateLimit.Backend.Redis.URL == "" { + return fmt.Errorf("empty ratelimit redis settings") + } + if _, err := url.Parse(rateLimit.Backend.Redis.URL); err != nil { + return fmt.Errorf("unknown ratelimit redis url format: %w", err) + } + return nil +} + +func validateEnvoyGatewayExtensionManager(extensionManager *v1alpha1.ExtensionManager) error { + if extensionManager == nil { + return nil + } + + if extensionManager.Hooks == nil || extensionManager.Hooks.XDSTranslator == nil { + return fmt.Errorf("registered extension has no hooks specified") + } + + if len(extensionManager.Hooks.XDSTranslator.Pre) == 0 && len(extensionManager.Hooks.XDSTranslator.Post) == 0 { + return fmt.Errorf("registered extension has no hooks specified") + } + + if extensionManager.Service == nil { + return fmt.Errorf("extension service config is empty") + } + + if extensionManager.Service.TLS != nil { + certificateRefKind := extensionManager.Service.TLS.CertificateRef.Kind + + if certificateRefKind == nil { + return fmt.Errorf("certificateRef empty in extension service server TLS settings") } - case eg.Telemetry != nil: - if eg.Telemetry.Metrics != nil { - for _, sink := range eg.Telemetry.Metrics.Sinks { - if sink.Type == v1alpha1.MetricSinkTypeOpenTelemetry { - if sink.OpenTelemetry == nil { - return fmt.Errorf("OpenTelemetry is required when sink Type is OpenTelemetry") - } + + if *certificateRefKind != "Secret" { + return fmt.Errorf("unsupported extension server TLS certificateRef %v", certificateRefKind) + } + } + return nil +} + +func validateEnvoyGatewayTelemetry(telemetry *v1alpha1.EnvoyGatewayTelemetry) error { + if telemetry == nil { + return nil + } + + if telemetry.Metrics != nil { + for _, sink := range telemetry.Metrics.Sinks { + if sink.Type == v1alpha1.MetricSinkTypeOpenTelemetry { + if sink.OpenTelemetry == nil { + return fmt.Errorf("OpenTelemetry is required when sink Type is OpenTelemetry") } } } diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index 612353ba88f..24acfc59ffa 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -68,11 +68,65 @@ func TestValidateEnvoyGateway(t *testing.T) { expect: false, }, { - name: "unsupported provider", + name: "supported file provider", eg: &v1alpha1.EnvoyGateway{ EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ - Gateway: v1alpha1.DefaultGateway(), - Provider: &v1alpha1.EnvoyGatewayProvider{Type: v1alpha1.ProviderTypeFile}, + Gateway: v1alpha1.DefaultGateway(), + Provider: &v1alpha1.EnvoyGatewayProvider{ + Type: v1alpha1.ProviderTypeFile, + Custom: &v1alpha1.EnvoyGatewayCustomProvider{ + Resource: v1alpha1.EnvoyGatewayResourceProvider{ + Type: v1alpha1.ResourceProviderTypeFile, + File: &v1alpha1.EnvoyGatewayFileResourceProvider{}, + }, + Infrastructure: v1alpha1.EnvoyGatewayInfrastructureProvider{ + Type: v1alpha1.InfrastructureProviderTypeHost, + Host: &v1alpha1.EnvoyGatewayHostInfrastructureProvider{}, + }, + }, + }, + }, + }, + expect: true, + }, + { + name: "file provider without file resource", + eg: &v1alpha1.EnvoyGateway{ + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Gateway: v1alpha1.DefaultGateway(), + Provider: &v1alpha1.EnvoyGatewayProvider{ + Type: v1alpha1.ProviderTypeFile, + Custom: &v1alpha1.EnvoyGatewayCustomProvider{ + Resource: v1alpha1.EnvoyGatewayResourceProvider{ + Type: v1alpha1.ResourceProviderTypeFile, + }, + Infrastructure: v1alpha1.EnvoyGatewayInfrastructureProvider{ + Type: v1alpha1.InfrastructureProviderTypeHost, + Host: &v1alpha1.EnvoyGatewayHostInfrastructureProvider{}, + }, + }, + }, + }, + }, + expect: false, + }, + { + name: "file provider without host infrastructure", + eg: &v1alpha1.EnvoyGateway{ + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Gateway: v1alpha1.DefaultGateway(), + Provider: &v1alpha1.EnvoyGatewayProvider{ + Type: v1alpha1.ProviderTypeFile, + Custom: &v1alpha1.EnvoyGatewayCustomProvider{ + Resource: v1alpha1.EnvoyGatewayResourceProvider{ + Type: v1alpha1.ResourceProviderTypeFile, + File: &v1alpha1.EnvoyGatewayFileResourceProvider{}, + }, + Infrastructure: v1alpha1.EnvoyGatewayInfrastructureProvider{ + Type: v1alpha1.InfrastructureProviderTypeHost, + }, + }, + }, }, }, expect: false, diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 6dc25a19946..7b04866aa11 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -124,7 +124,7 @@ func setupRunners(cfg *config.Server) error { pResources := new(message.ProviderResources) // Start the Provider Service // It fetches the resources from the configured provider type - // and publishes it + // and publishes it. // It also subscribes to status resources and once it receives // a status resource back, it writes it out. providerRunner := providerrunner.New(&providerrunner.Config{ diff --git a/internal/provider/runner/runner_test.go b/internal/provider/runner/runner_test.go index d7fcd1092af..6a6c91732bb 100644 --- a/internal/provider/runner/runner_test.go +++ b/internal/provider/runner/runner_test.go @@ -42,7 +42,7 @@ func TestStart(t *testing.T) { }, Logger: logger, }, - expect: false, + expect: true, }, } diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index a289a7530b8..a08ba568b9c 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -626,7 +626,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `paths` | _string array_ | true | Paths are the paths to a directory or file containing the resource configuration.
Recursive sub directories are not currently supported. | +| `paths` | _string array_ | true | Paths are the paths to a directory or file containing the resource configuration.
Recursive subdirectories are not currently supported. | #### EnvoyGatewayHostInfrastructureProvider @@ -773,7 +773,7 @@ _Appears in:_ | --- | --- | --- | --- | | `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes". | | `kubernetes` | _[EnvoyGatewayKubernetesProvider](#envoygatewaykubernetesprovider)_ | false | Kubernetes defines the configuration of the Kubernetes provider. Kubernetes
provides runtime configuration via the Kubernetes API. | -| `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and a infrastructure
provider. | +| `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and an infrastructure
provider. | #### EnvoyGatewayResourceProvider From 3956911b9d20f61a6a136b7388bfaa657e8d005c Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 2 Apr 2024 15:58:46 +0800 Subject: [PATCH 02/17] improve eg validation and add resource provider interface for various provider Signed-off-by: shawnh2 --- .../validation/envoygateway_validate.go | 8 ++- .../validation/envoygateway_validate_test.go | 26 ++++++- internal/infrastructure/manager.go | 11 ++- internal/provider/file/file.go | 42 ++++++++++++ internal/provider/file/watcher.go | 48 +++++++++++++ internal/provider/kubernetes/kubernetes.go | 5 ++ internal/provider/resource_provider.go | 20 ++++++ internal/provider/runner/runner.go | 39 +++++++---- internal/provider/runner/runner_test.go | 68 ------------------- 9 files changed, 179 insertions(+), 88 deletions(-) create mode 100644 internal/provider/file/file.go create mode 100644 internal/provider/file/watcher.go create mode 100644 internal/provider/resource_provider.go delete mode 100644 internal/provider/runner/runner_test.go diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go index 8ae0378c4f8..509312299c1 100644 --- a/api/v1alpha1/validation/envoygateway_validate.go +++ b/api/v1alpha1/validation/envoygateway_validate.go @@ -89,7 +89,7 @@ func validateEnvoyGatewayKubernetesProvider(provider *v1alpha1.EnvoyGatewayKuber func validateEnvoyGatewayFileProvider(provider *v1alpha1.EnvoyGatewayCustomProvider) error { if provider == nil { - return nil + return fmt.Errorf("empty custom provider settings for file provider") } rType, iType := provider.Resource.Type, provider.Infrastructure.Type @@ -101,12 +101,14 @@ func validateEnvoyGatewayFileProvider(provider *v1alpha1.EnvoyGatewayCustomProvi return fmt.Errorf("field 'file' should be specified when resource type is 'File'") } + if len(provider.Resource.File.Paths) == 0 { + return fmt.Errorf("no paths were assigned for file resource provider to watch") + } + if provider.Infrastructure.Host == nil { return fmt.Errorf("field 'host' should be specified when infrastructure type is 'Host'") } - // TODO(sh2): add more validations for infra.host - return nil } diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index 24acfc59ffa..00c74cce7e4 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -77,7 +77,9 @@ func TestValidateEnvoyGateway(t *testing.T) { Custom: &v1alpha1.EnvoyGatewayCustomProvider{ Resource: v1alpha1.EnvoyGatewayResourceProvider{ Type: v1alpha1.ResourceProviderTypeFile, - File: &v1alpha1.EnvoyGatewayFileResourceProvider{}, + File: &v1alpha1.EnvoyGatewayFileResourceProvider{ + Paths: []string{"foo", "bar"}, + }, }, Infrastructure: v1alpha1.EnvoyGatewayInfrastructureProvider{ Type: v1alpha1.InfrastructureProviderTypeHost, @@ -131,6 +133,28 @@ func TestValidateEnvoyGateway(t *testing.T) { }, expect: false, }, + { + name: "file provider without any paths assign in resource", + eg: &v1alpha1.EnvoyGateway{ + EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ + Gateway: v1alpha1.DefaultGateway(), + Provider: &v1alpha1.EnvoyGatewayProvider{ + Type: v1alpha1.ProviderTypeFile, + Custom: &v1alpha1.EnvoyGatewayCustomProvider{ + Resource: v1alpha1.EnvoyGatewayResourceProvider{ + Type: v1alpha1.ResourceProviderTypeFile, + File: &v1alpha1.EnvoyGatewayFileResourceProvider{}, + }, + Infrastructure: v1alpha1.EnvoyGatewayInfrastructureProvider{ + Type: v1alpha1.InfrastructureProviderTypeHost, + Host: &v1alpha1.EnvoyGatewayHostInfrastructureProvider{}, + }, + }, + }, + }, + }, + expect: false, + }, { name: "empty ratelimit", eg: &v1alpha1.EnvoyGateway{ diff --git a/internal/infrastructure/manager.go b/internal/infrastructure/manager.go index 2a5cae1062a..96627a688fc 100644 --- a/internal/infrastructure/manager.go +++ b/internal/infrastructure/manager.go @@ -36,14 +36,19 @@ type Manager interface { // NewManager returns a new infrastructure Manager. func NewManager(cfg *config.Server) (Manager, error) { var mgr Manager - if cfg.EnvoyGateway.Provider.Type == v1alpha1.ProviderTypeKubernetes { + + switch cfg.EnvoyGateway.Provider.Type { + case v1alpha1.ProviderTypeKubernetes: cli, err := client.New(clicfg.GetConfigOrDie(), client.Options{Scheme: envoygateway.GetScheme()}) if err != nil { return nil, err } mgr = kubernetes.NewInfra(cli, cfg) - } else { - // Kube is the only supported provider type for now. + + case v1alpha1.ProviderTypeFile: + // TODO(sh2): implement host infra for file provider + + default: return nil, fmt.Errorf("unsupported provider type %v", cfg.EnvoyGateway.Provider.Type) } diff --git a/internal/provider/file/file.go b/internal/provider/file/file.go new file mode 100644 index 00000000000..b48aa71430c --- /dev/null +++ b/internal/provider/file/file.go @@ -0,0 +1,42 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "context" + + "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/message" +) + +type Provider struct { + watcher *watcher +} + +func New(svr *config.Server, resources *message.ProviderResources) *Provider { + return &Provider{ + watcher: newWatcher(svr.EnvoyGateway.Provider.Custom.Resource.File.Paths), + } +} + +func (p *Provider) Type() v1alpha1.ProviderType { + return v1alpha1.ProviderTypeFile +} + +func (p *Provider) Start(ctx context.Context) error { + errChan := make(chan error) + go func() { + errChan <- p.watcher.Watch(ctx) + }() + + select { + case <-ctx.Done(): + return nil + case err := <-errChan: + return err + } +} diff --git a/internal/provider/file/watcher.go b/internal/provider/file/watcher.go new file mode 100644 index 00000000000..9e27bb9ea6c --- /dev/null +++ b/internal/provider/file/watcher.go @@ -0,0 +1,48 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "context" + "time" +) + +const ( + // defaultWatchTicker defines default ticker (in seconds) for watcher. + // TODO(sh2): make it configurable + defaultWatchTicker = 3 +) + +type watcher struct { + paths []string + + ticker *time.Ticker +} + +func newWatcher(paths []string) *watcher { + return &watcher{ + paths: paths, + ticker: time.NewTicker(defaultWatchTicker * time.Second), + } +} + +// Watch watches and loads files from paths. +func (w *watcher) Watch(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + w.ticker.Stop() + return nil + case <-w.ticker.C: + w.watch() + default: + } + } +} + +func (w *watcher) watch() { + // TODO: implement watch logic +} diff --git a/internal/provider/kubernetes/kubernetes.go b/internal/provider/kubernetes/kubernetes.go index 3a4bfb7c793..1d201f53837 100644 --- a/internal/provider/kubernetes/kubernetes.go +++ b/internal/provider/kubernetes/kubernetes.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/message" @@ -79,6 +80,10 @@ func New(cfg *rest.Config, svr *config.Server, resources *message.ProviderResour }, nil } +func (p *Provider) Type() v1alpha1.ProviderType { + return v1alpha1.ProviderTypeKubernetes +} + // Start starts the Provider synchronously until a message is received from ctx. func (p *Provider) Start(ctx context.Context) error { errChan := make(chan error) diff --git a/internal/provider/resource_provider.go b/internal/provider/resource_provider.go new file mode 100644 index 00000000000..0ce24940d11 --- /dev/null +++ b/internal/provider/resource_provider.go @@ -0,0 +1,20 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package provider + +import ( + "context" + + "github.com/envoyproxy/gateway/api/v1alpha1" +) + +type Provider interface { + // Start starts the resource provider. + Start(ctx context.Context) error + + // Type returns the type of resource provider. + Type() v1alpha1.ProviderType +} diff --git a/internal/provider/runner/runner.go b/internal/provider/runner/runner.go index 7e298948321..7a561a3440d 100644 --- a/internal/provider/runner/runner.go +++ b/internal/provider/runner/runner.go @@ -9,11 +9,14 @@ import ( "context" "fmt" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/message" + "github.com/envoyproxy/gateway/internal/provider" + "github.com/envoyproxy/gateway/internal/provider/file" "github.com/envoyproxy/gateway/internal/provider/kubernetes" ) @@ -37,24 +40,34 @@ func (r *Runner) Name() string { // Start the provider runner func (r *Runner) Start(ctx context.Context) (err error) { r.Logger = r.Logger.WithName(r.Name()).WithValues("runner", r.Name()) - if r.EnvoyGateway.Provider.Type == v1alpha1.ProviderTypeKubernetes { - r.Logger.Info("Using provider", "type", v1alpha1.ProviderTypeKubernetes) - cfg, err := ctrl.GetConfig() + + var p provider.Provider + switch r.EnvoyGateway.Provider.Type { + case v1alpha1.ProviderTypeKubernetes: + var cfg *rest.Config + cfg, err = ctrl.GetConfig() if err != nil { return fmt.Errorf("failed to get kubeconfig: %w", err) } - p, err := kubernetes.New(cfg, &r.Config.Server, r.ProviderResources) + p, err = kubernetes.New(cfg, &r.Config.Server, r.ProviderResources) if err != nil { return fmt.Errorf("failed to create provider %s: %w", v1alpha1.ProviderTypeKubernetes, err) } - go func() { - err := p.Start(ctx) - if err != nil { - r.Logger.Error(err, "unable to start provider") - } - }() - return nil + + case v1alpha1.ProviderTypeFile: + p = file.New(&r.Config.Server, r.ProviderResources) + + default: + // Unsupported provider. + return fmt.Errorf("unsupported provider type %v", r.EnvoyGateway.Provider.Type) } - // Unsupported provider. - return fmt.Errorf("unsupported provider type %v", r.EnvoyGateway.Provider.Type) + + r.Logger.Info("Using provider", "type", p.Type()) + go func() { + if err = p.Start(ctx); err != nil { + r.Logger.Error(err, "unable to start provider") + } + }() + + return nil } diff --git a/internal/provider/runner/runner_test.go b/internal/provider/runner/runner_test.go deleted file mode 100644 index 6a6c91732bb..00000000000 --- a/internal/provider/runner/runner_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package runner - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/envoyproxy/gateway/api/v1alpha1" - "github.com/envoyproxy/gateway/internal/envoygateway/config" - "github.com/envoyproxy/gateway/internal/logging" - "github.com/envoyproxy/gateway/internal/message" -) - -func TestStart(t *testing.T) { - logger := logging.DefaultLogger(v1alpha1.LogLevelInfo) - - testCases := []struct { - name string - cfg *config.Server - expect bool - }{ - { - name: "file provider", - cfg: &config.Server{ - EnvoyGateway: &v1alpha1.EnvoyGateway{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1alpha1.GroupVersion.String(), - Kind: v1alpha1.KindEnvoyGateway, - }, - EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{ - Provider: &v1alpha1.EnvoyGatewayProvider{ - Type: v1alpha1.ProviderTypeFile, - }, - }, - }, - Logger: logger, - }, - expect: true, - }, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - runner := &Runner{ - Config: Config{ - Server: *tc.cfg, - ProviderResources: new(message.ProviderResources), - }, - } - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - err := runner.Start(ctx) - if tc.expect { - require.NoError(t, err) - } else { - require.Error(t, err, "An error was expected") - } - }) - } -} From 7de91b31740844cc959521769e44e766e6814630 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Wed, 3 Apr 2024 16:55:36 +0800 Subject: [PATCH 03/17] extract common gatewayapi layer translate logic in egctl translate Signed-off-by: shawnh2 --- internal/cmd/egctl/translate.go | 458 +--------------------------- internal/gatewayapi/convert.go | 460 +++++++++++++++++++++++++++++ internal/provider/runner/runner.go | 2 +- 3 files changed, 467 insertions(+), 453 deletions(-) create mode 100644 internal/gatewayapi/convert.go diff --git a/internal/cmd/egctl/translate.go b/internal/cmd/egctl/translate.go index 7935950bcbc..61161cca277 100644 --- a/internal/cmd/egctl/translate.go +++ b/internal/cmd/egctl/translate.go @@ -6,35 +6,23 @@ package egctl import ( - "bufio" "encoding/json" "fmt" "io" - "os" - "reflect" "sort" - "strings" + adminv3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" + bootstrapv3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" + resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" "github.com/spf13/cobra" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/known/anypb" - "sigs.k8s.io/yaml" - - adminv3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" - bootstrapv3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" - resourcev3 "github.com/envoyproxy/go-control-plane/pkg/resource/v3" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/util/sets" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" - gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + "sigs.k8s.io/yaml" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/api/v1alpha1/validation" - "github.com/envoyproxy/gateway/internal/envoygateway" - "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/status" @@ -182,23 +170,6 @@ func getValidResourceTypesStr() string { return fmt.Sprintf("Valid types are %v.", validResourceTypes()) } -func getInputBytes(inFile string) ([]byte, error) { - // Get input from stdin - if inFile == "-" { - scanner := bufio.NewScanner(os.Stdin) - var input string - for { - if !scanner.Scan() { - break - } - input += scanner.Text() + "\n" - } - return []byte(input), nil - } - // Get input from file - return os.ReadFile(inFile) -} - func validate(inFile, inType string, outTypes []string, resourceType string) error { if !isValidInputType(inType) { return fmt.Errorf("%s is not a valid input type. %s", inType, getValidInputTypesStr()) @@ -222,14 +193,14 @@ func translate(w io.Writer, inFile, inType string, outTypes []string, output, re return err } - inBytes, err := getInputBytes(inFile) + inBytes, err := gatewayapi.ReadKubernetesYAMLBytes(inFile) if err != nil { return fmt.Errorf("unable to read input file: %w", err) } if inType == gatewayAPIType { // Unmarshal input - resources, err := kubernetesYAMLToResources(string(inBytes), addMissingResources) + resources, err := gatewayapi.ConvertKubernetesYAMLToResources(string(inBytes), addMissingResources) if err != nil { return fmt.Errorf("unable to unmarshal input: %w", err) } @@ -537,420 +508,3 @@ func constructConfigDump(resources *gatewayapi.Resources, tCtx *xds_types.Resour return globalConfigs, nil } - -func addMissingServices(requiredServices map[string]*v1.Service, obj interface{}) { - var objNamespace string - protocol := v1.Protocol(gatewayapi.TCPProtocol) - - refs := []gwapiv1.BackendRef{} - switch route := obj.(type) { - case *gwapiv1.HTTPRoute: - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - for _, httpBakcendRef := range rule.BackendRefs { - refs = append(refs, httpBakcendRef.BackendRef) - } - } - case *gwapiv1a2.GRPCRoute: - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - for _, gRPCBakcendRef := range rule.BackendRefs { - refs = append(refs, gRPCBakcendRef.BackendRef) - } - } - case *gwapiv1a2.TLSRoute: - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - refs = append(refs, rule.BackendRefs...) - } - case *gwapiv1a2.TCPRoute: - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - refs = append(refs, rule.BackendRefs...) - } - case *gwapiv1a2.UDPRoute: - protocol = v1.Protocol(gatewayapi.UDPProtocol) - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - refs = append(refs, rule.BackendRefs...) - } - } - - for _, ref := range refs { - if ref.Kind == nil || *ref.Kind != gatewayapi.KindService { - continue - } - - ns := objNamespace - if ref.Namespace != nil { - ns = string(*ref.Namespace) - } - name := string(ref.Name) - key := ns + "/" + name - - port := int32(*ref.Port) - servicePort := v1.ServicePort{ - Name: fmt.Sprintf("%s-%d", protocol, port), - Protocol: protocol, - Port: port, - } - if service, found := requiredServices[key]; !found { - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - }, - Spec: v1.ServiceSpec{ - // Just a dummy IP - ClusterIP: "127.0.0.1", - Ports: []v1.ServicePort{servicePort}, - }, - } - requiredServices[key] = service - - } else { - inserted := false - for _, port := range service.Spec.Ports { - if port.Protocol == servicePort.Protocol && port.Port == servicePort.Port { - inserted = true - break - } - } - - if !inserted { - service.Spec.Ports = append(service.Spec.Ports, servicePort) - } - } - } -} - -// kubernetesYAMLToResources converts a Kubernetes YAML string into GatewayAPI Resources -func kubernetesYAMLToResources(str string, addMissingResources bool) (*gatewayapi.Resources, error) { - resources := gatewayapi.NewResources() - var useDefaultNamespace bool - providedNamespaceMap := map[string]struct{}{} - requiredNamespaceMap := map[string]struct{}{} - yamls := strings.Split(str, "\n---") - combinedScheme := envoygateway.GetScheme() - for _, y := range yamls { - if strings.TrimSpace(y) == "" { - continue - } - var obj map[string]interface{} - err := yaml.Unmarshal([]byte(y), &obj) - if err != nil { - return nil, err - } - un := unstructured.Unstructured{Object: obj} - gvk := un.GroupVersionKind() - name, namespace := un.GetName(), un.GetNamespace() - if namespace == "" { - // When kubectl applies a resource in yaml which doesn't have a namespace, - // the current namespace is applied. Here we do the same thing before translating - // the GatewayAPI resource. Otherwise, the resource can't pass the namespace validation - useDefaultNamespace = true - namespace = config.DefaultNamespace - } - requiredNamespaceMap[namespace] = struct{}{} - kobj, err := combinedScheme.New(gvk) - if err != nil { - return nil, err - } - err = combinedScheme.Convert(&un, kobj, nil) - if err != nil { - return nil, err - } - - objType := reflect.TypeOf(kobj) - if objType.Kind() != reflect.Ptr { - return nil, fmt.Errorf("expected pointer type, but got %s", objType.Kind().String()) - } - kobjVal := reflect.ValueOf(kobj).Elem() - spec := kobjVal.FieldByName("Spec") - - switch gvk.Kind { - case gatewayapi.KindEnvoyProxy: - typedSpec := spec.Interface() - envoyProxy := &egv1a1.EnvoyProxy{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(egv1a1.EnvoyProxySpec), - } - resources.EnvoyProxy = envoyProxy - case gatewayapi.KindGatewayClass: - typedSpec := spec.Interface() - gatewayClass := &gwapiv1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1.GatewayClassSpec), - } - resources.GatewayClass = gatewayClass - case gatewayapi.KindGateway: - typedSpec := spec.Interface() - gateway := &gwapiv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1.GatewaySpec), - } - resources.Gateways = append(resources.Gateways, gateway) - case gatewayapi.KindTCPRoute: - typedSpec := spec.Interface() - tcpRoute := &gwapiv1a2.TCPRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: gatewayapi.KindTCPRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1a2.TCPRouteSpec), - } - resources.TCPRoutes = append(resources.TCPRoutes, tcpRoute) - case gatewayapi.KindUDPRoute: - typedSpec := spec.Interface() - udpRoute := &gwapiv1a2.UDPRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: gatewayapi.KindUDPRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1a2.UDPRouteSpec), - } - resources.UDPRoutes = append(resources.UDPRoutes, udpRoute) - case gatewayapi.KindTLSRoute: - typedSpec := spec.Interface() - tlsRoute := &gwapiv1a2.TLSRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: gatewayapi.KindTLSRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1a2.TLSRouteSpec), - } - resources.TLSRoutes = append(resources.TLSRoutes, tlsRoute) - case gatewayapi.KindHTTPRoute: - typedSpec := spec.Interface() - httpRoute := &gwapiv1.HTTPRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: gatewayapi.KindHTTPRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1.HTTPRouteSpec), - } - resources.HTTPRoutes = append(resources.HTTPRoutes, httpRoute) - case gatewayapi.KindGRPCRoute: - typedSpec := spec.Interface() - grpcRoute := &gwapiv1a2.GRPCRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: gatewayapi.KindGRPCRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1a2.GRPCRouteSpec), - } - resources.GRPCRoutes = append(resources.GRPCRoutes, grpcRoute) - case gatewayapi.KindNamespace: - namespace := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - } - resources.Namespaces = append(resources.Namespaces, namespace) - providedNamespaceMap[name] = struct{}{} - case gatewayapi.KindService: - typedSpec := spec.Interface() - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(v1.ServiceSpec), - } - resources.Services = append(resources.Services, service) - case egv1a1.KindEnvoyPatchPolicy: - typedSpec := spec.Interface() - envoyPatchPolicy := &egv1a1.EnvoyPatchPolicy{ - TypeMeta: metav1.TypeMeta{ - Kind: egv1a1.KindEnvoyPatchPolicy, - APIVersion: egv1a1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Spec: typedSpec.(egv1a1.EnvoyPatchPolicySpec), - } - resources.EnvoyPatchPolicies = append(resources.EnvoyPatchPolicies, envoyPatchPolicy) - case egv1a1.KindClientTrafficPolicy: - typedSpec := spec.Interface() - clientTrafficPolicy := &egv1a1.ClientTrafficPolicy{ - TypeMeta: metav1.TypeMeta{ - Kind: egv1a1.KindClientTrafficPolicy, - APIVersion: egv1a1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Spec: typedSpec.(egv1a1.ClientTrafficPolicySpec), - } - resources.ClientTrafficPolicies = append(resources.ClientTrafficPolicies, clientTrafficPolicy) - case egv1a1.KindBackendTrafficPolicy: - typedSpec := spec.Interface() - backendTrafficPolicy := &egv1a1.BackendTrafficPolicy{ - TypeMeta: metav1.TypeMeta{ - Kind: egv1a1.KindBackendTrafficPolicy, - APIVersion: egv1a1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Spec: typedSpec.(egv1a1.BackendTrafficPolicySpec), - } - resources.BackendTrafficPolicies = append(resources.BackendTrafficPolicies, backendTrafficPolicy) - case egv1a1.KindSecurityPolicy: - typedSpec := spec.Interface() - securityPolicy := &egv1a1.SecurityPolicy{ - TypeMeta: metav1.TypeMeta{ - Kind: egv1a1.KindSecurityPolicy, - APIVersion: egv1a1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Spec: typedSpec.(egv1a1.SecurityPolicySpec), - } - resources.SecurityPolicies = append(resources.SecurityPolicies, securityPolicy) - } - } - - if useDefaultNamespace { - if _, found := providedNamespaceMap[config.DefaultNamespace]; !found { - namespace := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: config.DefaultNamespace, - }, - } - resources.Namespaces = append(resources.Namespaces, namespace) - providedNamespaceMap[config.DefaultNamespace] = struct{}{} - } - } - - if addMissingResources { - for ns := range requiredNamespaceMap { - if _, found := providedNamespaceMap[ns]; !found { - namespace := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: ns, - }, - } - resources.Namespaces = append(resources.Namespaces, namespace) - } - } - - requiredServiceMap := map[string]*v1.Service{} - for _, route := range resources.TCPRoutes { - addMissingServices(requiredServiceMap, route) - } - for _, route := range resources.UDPRoutes { - addMissingServices(requiredServiceMap, route) - } - for _, route := range resources.TLSRoutes { - addMissingServices(requiredServiceMap, route) - } - for _, route := range resources.HTTPRoutes { - addMissingServices(requiredServiceMap, route) - } - for _, route := range resources.GRPCRoutes { - addMissingServices(requiredServiceMap, route) - } - - providedServiceMap := map[string]*v1.Service{} - for _, service := range resources.Services { - providedServiceMap[service.Namespace+"/"+service.Name] = service - } - - for key, service := range requiredServiceMap { - if provided, found := providedServiceMap[key]; !found { - resources.Services = append(resources.Services, service) - } else { - providedPorts := sets.NewString() - for _, port := range provided.Spec.Ports { - portKey := fmt.Sprintf("%s-%d", port.Protocol, port.Port) - providedPorts.Insert(portKey) - } - - for _, port := range service.Spec.Ports { - name := fmt.Sprintf("%s-%d", port.Protocol, port.Port) - if !providedPorts.Has(name) { - servicePort := v1.ServicePort{ - Name: name, - Protocol: port.Protocol, - Port: port.Port, - } - provided.Spec.Ports = append(provided.Spec.Ports, servicePort) - } - } - } - } - - // Add EnvoyProxy if it does not exist - if resources.EnvoyProxy == nil { - if err := addDefaultEnvoyProxy(resources); err != nil { - return nil, err - } - } - } - - return resources, nil -} - -func addDefaultEnvoyProxy(resources *gatewayapi.Resources) error { - if resources.GatewayClass == nil { - return fmt.Errorf("the GatewayClass resource is required") - } - - defaultEnvoyProxyName := "default-envoy-proxy" - namespace := resources.GatewayClass.Namespace - defaultBootstrapStr, err := bootstrap.GetRenderedBootstrapConfig(nil) - if err != nil { - return err - } - ep := &egv1a1.EnvoyProxy{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: defaultEnvoyProxyName, - }, - Spec: egv1a1.EnvoyProxySpec{ - Bootstrap: &egv1a1.ProxyBootstrap{ - Value: defaultBootstrapStr, - }, - }, - } - resources.EnvoyProxy = ep - ns := gwapiv1.Namespace(namespace) - resources.GatewayClass.Spec.ParametersRef = &gwapiv1.ParametersReference{ - Group: gwapiv1.Group(egv1a1.GroupVersion.Group), - Kind: gatewayapi.KindEnvoyProxy, - Name: defaultEnvoyProxyName, - Namespace: &ns, - } - return nil -} diff --git a/internal/gatewayapi/convert.go b/internal/gatewayapi/convert.go new file mode 100644 index 00000000000..98e2fe53ff2 --- /dev/null +++ b/internal/gatewayapi/convert.go @@ -0,0 +1,460 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package gatewayapi + +import ( + "bufio" + "fmt" + "os" + "reflect" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + "sigs.k8s.io/yaml" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/xds/bootstrap" +) + +func ReadKubernetesYAMLBytes(filePath string) ([]byte, error) { + // Get input from stdin + if filePath == "-" { + scanner := bufio.NewScanner(os.Stdin) + var input string + for { + if !scanner.Scan() { + break + } + input += scanner.Text() + "\n" + } + return []byte(input), nil + } + // Get input from file + return os.ReadFile(filePath) +} + +// ConvertKubernetesYAMLToResources converts a Kubernetes YAML string into GatewayAPI Resources +func ConvertKubernetesYAMLToResources(str string, addMissingResources bool) (*Resources, error) { + resources := NewResources() + var useDefaultNamespace bool + providedNamespaceMap := map[string]struct{}{} + requiredNamespaceMap := map[string]struct{}{} + yamls := strings.Split(str, "\n---") + combinedScheme := envoygateway.GetScheme() + for _, y := range yamls { + if strings.TrimSpace(y) == "" { + continue + } + var obj map[string]interface{} + err := yaml.Unmarshal([]byte(y), &obj) + if err != nil { + return nil, err + } + un := unstructured.Unstructured{Object: obj} + gvk := un.GroupVersionKind() + name, namespace := un.GetName(), un.GetNamespace() + if namespace == "" { + // When kubectl applies a resource in yaml which doesn't have a namespace, + // the current namespace is applied. Here we do the same thing before translating + // the GatewayAPI resource. Otherwise, the resource can't pass the namespace validation + useDefaultNamespace = true + namespace = config.DefaultNamespace + } + requiredNamespaceMap[namespace] = struct{}{} + kobj, err := combinedScheme.New(gvk) + if err != nil { + return nil, err + } + err = combinedScheme.Convert(&un, kobj, nil) + if err != nil { + return nil, err + } + + objType := reflect.TypeOf(kobj) + if objType.Kind() != reflect.Ptr { + return nil, fmt.Errorf("expected pointer type, but got %s", objType.Kind().String()) + } + kobjVal := reflect.ValueOf(kobj).Elem() + spec := kobjVal.FieldByName("Spec") + + switch gvk.Kind { + case KindEnvoyProxy: + typedSpec := spec.Interface() + envoyProxy := &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(egv1a1.EnvoyProxySpec), + } + resources.EnvoyProxy = envoyProxy + case KindGatewayClass: + typedSpec := spec.Interface() + gatewayClass := &gwapiv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.GatewayClassSpec), + } + resources.GatewayClass = gatewayClass + case KindGateway: + typedSpec := spec.Interface() + gateway := &gwapiv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.GatewaySpec), + } + resources.Gateways = append(resources.Gateways, gateway) + case KindTCPRoute: + typedSpec := spec.Interface() + tcpRoute := &gwapiv1a2.TCPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: KindTCPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.TCPRouteSpec), + } + resources.TCPRoutes = append(resources.TCPRoutes, tcpRoute) + case KindUDPRoute: + typedSpec := spec.Interface() + udpRoute := &gwapiv1a2.UDPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: KindUDPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.UDPRouteSpec), + } + resources.UDPRoutes = append(resources.UDPRoutes, udpRoute) + case KindTLSRoute: + typedSpec := spec.Interface() + tlsRoute := &gwapiv1a2.TLSRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: KindTLSRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.TLSRouteSpec), + } + resources.TLSRoutes = append(resources.TLSRoutes, tlsRoute) + case KindHTTPRoute: + typedSpec := spec.Interface() + httpRoute := &gwapiv1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: KindHTTPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.HTTPRouteSpec), + } + resources.HTTPRoutes = append(resources.HTTPRoutes, httpRoute) + case KindGRPCRoute: + typedSpec := spec.Interface() + grpcRoute := &gwapiv1a2.GRPCRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: KindGRPCRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.GRPCRouteSpec), + } + resources.GRPCRoutes = append(resources.GRPCRoutes, grpcRoute) + case KindNamespace: + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + resources.Namespaces = append(resources.Namespaces, ns) + providedNamespaceMap[name] = struct{}{} + case KindService: + typedSpec := spec.Interface() + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(corev1.ServiceSpec), + } + resources.Services = append(resources.Services, service) + case egv1a1.KindEnvoyPatchPolicy: + typedSpec := spec.Interface() + envoyPatchPolicy := &egv1a1.EnvoyPatchPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindEnvoyPatchPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.EnvoyPatchPolicySpec), + } + resources.EnvoyPatchPolicies = append(resources.EnvoyPatchPolicies, envoyPatchPolicy) + case egv1a1.KindClientTrafficPolicy: + typedSpec := spec.Interface() + clientTrafficPolicy := &egv1a1.ClientTrafficPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindClientTrafficPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.ClientTrafficPolicySpec), + } + resources.ClientTrafficPolicies = append(resources.ClientTrafficPolicies, clientTrafficPolicy) + case egv1a1.KindBackendTrafficPolicy: + typedSpec := spec.Interface() + backendTrafficPolicy := &egv1a1.BackendTrafficPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindBackendTrafficPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.BackendTrafficPolicySpec), + } + resources.BackendTrafficPolicies = append(resources.BackendTrafficPolicies, backendTrafficPolicy) + case egv1a1.KindSecurityPolicy: + typedSpec := spec.Interface() + securityPolicy := &egv1a1.SecurityPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindSecurityPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.SecurityPolicySpec), + } + resources.SecurityPolicies = append(resources.SecurityPolicies, securityPolicy) + } + } + + if useDefaultNamespace { + if _, found := providedNamespaceMap[config.DefaultNamespace]; !found { + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.DefaultNamespace, + }, + } + resources.Namespaces = append(resources.Namespaces, namespace) + providedNamespaceMap[config.DefaultNamespace] = struct{}{} + } + } + + if addMissingResources { + for ns := range requiredNamespaceMap { + if _, found := providedNamespaceMap[ns]; !found { + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + }, + } + resources.Namespaces = append(resources.Namespaces, namespace) + } + } + + requiredServiceMap := map[string]*corev1.Service{} + for _, route := range resources.TCPRoutes { + addMissingServices(requiredServiceMap, route) + } + for _, route := range resources.UDPRoutes { + addMissingServices(requiredServiceMap, route) + } + for _, route := range resources.TLSRoutes { + addMissingServices(requiredServiceMap, route) + } + for _, route := range resources.HTTPRoutes { + addMissingServices(requiredServiceMap, route) + } + for _, route := range resources.GRPCRoutes { + addMissingServices(requiredServiceMap, route) + } + + providedServiceMap := map[string]*corev1.Service{} + for _, service := range resources.Services { + providedServiceMap[service.Namespace+"/"+service.Name] = service + } + + for key, service := range requiredServiceMap { + if provided, found := providedServiceMap[key]; !found { + resources.Services = append(resources.Services, service) + } else { + providedPorts := sets.NewString() + for _, port := range provided.Spec.Ports { + portKey := fmt.Sprintf("%s-%d", port.Protocol, port.Port) + providedPorts.Insert(portKey) + } + + for _, port := range service.Spec.Ports { + name := fmt.Sprintf("%s-%d", port.Protocol, port.Port) + if !providedPorts.Has(name) { + servicePort := corev1.ServicePort{ + Name: name, + Protocol: port.Protocol, + Port: port.Port, + } + provided.Spec.Ports = append(provided.Spec.Ports, servicePort) + } + } + } + } + + // Add EnvoyProxy if it does not exist + if resources.EnvoyProxy == nil { + if err := addDefaultEnvoyProxy(resources); err != nil { + return nil, err + } + } + } + + return resources, nil +} + +func addMissingServices(requiredServices map[string]*corev1.Service, obj interface{}) { + var objNamespace string + protocol := corev1.Protocol(TCPProtocol) + + var refs []gwapiv1.BackendRef + switch route := obj.(type) { + case *gwapiv1.HTTPRoute: + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + for _, httpBakcendRef := range rule.BackendRefs { + refs = append(refs, httpBakcendRef.BackendRef) + } + } + case *gwapiv1a2.GRPCRoute: + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + for _, gRPCBakcendRef := range rule.BackendRefs { + refs = append(refs, gRPCBakcendRef.BackendRef) + } + } + case *gwapiv1a2.TLSRoute: + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + refs = append(refs, rule.BackendRefs...) + } + case *gwapiv1a2.TCPRoute: + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + refs = append(refs, rule.BackendRefs...) + } + case *gwapiv1a2.UDPRoute: + protocol = UDPProtocol + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + refs = append(refs, rule.BackendRefs...) + } + } + + for _, ref := range refs { + if ref.Kind == nil || *ref.Kind != KindService { + continue + } + + ns := objNamespace + if ref.Namespace != nil { + ns = string(*ref.Namespace) + } + name := string(ref.Name) + key := ns + "/" + name + + port := int32(*ref.Port) + servicePort := corev1.ServicePort{ + Name: fmt.Sprintf("%s-%d", protocol, port), + Protocol: protocol, + Port: port, + } + if service, found := requiredServices[key]; !found { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: corev1.ServiceSpec{ + // Just a dummy IP + ClusterIP: "127.0.0.1", + Ports: []corev1.ServicePort{servicePort}, + }, + } + requiredServices[key] = svc + } else { + inserted := false + for _, p := range service.Spec.Ports { + if p.Protocol == servicePort.Protocol && p.Port == servicePort.Port { + inserted = true + break + } + } + + if !inserted { + service.Spec.Ports = append(service.Spec.Ports, servicePort) + } + } + } +} + +func addDefaultEnvoyProxy(resources *Resources) error { + if resources.GatewayClass == nil { + return fmt.Errorf("the GatewayClass resource is required") + } + + defaultEnvoyProxyName := "default-envoy-proxy" + namespace := resources.GatewayClass.Namespace + defaultBootstrapStr, err := bootstrap.GetRenderedBootstrapConfig(nil) + if err != nil { + return err + } + ep := &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: defaultEnvoyProxyName, + }, + Spec: egv1a1.EnvoyProxySpec{ + Bootstrap: &egv1a1.ProxyBootstrap{ + Value: defaultBootstrapStr, + }, + }, + } + resources.EnvoyProxy = ep + ns := gwapiv1.Namespace(namespace) + resources.GatewayClass.Spec.ParametersRef = &gwapiv1.ParametersReference{ + Group: gwapiv1.Group(egv1a1.GroupVersion.Group), + Kind: KindEnvoyProxy, + Name: defaultEnvoyProxyName, + Namespace: &ns, + } + return nil +} diff --git a/internal/provider/runner/runner.go b/internal/provider/runner/runner.go index 7a561a3440d..03de6443480 100644 --- a/internal/provider/runner/runner.go +++ b/internal/provider/runner/runner.go @@ -62,7 +62,7 @@ func (r *Runner) Start(ctx context.Context) (err error) { return fmt.Errorf("unsupported provider type %v", r.EnvoyGateway.Provider.Type) } - r.Logger.Info("Using provider", "type", p.Type()) + r.Logger.Info("Running provider", "type", p.Type()) go func() { if err = p.Start(ctx); err != nil { r.Logger.Error(err, "unable to start provider") From 3419af6a1be4c60a7de770c1f5ccdd0eebb95a93 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Wed, 10 Apr 2024 11:46:42 +0800 Subject: [PATCH 04/17] add notifier support Signed-off-by: shawnh2 --- go.mod | 2 +- internal/provider/file/file.go | 42 ++-- internal/provider/file/notifier.go | 319 +++++++++++++++++++++++++++++ internal/provider/file/path.go | 46 +++++ internal/provider/file/watcher.go | 48 ----- internal/provider/runner/runner.go | 5 +- 6 files changed, 399 insertions(+), 63 deletions(-) create mode 100644 internal/provider/file/notifier.go create mode 100644 internal/provider/file/path.go delete mode 100644 internal/provider/file/watcher.go diff --git a/go.mod b/go.mod index b9f4885e373..1390d0e306c 100644 --- a/go.mod +++ b/go.mod @@ -125,7 +125,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/evanphx/json-patch v5.9.0+incompatible github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/internal/provider/file/file.go b/internal/provider/file/file.go index b48aa71430c..dda8ba3a8bf 100644 --- a/internal/provider/file/file.go +++ b/internal/provider/file/file.go @@ -14,13 +14,22 @@ import ( ) type Provider struct { - watcher *watcher + paths []string + notifier *Notifier + resources *message.ProviderResources } -func New(svr *config.Server, resources *message.ProviderResources) *Provider { - return &Provider{ - watcher: newWatcher(svr.EnvoyGateway.Provider.Custom.Resource.File.Paths), +func New(svr *config.Server, resources *message.ProviderResources) (*Provider, error) { + notifier, err := NewNotifier(svr.Logger.Logger) + if err != nil { + return nil, err } + + return &Provider{ + paths: svr.EnvoyGateway.Provider.Custom.Resource.File.Paths, + notifier: notifier, + resources: resources, + }, nil } func (p *Provider) Type() v1alpha1.ProviderType { @@ -28,15 +37,22 @@ func (p *Provider) Type() v1alpha1.ProviderType { } func (p *Provider) Start(ctx context.Context) error { - errChan := make(chan error) - go func() { - errChan <- p.watcher.Watch(ctx) - }() - - select { - case <-ctx.Done(): - return nil - case err := <-errChan: + dirs, files, err := getDirsAndFilesForWatcher(p.paths) + if err != nil { return err } + + // TODO: initial load for resources-store + + p.notifier.Watch(ctx, dirs, files) + defer p.notifier.Close() + + for { + select { + case <-ctx.Done(): + return nil + case <-p.notifier.Events: + // TODO: ask resources-store to update according to the recv event + } + } } diff --git a/internal/provider/file/notifier.go b/internal/provider/file/notifier.go new file mode 100644 index 00000000000..0546d52ee3b --- /dev/null +++ b/internal/provider/file/notifier.go @@ -0,0 +1,319 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "context" + "os" + "path/filepath" + "strings" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/sets" +) + +const ( + defaultCleanUpRemoveEventsPeriod = 300 * time.Millisecond + defaultWaitForWriteEventsPeriod = 100 * time.Millisecond +) + +type Notifier struct { + // Events record events used to update ResourcesStore, + // which only include two types of events: Write/Remove. + Events chan fsnotify.Event + + filesWatcher *fsnotify.Watcher + dirsWatcher *fsnotify.Watcher + cleanUpRemoveEventsPeriod time.Duration + waitForWriteEventsPeriod time.Duration + + logger logr.Logger +} + +func NewNotifier(logger logr.Logger) (*Notifier, error) { + fw, err := fsnotify.NewBufferedWatcher(10) + if err != nil { + return nil, err + } + + dw, err := fsnotify.NewBufferedWatcher(10) + if err != nil { + return nil, err + } + + return &Notifier{ + Events: make(chan fsnotify.Event, 0), + filesWatcher: fw, + dirsWatcher: dw, + cleanUpRemoveEventsPeriod: defaultCleanUpRemoveEventsPeriod, + waitForWriteEventsPeriod: defaultWaitForWriteEventsPeriod, + logger: logger, + }, nil +} + +func (n *Notifier) Watch(ctx context.Context, dirs, files sets.Set[string]) { + n.watchDirs(ctx, dirs) + n.watchFiles(ctx, files) +} + +func (n *Notifier) Close() error { + if err := n.filesWatcher.Close(); err != nil { + return err + } + if err := n.dirsWatcher.Close(); err != nil { + return err + } + return nil +} + +// watchFiles watches one or more files, but instead of watching the file directly, +// it watches its parent directory. This solves various issues where files are +// frequently renamed. +func (n *Notifier) watchFiles(ctx context.Context, files sets.Set[string]) { + if len(files) < 1 { + return + } + + go n.runFilesWatcher(ctx, files) + + for p := range files { + if err := n.filesWatcher.Add(filepath.Dir(p)); err != nil { + n.logger.Error(err, "error adding file to notifier", "path", p) + + continue + } + } +} + +func (n *Notifier) runFilesWatcher(ctx context.Context, files sets.Set[string]) { + var ( + cleanUpTicker = time.NewTicker(n.cleanUpRemoveEventsPeriod) + + // This map records the exact previous Op of one event. + preEventOp = make(map[string]fsnotify.Op) + // This set records the name of event that related to Remove Op. + curRemoveEvents = sets.NewString() + ) + + for { + select { + case <-ctx.Done(): + return + + case err, ok := <-n.filesWatcher.Errors: + if !ok { + return + } + n.logger.Error(err, "error from files watcher in notifier") + + case event, ok := <-n.filesWatcher.Events: + if !ok { + return + } + + // Ignore file and operation the watcher not interested in. + if !files.Has(event.Name) || event.Has(fsnotify.Chmod) { + continue + } + + // This logic is trying to avoid files be removed and then created + // frequently by considering Remove/Rename and the follow Create + // Op as one Write Notifier.Event. + // + // Actually, this approach is also suitable for commands like vi/vim. + // It creates a temporary file, removes the existing one and replace + // it with the temporary file when file is saved. So instead of Write + // Op, the watcher will receive Rename and Create Op. + + var writeEvent bool + switch event.Op { + case fsnotify.Create: + if op, ok := preEventOp[event.Name]; ok && + op.Has(fsnotify.Rename) || op.Has(fsnotify.Remove) { + writeEvent = true + // If the exact previous Op of Create is Rename/Remove, + // then consider them as a Write Notifier.Event instead of Remove. + curRemoveEvents.Delete(event.Name) + } + case fsnotify.Write: + writeEvent = true + case fsnotify.Remove, fsnotify.Rename: + curRemoveEvents.Insert(event.Name) + } + + if writeEvent { + n.logger.Info("sending write event", + "name", event.Name, "watcher", "files") + + n.Events <- fsnotify.Event{ + Name: event.Name, + Op: fsnotify.Write, + } + } + preEventOp[event.Name] = event.Op + + case <-cleanUpTicker.C: + // As for collected Remove Notifier.Event, clean them up + // in a period of time to avoid neglect of dealing with + // Remove/Rename Op. + for e := range curRemoveEvents { + n.logger.Info("sending remove event", + "name", e, "watcher", "files") + + n.Events <- fsnotify.Event{ + Name: e, + Op: fsnotify.Remove, + } + } + curRemoveEvents = sets.NewString() + } + } +} + +// watchDirs watches one or more directories. +func (n *Notifier) watchDirs(ctx context.Context, dirs sets.Set[string]) { + if len(dirs) < 1 { + return + } + + // This map maintains the subdirectories ignored by each directory. + ignoredSubDirs := make(map[string]sets.Set[string]) + + for p := range dirs { + if err := n.dirsWatcher.Add(p); err != nil { + n.logger.Error(err, "error adding dir to notifier", "path", p) + + continue + } + + // Find current exist subdirectories to init ignored subdirectories set. + entries, err := os.ReadDir(p) + if err != nil { + n.logger.Error(err, "error reading dir in notifier", "path", p) + + if err = n.dirsWatcher.Remove(p); err != nil { + n.logger.Error(err, "error removing dir from notifier", "path", p) + } + + continue + } + + ignoredSubDirs[p] = sets.New[string]() + for _, entry := range entries { + if entry.IsDir() { + // The entry name is dir name, not dir path. + ignoredSubDirs[p].Insert(entry.Name()) + } + } + } + + go n.runDirsWatcher(ctx, ignoredSubDirs) +} + +func (n *Notifier) runDirsWatcher(ctx context.Context, ignoredSubDirs map[string]sets.Set[string]) { + var ( + cleanUpTicker = time.NewTicker(n.cleanUpRemoveEventsPeriod) + + // This map records the exact previous Op of one event. + preEventOp = make(map[string]fsnotify.Op) + // This set records the name of event that related to Remove Op. + curRemoveEvents = sets.NewString() + ) + + for { + select { + case <-ctx.Done(): + return + + case err, ok := <-n.dirsWatcher.Errors: + if !ok { + return + } + n.logger.Error(err, "error from dirs watcher in notifier") + + case event, ok := <-n.dirsWatcher.Events: + if !ok { + return + } + + // Ignore the hidden or temporary file related event. + _, name := filepath.Split(event.Name) + if event.Has(fsnotify.Chmod) || + strings.HasPrefix(name, ".") || + strings.HasSuffix(name, "~") { + continue + } + + // Ignore any subdirectory related event. + switch event.Op { + case fsnotify.Create: + if fi, err := os.Lstat(event.Name); err == nil && fi.IsDir() { + parentDir := filepath.Dir(event.Name) + if _, ok := ignoredSubDirs[parentDir]; ok { + ignoredSubDirs[parentDir].Insert(name) + continue + } + } + case fsnotify.Remove, fsnotify.Rename: + parentDir := filepath.Dir(event.Name) + if sub, ok := ignoredSubDirs[parentDir]; ok && sub.Has(name) { + ignoredSubDirs[parentDir].Delete(name) + continue + } + } + + // Share the similar logic as in files watcher. + var writeEvent bool + switch event.Op { + case fsnotify.Create: + if op, ok := preEventOp[event.Name]; ok && + op.Has(fsnotify.Rename) || op.Has(fsnotify.Remove) { + curRemoveEvents.Delete(event.Name) + } + // Since the watcher watches the whole dir, the creation of file + // should also be able to trigger the Write event. + writeEvent = true + case fsnotify.Write: + writeEvent = true + case fsnotify.Remove, fsnotify.Rename: + curRemoveEvents.Insert(event.Name) + } + + if writeEvent { + n.logger.Info("sending write event", + "name", event.Name, "watcher", "dirs") + + n.Events <- fsnotify.Event{ + Name: event.Name, + Op: fsnotify.Write, + } + } + preEventOp[event.Name] = event.Op + + case <-cleanUpTicker.C: + // Merge files to be removed in the same parent directory + // to suppress events, because the file has already been + // removed and is unnecessary to send event for each of them. + parentDirs := sets.NewString() + for e := range curRemoveEvents { + parentDirs.Insert(filepath.Dir(e)) + } + + for parentDir := range parentDirs { + n.logger.Info("sending remove event", + "name", parentDir, "watcher", "dirs") + + n.Events <- fsnotify.Event{ + Name: parentDir, + Op: fsnotify.Remove, + } + } + curRemoveEvents = sets.NewString() + } + } +} diff --git a/internal/provider/file/path.go b/internal/provider/file/path.go new file mode 100644 index 00000000000..0e687421b0a --- /dev/null +++ b/internal/provider/file/path.go @@ -0,0 +1,46 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "os" + "path/filepath" + + "k8s.io/apimachinery/pkg/util/sets" +) + +// getDirsAndFilesForWatcher prepares dirs and files for the watcher in notifier. +func getDirsAndFilesForWatcher(paths []string) ( + dirs sets.Set[string], files sets.Set[string], err error) { + dirs, files = sets.New[string](), sets.New[string]() + + // Separate paths by whether is a directory or not. + paths = sets.NewString(paths...).List() + for _, path := range paths { + var p os.FileInfo + p, err = os.Lstat(path) + if err != nil { + return + } + + if p.IsDir() { + dirs.Insert(path) + } else { + files.Insert(path) + } + } + + // Ignore filepath if its parent directory is also be watched. + var ignoreFiles []string + for fp := range files { + if dirs.Has(filepath.Dir(fp)) { + ignoreFiles = append(ignoreFiles, fp) + } + } + files.Delete(ignoreFiles...) + + return +} diff --git a/internal/provider/file/watcher.go b/internal/provider/file/watcher.go deleted file mode 100644 index 9e27bb9ea6c..00000000000 --- a/internal/provider/file/watcher.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package file - -import ( - "context" - "time" -) - -const ( - // defaultWatchTicker defines default ticker (in seconds) for watcher. - // TODO(sh2): make it configurable - defaultWatchTicker = 3 -) - -type watcher struct { - paths []string - - ticker *time.Ticker -} - -func newWatcher(paths []string) *watcher { - return &watcher{ - paths: paths, - ticker: time.NewTicker(defaultWatchTicker * time.Second), - } -} - -// Watch watches and loads files from paths. -func (w *watcher) Watch(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - w.ticker.Stop() - return nil - case <-w.ticker.C: - w.watch() - default: - } - } -} - -func (w *watcher) watch() { - // TODO: implement watch logic -} diff --git a/internal/provider/runner/runner.go b/internal/provider/runner/runner.go index 03de6443480..8cae7482d1e 100644 --- a/internal/provider/runner/runner.go +++ b/internal/provider/runner/runner.go @@ -55,7 +55,10 @@ func (r *Runner) Start(ctx context.Context) (err error) { } case v1alpha1.ProviderTypeFile: - p = file.New(&r.Config.Server, r.ProviderResources) + p, err = file.New(&r.Config.Server, r.ProviderResources) + if err != nil { + return fmt.Errorf("failed to create provider %s: %w", v1alpha1.ProviderTypeFile, err) + } default: // Unsupported provider. From e709eae146a838d60a49316f5fe181a1e735a141 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Wed, 10 Apr 2024 21:07:39 +0800 Subject: [PATCH 05/17] fix lint and move read yaml bytes function back to translate Signed-off-by: shawnh2 --- internal/cmd/egctl/translate.go | 21 ++++++++++++++++++++- internal/gatewayapi/convert.go | 19 ------------------- internal/provider/file/notifier.go | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/internal/cmd/egctl/translate.go b/internal/cmd/egctl/translate.go index 61161cca277..e0dc636899a 100644 --- a/internal/cmd/egctl/translate.go +++ b/internal/cmd/egctl/translate.go @@ -6,9 +6,11 @@ package egctl import ( + "bufio" "encoding/json" "fmt" "io" + "os" "sort" adminv3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" @@ -170,6 +172,23 @@ func getValidResourceTypesStr() string { return fmt.Sprintf("Valid types are %v.", validResourceTypes()) } +func getInputBytes(inFile string) ([]byte, error) { + // Get input from stdin + if inFile == "-" { + scanner := bufio.NewScanner(os.Stdin) + var input string + for { + if !scanner.Scan() { + break + } + input += scanner.Text() + "\n" + } + return []byte(input), nil + } + // Get input from file + return os.ReadFile(inFile) +} + func validate(inFile, inType string, outTypes []string, resourceType string) error { if !isValidInputType(inType) { return fmt.Errorf("%s is not a valid input type. %s", inType, getValidInputTypesStr()) @@ -193,7 +212,7 @@ func translate(w io.Writer, inFile, inType string, outTypes []string, output, re return err } - inBytes, err := gatewayapi.ReadKubernetesYAMLBytes(inFile) + inBytes, err := getInputBytes(inFile) if err != nil { return fmt.Errorf("unable to read input file: %w", err) } diff --git a/internal/gatewayapi/convert.go b/internal/gatewayapi/convert.go index 98e2fe53ff2..3d8b40f862d 100644 --- a/internal/gatewayapi/convert.go +++ b/internal/gatewayapi/convert.go @@ -6,9 +6,7 @@ package gatewayapi import ( - "bufio" "fmt" - "os" "reflect" "strings" @@ -26,23 +24,6 @@ import ( "github.com/envoyproxy/gateway/internal/xds/bootstrap" ) -func ReadKubernetesYAMLBytes(filePath string) ([]byte, error) { - // Get input from stdin - if filePath == "-" { - scanner := bufio.NewScanner(os.Stdin) - var input string - for { - if !scanner.Scan() { - break - } - input += scanner.Text() + "\n" - } - return []byte(input), nil - } - // Get input from file - return os.ReadFile(filePath) -} - // ConvertKubernetesYAMLToResources converts a Kubernetes YAML string into GatewayAPI Resources func ConvertKubernetesYAMLToResources(str string, addMissingResources bool) (*Resources, error) { resources := NewResources() diff --git a/internal/provider/file/notifier.go b/internal/provider/file/notifier.go index 0546d52ee3b..7851950397b 100644 --- a/internal/provider/file/notifier.go +++ b/internal/provider/file/notifier.go @@ -47,7 +47,7 @@ func NewNotifier(logger logr.Logger) (*Notifier, error) { } return &Notifier{ - Events: make(chan fsnotify.Event, 0), + Events: make(chan fsnotify.Event), filesWatcher: fw, dirsWatcher: dw, cleanUpRemoveEventsPeriod: defaultCleanUpRemoveEventsPeriod, From 6f3b400fc0bb554d436b52103008c55a364a2841 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Thu, 18 Apr 2024 15:07:55 +0800 Subject: [PATCH 06/17] add resources store support Signed-off-by: shawnh2 --- internal/cmd/egctl/translate.go | 432 ++++++++++++++++++++++++++- internal/cmd/server.go | 21 +- internal/gatewayapi/convert.go | 441 ---------------------------- internal/provider/file/file.go | 37 ++- internal/provider/file/notifier.go | 3 - internal/provider/file/resources.go | 329 +++++++++++++++++++++ internal/provider/file/store.go | 63 ++++ 7 files changed, 864 insertions(+), 462 deletions(-) delete mode 100644 internal/gatewayapi/convert.go create mode 100644 internal/provider/file/resources.go create mode 100644 internal/provider/file/store.go diff --git a/internal/cmd/egctl/translate.go b/internal/cmd/egctl/translate.go index e0dc636899a..bf81013f033 100644 --- a/internal/cmd/egctl/translate.go +++ b/internal/cmd/egctl/translate.go @@ -11,7 +11,9 @@ import ( "fmt" "io" "os" + "reflect" "sort" + "strings" adminv3 "github.com/envoyproxy/go-control-plane/envoy/admin/v3" bootstrapv3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3" @@ -20,11 +22,18 @@ import ( "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/known/anypb" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/yaml" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/api/v1alpha1/validation" + "github.com/envoyproxy/gateway/internal/envoygateway" + "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/gatewayapi" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/ratelimit" "github.com/envoyproxy/gateway/internal/status" @@ -219,7 +228,7 @@ func translate(w io.Writer, inFile, inType string, outTypes []string, output, re if inType == gatewayAPIType { // Unmarshal input - resources, err := gatewayapi.ConvertKubernetesYAMLToResources(string(inBytes), addMissingResources) + resources, err := kubernetesYAMLToResources(string(inBytes), addMissingResources) if err != nil { return fmt.Errorf("unable to unmarshal input: %w", err) } @@ -527,3 +536,424 @@ func constructConfigDump(resources *gatewayapi.Resources, tCtx *xds_types.Resour return globalConfigs, nil } + +func addMissingServices(requiredServices map[string]*v1.Service, obj interface{}) { + var objNamespace string + protocol := v1.Protocol(gatewayapi.TCPProtocol) + + refs := []gwapiv1.BackendRef{} + switch route := obj.(type) { + case *gwapiv1.HTTPRoute: + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + for _, httpBakcendRef := range rule.BackendRefs { + refs = append(refs, httpBakcendRef.BackendRef) + } + } + case *gwapiv1a2.GRPCRoute: + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + for _, gRPCBakcendRef := range rule.BackendRefs { + refs = append(refs, gRPCBakcendRef.BackendRef) + } + } + case *gwapiv1a2.TLSRoute: + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + refs = append(refs, rule.BackendRefs...) + } + case *gwapiv1a2.TCPRoute: + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + refs = append(refs, rule.BackendRefs...) + } + case *gwapiv1a2.UDPRoute: + protocol = v1.Protocol(gatewayapi.UDPProtocol) + objNamespace = route.Namespace + for _, rule := range route.Spec.Rules { + refs = append(refs, rule.BackendRefs...) + } + } + + for _, ref := range refs { + if ref.Kind == nil || *ref.Kind != gatewayapi.KindService { + continue + } + + ns := objNamespace + if ref.Namespace != nil { + ns = string(*ref.Namespace) + } + name := string(ref.Name) + key := ns + "/" + name + + port := int32(*ref.Port) + servicePort := v1.ServicePort{ + Name: fmt.Sprintf("%s-%d", protocol, port), + Protocol: protocol, + Port: port, + } + if service, found := requiredServices[key]; !found { + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: v1.ServiceSpec{ + // Just a dummy IP + ClusterIP: "127.0.0.1", + Ports: []v1.ServicePort{servicePort}, + }, + } + requiredServices[key] = service + + } else { + inserted := false + for _, port := range service.Spec.Ports { + if port.Protocol == servicePort.Protocol && port.Port == servicePort.Port { + inserted = true + break + } + } + + if !inserted { + service.Spec.Ports = append(service.Spec.Ports, servicePort) + } + } + } +} + +// kubernetesYAMLToResources converts a Kubernetes YAML string into GatewayAPI Resources +func kubernetesYAMLToResources(str string, addMissingResources bool) (*gatewayapi.Resources, error) { + resources := gatewayapi.NewResources() + var useDefaultNamespace bool + providedNamespaceMap := map[string]struct{}{} + requiredNamespaceMap := map[string]struct{}{} + yamls := strings.Split(str, "\n---") + combinedScheme := envoygateway.GetScheme() + for _, y := range yamls { + if strings.TrimSpace(y) == "" { + continue + } + var obj map[string]interface{} + err := yaml.Unmarshal([]byte(y), &obj) + if err != nil { + return nil, err + } + un := unstructured.Unstructured{Object: obj} + gvk := un.GroupVersionKind() + name, namespace := un.GetName(), un.GetNamespace() + if namespace == "" { + // When kubectl applies a resource in yaml which doesn't have a namespace, + // the current namespace is applied. Here we do the same thing before translating + // the GatewayAPI resource. Otherwise, the resource can't pass the namespace validation + useDefaultNamespace = true + namespace = config.DefaultNamespace + } + requiredNamespaceMap[namespace] = struct{}{} + kobj, err := combinedScheme.New(gvk) + if err != nil { + return nil, err + } + err = combinedScheme.Convert(&un, kobj, nil) + if err != nil { + return nil, err + } + + objType := reflect.TypeOf(kobj) + if objType.Kind() != reflect.Ptr { + return nil, fmt.Errorf("expected pointer type, but got %s", objType.Kind().String()) + } + kobjVal := reflect.ValueOf(kobj).Elem() + spec := kobjVal.FieldByName("Spec") + + switch gvk.Kind { + case gatewayapi.KindEnvoyProxy: + typedSpec := spec.Interface() + envoyProxy := &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(egv1a1.EnvoyProxySpec), + } + resources.EnvoyProxy = envoyProxy + case gatewayapi.KindGatewayClass: + typedSpec := spec.Interface() + gatewayClass := &gwapiv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.GatewayClassSpec), + } + // fill controller name by default controller name when gatewayclass controller name empty. + if gatewayClass.Spec.ControllerName == "" { + gatewayClass.Spec.ControllerName = egv1a1.GatewayControllerName + } + resources.GatewayClass = gatewayClass + case gatewayapi.KindGateway: + typedSpec := spec.Interface() + gateway := &gwapiv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.GatewaySpec), + } + resources.Gateways = append(resources.Gateways, gateway) + case gatewayapi.KindTCPRoute: + typedSpec := spec.Interface() + tcpRoute := &gwapiv1a2.TCPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindTCPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.TCPRouteSpec), + } + resources.TCPRoutes = append(resources.TCPRoutes, tcpRoute) + case gatewayapi.KindUDPRoute: + typedSpec := spec.Interface() + udpRoute := &gwapiv1a2.UDPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindUDPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.UDPRouteSpec), + } + resources.UDPRoutes = append(resources.UDPRoutes, udpRoute) + case gatewayapi.KindTLSRoute: + typedSpec := spec.Interface() + tlsRoute := &gwapiv1a2.TLSRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindTLSRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.TLSRouteSpec), + } + resources.TLSRoutes = append(resources.TLSRoutes, tlsRoute) + case gatewayapi.KindHTTPRoute: + typedSpec := spec.Interface() + httpRoute := &gwapiv1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindHTTPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.HTTPRouteSpec), + } + resources.HTTPRoutes = append(resources.HTTPRoutes, httpRoute) + case gatewayapi.KindGRPCRoute: + typedSpec := spec.Interface() + grpcRoute := &gwapiv1a2.GRPCRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindGRPCRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.GRPCRouteSpec), + } + resources.GRPCRoutes = append(resources.GRPCRoutes, grpcRoute) + case gatewayapi.KindNamespace: + namespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + resources.Namespaces = append(resources.Namespaces, namespace) + providedNamespaceMap[name] = struct{}{} + case gatewayapi.KindService: + typedSpec := spec.Interface() + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(v1.ServiceSpec), + } + resources.Services = append(resources.Services, service) + case egv1a1.KindEnvoyPatchPolicy: + typedSpec := spec.Interface() + envoyPatchPolicy := &egv1a1.EnvoyPatchPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindEnvoyPatchPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.EnvoyPatchPolicySpec), + } + resources.EnvoyPatchPolicies = append(resources.EnvoyPatchPolicies, envoyPatchPolicy) + case egv1a1.KindClientTrafficPolicy: + typedSpec := spec.Interface() + clientTrafficPolicy := &egv1a1.ClientTrafficPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindClientTrafficPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.ClientTrafficPolicySpec), + } + resources.ClientTrafficPolicies = append(resources.ClientTrafficPolicies, clientTrafficPolicy) + case egv1a1.KindBackendTrafficPolicy: + typedSpec := spec.Interface() + backendTrafficPolicy := &egv1a1.BackendTrafficPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindBackendTrafficPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.BackendTrafficPolicySpec), + } + resources.BackendTrafficPolicies = append(resources.BackendTrafficPolicies, backendTrafficPolicy) + case egv1a1.KindSecurityPolicy: + typedSpec := spec.Interface() + securityPolicy := &egv1a1.SecurityPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindSecurityPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.SecurityPolicySpec), + } + resources.SecurityPolicies = append(resources.SecurityPolicies, securityPolicy) + } + } + + if useDefaultNamespace { + if _, found := providedNamespaceMap[config.DefaultNamespace]; !found { + namespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.DefaultNamespace, + }, + } + resources.Namespaces = append(resources.Namespaces, namespace) + providedNamespaceMap[config.DefaultNamespace] = struct{}{} + } + } + + if addMissingResources { + for ns := range requiredNamespaceMap { + if _, found := providedNamespaceMap[ns]; !found { + namespace := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + }, + } + resources.Namespaces = append(resources.Namespaces, namespace) + } + } + + requiredServiceMap := map[string]*v1.Service{} + for _, route := range resources.TCPRoutes { + addMissingServices(requiredServiceMap, route) + } + for _, route := range resources.UDPRoutes { + addMissingServices(requiredServiceMap, route) + } + for _, route := range resources.TLSRoutes { + addMissingServices(requiredServiceMap, route) + } + for _, route := range resources.HTTPRoutes { + addMissingServices(requiredServiceMap, route) + } + for _, route := range resources.GRPCRoutes { + addMissingServices(requiredServiceMap, route) + } + + providedServiceMap := map[string]*v1.Service{} + for _, service := range resources.Services { + providedServiceMap[service.Namespace+"/"+service.Name] = service + } + + for key, service := range requiredServiceMap { + if provided, found := providedServiceMap[key]; !found { + resources.Services = append(resources.Services, service) + } else { + providedPorts := sets.NewString() + for _, port := range provided.Spec.Ports { + portKey := fmt.Sprintf("%s-%d", port.Protocol, port.Port) + providedPorts.Insert(portKey) + } + + for _, port := range service.Spec.Ports { + name := fmt.Sprintf("%s-%d", port.Protocol, port.Port) + if !providedPorts.Has(name) { + servicePort := v1.ServicePort{ + Name: name, + Protocol: port.Protocol, + Port: port.Port, + } + provided.Spec.Ports = append(provided.Spec.Ports, servicePort) + } + } + } + } + + // Add EnvoyProxy if it does not exist + if resources.EnvoyProxy == nil { + if err := addDefaultEnvoyProxy(resources); err != nil { + return nil, err + } + } + } + + return resources, nil +} + +func addDefaultEnvoyProxy(resources *gatewayapi.Resources) error { + if resources.GatewayClass == nil { + return fmt.Errorf("the GatewayClass resource is required") + } + + defaultEnvoyProxyName := "default-envoy-proxy" + namespace := resources.GatewayClass.Namespace + defaultBootstrapStr, err := bootstrap.GetRenderedBootstrapConfig(nil) + if err != nil { + return err + } + ep := &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: defaultEnvoyProxyName, + }, + Spec: egv1a1.EnvoyProxySpec{ + Bootstrap: &egv1a1.ProxyBootstrap{ + Value: defaultBootstrapStr, + }, + }, + } + resources.EnvoyProxy = ep + ns := gwapiv1.Namespace(namespace) + resources.GatewayClass.Spec.ParametersRef = &gwapiv1.ParametersReference{ + Group: gwapiv1.Group(egv1a1.GroupVersion.Group), + Kind: gatewayapi.KindEnvoyProxy, + Name: defaultEnvoyProxyName, + Namespace: &ns, + } + return nil +} diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 7b04866aa11..d2a5c2a97b0 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -9,9 +9,11 @@ import ( "github.com/spf13/cobra" ctrl "sigs.k8s.io/controller-runtime" + "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/admin" "github.com/envoyproxy/gateway/internal/envoygateway/config" extensionregistry "github.com/envoyproxy/gateway/internal/extension/registry" + "github.com/envoyproxy/gateway/internal/extension/types" gatewayapirunner "github.com/envoyproxy/gateway/internal/gatewayapi/runner" ratelimitrunner "github.com/envoyproxy/gateway/internal/globalratelimit/runner" infrarunner "github.com/envoyproxy/gateway/internal/infrastructure/runner" @@ -110,15 +112,18 @@ func getConfigByPath(cfgPath string) (*config.Server, error) { // setupRunners starts all the runners required for the Envoy Gateway to // fulfill its tasks. -func setupRunners(cfg *config.Server) error { +func setupRunners(cfg *config.Server) (err error) { // TODO - Setup a Config Manager // https://github.com/envoyproxy/gateway/issues/43 ctx := ctrl.SetupSignalHandler() - // Setup the Extension Manager - extMgr, err := extensionregistry.NewManager(cfg) - if err != nil { - return err + // Setup the Extension Manager for Kubernetes provider. + var extMgr types.Manager + if cfg.EnvoyGateway.Provider.Type == v1alpha1.ProviderTypeKubernetes { + extMgr, err = extensionregistry.NewManager(cfg) + if err != nil { + return err + } } pResources := new(message.ProviderResources) @@ -212,8 +217,10 @@ func setupRunners(cfg *config.Server) error { cfg.Logger.Info("shutting down") // Close connections to extension services - if mgr, ok := extMgr.(*extensionregistry.Manager); ok { - mgr.CleanupHookConns() + if extMgr != nil { + if mgr, ok := extMgr.(*extensionregistry.Manager); ok { + mgr.CleanupHookConns() + } } return nil diff --git a/internal/gatewayapi/convert.go b/internal/gatewayapi/convert.go deleted file mode 100644 index 3d8b40f862d..00000000000 --- a/internal/gatewayapi/convert.go +++ /dev/null @@ -1,441 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -package gatewayapi - -import ( - "fmt" - "reflect" - "strings" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/util/sets" - gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" - gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" - "sigs.k8s.io/yaml" - - egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" - "github.com/envoyproxy/gateway/internal/envoygateway" - "github.com/envoyproxy/gateway/internal/envoygateway/config" - "github.com/envoyproxy/gateway/internal/xds/bootstrap" -) - -// ConvertKubernetesYAMLToResources converts a Kubernetes YAML string into GatewayAPI Resources -func ConvertKubernetesYAMLToResources(str string, addMissingResources bool) (*Resources, error) { - resources := NewResources() - var useDefaultNamespace bool - providedNamespaceMap := map[string]struct{}{} - requiredNamespaceMap := map[string]struct{}{} - yamls := strings.Split(str, "\n---") - combinedScheme := envoygateway.GetScheme() - for _, y := range yamls { - if strings.TrimSpace(y) == "" { - continue - } - var obj map[string]interface{} - err := yaml.Unmarshal([]byte(y), &obj) - if err != nil { - return nil, err - } - un := unstructured.Unstructured{Object: obj} - gvk := un.GroupVersionKind() - name, namespace := un.GetName(), un.GetNamespace() - if namespace == "" { - // When kubectl applies a resource in yaml which doesn't have a namespace, - // the current namespace is applied. Here we do the same thing before translating - // the GatewayAPI resource. Otherwise, the resource can't pass the namespace validation - useDefaultNamespace = true - namespace = config.DefaultNamespace - } - requiredNamespaceMap[namespace] = struct{}{} - kobj, err := combinedScheme.New(gvk) - if err != nil { - return nil, err - } - err = combinedScheme.Convert(&un, kobj, nil) - if err != nil { - return nil, err - } - - objType := reflect.TypeOf(kobj) - if objType.Kind() != reflect.Ptr { - return nil, fmt.Errorf("expected pointer type, but got %s", objType.Kind().String()) - } - kobjVal := reflect.ValueOf(kobj).Elem() - spec := kobjVal.FieldByName("Spec") - - switch gvk.Kind { - case KindEnvoyProxy: - typedSpec := spec.Interface() - envoyProxy := &egv1a1.EnvoyProxy{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(egv1a1.EnvoyProxySpec), - } - resources.EnvoyProxy = envoyProxy - case KindGatewayClass: - typedSpec := spec.Interface() - gatewayClass := &gwapiv1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1.GatewayClassSpec), - } - resources.GatewayClass = gatewayClass - case KindGateway: - typedSpec := spec.Interface() - gateway := &gwapiv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1.GatewaySpec), - } - resources.Gateways = append(resources.Gateways, gateway) - case KindTCPRoute: - typedSpec := spec.Interface() - tcpRoute := &gwapiv1a2.TCPRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: KindTCPRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1a2.TCPRouteSpec), - } - resources.TCPRoutes = append(resources.TCPRoutes, tcpRoute) - case KindUDPRoute: - typedSpec := spec.Interface() - udpRoute := &gwapiv1a2.UDPRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: KindUDPRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1a2.UDPRouteSpec), - } - resources.UDPRoutes = append(resources.UDPRoutes, udpRoute) - case KindTLSRoute: - typedSpec := spec.Interface() - tlsRoute := &gwapiv1a2.TLSRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: KindTLSRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1a2.TLSRouteSpec), - } - resources.TLSRoutes = append(resources.TLSRoutes, tlsRoute) - case KindHTTPRoute: - typedSpec := spec.Interface() - httpRoute := &gwapiv1.HTTPRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: KindHTTPRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1.HTTPRouteSpec), - } - resources.HTTPRoutes = append(resources.HTTPRoutes, httpRoute) - case KindGRPCRoute: - typedSpec := spec.Interface() - grpcRoute := &gwapiv1a2.GRPCRoute{ - TypeMeta: metav1.TypeMeta{ - Kind: KindGRPCRoute, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(gwapiv1a2.GRPCRouteSpec), - } - resources.GRPCRoutes = append(resources.GRPCRoutes, grpcRoute) - case KindNamespace: - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - } - resources.Namespaces = append(resources.Namespaces, ns) - providedNamespaceMap[name] = struct{}{} - case KindService: - typedSpec := spec.Interface() - service := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: typedSpec.(corev1.ServiceSpec), - } - resources.Services = append(resources.Services, service) - case egv1a1.KindEnvoyPatchPolicy: - typedSpec := spec.Interface() - envoyPatchPolicy := &egv1a1.EnvoyPatchPolicy{ - TypeMeta: metav1.TypeMeta{ - Kind: egv1a1.KindEnvoyPatchPolicy, - APIVersion: egv1a1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Spec: typedSpec.(egv1a1.EnvoyPatchPolicySpec), - } - resources.EnvoyPatchPolicies = append(resources.EnvoyPatchPolicies, envoyPatchPolicy) - case egv1a1.KindClientTrafficPolicy: - typedSpec := spec.Interface() - clientTrafficPolicy := &egv1a1.ClientTrafficPolicy{ - TypeMeta: metav1.TypeMeta{ - Kind: egv1a1.KindClientTrafficPolicy, - APIVersion: egv1a1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Spec: typedSpec.(egv1a1.ClientTrafficPolicySpec), - } - resources.ClientTrafficPolicies = append(resources.ClientTrafficPolicies, clientTrafficPolicy) - case egv1a1.KindBackendTrafficPolicy: - typedSpec := spec.Interface() - backendTrafficPolicy := &egv1a1.BackendTrafficPolicy{ - TypeMeta: metav1.TypeMeta{ - Kind: egv1a1.KindBackendTrafficPolicy, - APIVersion: egv1a1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Spec: typedSpec.(egv1a1.BackendTrafficPolicySpec), - } - resources.BackendTrafficPolicies = append(resources.BackendTrafficPolicies, backendTrafficPolicy) - case egv1a1.KindSecurityPolicy: - typedSpec := spec.Interface() - securityPolicy := &egv1a1.SecurityPolicy{ - TypeMeta: metav1.TypeMeta{ - Kind: egv1a1.KindSecurityPolicy, - APIVersion: egv1a1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Spec: typedSpec.(egv1a1.SecurityPolicySpec), - } - resources.SecurityPolicies = append(resources.SecurityPolicies, securityPolicy) - } - } - - if useDefaultNamespace { - if _, found := providedNamespaceMap[config.DefaultNamespace]; !found { - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: config.DefaultNamespace, - }, - } - resources.Namespaces = append(resources.Namespaces, namespace) - providedNamespaceMap[config.DefaultNamespace] = struct{}{} - } - } - - if addMissingResources { - for ns := range requiredNamespaceMap { - if _, found := providedNamespaceMap[ns]; !found { - namespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: ns, - }, - } - resources.Namespaces = append(resources.Namespaces, namespace) - } - } - - requiredServiceMap := map[string]*corev1.Service{} - for _, route := range resources.TCPRoutes { - addMissingServices(requiredServiceMap, route) - } - for _, route := range resources.UDPRoutes { - addMissingServices(requiredServiceMap, route) - } - for _, route := range resources.TLSRoutes { - addMissingServices(requiredServiceMap, route) - } - for _, route := range resources.HTTPRoutes { - addMissingServices(requiredServiceMap, route) - } - for _, route := range resources.GRPCRoutes { - addMissingServices(requiredServiceMap, route) - } - - providedServiceMap := map[string]*corev1.Service{} - for _, service := range resources.Services { - providedServiceMap[service.Namespace+"/"+service.Name] = service - } - - for key, service := range requiredServiceMap { - if provided, found := providedServiceMap[key]; !found { - resources.Services = append(resources.Services, service) - } else { - providedPorts := sets.NewString() - for _, port := range provided.Spec.Ports { - portKey := fmt.Sprintf("%s-%d", port.Protocol, port.Port) - providedPorts.Insert(portKey) - } - - for _, port := range service.Spec.Ports { - name := fmt.Sprintf("%s-%d", port.Protocol, port.Port) - if !providedPorts.Has(name) { - servicePort := corev1.ServicePort{ - Name: name, - Protocol: port.Protocol, - Port: port.Port, - } - provided.Spec.Ports = append(provided.Spec.Ports, servicePort) - } - } - } - } - - // Add EnvoyProxy if it does not exist - if resources.EnvoyProxy == nil { - if err := addDefaultEnvoyProxy(resources); err != nil { - return nil, err - } - } - } - - return resources, nil -} - -func addMissingServices(requiredServices map[string]*corev1.Service, obj interface{}) { - var objNamespace string - protocol := corev1.Protocol(TCPProtocol) - - var refs []gwapiv1.BackendRef - switch route := obj.(type) { - case *gwapiv1.HTTPRoute: - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - for _, httpBakcendRef := range rule.BackendRefs { - refs = append(refs, httpBakcendRef.BackendRef) - } - } - case *gwapiv1a2.GRPCRoute: - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - for _, gRPCBakcendRef := range rule.BackendRefs { - refs = append(refs, gRPCBakcendRef.BackendRef) - } - } - case *gwapiv1a2.TLSRoute: - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - refs = append(refs, rule.BackendRefs...) - } - case *gwapiv1a2.TCPRoute: - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - refs = append(refs, rule.BackendRefs...) - } - case *gwapiv1a2.UDPRoute: - protocol = UDPProtocol - objNamespace = route.Namespace - for _, rule := range route.Spec.Rules { - refs = append(refs, rule.BackendRefs...) - } - } - - for _, ref := range refs { - if ref.Kind == nil || *ref.Kind != KindService { - continue - } - - ns := objNamespace - if ref.Namespace != nil { - ns = string(*ref.Namespace) - } - name := string(ref.Name) - key := ns + "/" + name - - port := int32(*ref.Port) - servicePort := corev1.ServicePort{ - Name: fmt.Sprintf("%s-%d", protocol, port), - Protocol: protocol, - Port: port, - } - if service, found := requiredServices[key]; !found { - svc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: ns, - }, - Spec: corev1.ServiceSpec{ - // Just a dummy IP - ClusterIP: "127.0.0.1", - Ports: []corev1.ServicePort{servicePort}, - }, - } - requiredServices[key] = svc - } else { - inserted := false - for _, p := range service.Spec.Ports { - if p.Protocol == servicePort.Protocol && p.Port == servicePort.Port { - inserted = true - break - } - } - - if !inserted { - service.Spec.Ports = append(service.Spec.Ports, servicePort) - } - } - } -} - -func addDefaultEnvoyProxy(resources *Resources) error { - if resources.GatewayClass == nil { - return fmt.Errorf("the GatewayClass resource is required") - } - - defaultEnvoyProxyName := "default-envoy-proxy" - namespace := resources.GatewayClass.Namespace - defaultBootstrapStr, err := bootstrap.GetRenderedBootstrapConfig(nil) - if err != nil { - return err - } - ep := &egv1a1.EnvoyProxy{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: defaultEnvoyProxyName, - }, - Spec: egv1a1.EnvoyProxySpec{ - Bootstrap: &egv1a1.ProxyBootstrap{ - Value: defaultBootstrapStr, - }, - }, - } - resources.EnvoyProxy = ep - ns := gwapiv1.Namespace(namespace) - resources.GatewayClass.Spec.ParametersRef = &gwapiv1.ParametersReference{ - Group: gwapiv1.Group(egv1a1.GroupVersion.Group), - Kind: KindEnvoyProxy, - Name: defaultEnvoyProxyName, - Namespace: &ns, - } - return nil -} diff --git a/internal/provider/file/file.go b/internal/provider/file/file.go index dda8ba3a8bf..ebdb1025642 100644 --- a/internal/provider/file/file.go +++ b/internal/provider/file/file.go @@ -8,27 +8,31 @@ package file import ( "context" + "github.com/fsnotify/fsnotify" + "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/message" ) type Provider struct { - paths []string - notifier *Notifier - resources *message.ProviderResources + paths []string + notifier *Notifier + resourcesStore *resourcesStore } func New(svr *config.Server, resources *message.ProviderResources) (*Provider, error) { - notifier, err := NewNotifier(svr.Logger.Logger) + logger := svr.Logger.Logger + + notifier, err := NewNotifier(logger) if err != nil { return nil, err } return &Provider{ - paths: svr.EnvoyGateway.Provider.Custom.Resource.File.Paths, - notifier: notifier, - resources: resources, + paths: svr.EnvoyGateway.Provider.Custom.Resource.File.Paths, + notifier: notifier, + resourcesStore: newResourcesStore(svr.EnvoyGateway.Gateway.ControllerName, resources, logger), }, nil } @@ -42,8 +46,12 @@ func (p *Provider) Start(ctx context.Context) error { return err } - // TODO: initial load for resources-store + // Initially load resources from paths on host. + if err = p.resourcesStore.LoadAndStore(files.UnsortedList(), dirs.UnsortedList()); err != nil { + return err + } + // Start watchers in notifier. p.notifier.Watch(ctx, dirs, files) defer p.notifier.Close() @@ -51,8 +59,17 @@ func (p *Provider) Start(ctx context.Context) error { select { case <-ctx.Done(): return nil - case <-p.notifier.Events: - // TODO: ask resources-store to update according to the recv event + case event := <-p.notifier.Events: + switch event.Op { + case fsnotify.Create: + dirs.Insert(event.Name) + files.Insert(event.Name) + case fsnotify.Remove: + dirs.Delete(event.Name) + files.Delete(event.Name) + } + + p.resourcesStore.HandleEvent(event, files.UnsortedList(), dirs.UnsortedList()) } } } diff --git a/internal/provider/file/notifier.go b/internal/provider/file/notifier.go index 7851950397b..fca8465e3af 100644 --- a/internal/provider/file/notifier.go +++ b/internal/provider/file/notifier.go @@ -19,7 +19,6 @@ import ( const ( defaultCleanUpRemoveEventsPeriod = 300 * time.Millisecond - defaultWaitForWriteEventsPeriod = 100 * time.Millisecond ) type Notifier struct { @@ -30,7 +29,6 @@ type Notifier struct { filesWatcher *fsnotify.Watcher dirsWatcher *fsnotify.Watcher cleanUpRemoveEventsPeriod time.Duration - waitForWriteEventsPeriod time.Duration logger logr.Logger } @@ -51,7 +49,6 @@ func NewNotifier(logger logr.Logger) (*Notifier, error) { filesWatcher: fw, dirsWatcher: dw, cleanUpRemoveEventsPeriod: defaultCleanUpRemoveEventsPeriod, - waitForWriteEventsPeriod: defaultWaitForWriteEventsPeriod, logger: logger, }, nil } diff --git a/internal/provider/file/resources.go b/internal/provider/file/resources.go new file mode 100644 index 00000000000..53c3090b149 --- /dev/null +++ b/internal/provider/file/resources.go @@ -0,0 +1,329 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + "sigs.k8s.io/yaml" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway" + "github.com/envoyproxy/gateway/internal/envoygateway/config" + "github.com/envoyproxy/gateway/internal/gatewayapi" +) + +// loadFromFilesAndDirs loads resources from specific files and directories. +func loadFromFilesAndDirs(files, dirs []string) ([]*gatewayapi.Resources, error) { + var rs []*gatewayapi.Resources + + for _, file := range files { + r, err := loadFromFile(file) + if err != nil { + return nil, err + } + rs = append(rs, r) + } + + for _, dir := range dirs { + r, err := loadFromDir(dir) + if err != nil { + return nil, err + } + rs = append(rs, r...) + } + + return rs, nil +} + +// loadFromFile loads resources from a specific file. +func loadFromFile(path string) (*gatewayapi.Resources, error) { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("file %s is not exist", path) + } + return nil, err + } + + bytes, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + return convertKubernetesYAMLToResources(string(bytes)) +} + +// loadFromDir loads resources from all the files under a specific directory excluding subdirectories. +func loadFromDir(path string) ([]*gatewayapi.Resources, error) { + entries, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + var rs []*gatewayapi.Resources + for _, entry := range entries { + if entry.IsDir() { + continue + } + + r, err := loadFromFile(filepath.Join(path, entry.Name())) + if err != nil { + return nil, err + } + + rs = append(rs, r) + } + + return rs, nil +} + +// TODO(sh2): This function is copied and updated from internal/cmd/egctl/translate.go. +// This function should be able to process arbitrary number of resources, so we +// need to come up with a way to extend the GatewayClass and EnvoyProxy field to array +// instead of single variable in gatewayapi.Resources structure. +// +// - This issue is tracked by https://github.com/envoyproxy/gateway/issues/3207 +// +// convertKubernetesYAMLToResources converts a Kubernetes YAML string into GatewayAPI Resources. +func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) { + res := gatewayapi.NewResources() + var useDefaultNamespace bool + providedNamespaceMap := map[string]struct{}{} + requiredNamespaceMap := map[string]struct{}{} + yamls := strings.Split(str, "\n---") + combinedScheme := envoygateway.GetScheme() + for _, y := range yamls { + if strings.TrimSpace(y) == "" { + continue + } + var obj map[string]interface{} + err := yaml.Unmarshal([]byte(y), &obj) + if err != nil { + return nil, err + } + un := unstructured.Unstructured{Object: obj} + gvk := un.GroupVersionKind() + name, namespace := un.GetName(), un.GetNamespace() + if namespace == "" { + // When kubectl applies a resource in yaml which doesn't have a namespace, + // the current namespace is applied. Here we do the same thing before translating + // the GatewayAPI resource. Otherwise, the resource can't pass the namespace validation + useDefaultNamespace = true + namespace = config.DefaultNamespace + } + requiredNamespaceMap[namespace] = struct{}{} + kobj, err := combinedScheme.New(gvk) + if err != nil { + return nil, err + } + err = combinedScheme.Convert(&un, kobj, nil) + if err != nil { + return nil, err + } + + objType := reflect.TypeOf(kobj) + if objType.Kind() != reflect.Ptr { + return nil, fmt.Errorf("expected pointer type, but got %s", objType.Kind().String()) + } + kobjVal := reflect.ValueOf(kobj).Elem() + spec := kobjVal.FieldByName("Spec") + + switch gvk.Kind { + case gatewayapi.KindEnvoyProxy: + typedSpec := spec.Interface() + envoyProxy := &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(egv1a1.EnvoyProxySpec), + } + res.EnvoyProxy = envoyProxy + case gatewayapi.KindGatewayClass: + typedSpec := spec.Interface() + gatewayClass := &gwapiv1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.GatewayClassSpec), + } + res.GatewayClass = gatewayClass + case gatewayapi.KindGateway: + typedSpec := spec.Interface() + gateway := &gwapiv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.GatewaySpec), + } + res.Gateways = append(res.Gateways, gateway) + case gatewayapi.KindTCPRoute: + typedSpec := spec.Interface() + tcpRoute := &gwapiv1a2.TCPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindTCPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.TCPRouteSpec), + } + res.TCPRoutes = append(res.TCPRoutes, tcpRoute) + case gatewayapi.KindUDPRoute: + typedSpec := spec.Interface() + udpRoute := &gwapiv1a2.UDPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindUDPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.UDPRouteSpec), + } + res.UDPRoutes = append(res.UDPRoutes, udpRoute) + case gatewayapi.KindTLSRoute: + typedSpec := spec.Interface() + tlsRoute := &gwapiv1a2.TLSRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindTLSRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.TLSRouteSpec), + } + res.TLSRoutes = append(res.TLSRoutes, tlsRoute) + case gatewayapi.KindHTTPRoute: + typedSpec := spec.Interface() + httpRoute := &gwapiv1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindHTTPRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1.HTTPRouteSpec), + } + res.HTTPRoutes = append(res.HTTPRoutes, httpRoute) + case gatewayapi.KindGRPCRoute: + typedSpec := spec.Interface() + grpcRoute := &gwapiv1a2.GRPCRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: gatewayapi.KindGRPCRoute, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(gwapiv1a2.GRPCRouteSpec), + } + res.GRPCRoutes = append(res.GRPCRoutes, grpcRoute) + case gatewayapi.KindNamespace: + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + res.Namespaces = append(res.Namespaces, ns) + providedNamespaceMap[name] = struct{}{} + case gatewayapi.KindService: + typedSpec := spec.Interface() + service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: typedSpec.(corev1.ServiceSpec), + } + res.Services = append(res.Services, service) + case egv1a1.KindEnvoyPatchPolicy: + typedSpec := spec.Interface() + envoyPatchPolicy := &egv1a1.EnvoyPatchPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindEnvoyPatchPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.EnvoyPatchPolicySpec), + } + res.EnvoyPatchPolicies = append(res.EnvoyPatchPolicies, envoyPatchPolicy) + case egv1a1.KindClientTrafficPolicy: + typedSpec := spec.Interface() + clientTrafficPolicy := &egv1a1.ClientTrafficPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindClientTrafficPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.ClientTrafficPolicySpec), + } + res.ClientTrafficPolicies = append(res.ClientTrafficPolicies, clientTrafficPolicy) + case egv1a1.KindBackendTrafficPolicy: + typedSpec := spec.Interface() + backendTrafficPolicy := &egv1a1.BackendTrafficPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindBackendTrafficPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.BackendTrafficPolicySpec), + } + res.BackendTrafficPolicies = append(res.BackendTrafficPolicies, backendTrafficPolicy) + case egv1a1.KindSecurityPolicy: + typedSpec := spec.Interface() + securityPolicy := &egv1a1.SecurityPolicy{ + TypeMeta: metav1.TypeMeta{ + Kind: egv1a1.KindSecurityPolicy, + APIVersion: egv1a1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + }, + Spec: typedSpec.(egv1a1.SecurityPolicySpec), + } + res.SecurityPolicies = append(res.SecurityPolicies, securityPolicy) + } + } + + if useDefaultNamespace { + if _, found := providedNamespaceMap[config.DefaultNamespace]; !found { + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.DefaultNamespace, + }, + } + res.Namespaces = append(res.Namespaces, namespace) + providedNamespaceMap[config.DefaultNamespace] = struct{}{} + } + } + + return res, nil +} diff --git a/internal/provider/file/store.go b/internal/provider/file/store.go new file mode 100644 index 00000000000..fb661d8e589 --- /dev/null +++ b/internal/provider/file/store.go @@ -0,0 +1,63 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" + + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/message" +) + +type resourcesStore struct { + name string + resources *message.ProviderResources + + logger logr.Logger +} + +func newResourcesStore(name string, resources *message.ProviderResources, logger logr.Logger) *resourcesStore { + return &resourcesStore{ + name: name, + resources: resources, + logger: logger, + } +} + +func (r *resourcesStore) HandleEvent(event fsnotify.Event, files, dirs []string) { + r.logger.Info("receive an event", "name", event.Name, "op", event.Op.String()) + + // TODO(sh2): Find a better way to process the event. For now, + // it only simply reload all the resources from files and + // directories despite the event type. + if err := r.LoadAndStore(files, dirs); err != nil { + r.logger.Error(err, "error processing resources") + } +} + +// LoadAndStore loads and stores resources from files and directories. +func (r *resourcesStore) LoadAndStore(files, dirs []string) error { + rs, err := loadFromFilesAndDirs(files, dirs) + if err != nil { + return err + } + + // TODO(sh2): For now, we assume that one file only contains one GatewayClass and all its other + // related resources, like Gateway, HTTPRoute, etc. If we manged to extend Resources structure, + // we also need to process all the resources and its relationship, like what is done in + // Kubernetes provider. However, this will cause us to maintain two places of the same logic + // in each provider. The ideal case is two different providers share the same resources process logic. + // + // - This issue is tracked by https://github.com/envoyproxy/gateway/issues/3213 + // + // So here we just simply Store each gatewayapi.Resources. + var gwcResources gatewayapi.ControllerResources = rs + r.resources.GatewayAPIResources.Store(r.name, &gwcResources) + + r.logger.Info("loaded and stored resources successfully") + return nil +} From 5726ee58c941818929b9d82bc955e83970480c8e Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Thu, 18 Apr 2024 15:18:09 +0800 Subject: [PATCH 07/17] fix lint Signed-off-by: shawnh2 --- internal/provider/file/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/provider/file/store.go b/internal/provider/file/store.go index fb661d8e589..1e3410701f9 100644 --- a/internal/provider/file/store.go +++ b/internal/provider/file/store.go @@ -47,7 +47,7 @@ func (r *resourcesStore) LoadAndStore(files, dirs []string) error { } // TODO(sh2): For now, we assume that one file only contains one GatewayClass and all its other - // related resources, like Gateway, HTTPRoute, etc. If we manged to extend Resources structure, + // related resources, like Gateway, HTTPRoute, etc. If we managed to extend Resources structure, // we also need to process all the resources and its relationship, like what is done in // Kubernetes provider. However, this will cause us to maintain two places of the same logic // in each provider. The ideal case is two different providers share the same resources process logic. From 2e1db1bd6a3b170466e7f3f3f03d8e5e178f15f2 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 13 Aug 2024 22:40:14 +0800 Subject: [PATCH 08/17] fix ci Signed-off-by: shawnh2 --- go.mod | 2 +- internal/cmd/server.go | 4 +- internal/provider/file/file.go | 6 +- internal/provider/file/path.go | 3 +- internal/provider/file/resources.go | 57 ++++++++++--------- internal/provider/kubernetes/kubernetes.go | 6 +- internal/provider/resource_provider.go | 4 +- site/content/zh/latest/api/extension_types.md | 4 +- 8 files changed, 46 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index e23ce14fff6..75651c767cd 100644 --- a/go.mod +++ b/go.mod @@ -229,7 +229,7 @@ require ( github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect github.com/evanphx/json-patch v5.9.0+incompatible github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 3c513c9360b..cfdd243c0c8 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ctrl "sigs.k8s.io/controller-runtime" - "github.com/envoyproxy/gateway/api/v1alpha1" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/admin" "github.com/envoyproxy/gateway/internal/envoygateway/config" extensionregistry "github.com/envoyproxy/gateway/internal/extension/registry" @@ -117,7 +117,7 @@ func setupRunners(cfg *config.Server) (err error) { // Setup the Extension Manager for Kubernetes provider. var extMgr types.Manager - if cfg.EnvoyGateway.Provider.Type == v1alpha1.ProviderTypeKubernetes { + if cfg.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes { extMgr, err = extensionregistry.NewManager(cfg) if err != nil { return err diff --git a/internal/provider/file/file.go b/internal/provider/file/file.go index ebdb1025642..de1f940c0e2 100644 --- a/internal/provider/file/file.go +++ b/internal/provider/file/file.go @@ -10,7 +10,7 @@ import ( "github.com/fsnotify/fsnotify" - "github.com/envoyproxy/gateway/api/v1alpha1" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/message" ) @@ -36,8 +36,8 @@ func New(svr *config.Server, resources *message.ProviderResources) (*Provider, e }, nil } -func (p *Provider) Type() v1alpha1.ProviderType { - return v1alpha1.ProviderTypeFile +func (p *Provider) Type() egv1a1.ProviderType { + return egv1a1.ProviderTypeFile } func (p *Provider) Start(ctx context.Context) error { diff --git a/internal/provider/file/path.go b/internal/provider/file/path.go index 0e687421b0a..fe3ad7539f6 100644 --- a/internal/provider/file/path.go +++ b/internal/provider/file/path.go @@ -14,7 +14,8 @@ import ( // getDirsAndFilesForWatcher prepares dirs and files for the watcher in notifier. func getDirsAndFilesForWatcher(paths []string) ( - dirs sets.Set[string], files sets.Set[string], err error) { + dirs sets.Set[string], files sets.Set[string], err error, +) { dirs, files = sets.New[string](), sets.New[string]() // Separate paths by whether is a directory or not. diff --git a/internal/provider/file/resources.go b/internal/provider/file/resources.go index 53c3090b149..5c0dbe45ea4 100644 --- a/internal/provider/file/resources.go +++ b/internal/provider/file/resources.go @@ -15,6 +15,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/yaml" @@ -98,10 +99,10 @@ func loadFromDir(path string) ([]*gatewayapi.Resources, error) { // // convertKubernetesYAMLToResources converts a Kubernetes YAML string into GatewayAPI Resources. func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) { - res := gatewayapi.NewResources() + resources := gatewayapi.NewResources() var useDefaultNamespace bool - providedNamespaceMap := map[string]struct{}{} - requiredNamespaceMap := map[string]struct{}{} + providedNamespaceMap := sets.New[string]() + requiredNamespaceMap := sets.New[string]() yamls := strings.Split(str, "\n---") combinedScheme := envoygateway.GetScheme() for _, y := range yamls { @@ -123,7 +124,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) useDefaultNamespace = true namespace = config.DefaultNamespace } - requiredNamespaceMap[namespace] = struct{}{} + requiredNamespaceMap.Insert(namespace) kobj, err := combinedScheme.New(gvk) if err != nil { return nil, err @@ -150,7 +151,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(egv1a1.EnvoyProxySpec), } - res.EnvoyProxy = envoyProxy + resources.EnvoyProxyForGatewayClass = envoyProxy case gatewayapi.KindGatewayClass: typedSpec := spec.Interface() gatewayClass := &gwapiv1.GatewayClass{ @@ -160,7 +161,11 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(gwapiv1.GatewayClassSpec), } - res.GatewayClass = gatewayClass + // fill controller name by default controller name when gatewayclass controller name empty. + if gatewayClass.Spec.ControllerName == "" { + gatewayClass.Spec.ControllerName = egv1a1.GatewayControllerName + } + resources.GatewayClass = gatewayClass case gatewayapi.KindGateway: typedSpec := spec.Interface() gateway := &gwapiv1.Gateway{ @@ -170,7 +175,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(gwapiv1.GatewaySpec), } - res.Gateways = append(res.Gateways, gateway) + resources.Gateways = append(resources.Gateways, gateway) case gatewayapi.KindTCPRoute: typedSpec := spec.Interface() tcpRoute := &gwapiv1a2.TCPRoute{ @@ -183,7 +188,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(gwapiv1a2.TCPRouteSpec), } - res.TCPRoutes = append(res.TCPRoutes, tcpRoute) + resources.TCPRoutes = append(resources.TCPRoutes, tcpRoute) case gatewayapi.KindUDPRoute: typedSpec := spec.Interface() udpRoute := &gwapiv1a2.UDPRoute{ @@ -196,7 +201,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(gwapiv1a2.UDPRouteSpec), } - res.UDPRoutes = append(res.UDPRoutes, udpRoute) + resources.UDPRoutes = append(resources.UDPRoutes, udpRoute) case gatewayapi.KindTLSRoute: typedSpec := spec.Interface() tlsRoute := &gwapiv1a2.TLSRoute{ @@ -209,7 +214,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(gwapiv1a2.TLSRouteSpec), } - res.TLSRoutes = append(res.TLSRoutes, tlsRoute) + resources.TLSRoutes = append(resources.TLSRoutes, tlsRoute) case gatewayapi.KindHTTPRoute: typedSpec := spec.Interface() httpRoute := &gwapiv1.HTTPRoute{ @@ -222,10 +227,10 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(gwapiv1.HTTPRouteSpec), } - res.HTTPRoutes = append(res.HTTPRoutes, httpRoute) + resources.HTTPRoutes = append(resources.HTTPRoutes, httpRoute) case gatewayapi.KindGRPCRoute: typedSpec := spec.Interface() - grpcRoute := &gwapiv1a2.GRPCRoute{ + grpcRoute := &gwapiv1.GRPCRoute{ TypeMeta: metav1.TypeMeta{ Kind: gatewayapi.KindGRPCRoute, }, @@ -233,17 +238,17 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) Name: name, Namespace: namespace, }, - Spec: typedSpec.(gwapiv1a2.GRPCRouteSpec), + Spec: typedSpec.(gwapiv1.GRPCRouteSpec), } - res.GRPCRoutes = append(res.GRPCRoutes, grpcRoute) + resources.GRPCRoutes = append(resources.GRPCRoutes, grpcRoute) case gatewayapi.KindNamespace: - ns := &corev1.Namespace{ + namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, } - res.Namespaces = append(res.Namespaces, ns) - providedNamespaceMap[name] = struct{}{} + resources.Namespaces = append(resources.Namespaces, namespace) + providedNamespaceMap.Insert(name) case gatewayapi.KindService: typedSpec := spec.Interface() service := &corev1.Service{ @@ -253,7 +258,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(corev1.ServiceSpec), } - res.Services = append(res.Services, service) + resources.Services = append(resources.Services, service) case egv1a1.KindEnvoyPatchPolicy: typedSpec := spec.Interface() envoyPatchPolicy := &egv1a1.EnvoyPatchPolicy{ @@ -267,7 +272,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(egv1a1.EnvoyPatchPolicySpec), } - res.EnvoyPatchPolicies = append(res.EnvoyPatchPolicies, envoyPatchPolicy) + resources.EnvoyPatchPolicies = append(resources.EnvoyPatchPolicies, envoyPatchPolicy) case egv1a1.KindClientTrafficPolicy: typedSpec := spec.Interface() clientTrafficPolicy := &egv1a1.ClientTrafficPolicy{ @@ -281,7 +286,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(egv1a1.ClientTrafficPolicySpec), } - res.ClientTrafficPolicies = append(res.ClientTrafficPolicies, clientTrafficPolicy) + resources.ClientTrafficPolicies = append(resources.ClientTrafficPolicies, clientTrafficPolicy) case egv1a1.KindBackendTrafficPolicy: typedSpec := spec.Interface() backendTrafficPolicy := &egv1a1.BackendTrafficPolicy{ @@ -295,7 +300,7 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(egv1a1.BackendTrafficPolicySpec), } - res.BackendTrafficPolicies = append(res.BackendTrafficPolicies, backendTrafficPolicy) + resources.BackendTrafficPolicies = append(resources.BackendTrafficPolicies, backendTrafficPolicy) case egv1a1.KindSecurityPolicy: typedSpec := spec.Interface() securityPolicy := &egv1a1.SecurityPolicy{ @@ -309,21 +314,21 @@ func convertKubernetesYAMLToResources(str string) (*gatewayapi.Resources, error) }, Spec: typedSpec.(egv1a1.SecurityPolicySpec), } - res.SecurityPolicies = append(res.SecurityPolicies, securityPolicy) + resources.SecurityPolicies = append(resources.SecurityPolicies, securityPolicy) } } if useDefaultNamespace { - if _, found := providedNamespaceMap[config.DefaultNamespace]; !found { + if !providedNamespaceMap.Has(config.DefaultNamespace) { namespace := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: config.DefaultNamespace, }, } - res.Namespaces = append(res.Namespaces, namespace) - providedNamespaceMap[config.DefaultNamespace] = struct{}{} + resources.Namespaces = append(resources.Namespaces, namespace) + providedNamespaceMap.Insert(config.DefaultNamespace) } } - return res, nil + return resources, nil } diff --git a/internal/provider/kubernetes/kubernetes.go b/internal/provider/kubernetes/kubernetes.go index 81209d0696d..b79499c529d 100644 --- a/internal/provider/kubernetes/kubernetes.go +++ b/internal/provider/kubernetes/kubernetes.go @@ -19,7 +19,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" - "github.com/envoyproxy/gateway/api/v1alpha1" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" ec "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/message" @@ -116,8 +116,8 @@ func New(cfg *rest.Config, svr *ec.Server, resources *message.ProviderResources) }, nil } -func (p *Provider) Type() v1alpha1.ProviderType { - return v1alpha1.ProviderTypeKubernetes +func (p *Provider) Type() egv1a1.ProviderType { + return egv1a1.ProviderTypeKubernetes } // Start starts the Provider synchronously until a message is received from ctx. diff --git a/internal/provider/resource_provider.go b/internal/provider/resource_provider.go index 0ce24940d11..d14f95d158d 100644 --- a/internal/provider/resource_provider.go +++ b/internal/provider/resource_provider.go @@ -8,7 +8,7 @@ package provider import ( "context" - "github.com/envoyproxy/gateway/api/v1alpha1" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" ) type Provider interface { @@ -16,5 +16,5 @@ type Provider interface { Start(ctx context.Context) error // Type returns the type of resource provider. - Type() v1alpha1.ProviderType + Type() egv1a1.ProviderType } diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md index 2d2b75a3da3..eceda8d7003 100644 --- a/site/content/zh/latest/api/extension_types.md +++ b/site/content/zh/latest/api/extension_types.md @@ -1053,7 +1053,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `paths` | _string array_ | true | Paths are the paths to a directory or file containing the resource configuration.
Recursive sub directories are not currently supported. | +| `paths` | _string array_ | true | Paths are the paths to a directory or file containing the resource configuration.
Recursive subdirectories are not currently supported. | #### EnvoyGatewayHostInfrastructureProvider @@ -1213,7 +1213,7 @@ _Appears in:_ | --- | --- | --- | --- | | `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes". | | `kubernetes` | _[EnvoyGatewayKubernetesProvider](#envoygatewaykubernetesprovider)_ | false | Kubernetes defines the configuration of the Kubernetes provider. Kubernetes
provides runtime configuration via the Kubernetes API. | -| `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and a infrastructure
provider. | +| `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and an infrastructure
provider. | #### EnvoyGatewayResourceProvider From 5fe295fc429ccee40931aa1d69af523a14fc6618 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Fri, 16 Aug 2024 22:33:27 +0800 Subject: [PATCH 09/17] update infra provider api and address comments Signed-off-by: shawnh2 --- api/v1alpha1/envoygateway_types.go | 6 +- api/v1alpha1/shared_types.go | 7 +-- .../validation/envoygateway_validate.go | 47 ++++++++++---- .../validation/envoygateway_validate_test.go | 61 ++++++++++++++----- .../validation/envoyproxy_validate_test.go | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 6 +- internal/infrastructure/manager.go | 17 +++--- internal/provider/file/file.go | 2 +- internal/provider/runner/runner.go | 45 ++++++++++---- site/content/en/latest/api/extension_types.md | 4 +- site/content/zh/latest/api/extension_types.md | 4 +- 11 files changed, 143 insertions(+), 58 deletions(-) diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 4e1b5903069..9d0dfe6b8db 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -271,7 +271,11 @@ type EnvoyGatewayCustomProvider struct { // This provider is used to specify the provider to be used // to provide an environment to deploy the out resources like // the Envoy Proxy data plane. - Infrastructure EnvoyGatewayInfrastructureProvider `json:"infrastructure"` + // + // Infrastructure is optional, if no provider is set, + // The Kubernetes will be used as the default infrastructure provider. + // +optional + Infrastructure *EnvoyGatewayInfrastructureProvider `json:"infrastructure,omitempty"` } // ResourceProviderType defines the types of custom resource providers supported by Envoy Gateway. diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index c24db0ee547..1e60e4dfb2b 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -47,16 +47,15 @@ type GroupVersionKind struct { // ProviderType defines the types of providers supported by Envoy Gateway. // -// +kubebuilder:validation:Enum=Kubernetes +// +kubebuilder:validation:Enum=Kubernetes;Custom type ProviderType string const ( // ProviderTypeKubernetes defines the "Kubernetes" provider. ProviderTypeKubernetes ProviderType = "Kubernetes" - // ProviderTypeFile defines the "File" provider. This type is not implemented - // until https://github.com/envoyproxy/gateway/issues/1001 is fixed. - ProviderTypeFile ProviderType = "File" + // ProviderTypeCustom defines the "Custom" provider. + ProviderTypeCustom ProviderType = "Custom" ) // KubernetesDeploymentSpec defines the desired state of the Kubernetes deployment resource. diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go index 3e601971f1a..278681c90b5 100644 --- a/api/v1alpha1/validation/envoygateway_validate.go +++ b/api/v1alpha1/validation/envoygateway_validate.go @@ -35,8 +35,8 @@ func ValidateEnvoyGateway(eg *egv1a1.EnvoyGateway) error { if err := validateEnvoyGatewayKubernetesProvider(eg.Provider.Kubernetes); err != nil { return err } - case egv1a1.ProviderTypeFile: - if err := validateEnvoyGatewayFileProvider(eg.Provider.Custom); err != nil { + case egv1a1.ProviderTypeCustom: + if err := validateEnvoyGatewayCustomProvider(eg.Provider.Custom); err != nil { return err } default: @@ -83,28 +83,51 @@ func validateEnvoyGatewayKubernetesProvider(provider *egv1a1.EnvoyGatewayKuberne return nil } -func validateEnvoyGatewayFileProvider(provider *egv1a1.EnvoyGatewayCustomProvider) error { +func validateEnvoyGatewayCustomProvider(provider *egv1a1.EnvoyGatewayCustomProvider) error { if provider == nil { return fmt.Errorf("empty custom provider settings for file provider") } - rType, iType := provider.Resource.Type, provider.Infrastructure.Type - if rType != egv1a1.ResourceProviderTypeFile || iType != egv1a1.InfrastructureProviderTypeHost { - return fmt.Errorf("file provider only supports 'File' resource type and 'Host' infra type") + if err := validateEnvoyGatewayCustomResourceProvider(provider.Resource); err != nil { + return err } - if provider.Resource.File == nil { - return fmt.Errorf("field 'file' should be specified when resource type is 'File'") + if err := validateEnvoyGatewayCustomInfrastructureProvider(provider.Infrastructure); err != nil { + return err } - if len(provider.Resource.File.Paths) == 0 { - return fmt.Errorf("no paths were assigned for file resource provider to watch") + return nil +} + +func validateEnvoyGatewayCustomResourceProvider(resource egv1a1.EnvoyGatewayResourceProvider) error { + switch resource.Type { + case egv1a1.ResourceProviderTypeFile: + if resource.File == nil { + return fmt.Errorf("field 'file' should be specified when resource type is 'File'") + } + + if len(resource.File.Paths) == 0 { + return fmt.Errorf("no paths were assigned for file resource provider to watch") + } + default: + return fmt.Errorf("unsupported resource provider: %s", resource.Type) } + return nil +} - if provider.Infrastructure.Host == nil { - return fmt.Errorf("field 'host' should be specified when infrastructure type is 'Host'") +func validateEnvoyGatewayCustomInfrastructureProvider(infra *egv1a1.EnvoyGatewayInfrastructureProvider) error { + if infra == nil { + return nil } + switch infra.Type { + case egv1a1.InfrastructureProviderTypeHost: + if infra.Host == nil { + return fmt.Errorf("field 'host' should be specified when infrastructure type is 'Host'") + } + default: + return fmt.Errorf("unsupported infrastructure provdier: %s", infra.Type) + } return nil } diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index a9b2286a3c7..fe6b9d06079 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -68,12 +68,25 @@ func TestValidateEnvoyGateway(t *testing.T) { expect: false, }, { - name: "supported file provider", + name: "empty custom provider", eg: &egv1a1.EnvoyGateway{ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ Gateway: egv1a1.DefaultGateway(), Provider: &egv1a1.EnvoyGatewayProvider{ - Type: egv1a1.ProviderTypeFile, + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{}, + }, + }, + }, + expect: false, + }, + { + name: "custom provider with file resource provider and host infra provider", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, Custom: &egv1a1.EnvoyGatewayCustomProvider{ Resource: egv1a1.EnvoyGatewayResourceProvider{ Type: egv1a1.ResourceProviderTypeFile, @@ -81,7 +94,7 @@ func TestValidateEnvoyGateway(t *testing.T) { Paths: []string{"foo", "bar"}, }, }, - Infrastructure: egv1a1.EnvoyGatewayInfrastructureProvider{ + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ Type: egv1a1.InfrastructureProviderTypeHost, Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{}, }, @@ -92,19 +105,35 @@ func TestValidateEnvoyGateway(t *testing.T) { expect: true, }, { - name: "file provider without file resource", + name: "custom provider with file provider ans k8s infra provider", eg: &egv1a1.EnvoyGateway{ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ Gateway: egv1a1.DefaultGateway(), Provider: &egv1a1.EnvoyGatewayProvider{ - Type: egv1a1.ProviderTypeFile, + Type: egv1a1.ProviderTypeCustom, Custom: &egv1a1.EnvoyGatewayCustomProvider{ Resource: egv1a1.EnvoyGatewayResourceProvider{ Type: egv1a1.ResourceProviderTypeFile, + File: &egv1a1.EnvoyGatewayFileResourceProvider{ + Paths: []string{"foo", "bar"}, + }, }, - Infrastructure: egv1a1.EnvoyGatewayInfrastructureProvider{ - Type: egv1a1.InfrastructureProviderTypeHost, - Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{}, + }, + }, + }, + }, + expect: true, + }, + { + name: "custom provider with file provider but no file struct", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: egv1a1.ResourceProviderTypeFile, }, }, }, @@ -113,18 +142,20 @@ func TestValidateEnvoyGateway(t *testing.T) { expect: false, }, { - name: "file provider without host infrastructure", + name: "custom provider with file provider and host infra provider but no host struct", eg: &egv1a1.EnvoyGateway{ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ Gateway: egv1a1.DefaultGateway(), Provider: &egv1a1.EnvoyGatewayProvider{ - Type: egv1a1.ProviderTypeFile, + Type: egv1a1.ProviderTypeCustom, Custom: &egv1a1.EnvoyGatewayCustomProvider{ Resource: egv1a1.EnvoyGatewayResourceProvider{ Type: egv1a1.ResourceProviderTypeFile, - File: &egv1a1.EnvoyGatewayFileResourceProvider{}, + File: &egv1a1.EnvoyGatewayFileResourceProvider{ + Paths: []string{"a", "b"}, + }, }, - Infrastructure: egv1a1.EnvoyGatewayInfrastructureProvider{ + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ Type: egv1a1.InfrastructureProviderTypeHost, }, }, @@ -134,18 +165,18 @@ func TestValidateEnvoyGateway(t *testing.T) { expect: false, }, { - name: "file provider without any paths assign in resource", + name: "custom provider with file provider and host infra provider but no paths assign in resource", eg: &egv1a1.EnvoyGateway{ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ Gateway: egv1a1.DefaultGateway(), Provider: &egv1a1.EnvoyGatewayProvider{ - Type: egv1a1.ProviderTypeFile, + Type: egv1a1.ProviderTypeCustom, Custom: &egv1a1.EnvoyGatewayCustomProvider{ Resource: egv1a1.EnvoyGatewayResourceProvider{ Type: egv1a1.ResourceProviderTypeFile, File: &egv1a1.EnvoyGatewayFileResourceProvider{}, }, - Infrastructure: egv1a1.EnvoyGatewayInfrastructureProvider{ + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ Type: egv1a1.InfrastructureProviderTypeHost, Host: &egv1a1.EnvoyGatewayHostInfrastructureProvider{}, }, diff --git a/api/v1alpha1/validation/envoyproxy_validate_test.go b/api/v1alpha1/validation/envoyproxy_validate_test.go index 591c184fdd5..2dbdbb0d399 100644 --- a/api/v1alpha1/validation/envoyproxy_validate_test.go +++ b/api/v1alpha1/validation/envoyproxy_validate_test.go @@ -67,7 +67,7 @@ func TestValidateEnvoyProxy(t *testing.T) { }, Spec: egv1a1.EnvoyProxySpec{ Provider: &egv1a1.EnvoyProxyProvider{ - Type: egv1a1.ProviderTypeFile, + Type: egv1a1.ProviderTypeCustom, }, }, }, diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6e9f41f7723..2eebd493469 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1362,7 +1362,11 @@ func (in *EnvoyGatewayAdminAddress) DeepCopy() *EnvoyGatewayAdminAddress { func (in *EnvoyGatewayCustomProvider) DeepCopyInto(out *EnvoyGatewayCustomProvider) { *out = *in in.Resource.DeepCopyInto(&out.Resource) - in.Infrastructure.DeepCopyInto(&out.Infrastructure) + if in.Infrastructure != nil { + in, out := &in.Infrastructure, &out.Infrastructure + *out = new(EnvoyGatewayInfrastructureProvider) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvoyGatewayCustomProvider. diff --git a/internal/infrastructure/manager.go b/internal/infrastructure/manager.go index 528863043f0..95ed0f1ef4b 100644 --- a/internal/infrastructure/manager.go +++ b/internal/infrastructure/manager.go @@ -37,20 +37,21 @@ type Manager interface { func NewManager(cfg *config.Server) (Manager, error) { var mgr Manager - switch cfg.EnvoyGateway.Provider.Type { - case egv1a1.ProviderTypeKubernetes: + if runKubernetesInfraProvider(cfg.EnvoyGateway.Provider) { cli, err := client.New(clicfg.GetConfigOrDie(), client.Options{Scheme: envoygateway.GetScheme()}) if err != nil { return nil, err } mgr = kubernetes.NewInfra(cli, cfg) - - case egv1a1.ProviderTypeFile: - // TODO(sh2): implement host infra for file provider - - default: - return nil, fmt.Errorf("unsupported provider type %v", cfg.EnvoyGateway.Provider.Type) + } else { + // TODO(sh2): implement host infra provider + return nil, fmt.Errorf("unsupported infrasture provider") } return mgr, nil } + +func runKubernetesInfraProvider(provider *egv1a1.EnvoyGatewayProvider) bool { + return provider.Type == egv1a1.ProviderTypeKubernetes || + (provider.Type == egv1a1.ProviderTypeCustom && provider.Custom.Infrastructure == nil) +} diff --git a/internal/provider/file/file.go b/internal/provider/file/file.go index de1f940c0e2..49e7d22e3b0 100644 --- a/internal/provider/file/file.go +++ b/internal/provider/file/file.go @@ -37,7 +37,7 @@ func New(svr *config.Server, resources *message.ProviderResources) (*Provider, e } func (p *Provider) Type() egv1a1.ProviderType { - return egv1a1.ProviderTypeFile + return egv1a1.ProviderTypeCustom } func (p *Provider) Start(ctx context.Context) error { diff --git a/internal/provider/runner/runner.go b/internal/provider/runner/runner.go index 3d16e8949b2..94488489376 100644 --- a/internal/provider/runner/runner.go +++ b/internal/provider/runner/runner.go @@ -9,7 +9,6 @@ import ( "context" "fmt" - "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -44,20 +43,15 @@ func (r *Runner) Start(ctx context.Context) (err error) { var p provider.Provider switch r.EnvoyGateway.Provider.Type { case egv1a1.ProviderTypeKubernetes: - var cfg *rest.Config - cfg, err = ctrl.GetConfig() + p, err = r.createKubernetesProvider() if err != nil { - return fmt.Errorf("failed to get kubeconfig: %w", err) - } - p, err = kubernetes.New(cfg, &r.Config.Server, r.ProviderResources) - if err != nil { - return fmt.Errorf("failed to create provider %s: %w", egv1a1.ProviderTypeKubernetes, err) + return fmt.Errorf("failed to create kubernetes provider: %w", err) } - case egv1a1.ProviderTypeFile: - p, err = file.New(&r.Config.Server, r.ProviderResources) + case egv1a1.ProviderTypeCustom: + p, err = r.createCustomResourceProvider() if err != nil { - return fmt.Errorf("failed to create provider %s: %w", egv1a1.ProviderTypeFile, err) + return fmt.Errorf("failed to create custom provider: %w", err) } default: @@ -74,3 +68,32 @@ func (r *Runner) Start(ctx context.Context) (err error) { return nil } + +func (r *Runner) createKubernetesProvider() (*kubernetes.Provider, error) { + cfg, err := ctrl.GetConfig() + if err != nil { + return nil, fmt.Errorf("failed to get kubeconfig: %w", err) + } + + p, err := kubernetes.New(cfg, &r.Config.Server, r.ProviderResources) + if err != nil { + return nil, fmt.Errorf("failed to create provider %s: %w", egv1a1.ProviderTypeKubernetes, err) + } + + return p, err +} + +func (r *Runner) createCustomResourceProvider() (p provider.Provider, err error) { + switch r.EnvoyGateway.Provider.Custom.Resource.Type { + case egv1a1.ResourceProviderTypeFile: + p, err = file.New(&r.Config.Server, r.ProviderResources) + if err != nil { + return nil, fmt.Errorf("failed to create provider %s: %w", egv1a1.ProviderTypeCustom, err) + } + + default: + return nil, fmt.Errorf("unsupported resource provider type") + } + + return +} diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 5e8996a2c19..bc31f09a65c 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -1041,7 +1041,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `resource` | _[EnvoyGatewayResourceProvider](#envoygatewayresourceprovider)_ | true | Resource defines the desired resource provider.
This provider is used to specify the provider to be used
to retrieve the resource configurations such as Gateway API
resources | -| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | true | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane. | +| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | false | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane.

Infrastructure is optional, if no provider is set,
The Kubernetes will be used as the default infrastructure provider. | #### EnvoyGatewayFileResourceProvider @@ -2695,7 +2695,7 @@ _Appears in:_ | Value | Description | | ----- | ----------- | | `Kubernetes` | ProviderTypeKubernetes defines the "Kubernetes" provider.
| -| `File` | ProviderTypeFile defines the "File" provider. This type is not implemented
until https://github.com/envoyproxy/gateway/issues/1001 is fixed.
| +| `Custom` | ProviderTypeCustom defines the "Custom" provider.
| #### ProxyAccessLog diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md index 5e8996a2c19..bc31f09a65c 100644 --- a/site/content/zh/latest/api/extension_types.md +++ b/site/content/zh/latest/api/extension_types.md @@ -1041,7 +1041,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `resource` | _[EnvoyGatewayResourceProvider](#envoygatewayresourceprovider)_ | true | Resource defines the desired resource provider.
This provider is used to specify the provider to be used
to retrieve the resource configurations such as Gateway API
resources | -| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | true | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane. | +| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | false | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane.

Infrastructure is optional, if no provider is set,
The Kubernetes will be used as the default infrastructure provider. | #### EnvoyGatewayFileResourceProvider @@ -2695,7 +2695,7 @@ _Appears in:_ | Value | Description | | ----- | ----------- | | `Kubernetes` | ProviderTypeKubernetes defines the "Kubernetes" provider.
| -| `File` | ProviderTypeFile defines the "File" provider. This type is not implemented
until https://github.com/envoyproxy/gateway/issues/1001 is fixed.
| +| `Custom` | ProviderTypeCustom defines the "Custom" provider.
| #### ProxyAccessLog From 21353e294dc99a894f5a94b9d91b8e768160f865 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sat, 24 Aug 2024 20:13:59 +0800 Subject: [PATCH 10/17] update custom provider comments and validate method test Signed-off-by: shawnh2 --- api/v1alpha1/envoygateway_types.go | 2 +- .../validation/envoygateway_validate_test.go | 55 ++++++++++++++++++- .../gateway.envoyproxy.io_envoyproxies.yaml | 1 + site/content/en/latest/api/extension_types.md | 2 +- site/content/zh/latest/api/extension_types.md | 2 +- 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 9d0dfe6b8db..59477f89ea0 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -174,7 +174,7 @@ type ExtensionAPISettings struct { // EnvoyGatewayProvider defines the desired configuration of a provider. // +union type EnvoyGatewayProvider struct { - // Type is the type of provider to use. Supported types are "Kubernetes". + // Type is the type of provider to use. Supported types are "Kubernetes", "Custom". // // +unionDiscriminator Type ProviderType `json:"type"` diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index fe6b9d06079..a0cbc7b059e 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -67,6 +67,19 @@ func TestValidateEnvoyGateway(t *testing.T) { }, expect: false, }, + { + name: "nil custom provider", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: nil, + }, + }, + }, + expect: false, + }, { name: "empty custom provider", eg: &egv1a1.EnvoyGateway{ @@ -105,7 +118,7 @@ func TestValidateEnvoyGateway(t *testing.T) { expect: true, }, { - name: "custom provider with file provider ans k8s infra provider", + name: "custom provider with file provider and k8s infra provider", eg: &egv1a1.EnvoyGateway{ EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ Gateway: egv1a1.DefaultGateway(), @@ -124,6 +137,23 @@ func TestValidateEnvoyGateway(t *testing.T) { }, expect: true, }, + { + name: "custom provider with unsupported resource provider", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: "foobar", + }, + }, + }, + }, + }, + expect: false, + }, { name: "custom provider with file provider but no file struct", eg: &egv1a1.EnvoyGateway{ @@ -164,6 +194,29 @@ func TestValidateEnvoyGateway(t *testing.T) { }, expect: false, }, + { + name: "custom provider with file provider and unsupported infra provider", + eg: &egv1a1.EnvoyGateway{ + EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{ + Gateway: egv1a1.DefaultGateway(), + Provider: &egv1a1.EnvoyGatewayProvider{ + Type: egv1a1.ProviderTypeCustom, + Custom: &egv1a1.EnvoyGatewayCustomProvider{ + Resource: egv1a1.EnvoyGatewayResourceProvider{ + Type: egv1a1.ResourceProviderTypeFile, + File: &egv1a1.EnvoyGatewayFileResourceProvider{ + Paths: []string{"a", "b"}, + }, + }, + Infrastructure: &egv1a1.EnvoyGatewayInfrastructureProvider{ + Type: "foobar", + }, + }, + }, + }, + }, + expect: false, + }, { name: "custom provider with file provider and host infra provider but no paths assign in resource", eg: &egv1a1.EnvoyGateway{ diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 436a3331430..28b5800f2e7 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -10213,6 +10213,7 @@ spec: optional auxiliary control planes. Supported types are "Kubernetes". enum: - Kubernetes + - Custom type: string required: - type diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index bc31f09a65c..f3203ca9f1f 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -1213,7 +1213,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes". | +| `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes", "Custom". | | `kubernetes` | _[EnvoyGatewayKubernetesProvider](#envoygatewaykubernetesprovider)_ | false | Kubernetes defines the configuration of the Kubernetes provider. Kubernetes
provides runtime configuration via the Kubernetes API. | | `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and an infrastructure
provider. | diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md index bc31f09a65c..f3203ca9f1f 100644 --- a/site/content/zh/latest/api/extension_types.md +++ b/site/content/zh/latest/api/extension_types.md @@ -1213,7 +1213,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | -| `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes". | +| `type` | _[ProviderType](#providertype)_ | true | Type is the type of provider to use. Supported types are "Kubernetes", "Custom". | | `kubernetes` | _[EnvoyGatewayKubernetesProvider](#envoygatewaykubernetesprovider)_ | false | Kubernetes defines the configuration of the Kubernetes provider. Kubernetes
provides runtime configuration via the Kubernetes API. | | `custom` | _[EnvoyGatewayCustomProvider](#envoygatewaycustomprovider)_ | false | Custom defines the configuration for the Custom provider. This provider
allows you to define a specific resource provider and an infrastructure
provider. | From 59e9c5957f57408d93972c680cb4319e5a288dd2 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sat, 24 Aug 2024 21:38:56 +0800 Subject: [PATCH 11/17] restore extension manager and add health probe server for file provider Signed-off-by: shawnh2 --- internal/cmd/server.go | 21 ++++------ internal/provider/file/file.go | 60 ++++++++++++++++++++++++++++- internal/provider/file/resources.go | 3 +- 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/internal/cmd/server.go b/internal/cmd/server.go index cfdd243c0c8..9056fd0dbca 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -9,11 +9,9 @@ import ( "github.com/spf13/cobra" ctrl "sigs.k8s.io/controller-runtime" - egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/admin" "github.com/envoyproxy/gateway/internal/envoygateway/config" extensionregistry "github.com/envoyproxy/gateway/internal/extension/registry" - "github.com/envoyproxy/gateway/internal/extension/types" gatewayapirunner "github.com/envoyproxy/gateway/internal/gatewayapi/runner" ratelimitrunner "github.com/envoyproxy/gateway/internal/globalratelimit/runner" infrarunner "github.com/envoyproxy/gateway/internal/infrastructure/runner" @@ -110,18 +108,15 @@ func getConfigByPath(cfgPath string) (*config.Server, error) { // setupRunners starts all the runners required for the Envoy Gateway to // fulfill its tasks. -func setupRunners(cfg *config.Server) (err error) { +func setupRunners(cfg *config.Server) error { // TODO - Setup a Config Manager // https://github.com/envoyproxy/gateway/issues/43 ctx := ctrl.SetupSignalHandler() - // Setup the Extension Manager for Kubernetes provider. - var extMgr types.Manager - if cfg.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes { - extMgr, err = extensionregistry.NewManager(cfg) - if err != nil { - return err - } + // Setup the Extension Manager + extMgr, err := extensionregistry.NewManager(cfg) + if err != nil { + return err } pResources := new(message.ProviderResources) @@ -215,10 +210,8 @@ func setupRunners(cfg *config.Server) (err error) { cfg.Logger.Info("shutting down") // Close connections to extension services - if extMgr != nil { - if mgr, ok := extMgr.(*extensionregistry.Manager); ok { - mgr.CleanupHookConns() - } + if mgr, ok := extMgr.(*extensionregistry.Manager); ok { + mgr.CleanupHookConns() } return nil diff --git a/internal/provider/file/file.go b/internal/provider/file/file.go index 49e7d22e3b0..79ccd04e763 100644 --- a/internal/provider/file/file.go +++ b/internal/provider/file/file.go @@ -7,8 +7,13 @@ package file import ( "context" + "fmt" + "net/http" + "time" "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/healthz" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway/config" @@ -17,6 +22,7 @@ import ( type Provider struct { paths []string + logger logr.Logger notifier *Notifier resourcesStore *resourcesStore } @@ -31,6 +37,7 @@ func New(svr *config.Server, resources *message.ProviderResources) (*Provider, e return &Provider{ paths: svr.EnvoyGateway.Provider.Custom.Resource.File.Paths, + logger: logger, notifier: notifier, resourcesStore: newResourcesStore(svr.EnvoyGateway.Gateway.ControllerName, resources, logger), }, nil @@ -43,12 +50,15 @@ func (p *Provider) Type() egv1a1.ProviderType { func (p *Provider) Start(ctx context.Context) error { dirs, files, err := getDirsAndFilesForWatcher(p.paths) if err != nil { - return err + return fmt.Errorf("failed to get directories and files for the watcher: %w", err) } + // Start runnable servers. + go p.startHealthProbeServer(ctx) + // Initially load resources from paths on host. if err = p.resourcesStore.LoadAndStore(files.UnsortedList(), dirs.UnsortedList()); err != nil { - return err + return fmt.Errorf("failed to load resources into store: %w", err) } // Start watchers in notifier. @@ -73,3 +83,49 @@ func (p *Provider) Start(ctx context.Context) error { } } } + +func (p *Provider) startHealthProbeServer(ctx context.Context) { + const ( + readyzEndpoint = "/readyz" + healthzEndpoint = "/healthz" + ) + + mux := http.NewServeMux() + srv := &http.Server{ + Addr: ":8081", + Handler: mux, + MaxHeaderBytes: 1 << 20, + IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout + ReadHeaderTimeout: 32 * time.Second, + } + + readyzHandler := &healthz.Handler{ + Checks: map[string]healthz.Checker{ + readyzEndpoint: healthz.Ping, + }, + } + mux.Handle(readyzEndpoint, http.StripPrefix(readyzEndpoint, readyzHandler)) + // Append '/' suffix to handle subpaths. + mux.Handle(readyzEndpoint+"/", http.StripPrefix(readyzEndpoint, readyzHandler)) + + healthzHandler := &healthz.Handler{ + Checks: map[string]healthz.Checker{ + healthzEndpoint: healthz.Ping, + }, + } + mux.Handle(healthzEndpoint, http.StripPrefix(healthzEndpoint, healthzHandler)) + // Append '/' suffix to handle subpaths. + mux.Handle(healthzEndpoint+"/", http.StripPrefix(healthzEndpoint, readyzHandler)) + + go func() { + <-ctx.Done() + if err := srv.Close(); err != nil { + p.logger.Error(err, "failed to close health probe server") + } + }() + + p.logger.Info("starting health probe server", "address", srv.Addr) + if err := srv.ListenAndServe(); err != nil { + p.logger.Error(err, "failed to start health probe server") + } +} diff --git a/internal/provider/file/resources.go b/internal/provider/file/resources.go index 5c0dbe45ea4..8dcd60ac78a 100644 --- a/internal/provider/file/resources.go +++ b/internal/provider/file/resources.go @@ -75,7 +75,8 @@ func loadFromDir(path string) ([]*gatewayapi.Resources, error) { var rs []*gatewayapi.Resources for _, entry := range entries { - if entry.IsDir() { + // Ignoring subdirectories and all hidden files and directories. + if entry.IsDir() || strings.HasPrefix(entry.Name(), ".") { continue } From be44b88290a7bbece224e96a5b6756d37551ce6d Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sat, 24 Aug 2024 22:51:12 +0800 Subject: [PATCH 12/17] update envoy gateway helper functions Signed-off-by: shawnh2 --- api/v1alpha1/envoygateway_helpers.go | 13 ++++++++++--- internal/infrastructure/manager.go | 8 +------- internal/infrastructure/runner/runner.go | 3 ++- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/api/v1alpha1/envoygateway_helpers.go b/api/v1alpha1/envoygateway_helpers.go index 0b1faf7e66a..b090ce14524 100644 --- a/api/v1alpha1/envoygateway_helpers.go +++ b/api/v1alpha1/envoygateway_helpers.go @@ -209,10 +209,9 @@ func DefaultEnvoyGatewayAdminAddress() *EnvoyGatewayAdminAddress { } // GetEnvoyGatewayKubeProvider returns the EnvoyGatewayKubernetesProvider of Provider or -// a default EnvoyGatewayKubernetesProvider if unspecified. If EnvoyGatewayProvider is not of -// type "Kubernetes", a nil EnvoyGatewayKubernetesProvider is returned. +// a default EnvoyGatewayKubernetesProvider if its Infrastructure provider is Kubernetes. func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubernetesProvider { - if r.Type != ProviderTypeKubernetes { + if !r.CanRunKubernetesInfraProvider() { return nil } @@ -237,9 +236,17 @@ func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubern if r.Kubernetes.ShutdownManager == nil { r.Kubernetes.ShutdownManager = &ShutdownManager{Image: ptr.To(DefaultShutdownManagerImage)} } + return r.Kubernetes } +// CanRunKubernetesInfraProvider returns true if the resource provider is "Kubernetes" +// or "Custom" (but with unspecific infrastructure provider). +func (r *EnvoyGatewayProvider) CanRunKubernetesInfraProvider() bool { + return r.Type == ProviderTypeKubernetes || + (r.Type == ProviderTypeCustom && r.Custom.Infrastructure == nil) +} + // DefaultEnvoyGatewayLoggingLevel returns a new EnvoyGatewayLogging with default configuration parameters. // When v1alpha1.LogComponentGatewayDefault specified, all other logging components are ignored. func (logging *EnvoyGatewayLogging) DefaultEnvoyGatewayLoggingLevel(level LogLevel) LogLevel { diff --git a/internal/infrastructure/manager.go b/internal/infrastructure/manager.go index 95ed0f1ef4b..d1785ec02f6 100644 --- a/internal/infrastructure/manager.go +++ b/internal/infrastructure/manager.go @@ -12,7 +12,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" clicfg "sigs.k8s.io/controller-runtime/pkg/client/config" - egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes" @@ -37,7 +36,7 @@ type Manager interface { func NewManager(cfg *config.Server) (Manager, error) { var mgr Manager - if runKubernetesInfraProvider(cfg.EnvoyGateway.Provider) { + if cfg.EnvoyGateway.Provider.CanRunKubernetesInfraProvider() { cli, err := client.New(clicfg.GetConfigOrDie(), client.Options{Scheme: envoygateway.GetScheme()}) if err != nil { return nil, err @@ -50,8 +49,3 @@ func NewManager(cfg *config.Server) (Manager, error) { return mgr, nil } - -func runKubernetesInfraProvider(provider *egv1a1.EnvoyGatewayProvider) bool { - return provider.Type == egv1a1.ProviderTypeKubernetes || - (provider.Type == egv1a1.ProviderTypeCustom && provider.Custom.Infrastructure == nil) -} diff --git a/internal/infrastructure/runner/runner.go b/internal/infrastructure/runner/runner.go index 7574c493090..0618e21ae87 100644 --- a/internal/infrastructure/runner/runner.go +++ b/internal/infrastructure/runner/runner.go @@ -56,7 +56,8 @@ func (r *Runner) Start(ctx context.Context) (err error) { // When leader election is active, infrastructure initialization occurs only upon acquiring leadership // to avoid multiple EG instances processing envoy proxy infra resources. - if !ptr.Deref(r.EnvoyGateway.Provider.Kubernetes.LeaderElection.Disable, false) { + if r.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes && + !ptr.Deref(r.EnvoyGateway.Provider.Kubernetes.LeaderElection.Disable, false) { go func() { select { case <-ctx.Done(): From ec357fa4e7e083e1742816fff0fcc372b6cad52c Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sun, 25 Aug 2024 15:49:51 +0800 Subject: [PATCH 13/17] add some unit tests Signed-off-by: shawnh2 --- internal/provider/file/path_test.go | 54 +++++++++++++++++++ internal/provider/file/testdata/paths/dir/bar | 1 + internal/provider/file/testdata/paths/foo | 1 + 3 files changed, 56 insertions(+) create mode 100644 internal/provider/file/path_test.go create mode 100644 internal/provider/file/testdata/paths/dir/bar create mode 100644 internal/provider/file/testdata/paths/foo diff --git a/internal/provider/file/path_test.go b/internal/provider/file/path_test.go new file mode 100644 index 00000000000..183c24efa97 --- /dev/null +++ b/internal/provider/file/path_test.go @@ -0,0 +1,54 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package file + +import ( + "path" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetDirsAndFilesForWatcher(t *testing.T) { + testPath := path.Join("testdata", "paths") + testCases := []struct { + name string + paths []string + expectDirs []string + expectFiles []string + }{ + { + name: "get file and dir path", + paths: []string{ + path.Join(testPath, "dir"), path.Join(testPath, "foo"), + }, + expectDirs: []string{ + path.Join(testPath, "dir"), + }, + expectFiles: []string{ + path.Join(testPath, "foo"), + }, + }, + { + name: "overlap file path will be ignored", + paths: []string{ + path.Join(testPath, "dir"), path.Join(testPath, "dir", "bar"), + }, + expectDirs: []string{ + path.Join(testPath, "dir"), + }, + expectFiles: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dirs, paths, _ := getDirsAndFilesForWatcher(tc.paths) + require.ElementsMatch(t, dirs.UnsortedList(), tc.expectDirs) + require.ElementsMatch(t, paths.UnsortedList(), tc.expectFiles) + }) + } +} diff --git a/internal/provider/file/testdata/paths/dir/bar b/internal/provider/file/testdata/paths/dir/bar new file mode 100644 index 00000000000..e1878797a7c --- /dev/null +++ b/internal/provider/file/testdata/paths/dir/bar @@ -0,0 +1 @@ +THIS FILE IS FOR TEST ONLY \ No newline at end of file diff --git a/internal/provider/file/testdata/paths/foo b/internal/provider/file/testdata/paths/foo new file mode 100644 index 00000000000..e1878797a7c --- /dev/null +++ b/internal/provider/file/testdata/paths/foo @@ -0,0 +1 @@ +THIS FILE IS FOR TEST ONLY \ No newline at end of file From 405261859717368c942cb0385259bbbcc4de089c Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sun, 25 Aug 2024 16:21:18 +0800 Subject: [PATCH 14/17] properly handle the remove event for the file provider Signed-off-by: shawnh2 --- internal/provider/file/store.go | 35 +++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/internal/provider/file/store.go b/internal/provider/file/store.go index 1e3410701f9..00e21fbaf32 100644 --- a/internal/provider/file/store.go +++ b/internal/provider/file/store.go @@ -31,15 +31,20 @@ func newResourcesStore(name string, resources *message.ProviderResources, logger func (r *resourcesStore) HandleEvent(event fsnotify.Event, files, dirs []string) { r.logger.Info("receive an event", "name", event.Name, "op", event.Op.String()) - // TODO(sh2): Find a better way to process the event. For now, - // it only simply reload all the resources from files and - // directories despite the event type. - if err := r.LoadAndStore(files, dirs); err != nil { - r.logger.Error(err, "error processing resources") + // TODO(sh2): Support multiple GatewayClass. + switch event.Op { + case fsnotify.Write: + if err := r.LoadAndStore(files, dirs); err != nil { + r.logger.Error(err, "failed to load and store resources") + } + case fsnotify.Remove: + // Under our current assumption, one file only contains one GatewayClass and + // all its other related resources, so we can remove them safely. + r.resources.GatewayAPIResources.Delete(r.name) } } -// LoadAndStore loads and stores resources from files and directories. +// LoadAndStore loads and stores all resources from files and directories. func (r *resourcesStore) LoadAndStore(files, dirs []string) error { rs, err := loadFromFilesAndDirs(files, dirs) if err != nil { @@ -53,11 +58,21 @@ func (r *resourcesStore) LoadAndStore(files, dirs []string) error { // in each provider. The ideal case is two different providers share the same resources process logic. // // - This issue is tracked by https://github.com/envoyproxy/gateway/issues/3213 - // - // So here we just simply Store each gatewayapi.Resources. - var gwcResources gatewayapi.ControllerResources = rs - r.resources.GatewayAPIResources.Store(r.name, &gwcResources) + // We cannot make sure by the time the Write event was triggerd, whether the GatewayClass exist, + // so here we just simply Store the first gatewayapi.Resources that has GatewayClass. + gwcResources := make(gatewayapi.ControllerResources, 0, 1) + for _, resource := range rs { + if resource.GatewayClass != nil { + gwcResources = append(gwcResources, resource) + } + } + if len(gwcResources) == 0 { + return nil + } + + r.resources.GatewayAPIResources.Store(r.name, &gwcResources) r.logger.Info("loaded and stored resources successfully") + return nil } From 381efd647f476907f0885ee2c3fee8d29d32bb64 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sun, 25 Aug 2024 16:28:30 +0800 Subject: [PATCH 15/17] fix lint Signed-off-by: shawnh2 --- internal/provider/file/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/provider/file/store.go b/internal/provider/file/store.go index 00e21fbaf32..5a45f1fd638 100644 --- a/internal/provider/file/store.go +++ b/internal/provider/file/store.go @@ -59,7 +59,7 @@ func (r *resourcesStore) LoadAndStore(files, dirs []string) error { // // - This issue is tracked by https://github.com/envoyproxy/gateway/issues/3213 - // We cannot make sure by the time the Write event was triggerd, whether the GatewayClass exist, + // We cannot make sure by the time the Write event was triggered, whether the GatewayClass exist, // so here we just simply Store the first gatewayapi.Resources that has GatewayClass. gwcResources := make(gatewayapi.ControllerResources, 0, 1) for _, resource := range rs { From a30b1eadf90257406e425e50a7bf697f664c8f95 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sun, 8 Sep 2024 17:08:13 +0800 Subject: [PATCH 16/17] no default to k8s for infra provider Signed-off-by: shawnh2 --- api/v1alpha1/envoygateway_helpers.go | 12 ++----- api/v1alpha1/envoygateway_types.go | 4 +-- .../validation/envoygateway_validate.go | 2 +- internal/cmd/server.go | 33 +++++++++++-------- internal/infrastructure/manager.go | 16 ++++++--- internal/infrastructure/runner/runner.go | 5 +++ site/content/en/latest/api/extension_types.md | 2 +- site/content/zh/latest/api/extension_types.md | 2 +- 8 files changed, 45 insertions(+), 31 deletions(-) diff --git a/api/v1alpha1/envoygateway_helpers.go b/api/v1alpha1/envoygateway_helpers.go index b090ce14524..fed2f6fa075 100644 --- a/api/v1alpha1/envoygateway_helpers.go +++ b/api/v1alpha1/envoygateway_helpers.go @@ -209,9 +209,10 @@ func DefaultEnvoyGatewayAdminAddress() *EnvoyGatewayAdminAddress { } // GetEnvoyGatewayKubeProvider returns the EnvoyGatewayKubernetesProvider of Provider or -// a default EnvoyGatewayKubernetesProvider if its Infrastructure provider is Kubernetes. +// a default EnvoyGatewayKubernetesProvider if unspecified. If EnvoyGatewayProvider is not of +// type "Kubernetes", a nil EnvoyGatewayKubernetesProvider is returned. func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubernetesProvider { - if !r.CanRunKubernetesInfraProvider() { + if r.Type != ProviderTypeKubernetes { return nil } @@ -240,13 +241,6 @@ func (r *EnvoyGatewayProvider) GetEnvoyGatewayKubeProvider() *EnvoyGatewayKubern return r.Kubernetes } -// CanRunKubernetesInfraProvider returns true if the resource provider is "Kubernetes" -// or "Custom" (but with unspecific infrastructure provider). -func (r *EnvoyGatewayProvider) CanRunKubernetesInfraProvider() bool { - return r.Type == ProviderTypeKubernetes || - (r.Type == ProviderTypeCustom && r.Custom.Infrastructure == nil) -} - // DefaultEnvoyGatewayLoggingLevel returns a new EnvoyGatewayLogging with default configuration parameters. // When v1alpha1.LogComponentGatewayDefault specified, all other logging components are ignored. func (logging *EnvoyGatewayLogging) DefaultEnvoyGatewayLoggingLevel(level LogLevel) LogLevel { diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 59477f89ea0..6cf8e334182 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -272,8 +272,8 @@ type EnvoyGatewayCustomProvider struct { // to provide an environment to deploy the out resources like // the Envoy Proxy data plane. // - // Infrastructure is optional, if no provider is set, - // The Kubernetes will be used as the default infrastructure provider. + // Infrastructure is optional, if provider is not specified, + // No infrastructure provider is available. // +optional Infrastructure *EnvoyGatewayInfrastructureProvider `json:"infrastructure,omitempty"` } diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go index 278681c90b5..d27e2e1e416 100644 --- a/api/v1alpha1/validation/envoygateway_validate.go +++ b/api/v1alpha1/validation/envoygateway_validate.go @@ -85,7 +85,7 @@ func validateEnvoyGatewayKubernetesProvider(provider *egv1a1.EnvoyGatewayKuberne func validateEnvoyGatewayCustomProvider(provider *egv1a1.EnvoyGatewayCustomProvider) error { if provider == nil { - return fmt.Errorf("empty custom provider settings for file provider") + return fmt.Errorf("empty custom provider settings") } if err := validateEnvoyGatewayCustomResourceProvider(provider.Resource); err != nil { diff --git a/internal/cmd/server.go b/internal/cmd/server.go index 9056fd0dbca..25add4c8541 100644 --- a/internal/cmd/server.go +++ b/internal/cmd/server.go @@ -9,9 +9,11 @@ import ( "github.com/spf13/cobra" ctrl "sigs.k8s.io/controller-runtime" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/admin" "github.com/envoyproxy/gateway/internal/envoygateway/config" extensionregistry "github.com/envoyproxy/gateway/internal/extension/registry" + "github.com/envoyproxy/gateway/internal/extension/types" gatewayapirunner "github.com/envoyproxy/gateway/internal/gatewayapi/runner" ratelimitrunner "github.com/envoyproxy/gateway/internal/globalratelimit/runner" infrarunner "github.com/envoyproxy/gateway/internal/infrastructure/runner" @@ -108,15 +110,18 @@ func getConfigByPath(cfgPath string) (*config.Server, error) { // setupRunners starts all the runners required for the Envoy Gateway to // fulfill its tasks. -func setupRunners(cfg *config.Server) error { +func setupRunners(cfg *config.Server) (err error) { // TODO - Setup a Config Manager // https://github.com/envoyproxy/gateway/issues/43 ctx := ctrl.SetupSignalHandler() // Setup the Extension Manager - extMgr, err := extensionregistry.NewManager(cfg) - if err != nil { - return err + var extMgr types.Manager + if cfg.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeKubernetes { + extMgr, err = extensionregistry.NewManager(cfg) + if err != nil { + return err + } } pResources := new(message.ProviderResources) @@ -129,7 +134,7 @@ func setupRunners(cfg *config.Server) error { Server: *cfg, ProviderResources: pResources, }) - if err := providerRunner.Start(ctx); err != nil { + if err = providerRunner.Start(ctx); err != nil { return err } @@ -145,7 +150,7 @@ func setupRunners(cfg *config.Server) error { InfraIR: infraIR, ExtensionManager: extMgr, }) - if err := gwRunner.Start(ctx); err != nil { + if err = gwRunner.Start(ctx); err != nil { return err } @@ -160,7 +165,7 @@ func setupRunners(cfg *config.Server) error { ExtensionManager: extMgr, ProviderResources: pResources, }) - if err := xdsTranslatorRunner.Start(ctx); err != nil { + if err = xdsTranslatorRunner.Start(ctx); err != nil { return err } @@ -171,7 +176,7 @@ func setupRunners(cfg *config.Server) error { Server: *cfg, InfraIR: infraIR, }) - if err := infraRunner.Start(ctx); err != nil { + if err = infraRunner.Start(ctx); err != nil { return err } @@ -182,7 +187,7 @@ func setupRunners(cfg *config.Server) error { Server: *cfg, Xds: xds, }) - if err := xdsServerRunner.Start(ctx); err != nil { + if err = xdsServerRunner.Start(ctx); err != nil { return err } @@ -194,7 +199,7 @@ func setupRunners(cfg *config.Server) error { Server: *cfg, XdsIR: xdsIR, }) - if err := rateLimitRunner.Start(ctx); err != nil { + if err = rateLimitRunner.Start(ctx); err != nil { return err } } @@ -209,9 +214,11 @@ func setupRunners(cfg *config.Server) error { cfg.Logger.Info("shutting down") - // Close connections to extension services - if mgr, ok := extMgr.(*extensionregistry.Manager); ok { - mgr.CleanupHookConns() + if extMgr != nil { + // Close connections to extension services + if mgr, ok := extMgr.(*extensionregistry.Manager); ok { + mgr.CleanupHookConns() + } } return nil diff --git a/internal/infrastructure/manager.go b/internal/infrastructure/manager.go index d1785ec02f6..198acef8708 100644 --- a/internal/infrastructure/manager.go +++ b/internal/infrastructure/manager.go @@ -12,6 +12,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" clicfg "sigs.k8s.io/controller-runtime/pkg/client/config" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/envoyproxy/gateway/internal/envoygateway" "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes" @@ -36,15 +37,22 @@ type Manager interface { func NewManager(cfg *config.Server) (Manager, error) { var mgr Manager - if cfg.EnvoyGateway.Provider.CanRunKubernetesInfraProvider() { + switch cfg.EnvoyGateway.Provider.Type { + case egv1a1.ProviderTypeKubernetes: cli, err := client.New(clicfg.GetConfigOrDie(), client.Options{Scheme: envoygateway.GetScheme()}) if err != nil { return nil, err } mgr = kubernetes.NewInfra(cli, cfg) - } else { - // TODO(sh2): implement host infra provider - return nil, fmt.Errorf("unsupported infrasture provider") + case egv1a1.ProviderTypeCustom: + infra := cfg.EnvoyGateway.Provider.Custom.Infrastructure + switch infra.Type { + case egv1a1.InfrastructureProviderTypeHost: + // TODO(sh2): implement host provider + return nil, fmt.Errorf("host provider is not available yet") + default: + return nil, fmt.Errorf("unsupported provider type: %s", infra.Type) + } } return mgr, nil diff --git a/internal/infrastructure/runner/runner.go b/internal/infrastructure/runner/runner.go index 0618e21ae87..bd512255e1a 100644 --- a/internal/infrastructure/runner/runner.go +++ b/internal/infrastructure/runner/runner.go @@ -38,6 +38,11 @@ func New(cfg *Config) *Runner { // Start starts the infrastructure runner func (r *Runner) Start(ctx context.Context) (err error) { r.Logger = r.Logger.WithName(r.Name()).WithValues("runner", r.Name()) + if r.Config.Server.EnvoyGateway.Provider.Custom.Infrastructure == nil { + r.Logger.Info("provider is not specified, no provider is available") + return nil + } + r.mgr, err = infrastructure.NewManager(&r.Config.Server) if err != nil { r.Logger.Error(err, "failed to create new manager") diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index f3203ca9f1f..f2bcdfc4473 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -1041,7 +1041,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `resource` | _[EnvoyGatewayResourceProvider](#envoygatewayresourceprovider)_ | true | Resource defines the desired resource provider.
This provider is used to specify the provider to be used
to retrieve the resource configurations such as Gateway API
resources | -| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | false | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane.

Infrastructure is optional, if no provider is set,
The Kubernetes will be used as the default infrastructure provider. | +| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | false | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane.

Infrastructure is optional, if provider is not specified,
No infrastructure provider is available. | #### EnvoyGatewayFileResourceProvider diff --git a/site/content/zh/latest/api/extension_types.md b/site/content/zh/latest/api/extension_types.md index f3203ca9f1f..f2bcdfc4473 100644 --- a/site/content/zh/latest/api/extension_types.md +++ b/site/content/zh/latest/api/extension_types.md @@ -1041,7 +1041,7 @@ _Appears in:_ | Field | Type | Required | Description | | --- | --- | --- | --- | | `resource` | _[EnvoyGatewayResourceProvider](#envoygatewayresourceprovider)_ | true | Resource defines the desired resource provider.
This provider is used to specify the provider to be used
to retrieve the resource configurations such as Gateway API
resources | -| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | false | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane.

Infrastructure is optional, if no provider is set,
The Kubernetes will be used as the default infrastructure provider. | +| `infrastructure` | _[EnvoyGatewayInfrastructureProvider](#envoygatewayinfrastructureprovider)_ | false | Infrastructure defines the desired infrastructure provider.
This provider is used to specify the provider to be used
to provide an environment to deploy the out resources like
the Envoy Proxy data plane.

Infrastructure is optional, if provider is not specified,
No infrastructure provider is available. | #### EnvoyGatewayFileResourceProvider From a635d03322a9f10ba08273b4039a7ddfadf23a94 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sun, 8 Sep 2024 18:21:24 +0800 Subject: [PATCH 17/17] fix runner Signed-off-by: shawnh2 --- internal/infrastructure/runner/runner.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/infrastructure/runner/runner.go b/internal/infrastructure/runner/runner.go index bd512255e1a..6c261aff3f3 100644 --- a/internal/infrastructure/runner/runner.go +++ b/internal/infrastructure/runner/runner.go @@ -38,7 +38,8 @@ func New(cfg *Config) *Runner { // Start starts the infrastructure runner func (r *Runner) Start(ctx context.Context) (err error) { r.Logger = r.Logger.WithName(r.Name()).WithValues("runner", r.Name()) - if r.Config.Server.EnvoyGateway.Provider.Custom.Infrastructure == nil { + if r.EnvoyGateway.Provider.Type == egv1a1.ProviderTypeCustom && + r.EnvoyGateway.Provider.Custom.Infrastructure == nil { r.Logger.Info("provider is not specified, no provider is available") return nil }