Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add file resource provider for EG standalone mode #3159

Merged
merged 23 commits into from
Sep 11, 2024
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
93e4dd3
add validations for envoy-gateway file resource type
shawnh2 Apr 1, 2024
3956911
improve eg validation and add resource provider interface for various…
shawnh2 Apr 2, 2024
9acf325
Merge branch 'main' of github.com:envoyproxy/gateway into host-file-p…
shawnh2 Apr 2, 2024
7de91b3
extract common gatewayapi layer translate logic in egctl translate
shawnh2 Apr 3, 2024
3419af6
add notifier support
shawnh2 Apr 10, 2024
e709eae
fix lint and move read yaml bytes function back to translate
shawnh2 Apr 10, 2024
6f3b400
add resources store support
shawnh2 Apr 18, 2024
739c8dc
Merge branch 'main' of github.com:envoyproxy/gateway into host-file-p…
shawnh2 Apr 18, 2024
5726ee5
fix lint
shawnh2 Apr 18, 2024
9460667
resolve conflicts
shawnh2 Aug 13, 2024
2e1db1b
fix ci
shawnh2 Aug 13, 2024
55de67f
resolve conflicts
shawnh2 Aug 16, 2024
5fe295f
update infra provider api and address comments
shawnh2 Aug 16, 2024
21353e2
update custom provider comments and validate method test
shawnh2 Aug 24, 2024
59e9c59
restore extension manager and add health probe server for file provider
shawnh2 Aug 24, 2024
be44b88
update envoy gateway helper functions
shawnh2 Aug 24, 2024
ec357fa
add some unit tests
shawnh2 Aug 25, 2024
4052618
properly handle the remove event for the file provider
shawnh2 Aug 25, 2024
381efd6
fix lint
shawnh2 Aug 25, 2024
a30b1ea
no default to k8s for infra provider
shawnh2 Sep 8, 2024
a635d03
fix runner
shawnh2 Sep 8, 2024
61638ee
Merge branch 'main' of github.com:envoyproxy/gateway into host-file-p…
shawnh2 Sep 8, 2024
3c5d0f1
Merge branch 'main' into host-file-provider
Xunzhuo Sep 11, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions api/v1alpha1/envoygateway_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ type EnvoyGatewayProvider struct {
Kubernetes *EnvoyGatewayKubernetesProvider `json:"kubernetes,omitempty"`

// Custom defines the configuration for the Custom provider. This provider
// allows you to define a specific resource provider and a infrastructure
// allows you to define a specific resource provider and an infrastructure
// provider.
//
// +optional
Expand Down Expand Up @@ -293,7 +293,7 @@ type EnvoyGatewayResourceProvider struct {
// EnvoyGatewayFileResourceProvider defines configuration for the File Resource provider.
type EnvoyGatewayFileResourceProvider struct {
// Paths are the paths to a directory or file containing the resource configuration.
// Recursive sub directories are not currently supported.
// Recursive subdirectories are not currently supported.
Paths []string `json:"paths"`
}

Expand Down
248 changes: 170 additions & 78 deletions api/v1alpha1/validation/envoygateway_validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,103 +6,195 @@
package validation

import (
"errors"
"fmt"
"net/url"

gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"

"github.com/envoyproxy/gateway/api/v1alpha1"
)

// ValidateEnvoyGateway validates the provided EnvoyGateway.
func ValidateEnvoyGateway(eg *v1alpha1.EnvoyGateway) error {
switch {
case eg == nil:
return errors.New("envoy gateway config is unspecified")
case eg.Gateway == nil:
return errors.New("gateway is unspecified")
case len(eg.Gateway.ControllerName) == 0:
return errors.New("gateway controllerName is unspecified")
case eg.Provider == nil:
return errors.New("provider is unspecified")
case eg.Provider.Type != v1alpha1.ProviderTypeKubernetes:
return fmt.Errorf("unsupported provider %v", eg.Provider.Type)
case eg.Provider.Kubernetes != nil && eg.Provider.Kubernetes.Watch != nil:
watch := eg.Provider.Kubernetes.Watch
switch watch.Type {
case v1alpha1.KubernetesWatchModeTypeNamespaces:
if len(watch.Namespaces) == 0 {
return errors.New("namespaces should be specified when envoy gateway watch mode is 'Namespaces'")
}
case v1alpha1.KubernetesWatchModeTypeNamespaceSelector:
if watch.NamespaceSelector == nil {
return errors.New("namespaceSelector should be specified when envoy gateway watch mode is 'NamespaceSelector'")
}
default:
return errors.New("envoy gateway watch mode invalid, should be 'Namespaces' or 'NamespaceSelector'")
}
case eg.Logging != nil && len(eg.Logging.Level) != 0:
level := eg.Logging.Level
for component, logLevel := range level {
switch component {
case v1alpha1.LogComponentGatewayDefault,
v1alpha1.LogComponentProviderRunner,
v1alpha1.LogComponentGatewayAPIRunner,
v1alpha1.LogComponentXdsTranslatorRunner,
v1alpha1.LogComponentXdsServerRunner,
v1alpha1.LogComponentInfrastructureRunner,
v1alpha1.LogComponentGlobalRateLimitRunner:
switch logLevel {
case v1alpha1.LogLevelDebug, v1alpha1.LogLevelError, v1alpha1.LogLevelWarn, v1alpha1.LogLevelInfo:
default:
return errors.New("envoy gateway logging level invalid. valid options: info/debug/warn/error")
}
default:
return errors.New("envoy gateway logging components invalid. valid options: system/provider/gateway-api/xds-translator/xds-server/infrastructure")
}
}
case eg.RateLimit != nil:
if eg.RateLimit.Backend.Type != v1alpha1.RedisBackendType {
return fmt.Errorf("unsupported ratelimit backend %v", eg.RateLimit.Backend.Type)
if eg == nil {
return fmt.Errorf("envoy gateway config is unspecified")
}

if eg.Gateway == nil {
return fmt.Errorf("gateway is unspecified")
}

if len(eg.Gateway.ControllerName) == 0 {
return fmt.Errorf("gateway controllerName is unspecified")
}

if eg.Provider == nil {
return fmt.Errorf("provider is unspecified")
}

switch eg.Provider.Type {
case v1alpha1.ProviderTypeKubernetes:
if err := validateEnvoyGatewayKubernetesProvider(eg.Provider.Kubernetes); err != nil {
return err
}
if eg.RateLimit.Backend.Redis == nil || eg.RateLimit.Backend.Redis.URL == "" {
return fmt.Errorf("empty ratelimit redis settings")
case v1alpha1.ProviderTypeFile:
if err := validateEnvoyGatewayFileProvider(eg.Provider.Custom); err != nil {
return err
}
if _, err := url.Parse(eg.RateLimit.Backend.Redis.URL); err != nil {
return fmt.Errorf("unknown ratelimit redis url format: %w", err)
default:
return fmt.Errorf("unsupported provider type")
}

if err := validateEnvoyGatewayLogging(eg.Logging); err != nil {
return err
}

if err := validateEnvoyGatewayRateLimit(eg.RateLimit); err != nil {
return err
}

if err := validateEnvoyGatewayExtensionManager(eg.ExtensionManager); err != nil {
return err
}

if err := validateEnvoyGatewayTelemetry(eg.Telemetry); err != nil {
return err
}

return nil
}

func validateEnvoyGatewayKubernetesProvider(provider *v1alpha1.EnvoyGatewayKubernetesProvider) error {
if provider == nil {
return nil
}

watch := provider.Watch
if watch == nil {
return nil
}

switch watch.Type {
case v1alpha1.KubernetesWatchModeTypeNamespaces:
if len(watch.Namespaces) == 0 {
return fmt.Errorf("namespaces should be specified when envoy gateway watch mode is 'Namespaces'")
}
case eg.ExtensionManager != nil:
if eg.ExtensionManager.Hooks == nil || eg.ExtensionManager.Hooks.XDSTranslator == nil {
return fmt.Errorf("registered extension has no hooks specified")
case v1alpha1.KubernetesWatchModeTypeNamespaceSelector:
if watch.NamespaceSelector == nil {
return fmt.Errorf("namespaceSelector should be specified when envoy gateway watch mode is 'NamespaceSelector'")
}
default:
return fmt.Errorf("envoy gateway watch mode invalid, should be 'Namespaces' or 'NamespaceSelector'")
}
return nil
}

if len(eg.ExtensionManager.Hooks.XDSTranslator.Pre) == 0 && len(eg.ExtensionManager.Hooks.XDSTranslator.Post) == 0 {
return fmt.Errorf("registered extension has no hooks specified")
}
func validateEnvoyGatewayFileProvider(provider *v1alpha1.EnvoyGatewayCustomProvider) error {
if provider == nil {
return fmt.Errorf("empty custom provider settings for file provider")
}

if eg.ExtensionManager.Service == nil {
return fmt.Errorf("extension service config is empty")
}
rType, iType := provider.Resource.Type, provider.Infrastructure.Type
if rType != v1alpha1.ResourceProviderTypeFile || iType != v1alpha1.InfrastructureProviderTypeHost {
return fmt.Errorf("file provider only supports 'File' resource type and 'Host' infra type")
}

if eg.ExtensionManager.Service.TLS != nil {
certificateRefKind := eg.ExtensionManager.Service.TLS.CertificateRef.Kind
if provider.Resource.File == nil {
return fmt.Errorf("field 'file' should be specified when resource type is 'File'")
}

if certificateRefKind == nil {
return fmt.Errorf("certificateRef empty in extension service server TLS settings")
}
if len(provider.Resource.File.Paths) == 0 {
return fmt.Errorf("no paths were assigned for file resource provider to watch")
}

if *certificateRefKind != gwapiv1.Kind("Secret") {
return fmt.Errorf("unsupported extension server TLS certificateRef %v", certificateRefKind)
if provider.Infrastructure.Host == nil {
shawnh2 marked this conversation as resolved.
Show resolved Hide resolved
return fmt.Errorf("field 'host' should be specified when infrastructure type is 'Host'")
}

return nil
}

func validateEnvoyGatewayLogging(logging *v1alpha1.EnvoyGatewayLogging) error {
if logging == nil || len(logging.Level) == 0 {
return nil
}

for component, logLevel := range logging.Level {
switch component {
case v1alpha1.LogComponentGatewayDefault,
v1alpha1.LogComponentProviderRunner,
v1alpha1.LogComponentGatewayAPIRunner,
v1alpha1.LogComponentXdsTranslatorRunner,
v1alpha1.LogComponentXdsServerRunner,
v1alpha1.LogComponentInfrastructureRunner,
v1alpha1.LogComponentGlobalRateLimitRunner:
switch logLevel {
case v1alpha1.LogLevelDebug, v1alpha1.LogLevelError, v1alpha1.LogLevelWarn, v1alpha1.LogLevelInfo:
default:
return fmt.Errorf("envoy gateway logging level invalid. valid options: info/debug/warn/error")
}
default:
return fmt.Errorf("envoy gateway logging components invalid. valid options: system/provider/gateway-api/xds-translator/xds-server/infrastructure")
}
case eg.Telemetry != nil:
if eg.Telemetry.Metrics != nil {
for _, sink := range eg.Telemetry.Metrics.Sinks {
if sink.Type == v1alpha1.MetricSinkTypeOpenTelemetry {
if sink.OpenTelemetry == nil {
return fmt.Errorf("OpenTelemetry is required when sink Type is OpenTelemetry")
}
}
return nil
}

func validateEnvoyGatewayRateLimit(rateLimit *v1alpha1.RateLimit) error {
if rateLimit == nil {
return nil
}
if rateLimit.Backend.Type != v1alpha1.RedisBackendType {
return fmt.Errorf("unsupported ratelimit backend %v", rateLimit.Backend.Type)
}
if rateLimit.Backend.Redis == nil || rateLimit.Backend.Redis.URL == "" {
return fmt.Errorf("empty ratelimit redis settings")
}
if _, err := url.Parse(rateLimit.Backend.Redis.URL); err != nil {
return fmt.Errorf("unknown ratelimit redis url format: %w", err)
}
return nil
}

func validateEnvoyGatewayExtensionManager(extensionManager *v1alpha1.ExtensionManager) error {
if extensionManager == nil {
return nil
}

if extensionManager.Hooks == nil || extensionManager.Hooks.XDSTranslator == nil {
return fmt.Errorf("registered extension has no hooks specified")
}

if len(extensionManager.Hooks.XDSTranslator.Pre) == 0 && len(extensionManager.Hooks.XDSTranslator.Post) == 0 {
return fmt.Errorf("registered extension has no hooks specified")
}

if extensionManager.Service == nil {
return fmt.Errorf("extension service config is empty")
}

if extensionManager.Service.TLS != nil {
certificateRefKind := extensionManager.Service.TLS.CertificateRef.Kind

if certificateRefKind == nil {
return fmt.Errorf("certificateRef empty in extension service server TLS settings")
}

if *certificateRefKind != "Secret" {
return fmt.Errorf("unsupported extension server TLS certificateRef %v", certificateRefKind)
}
}
return nil
}

func validateEnvoyGatewayTelemetry(telemetry *v1alpha1.EnvoyGatewayTelemetry) error {
if telemetry == nil {
return nil
}

if telemetry.Metrics != nil {
for _, sink := range telemetry.Metrics.Sinks {
if sink.Type == v1alpha1.MetricSinkTypeOpenTelemetry {
if sink.OpenTelemetry == nil {
return fmt.Errorf("OpenTelemetry is required when sink Type is OpenTelemetry")
}
}
}
Expand Down
84 changes: 81 additions & 3 deletions api/v1alpha1/validation/envoygateway_validate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,89 @@ func TestValidateEnvoyGateway(t *testing.T) {
expect: false,
},
{
name: "unsupported provider",
name: "supported file provider",
eg: &v1alpha1.EnvoyGateway{
EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{
Gateway: v1alpha1.DefaultGateway(),
Provider: &v1alpha1.EnvoyGatewayProvider{Type: v1alpha1.ProviderTypeFile},
Gateway: v1alpha1.DefaultGateway(),
Provider: &v1alpha1.EnvoyGatewayProvider{
Type: v1alpha1.ProviderTypeFile,
Custom: &v1alpha1.EnvoyGatewayCustomProvider{
Resource: v1alpha1.EnvoyGatewayResourceProvider{
Type: v1alpha1.ResourceProviderTypeFile,
File: &v1alpha1.EnvoyGatewayFileResourceProvider{
Paths: []string{"foo", "bar"},
},
},
Infrastructure: v1alpha1.EnvoyGatewayInfrastructureProvider{
Type: v1alpha1.InfrastructureProviderTypeHost,
Host: &v1alpha1.EnvoyGatewayHostInfrastructureProvider{},
},
},
},
},
},
expect: true,
},
{
name: "file provider without file resource",
eg: &v1alpha1.EnvoyGateway{
EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{
Gateway: v1alpha1.DefaultGateway(),
Provider: &v1alpha1.EnvoyGatewayProvider{
Type: v1alpha1.ProviderTypeFile,
Custom: &v1alpha1.EnvoyGatewayCustomProvider{
Resource: v1alpha1.EnvoyGatewayResourceProvider{
Type: v1alpha1.ResourceProviderTypeFile,
},
Infrastructure: v1alpha1.EnvoyGatewayInfrastructureProvider{
Type: v1alpha1.InfrastructureProviderTypeHost,
Host: &v1alpha1.EnvoyGatewayHostInfrastructureProvider{},
},
},
},
},
},
expect: false,
},
{
name: "file provider without host infrastructure",
eg: &v1alpha1.EnvoyGateway{
EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{
Gateway: v1alpha1.DefaultGateway(),
Provider: &v1alpha1.EnvoyGatewayProvider{
Type: v1alpha1.ProviderTypeFile,
Custom: &v1alpha1.EnvoyGatewayCustomProvider{
Resource: v1alpha1.EnvoyGatewayResourceProvider{
Type: v1alpha1.ResourceProviderTypeFile,
File: &v1alpha1.EnvoyGatewayFileResourceProvider{},
},
Infrastructure: v1alpha1.EnvoyGatewayInfrastructureProvider{
Type: v1alpha1.InfrastructureProviderTypeHost,
},
},
},
},
},
expect: false,
},
{
name: "file provider without any paths assign in resource",
eg: &v1alpha1.EnvoyGateway{
EnvoyGatewaySpec: v1alpha1.EnvoyGatewaySpec{
Gateway: v1alpha1.DefaultGateway(),
Provider: &v1alpha1.EnvoyGatewayProvider{
Type: v1alpha1.ProviderTypeFile,
Custom: &v1alpha1.EnvoyGatewayCustomProvider{
Resource: v1alpha1.EnvoyGatewayResourceProvider{
Type: v1alpha1.ResourceProviderTypeFile,
File: &v1alpha1.EnvoyGatewayFileResourceProvider{},
},
Infrastructure: v1alpha1.EnvoyGatewayInfrastructureProvider{
Type: v1alpha1.InfrastructureProviderTypeHost,
Host: &v1alpha1.EnvoyGatewayHostInfrastructureProvider{},
},
},
},
},
},
expect: false,
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ require (
github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
github.com/evanphx/json-patch v5.9.0+incompatible
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/fsnotify/fsnotify v1.7.0
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
Expand Down
Loading