diff --git a/pkg/xds/envoy/listeners/http_access_log_configurer_test.go b/pkg/xds/envoy/listeners/http_access_log_configurer_test.go index ed6df318a822..e70158d3eac3 100644 --- a/pkg/xds/envoy/listeners/http_access_log_configurer_test.go +++ b/pkg/xds/envoy/listeners/http_access_log_configurer_test.go @@ -62,7 +62,6 @@ var _ = Describe("HttpAccessLogConfigurer", func() { Configure(OutboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). Configure(FilterChain(NewFilterChainBuilder(). Configure(HttpConnectionManager(given.statsName)). - Configure(HttpOutboundRoute(given.routeName)). Configure(HttpAccessLog(mesh, TrafficDirectionOutbound, sourceService, destinationService, given.backend, proxy)))). Build() // then @@ -94,10 +93,6 @@ var _ = Describe("HttpAccessLogConfigurer", func() { '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager httpFilters: - name: envoy.router - rds: - configSource: - ads: {} - routeConfigName: outbound:backend statPrefix: backend trafficDirection: OUTBOUND `, @@ -136,10 +131,6 @@ var _ = Describe("HttpAccessLogConfigurer", func() { path: /tmp/log httpFilters: - name: envoy.router - rds: - configSource: - ads: {} - routeConfigName: outbound:backend statPrefix: backend trafficDirection: OUTBOUND `, @@ -194,10 +185,6 @@ var _ = Describe("HttpAccessLogConfigurer", func() { httpFilters: - name: envoy.router - rds: - configSource: - ads: {} - routeConfigName: outbound:backend statPrefix: backend trafficDirection: OUTBOUND `, diff --git a/pkg/xds/envoy/listeners/http_inbound_route_configurer_test.go b/pkg/xds/envoy/listeners/http_inbound_route_configurer_test.go index 25defb834425..d94951df4078 100644 --- a/pkg/xds/envoy/listeners/http_inbound_route_configurer_test.go +++ b/pkg/xds/envoy/listeners/http_inbound_route_configurer_test.go @@ -64,9 +64,9 @@ var _ = Describe("HttpInboundRouteConfigurer", func() { - name: envoy.router routeConfig: name: inbound:backend + validateClusters: false requestHeadersToRemove: - x-kuma-tags - validateClusters: true virtualHosts: - domains: - '*' diff --git a/pkg/xds/envoy/listeners/http_outbound_route_configurer.go b/pkg/xds/envoy/listeners/http_outbound_route_configurer.go index 4f381835d8b3..0f7336e22a18 100644 --- a/pkg/xds/envoy/listeners/http_outbound_route_configurer.go +++ b/pkg/xds/envoy/listeners/http_outbound_route_configurer.go @@ -1,34 +1,46 @@ package listeners import ( - envoy_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" + + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + envoy_common "github.com/kumahq/kuma/pkg/xds/envoy" + envoy_names "github.com/kumahq/kuma/pkg/xds/envoy/names" + envoy_routes "github.com/kumahq/kuma/pkg/xds/envoy/routes" ) -func HttpOutboundRoute(routeName string) FilterChainBuilderOpt { +func HttpOutboundRoute(service string, subsets []envoy_common.ClusterSubset, dpTags mesh_proto.MultiValueTagSet) FilterChainBuilderOpt { return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) { config.Add(&HttpOutboundRouteConfigurer{ - routeName: routeName, + service: service, + subsets: subsets, + dpTags: dpTags, }) }) } type HttpOutboundRouteConfigurer struct { - routeName string + service string + subsets []envoy_common.ClusterSubset + dpTags mesh_proto.MultiValueTagSet } func (c *HttpOutboundRouteConfigurer) Configure(filterChain *envoy_listener.FilterChain) error { + routeConfig, err := envoy_routes.NewRouteConfigurationBuilder(). + Configure(envoy_routes.CommonRouteConfiguration(envoy_names.GetOutboundRouteName(c.service))). + Configure(envoy_routes.TagsHeader(c.dpTags)). + Configure(envoy_routes.VirtualHost(envoy_routes.NewVirtualHostBuilder(). + Configure(envoy_routes.CommonVirtualHost(c.service)). + Configure(envoy_routes.DefaultRoute(c.subsets...)))). + Build() + if err != nil { + return err + } + return UpdateHTTPConnectionManager(filterChain, func(hcm *envoy_hcm.HttpConnectionManager) error { - hcm.RouteSpecifier = &envoy_hcm.HttpConnectionManager_Rds{ - Rds: &envoy_hcm.Rds{ - ConfigSource: &envoy_core.ConfigSource{ - ConfigSourceSpecifier: &envoy_core.ConfigSource_Ads{ - Ads: &envoy_core.AggregatedConfigSource{}, - }, - }, - RouteConfigName: c.routeName, - }, + hcm.RouteSpecifier = &envoy_hcm.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, } return nil }) diff --git a/pkg/xds/envoy/listeners/http_outbound_route_configurer_test.go b/pkg/xds/envoy/listeners/http_outbound_route_configurer_test.go index 00d454d319c2..60d1f11b7eba 100644 --- a/pkg/xds/envoy/listeners/http_outbound_route_configurer_test.go +++ b/pkg/xds/envoy/listeners/http_outbound_route_configurer_test.go @@ -5,6 +5,8 @@ import ( . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + envoy_common "github.com/kumahq/kuma/pkg/xds/envoy" . "github.com/kumahq/kuma/pkg/xds/envoy/listeners" util_proto "github.com/kumahq/kuma/pkg/util/proto" @@ -17,7 +19,9 @@ var _ = Describe("HttpOutboundRouteConfigurer", func() { listenerAddress string listenerPort uint32 statsName string - routeName string + service string + subsets []envoy_common.ClusterSubset + dpTags mesh_proto.MultiValueTagSet expected string } @@ -28,7 +32,7 @@ var _ = Describe("HttpOutboundRouteConfigurer", func() { Configure(OutboundListener(given.listenerName, given.listenerAddress, given.listenerPort)). Configure(FilterChain(NewFilterChainBuilder(). Configure(HttpConnectionManager(given.statsName)). - Configure(HttpOutboundRoute(given.routeName)))). + Configure(HttpOutboundRoute(given.service, given.subsets, given.dpTags)))). Build() // then Expect(err).ToNot(HaveOccurred()) @@ -44,10 +48,29 @@ var _ = Describe("HttpOutboundRouteConfigurer", func() { listenerAddress: "127.0.0.1", listenerPort: 18080, statsName: "127.0.0.1:18080", - routeName: "outbound:backend", + service: "backend", + subsets: []envoy_common.ClusterSubset{ + { + ClusterName: "backend", + Weight: 20, + Tags: map[string]string{ + "version": "v1", + }, + }, + { + ClusterName: "backend", + Weight: 80, + Tags: map[string]string{ + "version": "v2", + }, + }, + }, + dpTags: map[string]map[string]bool{ + "kuma.io/service": { + "web": true, + }, + }, expected: ` - name: outbound:127.0.0.1:18080 - trafficDirection: OUTBOUND address: socketAddress: address: 127.0.0.1 @@ -59,12 +82,39 @@ var _ = Describe("HttpOutboundRouteConfigurer", func() { '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager httpFilters: - name: envoy.router - rds: - configSource: - ads: {} - routeConfigName: outbound:backend + routeConfig: + name: outbound:backend + validateClusters: false + requestHeadersToAdd: + - header: + key: x-kuma-tags + value: '&kuma.io/service=web&' + virtualHosts: + - domains: + - '*' + name: backend + routes: + - match: + prefix: / + route: + weightedClusters: + clusters: + - metadataMatch: + filterMetadata: + envoy.lb: + version: v1 + name: backend + weight: 20 + - metadataMatch: + filterMetadata: + envoy.lb: + version: v2 + name: backend + weight: 80 + totalWeight: 100 statPrefix: "127_0_0_1_18080" -`, + name: outbound:127.0.0.1:18080 + trafficDirection: OUTBOUND`, }), ) }) diff --git a/pkg/xds/envoy/listeners/prometheus_endpoint_configurer.go b/pkg/xds/envoy/listeners/prometheus_endpoint_configurer.go index 422ede0649db..a4abb61d59ee 100644 --- a/pkg/xds/envoy/listeners/prometheus_endpoint_configurer.go +++ b/pkg/xds/envoy/listeners/prometheus_endpoint_configurer.go @@ -6,6 +6,7 @@ import ( envoy_route "github.com/envoyproxy/go-control-plane/envoy/api/v2/route" envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown" + "github.com/golang/protobuf/ptypes/wrappers" "github.com/kumahq/kuma/pkg/util/proto" util_xds "github.com/kumahq/kuma/pkg/util/xds" @@ -55,6 +56,9 @@ func (c *PrometheusEndpointConfigurer) Configure(filterChain *envoy_listener.Fil }, }}, }}, + ValidateClusters: &wrappers.BoolValue{ + Value: false, + }, }, }, } diff --git a/pkg/xds/envoy/listeners/prometheus_endpoint_configurer_test.go b/pkg/xds/envoy/listeners/prometheus_endpoint_configurer_test.go index c00b40bb2c45..764223e97482 100644 --- a/pkg/xds/envoy/listeners/prometheus_endpoint_configurer_test.go +++ b/pkg/xds/envoy/listeners/prometheus_endpoint_configurer_test.go @@ -59,6 +59,7 @@ var _ = Describe("PrometheusEndpointConfigurer", func() { httpFilters: - name: envoy.router routeConfig: + validateClusters: false virtualHosts: - domains: - '*' diff --git a/pkg/xds/envoy/routes/common_route_configuration_configurer.go b/pkg/xds/envoy/routes/common_route_configuration_configurer.go index 30d27e40fbcb..2bc749c94cb6 100644 --- a/pkg/xds/envoy/routes/common_route_configuration_configurer.go +++ b/pkg/xds/envoy/routes/common_route_configuration_configurer.go @@ -21,7 +21,7 @@ type CommonRouteConfigurationConfigurer struct { func (c CommonRouteConfigurationConfigurer) Configure(routeConfiguration *v2.RouteConfiguration) error { routeConfiguration.Name = c.name routeConfiguration.ValidateClusters = &wrappers.BoolValue{ - Value: true, + Value: false, } return nil } diff --git a/pkg/xds/envoy/routes/common_route_configuration_configurer_test.go b/pkg/xds/envoy/routes/common_route_configuration_configurer_test.go index b0e9becdb8bb..06a994e156ce 100644 --- a/pkg/xds/envoy/routes/common_route_configuration_configurer_test.go +++ b/pkg/xds/envoy/routes/common_route_configuration_configurer_test.go @@ -37,7 +37,7 @@ var _ = Describe("CommonRouteConfigurationConfigurer", func() { routeName: "outbound:backend", expected: ` name: outbound:backend - validateClusters: true + validateClusters: false `, }), ) diff --git a/pkg/xds/generator/modifications/virtual_host.go b/pkg/xds/generator/modifications/virtual_host.go index 4e951aea30d3..5374fd184cb8 100644 --- a/pkg/xds/generator/modifications/virtual_host.go +++ b/pkg/xds/generator/modifications/virtual_host.go @@ -2,9 +2,13 @@ package modifications import ( envoy_api "github.com/envoyproxy/go-control-plane/envoy/api/v2" + envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener" envoy_route "github.com/envoyproxy/go-control-plane/envoy/api/v2/route" + envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" envoy_resource "github.com/envoyproxy/go-control-plane/pkg/resource/v2" + envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown" "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" "github.com/pkg/errors" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" @@ -12,6 +16,8 @@ import ( util_proto "github.com/kumahq/kuma/pkg/util/proto" ) +// virtualHostModificator assumes that the routes are specified as `routeConfig` in Listeners, not through RDS +// If we ever change it to RDS we need to modify RouteConfiguration objects type virtualHostModificator mesh_proto.ProxyTemplate_Modifications_VirtualHost func (c *virtualHostModificator) apply(resources *core_xds.ResourceSet) error { @@ -20,24 +26,55 @@ func (c *virtualHostModificator) apply(resources *core_xds.ResourceSet) error { return err } - for _, resource := range resources.Resources(envoy_resource.RouteType) { - if c.routeConfigurationMatches(resource) { - routeCfg := resource.Resource.(*envoy_api.RouteConfiguration) - switch c.Operation { - case mesh_proto.OpAdd: - c.add(routeCfg, virtualHost) - case mesh_proto.OpRemove: - c.remove(routeCfg) - case mesh_proto.OpPatch: - c.patch(routeCfg, virtualHost) - default: - return errors.Errorf("invalid operation: %s", c.Operation) + for _, resource := range resources.Resources(envoy_resource.ListenerType) { + listener := resource.Resource.(*envoy_api.Listener) + if !c.originMatches(resource) { + continue + } + for _, chain := range listener.FilterChains { // apply on all filter chains. We could introduce filter chain matcher as an improvement. + for _, networkFilter := range chain.Filters { + if networkFilter.Name == envoy_wellknown.HTTPConnectionManager { + hcm := &envoy_hcm.HttpConnectionManager{} + err := ptypes.UnmarshalAny(networkFilter.ConfigType.(*envoy_listener.Filter_TypedConfig).TypedConfig, hcm) + if err != nil { + return err + } + if err := c.applyHCMModification(hcm, virtualHost); err != nil { + return err + } + any, err := util_proto.MarshalAnyDeterministic(hcm) + if err != nil { + return err + } + networkFilter.ConfigType.(*envoy_listener.Filter_TypedConfig).TypedConfig = any + } } } } return nil } +func (c *virtualHostModificator) applyHCMModification(hcm *envoy_hcm.HttpConnectionManager, virtualHost *envoy_route.VirtualHost) error { + routeCfg := hcm.GetRouteConfig() + if routeCfg == nil { + return nil // ignore HCMs without embedded routes + } + if !c.routeConfigurationMatches(routeCfg) { + return nil + } + switch c.Operation { + case mesh_proto.OpAdd: + c.add(routeCfg, virtualHost) + case mesh_proto.OpRemove: + c.remove(routeCfg) + case mesh_proto.OpPatch: + c.patch(routeCfg, virtualHost) + default: + return errors.Errorf("invalid operation: %s", c.Operation) + } + return nil +} + func (c *virtualHostModificator) patch(routeCfg *envoy_api.RouteConfiguration, vHostPatch *envoy_route.VirtualHost) { for _, vHost := range routeCfg.VirtualHosts { if c.virtualHostMatches(vHost) { @@ -67,10 +104,11 @@ func (c *virtualHostModificator) virtualHostMatches(vHost *envoy_route.VirtualHo return true } -func (c *virtualHostModificator) routeConfigurationMatches(routeCfg *core_xds.Resource) bool { - if c.Match.GetOrigin() != "" && c.Match.GetOrigin() != routeCfg.Origin { - return false - } +func (c *virtualHostModificator) originMatches(routeCfg *core_xds.Resource) bool { + return c.Match.GetOrigin() == "" || (c.Match.GetOrigin() == routeCfg.Origin) +} + +func (c *virtualHostModificator) routeConfigurationMatches(routeCfg *envoy_api.RouteConfiguration) bool { if c.Match.GetRouteConfigurationName() != "" && c.Match.GetRouteConfigurationName() != routeCfg.Name { return false } diff --git a/pkg/xds/generator/modifications/virtual_host_test.go b/pkg/xds/generator/modifications/virtual_host_test.go index 41e7ad1a2805..ddd96a79723f 100644 --- a/pkg/xds/generator/modifications/virtual_host_test.go +++ b/pkg/xds/generator/modifications/virtual_host_test.go @@ -27,7 +27,7 @@ var _ = Describe("Virtual Host modifications", func() { // given set := core_xds.NewResourceSet() for _, routeCfgYAML := range given.routeCfgs { - routeCfg := &envoy_api.RouteConfiguration{} + routeCfg := &envoy_api.Listener{} err := util_proto.FromYAML([]byte(routeCfgYAML), routeCfg) Expect(err).ToNot(HaveOccurred()) set.Add(&core_xds.Resource{ @@ -59,16 +59,22 @@ var _ = Describe("Virtual Host modifications", func() { Entry("should add virtual host", testCase{ routeCfgs: []string{ ` - name: outbound:backend - virtualHosts: - - name: backend - domains: - - "*" - routes: - - match: - prefix: / - route: - cluster: backend + name: outbound:192.168.0.1:8080 + trafficDirection: INBOUND + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + statPrefix: localhost_8080 + httpFilters: + - name: envoy.router + routeConfig: + name: outbound:backend `, }, modifications: []string{` @@ -85,51 +91,65 @@ var _ = Describe("Virtual Host modifications", func() { cluster: backend`, }, expected: ` - resources: - - name: outbound:backend - resource: - '@type': type.googleapis.com/envoy.api.v2.RouteConfiguration - name: outbound:backend - virtualHosts: - - domains: - - '*' - name: backend - routes: - - match: - prefix: / - route: - cluster: backend - - domains: - - backend.com - name: backend - routes: - - match: - prefix: / - route: - cluster: backend`, + resources: + - name: outbound:192.168.0.1:8080 + resource: + '@type': type.googleapis.com/envoy.api.v2.Listener + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + httpFilters: + - name: envoy.router + routeConfig: + name: outbound:backend + virtualHosts: + - domains: + - backend.com + name: backend + routes: + - match: + prefix: / + route: + cluster: backend + statPrefix: localhost_8080 + name: outbound:192.168.0.1:8080 + trafficDirection: INBOUND +`, }), Entry("should remove virtual host", testCase{ routeCfgs: []string{ ` - name: outbound:backend - virtualHosts: - - name: backend - domains: - - "*" - routes: - - match: - prefix: /backend - route: - cluster: backend - virtualHosts: - - name: web - domains: - - "*" - routes: - - match: - prefix: /web - route: - cluster: web + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + httpFilters: + - name: envoy.router + routeConfig: + name: outbound:backend + virtualHosts: + - domains: + - backend.com + name: backend + routes: + - match: + prefix: / + route: + cluster: backend + statPrefix: localhost_8080 + name: outbound:192.168.0.1:8080 + trafficDirection: INBOUND `, }, modifications: []string{` @@ -139,34 +159,55 @@ var _ = Describe("Virtual Host modifications", func() { name: backend`, }, expected: ` - resources: - - name: outbound:backend - resource: - '@type': type.googleapis.com/envoy.api.v2.RouteConfiguration - name: outbound:backend - virtualHosts: - - domains: - - '*' - name: web - routes: - - match: - prefix: /web - route: - cluster: web`, + resources: + - name: outbound:192.168.0.1:8080 + resource: + '@type': type.googleapis.com/envoy.api.v2.Listener + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + httpFilters: + - name: envoy.router + routeConfig: + name: outbound:backend + statPrefix: localhost_8080 + name: outbound:192.168.0.1:8080 + trafficDirection: INBOUND`, }), Entry("should patch a virtual host", testCase{ routeCfgs: []string{ ` - name: outbound:backend - virtualHosts: - - name: backend - domains: - - "*" - routes: - - match: - prefix: / - route: - cluster: backend + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + httpFilters: + - name: envoy.router + routeConfig: + name: outbound:backend + virtualHosts: + - domains: + - backend.com + name: backend + routes: + - match: + prefix: / + route: + cluster: backend + statPrefix: localhost_8080 + name: outbound:192.168.0.1:8080 + trafficDirection: INBOUND `, }, modifications: []string{` @@ -180,37 +221,67 @@ var _ = Describe("Virtual Host modifications", func() { numRetries: 3`, }, expected: ` - resources: - - name: outbound:backend - resource: - '@type': type.googleapis.com/envoy.api.v2.RouteConfiguration - name: outbound:backend - virtualHosts: - - domains: - - '*' - name: backend - retryPolicy: - numRetries: 3 - retryOn: 5xx - routes: - - match: - prefix: / - route: - cluster: backend`, + resources: + - name: outbound:192.168.0.1:8080 + resource: + '@type': type.googleapis.com/envoy.api.v2.Listener + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + httpFilters: + - name: envoy.router + routeConfig: + name: outbound:backend + virtualHosts: + - domains: + - backend.com + name: backend + retryPolicy: + numRetries: 3 + retryOn: 5xx + routes: + - match: + prefix: / + route: + cluster: backend + statPrefix: localhost_8080 + name: outbound:192.168.0.1:8080 + trafficDirection: INBOUND`, }), Entry("should patch a virtual host adding new route", testCase{ routeCfgs: []string{ ` - name: outbound:backend - virtualHosts: - - name: backend - domains: - - "*" - routes: - - match: - prefix: / - route: - cluster: backend + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + httpFilters: + - name: envoy.router + routeConfig: + name: outbound:backend + virtualHosts: + - domains: + - backend.com + name: backend + routes: + - match: + prefix: / + route: + cluster: backend + statPrefix: localhost_8080 + name: outbound:192.168.0.1:8080 + trafficDirection: INBOUND `, }, modifications: []string{` @@ -226,24 +297,39 @@ var _ = Describe("Virtual Host modifications", func() { cluster: web`, }, expected: ` - resources: - - name: outbound:backend - resource: - '@type': type.googleapis.com/envoy.api.v2.RouteConfiguration - name: outbound:backend - virtualHosts: - - domains: - - '*' - name: backend - routes: - - match: - prefix: / - route: - cluster: backend - - match: - prefix: /web - route: - cluster: web`, + resources: + - name: outbound:192.168.0.1:8080 + resource: + '@type': type.googleapis.com/envoy.api.v2.Listener + address: + socketAddress: + address: 192.168.0.1 + portValue: 8080 + filterChains: + - filters: + - name: envoy.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + httpFilters: + - name: envoy.router + routeConfig: + name: outbound:backend + virtualHosts: + - domains: + - backend.com + name: backend + routes: + - match: + prefix: / + route: + cluster: backend + - match: + prefix: /web + route: + cluster: web + statPrefix: localhost_8080 + name: outbound:192.168.0.1:8080 + trafficDirection: INBOUND`, }), ) }) diff --git a/pkg/xds/generator/outbound_proxy_generator.go b/pkg/xds/generator/outbound_proxy_generator.go index 77a1c59105a9..b9ef64d21928 100644 --- a/pkg/xds/generator/outbound_proxy_generator.go +++ b/pkg/xds/generator/outbound_proxy_generator.go @@ -5,12 +5,11 @@ import ( envoy_api_v2 "github.com/envoyproxy/go-control-plane/envoy/api/v2" + "github.com/pkg/errors" + "github.com/kumahq/kuma/pkg/core/validators" envoy_endpoints "github.com/kumahq/kuma/pkg/xds/envoy/endpoints" envoy_names "github.com/kumahq/kuma/pkg/xds/envoy/names" - envoy_routes "github.com/kumahq/kuma/pkg/xds/envoy/routes" - - "github.com/pkg/errors" kuma_mesh "github.com/kumahq/kuma/api/mesh/v1alpha1" mesh_core "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" @@ -57,22 +56,6 @@ func (g OutboundProxyGenerator) Generate(ctx xds_context.Context, proxy *model.P Origin: OriginOutbound, Resource: listener, }) - - // Generate route, routes are only applicable to the HTTP, HTTP/2 and GRPC protocols - switch protocol { - case mesh_core.ProtocolHTTP, mesh_core.ProtocolHTTP2: - fallthrough - case mesh_core.ProtocolGRPC: - route, err := g.generateRDS(proxy, subsets, outbound) - if err != nil { - return nil, err - } - resources.Add(&model.Resource{ - Name: route.Name, - Origin: OriginOutbound, - Resource: route, - }) - } } // Generate clusters. It cannot be generated on the fly with outbound loop because we need to know all subsets of the cluster for every service. @@ -102,14 +85,14 @@ func (_ OutboundProxyGenerator) generateLDS(proxy *model.Proxy, subsets []envoy_ Configure(envoy_listeners.HttpConnectionManager(serviceName)). Configure(envoy_listeners.Tracing(proxy.TracingBackend)). Configure(envoy_listeners.HttpAccessLog(meshName, envoy_listeners.TrafficDirectionOutbound, sourceService, serviceName, proxy.Logs[serviceName], proxy)). - Configure(envoy_listeners.HttpOutboundRoute(envoy_names.GetOutboundRouteName(serviceName))). + Configure(envoy_listeners.HttpOutboundRoute(serviceName, subsets, proxy.Dataplane.Spec.Tags())). Configure(envoy_listeners.GrpcStats()) case mesh_core.ProtocolHTTP, mesh_core.ProtocolHTTP2: filterChainBuilder. Configure(envoy_listeners.HttpConnectionManager(serviceName)). Configure(envoy_listeners.Tracing(proxy.TracingBackend)). Configure(envoy_listeners.HttpAccessLog(meshName, envoy_listeners.TrafficDirectionOutbound, sourceService, serviceName, proxy.Logs[serviceName], proxy)). - Configure(envoy_listeners.HttpOutboundRoute(envoy_names.GetOutboundRouteName(serviceName))) + Configure(envoy_listeners.HttpOutboundRoute(serviceName, subsets, proxy.Dataplane.Spec.Tags())) case mesh_core.ProtocolTCP: fallthrough default: @@ -221,15 +204,3 @@ func (_ OutboundProxyGenerator) determineSubsets(proxy *model.Proxy, outbound *k } return } - -func (_ OutboundProxyGenerator) generateRDS(proxy *model.Proxy, subsets []envoy_common.ClusterSubset, outbound *kuma_mesh.Dataplane_Networking_Outbound) (*envoy_api_v2.RouteConfiguration, error) { - serviceName := outbound.GetTagsIncludingLegacy()[kuma_mesh.ServiceTag] - - return envoy_routes.NewRouteConfigurationBuilder(). - Configure(envoy_routes.CommonRouteConfiguration(envoy_names.GetOutboundRouteName(serviceName))). - Configure(envoy_routes.TagsHeader(proxy.Dataplane.Spec.Tags())). - Configure(envoy_routes.VirtualHost(envoy_routes.NewVirtualHostBuilder(). - Configure(envoy_routes.CommonVirtualHost(serviceName)). - Configure(envoy_routes.DefaultRoute(subsets...)))). - Build() -} diff --git a/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml index fa54a6ce91f1..383b75ae5e0a 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/3-envoy-config.golden.yaml @@ -130,9 +130,9 @@ resources: - name: envoy.router routeConfig: name: inbound:backend1 + validateClusters: false requestHeadersToRemove: - x-kuma-tags - validateClusters: true virtualHosts: - domains: - '*' @@ -259,9 +259,9 @@ resources: - name: envoy.router routeConfig: name: inbound:backend3 + validateClusters: false requestHeadersToRemove: - x-kuma-tags - validateClusters: true virtualHosts: - domains: - '*' diff --git a/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml b/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml index bccb43335792..5cb301cc5209 100644 --- a/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/inbound-proxy/4-envoy-config.golden.yaml @@ -134,9 +134,9 @@ resources: - name: envoy.router routeConfig: name: inbound:backend1 + validateClusters: false requestHeadersToRemove: - x-kuma-tags - validateClusters: true virtualHosts: - domains: - '*' @@ -267,9 +267,9 @@ resources: - name: envoy.router routeConfig: name: inbound:backend3 + validateClusters: false requestHeadersToRemove: - x-kuma-tags - validateClusters: true virtualHosts: - domains: - '*' diff --git a/pkg/xds/generator/testdata/outbound-proxy/03.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/03.envoy.golden.yaml index fd37c018545f..2c7d88d36360 100644 --- a/pkg/xds/generator/testdata/outbound-proxy/03.envoy.golden.yaml +++ b/pkg/xds/generator/testdata/outbound-proxy/03.envoy.golden.yaml @@ -175,8 +175,8 @@ resources: connectTimeout: 5s edsClusterConfig: edsConfig: - ads: { } - http2ProtocolOptions: { } + ads: {} + http2ProtocolOptions: {} name: api-http2 type: EDS - name: api-tcp @@ -185,8 +185,8 @@ resources: connectTimeout: 5s edsClusterConfig: edsConfig: - ads: { } - http2ProtocolOptions: { } + ads: {} + http2ProtocolOptions: {} name: api-tcp type: EDS - name: backend @@ -195,8 +195,8 @@ resources: connectTimeout: 5s edsClusterConfig: edsConfig: - ads: { } - http2ProtocolOptions: { } + ads: {} + http2ProtocolOptions: {} name: backend type: EDS - name: db @@ -205,8 +205,8 @@ resources: connectTimeout: 5s edsClusterConfig: edsConfig: - ads: { } - http2ProtocolOptions: { } + ads: {} + http2ProtocolOptions: {} lbSubsetConfig: fallbackPolicy: ANY_ENDPOINT subsetSelectors: @@ -215,60 +215,6 @@ resources: - role name: db type: EDS - - name: outbound:api-grpc - resource: - '@type': type.googleapis.com/envoy.api.v2.RouteConfiguration - name: outbound:api-grpc - requestHeadersToAdd: - - header: - key: x-kuma-tags - value: '&kuma.io/service=gateway&' - validateClusters: true - virtualHosts: - - domains: - - '*' - name: api-grpc - routes: - - match: - prefix: / - route: - cluster: api-grpc - - name: outbound:api-http - resource: - '@type': type.googleapis.com/envoy.api.v2.RouteConfiguration - name: outbound:api-http - requestHeadersToAdd: - - header: - key: x-kuma-tags - value: '&kuma.io/service=gateway&' - validateClusters: true - virtualHosts: - - domains: - - '*' - name: api-http - routes: - - match: - prefix: / - route: - cluster: api-http - - name: outbound:api-http2 - resource: - '@type': type.googleapis.com/envoy.api.v2.RouteConfiguration - name: outbound:api-http2 - requestHeadersToAdd: - - header: - key: x-kuma-tags - value: '&kuma.io/service=gateway&' - validateClusters: true - virtualHosts: - - domains: - - '*' - name: api-http2 - routes: - - match: - prefix: / - route: - cluster: api-http2 - name: outbound:127.0.0.1:18080 resource: '@type': type.googleapis.com/envoy.api.v2.Listener @@ -307,10 +253,22 @@ resources: path: /var/log httpFilters: - name: envoy.router - rds: - configSource: - ads: { } - routeConfigName: outbound:api-http + routeConfig: + name: outbound:api-http + validateClusters: false + requestHeadersToAdd: + - header: + key: x-kuma-tags + value: '&kuma.io/service=gateway&' + virtualHosts: + - domains: + - '*' + name: api-http + routes: + - match: + prefix: / + route: + cluster: api-http statPrefix: api-http name: outbound:127.0.0.1:40001 trafficDirection: OUTBOUND @@ -355,10 +313,22 @@ resources: '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager httpFilters: - name: envoy.router - rds: - configSource: - ads: { } - routeConfigName: outbound:api-http2 + routeConfig: + name: outbound:api-http2 + validateClusters: false + requestHeadersToAdd: + - header: + key: x-kuma-tags + value: '&kuma.io/service=gateway&' + virtualHosts: + - domains: + - '*' + name: api-http2 + routes: + - match: + prefix: / + route: + cluster: api-http2 statPrefix: api-http2 name: outbound:127.0.0.1:40003 trafficDirection: OUTBOUND @@ -381,10 +351,22 @@ resources: emitFilterState: true statsForAllMethods: true - name: envoy.router - rds: - configSource: - ads: { } - routeConfigName: outbound:api-grpc + routeConfig: + name: outbound:api-grpc + validateClusters: false + requestHeadersToAdd: + - header: + key: x-kuma-tags + value: '&kuma.io/service=gateway&' + virtualHosts: + - domains: + - '*' + name: api-grpc + routes: + - match: + prefix: / + route: + cluster: api-grpc statPrefix: api-grpc name: outbound:127.0.0.1:40004 trafficDirection: OUTBOUND diff --git a/pkg/xds/generator/testdata/outbound-proxy/04.envoy.golden.yaml b/pkg/xds/generator/testdata/outbound-proxy/04.envoy.golden.yaml index 2376821d91db..843ce6cfc444 100644 --- a/pkg/xds/generator/testdata/outbound-proxy/04.envoy.golden.yaml +++ b/pkg/xds/generator/testdata/outbound-proxy/04.envoy.golden.yaml @@ -346,24 +346,6 @@ resources: targetUri: kuma-system:5677 sni: db{mesh=mesh1,role=replica} type: EDS - - name: outbound:api-http - resource: - '@type': type.googleapis.com/envoy.api.v2.RouteConfiguration - name: outbound:api-http - requestHeadersToAdd: - - header: - key: x-kuma-tags - value: '&kuma.io/service=web&' - validateClusters: true - virtualHosts: - - domains: - - '*' - name: api-http - routes: - - match: - prefix: / - route: - cluster: api-http - name: outbound:127.0.0.1:18080 resource: '@type': type.googleapis.com/envoy.api.v2.Listener @@ -406,10 +388,22 @@ resources: path: /var/log httpFilters: - name: envoy.router - rds: - configSource: - ads: {} - routeConfigName: outbound:api-http + routeConfig: + name: outbound:api-http + validateClusters: false + requestHeadersToAdd: + - header: + key: x-kuma-tags + value: '&kuma.io/service=web&' + virtualHosts: + - domains: + - '*' + name: api-http + routes: + - match: + prefix: / + route: + cluster: api-http statPrefix: api-http name: outbound:127.0.0.1:40001 trafficDirection: OUTBOUND diff --git a/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml index 6865dc9e1151..6285655366ab 100644 --- a/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/3-envoy-config.golden.yaml @@ -180,9 +180,9 @@ resources: - name: envoy.router routeConfig: name: inbound:backend + validateClusters: false requestHeadersToRemove: - x-kuma-tags - validateClusters: true virtualHosts: - domains: - '*' @@ -250,6 +250,7 @@ resources: httpFilters: - name: envoy.router routeConfig: + validateClusters: false virtualHosts: - domains: - '*' @@ -273,6 +274,7 @@ resources: httpFilters: - name: envoy.router routeConfig: + validateClusters: false virtualHosts: - domains: - '*' diff --git a/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml b/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml index a9d9a9a33905..25e33affe33c 100644 --- a/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/profile-source/4-envoy-config.golden.yaml @@ -202,9 +202,9 @@ resources: - name: envoy.router routeConfig: name: inbound:backend + validateClusters: false requestHeadersToRemove: - x-kuma-tags - validateClusters: true virtualHosts: - domains: - '*' @@ -289,6 +289,7 @@ resources: httpFilters: - name: envoy.router routeConfig: + validateClusters: false virtualHosts: - domains: - '*' @@ -312,6 +313,7 @@ resources: httpFilters: - name: envoy.router routeConfig: + validateClusters: false virtualHosts: - domains: - '*' diff --git a/pkg/xds/generator/testdata/prometheus-endpoint/custom.envoy-config.golden.yaml b/pkg/xds/generator/testdata/prometheus-endpoint/custom.envoy-config.golden.yaml index 2ee94759a254..49628d9a2723 100644 --- a/pkg/xds/generator/testdata/prometheus-endpoint/custom.envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/prometheus-endpoint/custom.envoy-config.golden.yaml @@ -31,6 +31,7 @@ resources: httpFilters: - name: envoy.router routeConfig: + validateClusters: false virtualHosts: - domains: - '*' diff --git a/pkg/xds/generator/testdata/prometheus-endpoint/default-mtls.envoy-config.golden.yaml b/pkg/xds/generator/testdata/prometheus-endpoint/default-mtls.envoy-config.golden.yaml index ed1883fbf70e..a3d74cbaa3ff 100644 --- a/pkg/xds/generator/testdata/prometheus-endpoint/default-mtls.envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/prometheus-endpoint/default-mtls.envoy-config.golden.yaml @@ -34,6 +34,7 @@ resources: httpFilters: - name: envoy.router routeConfig: + validateClusters: false virtualHosts: - domains: - '*' @@ -57,6 +58,7 @@ resources: httpFilters: - name: envoy.router routeConfig: + validateClusters: false virtualHosts: - domains: - '*' diff --git a/pkg/xds/generator/testdata/prometheus-endpoint/default.envoy-config.golden.yaml b/pkg/xds/generator/testdata/prometheus-endpoint/default.envoy-config.golden.yaml index 1341048684f0..e0bc0febb800 100644 --- a/pkg/xds/generator/testdata/prometheus-endpoint/default.envoy-config.golden.yaml +++ b/pkg/xds/generator/testdata/prometheus-endpoint/default.envoy-config.golden.yaml @@ -31,6 +31,7 @@ resources: httpFilters: - name: envoy.router routeConfig: + validateClusters: false virtualHosts: - domains: - '*'