chore(kuma-cp) generate static outbound routes (#1098)

Signed-off-by: Jakub Dyszkiewicz <[email protected]>

pkg/xds/envoy/listeners/http_access_log_configurer_test.go

@@ -62,7 +62,6 @@ var _ = Describe("HttpAccessLogConfigurer", func() {
 				Configure(OutboundListener(given.listenerName, given.listenerAddress, given.listenerPort)).
 				Configure(FilterChain(NewFilterChainBuilder().
 					Configure(HttpConnectionManager(given.statsName)).
-					Configure(HttpOutboundRoute(given.routeName)).
 					Configure(HttpAccessLog(mesh, TrafficDirectionOutbound, sourceService, destinationService, given.backend, proxy)))).
 				Build()
 			// then
@@ -94,10 +93,6 @@ var _ = Describe("HttpAccessLogConfigurer", func() {
                   '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
                   httpFilters:
                   - name: envoy.router
-                  rds:
-                    configSource:
-                      ads: {}
-                    routeConfigName: outbound:backend
                   statPrefix: backend
             trafficDirection: OUTBOUND
 `,
@@ -136,10 +131,6 @@ var _ = Describe("HttpAccessLogConfigurer", func() {
                       path: /tmp/log
                   httpFilters:
                   - name: envoy.router
-                  rds:
-                    configSource:
-                      ads: {}
-                    routeConfigName: outbound:backend
                   statPrefix: backend
             trafficDirection: OUTBOUND
 `,
@@ -194,10 +185,6 @@ var _ = Describe("HttpAccessLogConfigurer", func() {
 
                   httpFilters:
                   - name: envoy.router
-                  rds:
-                    configSource:
-                      ads: {}
-                    routeConfigName: outbound:backend
                   statPrefix: backend
             trafficDirection: OUTBOUND
 `,

pkg/xds/envoy/listeners/http_inbound_route_configurer_test.go

@@ -64,9 +64,9 @@ var _ = Describe("HttpInboundRouteConfigurer", func() {
                   - name: envoy.router
                   routeConfig:
                     name: inbound:backend
+                    validateClusters: false
                     requestHeadersToRemove:
                     - x-kuma-tags
-                    validateClusters: true
                     virtualHosts:
                     - domains:
                       - '*'

pkg/xds/envoy/listeners/http_outbound_route_configurer.go

@@ -1,34 +1,46 @@
 package listeners
 
 import (
-	envoy_core "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener"
 	envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
+
+	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
+	envoy_common "github.com/kumahq/kuma/pkg/xds/envoy"
+	envoy_names "github.com/kumahq/kuma/pkg/xds/envoy/names"
+	envoy_routes "github.com/kumahq/kuma/pkg/xds/envoy/routes"
 )
 
-func HttpOutboundRoute(routeName string) FilterChainBuilderOpt {
+func HttpOutboundRoute(service string, subsets []envoy_common.ClusterSubset, dpTags mesh_proto.MultiValueTagSet) FilterChainBuilderOpt {
 	return FilterChainBuilderOptFunc(func(config *FilterChainBuilderConfig) {
 		config.Add(&HttpOutboundRouteConfigurer{
-			routeName: routeName,
+			service: service,
+			subsets: subsets,
+			dpTags:  dpTags,
 		})
 	})
 }
 
 type HttpOutboundRouteConfigurer struct {
-	routeName string
+	service string
+	subsets []envoy_common.ClusterSubset
+	dpTags  mesh_proto.MultiValueTagSet
 }
 
 func (c *HttpOutboundRouteConfigurer) Configure(filterChain *envoy_listener.FilterChain) error {
+	routeConfig, err := envoy_routes.NewRouteConfigurationBuilder().
+		Configure(envoy_routes.CommonRouteConfiguration(envoy_names.GetOutboundRouteName(c.service))).
+		Configure(envoy_routes.TagsHeader(c.dpTags)).
+		Configure(envoy_routes.VirtualHost(envoy_routes.NewVirtualHostBuilder().
+			Configure(envoy_routes.CommonVirtualHost(c.service)).
+			Configure(envoy_routes.DefaultRoute(c.subsets...)))).
+		Build()
+	if err != nil {
+		return err
+	}
+
 	return UpdateHTTPConnectionManager(filterChain, func(hcm *envoy_hcm.HttpConnectionManager) error {
-		hcm.RouteSpecifier = &envoy_hcm.HttpConnectionManager_Rds{
-			Rds: &envoy_hcm.Rds{
-				ConfigSource: &envoy_core.ConfigSource{
-					ConfigSourceSpecifier: &envoy_core.ConfigSource_Ads{
-						Ads: &envoy_core.AggregatedConfigSource{},
-					},
-				},
-				RouteConfigName: c.routeName,
-			},
+		hcm.RouteSpecifier = &envoy_hcm.HttpConnectionManager_RouteConfig{
+			RouteConfig: routeConfig,
 		}
 		return nil
 	})

pkg/xds/envoy/listeners/http_outbound_route_configurer_test.go

@@ -5,6 +5,8 @@ import (
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
 
+	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
+	envoy_common "github.com/kumahq/kuma/pkg/xds/envoy"
 	. "github.com/kumahq/kuma/pkg/xds/envoy/listeners"
 
 	util_proto "github.com/kumahq/kuma/pkg/util/proto"
@@ -17,7 +19,9 @@ var _ = Describe("HttpOutboundRouteConfigurer", func() {
 		listenerAddress string
 		listenerPort    uint32
 		statsName       string
-		routeName       string
+		service         string
+		subsets         []envoy_common.ClusterSubset
+		dpTags          mesh_proto.MultiValueTagSet
 		expected        string
 	}
 
@@ -28,7 +32,7 @@ var _ = Describe("HttpOutboundRouteConfigurer", func() {
 				Configure(OutboundListener(given.listenerName, given.listenerAddress, given.listenerPort)).
 				Configure(FilterChain(NewFilterChainBuilder().
 					Configure(HttpConnectionManager(given.statsName)).
-					Configure(HttpOutboundRoute(given.routeName)))).
+					Configure(HttpOutboundRoute(given.service, given.subsets, given.dpTags)))).
 				Build()
 			// then
 			Expect(err).ToNot(HaveOccurred())
@@ -44,10 +48,29 @@ var _ = Describe("HttpOutboundRouteConfigurer", func() {
 			listenerAddress: "127.0.0.1",
 			listenerPort:    18080,
 			statsName:       "127.0.0.1:18080",
-			routeName:       "outbound:backend",
+			service:         "backend",
+			subsets: []envoy_common.ClusterSubset{
+				{
+					ClusterName: "backend",
+					Weight:      20,
+					Tags: map[string]string{
+						"version": "v1",
+					},
+				},
+				{
+					ClusterName: "backend",
+					Weight:      80,
+					Tags: map[string]string{
+						"version": "v2",
+					},
+				},
+			},
+			dpTags: map[string]map[string]bool{
+				"kuma.io/service": {
+					"web": true,
+				},
+			},
 			expected: `
-            name: outbound:127.0.0.1:18080
-            trafficDirection: OUTBOUND
             address:
               socketAddress:
                 address: 127.0.0.1
@@ -59,12 +82,39 @@ var _ = Describe("HttpOutboundRouteConfigurer", func() {
                   '@type': type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
                   httpFilters:
                   - name: envoy.router
-                  rds:
-                    configSource:
-                      ads: {}
-                    routeConfigName: outbound:backend
+                  routeConfig:
+                    name: outbound:backend
+                    validateClusters: false
+                    requestHeadersToAdd:
+                    - header:
+                        key: x-kuma-tags
+                        value: '&kuma.io/service=web&'
+                    virtualHosts:
+                    - domains:
+                      - '*'
+                      name: backend
+                      routes:
+                      - match:
+                          prefix: /
+                        route:
+                          weightedClusters:
+                            clusters:
+                            - metadataMatch:
+                                filterMetadata:
+                                  envoy.lb:
+                                    version: v1
+                              name: backend
+                              weight: 20
+                            - metadataMatch:
+                                filterMetadata:
+                                  envoy.lb:
+                                    version: v2
+                              name: backend
+                              weight: 80
+                            totalWeight: 100
                   statPrefix: "127_0_0_1_18080"
-`,
+            name: outbound:127.0.0.1:18080
+            trafficDirection: OUTBOUND`,
 		}),
 	)
 })

pkg/xds/envoy/listeners/prometheus_endpoint_configurer.go

@@ -6,6 +6,7 @@ import (
 	envoy_route "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
 	envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
 	envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown"
+	"github.com/golang/protobuf/ptypes/wrappers"
 
 	"github.com/kumahq/kuma/pkg/util/proto"
 	util_xds "github.com/kumahq/kuma/pkg/util/xds"
@@ -55,6 +56,9 @@ func (c *PrometheusEndpointConfigurer) Configure(filterChain *envoy_listener.Fil
 						},
 					}},
 				}},
+				ValidateClusters: &wrappers.BoolValue{
+					Value: false,
+				},
 			},
 		},
 	}

pkg/xds/envoy/listeners/prometheus_endpoint_configurer_test.go

@@ -59,6 +59,7 @@ var _ = Describe("PrometheusEndpointConfigurer", func() {
                   httpFilters:
                   - name: envoy.router
                   routeConfig:
+                    validateClusters: false
                     virtualHosts:
                     - domains:
                       - '*'

pkg/xds/envoy/routes/common_route_configuration_configurer.go

@@ -21,7 +21,7 @@ type CommonRouteConfigurationConfigurer struct {
 func (c CommonRouteConfigurationConfigurer) Configure(routeConfiguration *v2.RouteConfiguration) error {
 	routeConfiguration.Name = c.name
 	routeConfiguration.ValidateClusters = &wrappers.BoolValue{
-		Value: true,
+		Value: false,
 	}
 	return nil
 }

pkg/xds/envoy/routes/common_route_configuration_configurer_test.go

@@ -37,7 +37,7 @@ var _ = Describe("CommonRouteConfigurationConfigurer", func() {
 			routeName: "outbound:backend",
 			expected: `
             name: outbound:backend
-            validateClusters: true
+            validateClusters: false
 `,
 		}),
 	)

pkg/xds/generator/modifications/virtual_host.go

@@ -2,16 +2,22 @@ package modifications
 
 import (
 	envoy_api "github.com/envoyproxy/go-control-plane/envoy/api/v2"
+	envoy_listener "github.com/envoyproxy/go-control-plane/envoy/api/v2/listener"
 	envoy_route "github.com/envoyproxy/go-control-plane/envoy/api/v2/route"
+	envoy_hcm "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2"
 	envoy_resource "github.com/envoyproxy/go-control-plane/pkg/resource/v2"
+	envoy_wellknown "github.com/envoyproxy/go-control-plane/pkg/wellknown"
 	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/ptypes"
 	"github.com/pkg/errors"
 
 	mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1"
 	core_xds "github.com/kumahq/kuma/pkg/core/xds"
 	util_proto "github.com/kumahq/kuma/pkg/util/proto"
 )
 
+// virtualHostModificator assumes that the routes are specified as `routeConfig` in Listeners, not through RDS
+// If we ever change it to RDS we need to modify RouteConfiguration objects
 type virtualHostModificator mesh_proto.ProxyTemplate_Modifications_VirtualHost
 
 func (c *virtualHostModificator) apply(resources *core_xds.ResourceSet) error {
@@ -20,24 +26,55 @@ func (c *virtualHostModificator) apply(resources *core_xds.ResourceSet) error {
 		return err
 	}
 
-	for _, resource := range resources.Resources(envoy_resource.RouteType) {
-		if c.routeConfigurationMatches(resource) {
-			routeCfg := resource.Resource.(*envoy_api.RouteConfiguration)
-			switch c.Operation {
-			case mesh_proto.OpAdd:
-				c.add(routeCfg, virtualHost)
-			case mesh_proto.OpRemove:
-				c.remove(routeCfg)
-			case mesh_proto.OpPatch:
-				c.patch(routeCfg, virtualHost)
-			default:
-				return errors.Errorf("invalid operation: %s", c.Operation)
+	for _, resource := range resources.Resources(envoy_resource.ListenerType) {
+		listener := resource.Resource.(*envoy_api.Listener)
+		if !c.originMatches(resource) {
+			continue
+		}
+		for _, chain := range listener.FilterChains { // apply on all filter chains. We could introduce filter chain matcher as an improvement.
+			for _, networkFilter := range chain.Filters {
+				if networkFilter.Name == envoy_wellknown.HTTPConnectionManager {
+					hcm := &envoy_hcm.HttpConnectionManager{}
+					err := ptypes.UnmarshalAny(networkFilter.ConfigType.(*envoy_listener.Filter_TypedConfig).TypedConfig, hcm)
+					if err != nil {
+						return err
+					}
+					if err := c.applyHCMModification(hcm, virtualHost); err != nil {
+						return err
+					}
+					any, err := util_proto.MarshalAnyDeterministic(hcm)
+					if err != nil {
+						return err
+					}
+					networkFilter.ConfigType.(*envoy_listener.Filter_TypedConfig).TypedConfig = any
+				}
 			}
 		}
 	}
 	return nil
 }
 
+func (c *virtualHostModificator) applyHCMModification(hcm *envoy_hcm.HttpConnectionManager, virtualHost *envoy_route.VirtualHost) error {
+	routeCfg := hcm.GetRouteConfig()
+	if routeCfg == nil {
+		return nil // ignore HCMs without embedded routes
+	}
+	if !c.routeConfigurationMatches(routeCfg) {
+		return nil
+	}
+	switch c.Operation {
+	case mesh_proto.OpAdd:
+		c.add(routeCfg, virtualHost)
+	case mesh_proto.OpRemove:
+		c.remove(routeCfg)
+	case mesh_proto.OpPatch:
+		c.patch(routeCfg, virtualHost)
+	default:
+		return errors.Errorf("invalid operation: %s", c.Operation)
+	}
+	return nil
+}
+
 func (c *virtualHostModificator) patch(routeCfg *envoy_api.RouteConfiguration, vHostPatch *envoy_route.VirtualHost) {
 	for _, vHost := range routeCfg.VirtualHosts {
 		if c.virtualHostMatches(vHost) {
@@ -67,10 +104,11 @@ func (c *virtualHostModificator) virtualHostMatches(vHost *envoy_route.VirtualHo
 	return true
 }
 
-func (c *virtualHostModificator) routeConfigurationMatches(routeCfg *core_xds.Resource) bool {
-	if c.Match.GetOrigin() != "" && c.Match.GetOrigin() != routeCfg.Origin {
-		return false
-	}
+func (c *virtualHostModificator) originMatches(routeCfg *core_xds.Resource) bool {
+	return c.Match.GetOrigin() == "" || (c.Match.GetOrigin() == routeCfg.Origin)
+}
+
+func (c *virtualHostModificator) routeConfigurationMatches(routeCfg *envoy_api.RouteConfiguration) bool {
 	if c.Match.GetRouteConfigurationName() != "" && c.Match.GetRouteConfigurationName() != routeCfg.Name {
 		return false
 	}