feat(kuma-cp) Direct access with forward cluster

api/mesh/v1alpha1/dataplane.proto

@@ -90,6 +90,9 @@ message Dataplane {
 
       // Port on which all traffic is being transparently redirected.
       uint32 redirect_port = 1 [ (validate.rules).uint32 = {lte : 65535} ];
+
+      // List of services that will be access directly via IP:PORT
+      repeated string direct_access_services = 2;
     }
 
     // Gateway describes configuration of gateway of the dataplane.

go.mod

@@ -31,7 +31,6 @@ require (
 	github.com/prometheus/common v0.9.1
 	github.com/prometheus/prometheus v0.0.0-00010101000000-000000000000
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
-	github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spiffe/go-spiffe v0.0.0-20190820222348-6adcf1eecbcc
 	github.com/spiffe/spire v0.10.0

go.sum

@@ -472,8 +472,6 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz
 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk=
 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0=
-github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=

pkg/plugins/discovery/k8s/controllers/outbound_converter.go

@@ -2,7 +2,6 @@ package controllers
 
 import (
 	"context"
-	"sort"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -11,12 +10,10 @@ import (
 
 	mesh_proto "github.com/Kong/kuma/api/mesh/v1alpha1"
 	mesh_k8s "github.com/Kong/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1"
-	injector_metadata "github.com/Kong/kuma/pkg/plugins/runtime/k8s/webhooks/injector/metadata"
 )
 
 func (p *PodConverter) OutboundInterfacesFor(pod *kube_core.Pod, others []*mesh_k8s.Dataplane) ([]*mesh_proto.Dataplane_Networking_Outbound, error) {
 	var outbounds []*mesh_proto.Dataplane_Networking_Outbound
-	directAccessServices := directAccessServices(pod)
 	endpoints := endpointsByService(others)
 	for _, serviceTag := range endpoints.Services() {
 		service, port, err := p.k8sService(serviceTag)
@@ -25,7 +22,14 @@ func (p *PodConverter) OutboundInterfacesFor(pod *kube_core.Pod, others []*mesh_
 			continue // one invalid Dataplane definition should not break the entire mesh
 		}
 		if isHeadlessService(service) {
-			directAccessServices[serviceTag] = true
+			// Generate outbound listeners for every endpoint of services.
+			for _, endpoint := range endpoints[serviceTag] {
+				outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
+					Address: endpoint.Address,
+					Port:    endpoint.Port,
+					Service: serviceTag,
+				})
+			}
 		} else {
 			// generate outbound based on ClusterIP. Transparent Proxy will work only if DNS name that resolves to ClusterIP is used
 			outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
@@ -35,44 +39,7 @@ func (p *PodConverter) OutboundInterfacesFor(pod *kube_core.Pod, others []*mesh_
 			})
 		}
 	}
-
-	directAccessOutbounds := directAccessOutbounds(directAccessServices, endpoints)
-	return append(outbounds, directAccessOutbounds...), nil
-}
-
-func directAccessServices(pod *kube_core.Pod) map[string]bool {
-	result := map[string]bool{}
-	servicesRaw := pod.GetAnnotations()[injector_metadata.KumaDirectAccess]
-	services := strings.Split(servicesRaw, ",")
-	for _, service := range services {
-		result[service] = true
-	}
-	return result
-}
-
-// Generate outbound listeners for every endpoint of services.
-// This will enable consuming applications via transparent proxy by PodIP instead of ClusterIP of its service
-// Generating listener for every endpoint will cause XDS snapshot to be huge therefore it should be used only if really needed
-func directAccessOutbounds(services map[string]bool, endpointsByService EndpointsByService) []*mesh_proto.Dataplane_Networking_Outbound {
-	var sortedServices []string // service should be sorted so we generate consistent every time
-	if services[injector_metadata.KumaDirectAccessAll] {
-		sortedServices = endpointsByService.Services()
-	} else {
-		sortedServices = stringSetToSortedList(services)
-	}
-
-	var outbounds []*mesh_proto.Dataplane_Networking_Outbound
-	for _, service := range sortedServices {
-		// services that are not found will be ignored
-		for _, endpoint := range endpointsByService[service] {
-			outbounds = append(outbounds, &mesh_proto.Dataplane_Networking_Outbound{
-				Address: endpoint.Address,
-				Port:    endpoint.Port,
-				Service: service,
-			})
-		}
-	}
-	return outbounds
+	return outbounds, nil
 }
 
 func isHeadlessService(svc *kube_core.Service) bool {
@@ -106,12 +73,3 @@ func ParseServiceFQDN(host string) (name string, namespace string, err error) {
 	name, namespace = segments[0], segments[1]
 	return
 }
-
-func stringSetToSortedList(set map[string]bool) []string {
-	list := make([]string, 0, len(set))
-	for key := range set {
-		list = append(list, key)
-	}
-	sort.Strings(list)
-	return list
-}

pkg/plugins/discovery/k8s/controllers/pod_converter.go

@@ -1,6 +1,8 @@
 package controllers
 
 import (
+	"strings"
+
 	mesh_proto "github.com/Kong/kuma/api/mesh/v1alpha1"
 	"github.com/Kong/kuma/pkg/core"
 	mesh_k8s "github.com/Kong/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1"
@@ -42,8 +44,10 @@ func (p *PodConverter) DataplaneFor(pod *kube_core.Pod, services []*kube_core.Se
 		Networking: &mesh_proto.Dataplane_Networking{},
 	}
 	if injector_metadata.HasTransparentProxyingEnabled(pod) {
+		services := pod.GetAnnotations()[injector_metadata.KumaDirectAccess]
 		dataplane.Networking.TransparentProxying = &mesh_proto.Dataplane_Networking_TransparentProxying{
-			RedirectPort: injector_metadata.GetTransparentProxyingPort(pod),
+			RedirectPort:         injector_metadata.GetTransparentProxyingPort(pod),
+			DirectAccessServices: strings.Split(services, ","),
 		}
 	}
 

pkg/plugins/discovery/k8s/controllers/pod_converter_test.go

@@ -196,13 +196,6 @@ var _ = Describe("PodToDataplane(..)", func() {
 			otherServices:   "06.other-services.yaml",
 			dataplane:       "06.dataplane.yaml",
 		}),
-		Entry("07. Pod with communication to headless services and direct access to this service should generate direct listener once", testCase{
-			pod:             "07.pod.yaml",
-			servicesForPod:  "07.services-for-pod.yaml",
-			otherDataplanes: "07.other-dataplanes.yaml",
-			otherServices:   "07.other-services.yaml",
-			dataplane:       "07.dataplane.yaml",
-		}),
 	)
 
 	Context("when Dataplane cannot be generated", func() {

pkg/plugins/discovery/k8s/controllers/testdata/04.dataplane.yaml

@@ -21,12 +21,6 @@ spec:
       - address: 10.108.144.24
         port: 80
         service: test-app.playground.svc:80
-      - address: 10.244.0.26
-        port: 80
-        service: second-test-app.playground.svc:80
-      - address: 10.244.0.25
-        port: 443
-        service: test-app.playground.svc:443
-      - address: 10.244.0.25
-        port: 80
-        service: test-app.playground.svc:80
+    transparentProxying:
+      directAccessServices:
+        - '*'

pkg/plugins/discovery/k8s/controllers/testdata/04.pod.yaml

@@ -6,6 +6,7 @@ metadata:
     version: "0.1"
   annotations:
     kuma.io/direct-access-services: "*"
+    kuma.io/transparent-proxying: "enabled"
 spec:
   containers:
     - ports:

pkg/plugins/discovery/k8s/controllers/testdata/05.dataplane.yaml

@@ -21,9 +21,7 @@ spec:
       - address: 10.108.144.24
         port: 80
         service: test-app.playground.svc:80
-      - address: 10.244.0.25
-        port: 443
-        service: test-app.playground.svc:443
-      - address: 10.244.0.25
-        port: 80
-        service: test-app.playground.svc:80
+    transparentProxying:
+      directAccessServices:
+        - test-app.playground.svc:80
+        - test-app.playground.svc:443

pkg/plugins/discovery/k8s/controllers/testdata/05.pod.yaml

@@ -6,6 +6,7 @@ metadata:
     version: "0.1"
   annotations:
     kuma.io/direct-access-services: "test-app.playground.svc:80,test-app.playground.svc:443"
+    kuma.io/transparent-proxying: "enabled"
 spec:
   containers:
     - ports: