Skip to content

Commit

Permalink
feat: add MeshGateway support to MeshAccessLog (#5101)
Browse files Browse the repository at this point in the history
* feat: add MeshGateway support to MeshAccessLog
* test: add MeshGateway tests for MeshAccessLog
* test(e2e): MeshAccessLog with MeshGateway

Signed-off-by: Mike Beaumont <[email protected]>
  • Loading branch information
michaelbeaumont authored Oct 7, 2022
1 parent ad7cc51 commit 6999678
Show file tree
Hide file tree
Showing 5 changed files with 298 additions and 30 deletions.
52 changes: 52 additions & 0 deletions pkg/plugins/policies/meshaccesslog/plugin/v1alpha1/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@ import (
"github.com/kumahq/kuma/pkg/plugins/policies/matchers"
api "github.com/kumahq/kuma/pkg/plugins/policies/meshaccesslog/api/v1alpha1"
plugin_xds "github.com/kumahq/kuma/pkg/plugins/policies/meshaccesslog/plugin/xds"
"github.com/kumahq/kuma/pkg/plugins/runtime/gateway"
xds_context "github.com/kumahq/kuma/pkg/xds/context"
"github.com/kumahq/kuma/pkg/xds/envoy"
"github.com/kumahq/kuma/pkg/xds/generator"
xds_topology "github.com/kumahq/kuma/pkg/xds/topology"
)

var _ core_plugins.PolicyPlugin = &plugin{}
Expand Down Expand Up @@ -50,6 +52,9 @@ func (p plugin) Apply(rs *core_xds.ResourceSet, ctx xds_context.Context, proxy *
if err := applyToDirectAccess(policies.ToRules, listeners.directAccess, proxy.Dataplane); err != nil {
return err
}
if err := applyToGateway(policies.ToRules, listeners.gateway, ctx.Mesh.Resources.MeshLocalResources, proxy.Dataplane); err != nil {
return err
}

return nil
}
Expand Down Expand Up @@ -145,9 +150,50 @@ func applyToDirectAccess(
return nil
}

func applyToGateway(
rules xds.ToRules, gatewayListeners map[xds.InboundListener]*envoy_listener.Listener, resources xds_context.ResourceMap, dataplane *core_mesh.DataplaneResource,
) error {
var gateways *core_mesh.MeshGatewayResourceList
if rawList := resources[core_mesh.MeshGatewayType]; rawList != nil {
gateways = rawList.(*core_mesh.MeshGatewayResourceList)
} else {
return nil
}

gateway := xds_topology.SelectGateway(gateways.Items, dataplane.Spec.Matches)
if gateway == nil {
return nil
}

for _, listener := range gateway.Spec.GetConf().GetListeners() {
address := dataplane.Spec.GetNetworking().Address
port := listener.GetPort()
listener, ok := gatewayListeners[xds.InboundListener{
Address: address,
Port: port,
}]
if !ok {
continue
}

if err := configureOutbound(
rules,
dataplane,
xds.Subset{},
mesh_proto.MatchAllTag,
listener,
); err != nil {
return err
}
}

return nil
}

type listeners struct {
inbound map[xds.InboundListener]*envoy_listener.Listener
outbound map[mesh_proto.OutboundInterface]*envoy_listener.Listener
gateway map[xds.InboundListener]*envoy_listener.Listener
ipv4Passthrough *envoy_listener.Listener
ipv6Passthrough *envoy_listener.Listener
directAccess map[generator.Endpoint]*envoy_listener.Listener
Expand All @@ -157,6 +203,7 @@ func gatherListeners(rs *core_xds.ResourceSet) listeners {
listeners := listeners{
inbound: map[xds.InboundListener]*envoy_listener.Listener{},
outbound: map[mesh_proto.OutboundInterface]*envoy_listener.Listener{},
gateway: map[xds.InboundListener]*envoy_listener.Listener{},
directAccess: map[generator.Endpoint]*envoy_listener.Listener{},
}

Expand Down Expand Up @@ -187,6 +234,11 @@ func gatherListeners(rs *core_xds.ResourceSet) listeners {
Address: address.GetAddress(),
Port: address.GetPortValue(),
}] = listener
case gateway.OriginGateway:
listeners.gateway[xds.InboundListener{
Address: address.GetAddress(),
Port: address.GetPortValue(),
}] = listener
default:
continue
}
Expand Down
193 changes: 178 additions & 15 deletions pkg/plugins/policies/meshaccesslog/plugin/v1alpha1/plugin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,30 +13,40 @@ import (
core_xds "github.com/kumahq/kuma/pkg/core/xds"
api "github.com/kumahq/kuma/pkg/plugins/policies/meshaccesslog/api/v1alpha1"
plugin "github.com/kumahq/kuma/pkg/plugins/policies/meshaccesslog/plugin/v1alpha1"
"github.com/kumahq/kuma/pkg/plugins/runtime/gateway"
test_model "github.com/kumahq/kuma/pkg/test/resources/model"
util_proto "github.com/kumahq/kuma/pkg/util/proto"
xds_context "github.com/kumahq/kuma/pkg/xds/context"
envoy_common "github.com/kumahq/kuma/pkg/xds/envoy"
. "github.com/kumahq/kuma/pkg/xds/envoy/listeners"
envoy_names "github.com/kumahq/kuma/pkg/xds/envoy/names"
"github.com/kumahq/kuma/pkg/xds/generator"
)

var _ = Describe("MeshAccessLog", func() {
type testCase struct {
type sidecarTestCase struct {
resources []core_xds.Resource
toRules core_xds.ToRules
fromRules core_xds.FromRules
expectedListeners []string
}
DescribeTable("should generate proper Envoy config",
func(given testCase) {
resources := core_xds.NewResourceSet()
func(given sidecarTestCase) {
resourceSet := core_xds.NewResourceSet()
for _, res := range given.resources {
r := res
resources.Add(&r)
resourceSet.Add(&r)
}

context := xds_context.Context{}
context := xds_context.Context{
Mesh: xds_context.MeshContext{
Resource: &core_mesh.MeshResource{
Meta: &test_model.ResourceMeta{
Name: "default",
},
},
},
}
proxy := xds.Proxy{
Dataplane: &core_mesh.DataplaneResource{
Meta: &test_model.ResourceMeta{
Expand Down Expand Up @@ -78,16 +88,16 @@ var _ = Describe("MeshAccessLog", func() {
}
plugin := plugin.NewPlugin().(core_plugins.PolicyPlugin)

Expect(plugin.Apply(resources, context, &proxy)).To(Succeed())
Expect(plugin.Apply(resourceSet, context, &proxy)).To(Succeed())

for i, r := range resources.ListOf(envoy_resource.ListenerType) {
for i, r := range resourceSet.ListOf(envoy_resource.ListenerType) {
actual, err := util_proto.ToYAML(r.Resource)
Expect(err).ToNot(HaveOccurred())

Expect(actual).To(MatchYAML(given.expectedListeners[i]))
}
},
Entry("basic outbound route", testCase{
Entry("basic outbound route", sidecarTestCase{
resources: []core_xds.Resource{{
Name: "outbound",
Origin: generator.OriginOutbound,
Expand Down Expand Up @@ -116,7 +126,7 @@ var _ = Describe("MeshAccessLog", func() {
toRules: core_xds.ToRules{
Rules: []*core_xds.Rule{
{
Subset: []core_xds.Tag{},
Subset: core_xds.Subset{},
Conf: &api.MeshAccessLog_Conf{
Backends: []*api.MeshAccessLog_Backend{{
File: &api.MeshAccessLog_FileBackend{
Expand Down Expand Up @@ -173,7 +183,7 @@ var _ = Describe("MeshAccessLog", func() {
name: outbound:127.0.0.1:27777
trafficDirection: OUTBOUND`,
}}),
Entry("basic outbound tcp", testCase{
Entry("basic outbound tcp", sidecarTestCase{
resources: []core_xds.Resource{{
Name: "outbound",
Origin: generator.OriginOutbound,
Expand All @@ -192,7 +202,7 @@ var _ = Describe("MeshAccessLog", func() {
toRules: core_xds.ToRules{
Rules: []*core_xds.Rule{
{
Subset: []core_xds.Tag{},
Subset: core_xds.Subset{},
Conf: &api.MeshAccessLog_Conf{
Backends: []*api.MeshAccessLog_Backend{{
File: &api.MeshAccessLog_FileBackend{
Expand Down Expand Up @@ -229,7 +239,7 @@ var _ = Describe("MeshAccessLog", func() {
name: outbound:127.0.0.1:27777
trafficDirection: OUTBOUND`,
}}),
Entry("basic outbound route without match", testCase{
Entry("basic outbound route without match", sidecarTestCase{
resources: []core_xds.Resource{{
Name: "outbound",
Origin: generator.OriginOutbound,
Expand Down Expand Up @@ -260,7 +270,7 @@ var _ = Describe("MeshAccessLog", func() {
toRules: core_xds.ToRules{
Rules: []*core_xds.Rule{
{
Subset: []core_xds.Tag{{
Subset: core_xds.Subset{{
Key: mesh_proto.ServiceTag,
Value: "other",
}},
Expand Down Expand Up @@ -311,7 +321,7 @@ var _ = Describe("MeshAccessLog", func() {
name: outbound:127.0.0.1:27777
trafficDirection: OUTBOUND`,
}}),
Entry("basic inbound route", testCase{
Entry("basic inbound route", sidecarTestCase{
resources: []core_xds.Resource{{
Name: "inbound",
Origin: generator.OriginInbound,
Expand All @@ -337,7 +347,7 @@ var _ = Describe("MeshAccessLog", func() {
fromRules: core_xds.FromRules{
Rules: map[xds.InboundListener]xds.Rules{
{Address: "127.0.0.1", Port: 17777}: {{
Subset: []core_xds.Tag{},
Subset: core_xds.Subset{},
Conf: &api.MeshAccessLog_Conf{
Backends: []*api.MeshAccessLog_Backend{{
File: &api.MeshAccessLog_FileBackend{
Expand Down Expand Up @@ -394,4 +404,157 @@ var _ = Describe("MeshAccessLog", func() {
trafficDirection: INBOUND`,
}}),
)
type gatewayTestCase struct {
resources []core_xds.Resource
toRules core_xds.ToRules
expectedListeners []string
}
DescribeTable("should generate proper Envoy config for MeshGateway Dataplanes",
func(given gatewayTestCase) {
resourceSet := core_xds.NewResourceSet()
for _, res := range given.resources {
r := res
resourceSet.Add(&r)
}

gateways := core_mesh.MeshGatewayResourceList{
Items: []*core_mesh.MeshGatewayResource{{
Meta: &test_model.ResourceMeta{Name: "gateway", Mesh: "default"},
Spec: &mesh_proto.MeshGateway{
Selectors: []*mesh_proto.Selector{{
Match: map[string]string{
mesh_proto.ServiceTag: "gateway",
}},
},
Conf: &mesh_proto.MeshGateway_Conf{
Listeners: []*mesh_proto.MeshGateway_Listener{
{
Protocol: mesh_proto.MeshGateway_Listener_HTTP,
Port: 8080,
},
},
},
},
}},
}
resources := xds_context.NewResources()
resources.MeshLocalResources[core_mesh.MeshGatewayType] = &gateways

context := xds_context.Context{
Mesh: xds_context.MeshContext{
Resource: &core_mesh.MeshResource{
Meta: &test_model.ResourceMeta{
Name: "default",
},
},
Resources: resources,
},
}
proxy := xds.Proxy{
Dataplane: &core_mesh.DataplaneResource{
Meta: &test_model.ResourceMeta{
Mesh: "default",
Name: "gateway",
},
Spec: &mesh_proto.Dataplane{
Networking: &mesh_proto.Dataplane_Networking{
Address: "127.0.0.1",
Gateway: &mesh_proto.Dataplane_Networking_Gateway{
Tags: map[string]string{
mesh_proto.ServiceTag: "gateway",
},
Type: mesh_proto.Dataplane_Networking_Gateway_BUILTIN,
},
},
},
},
Policies: xds.MatchedPolicies{
Dynamic: map[core_model.ResourceType]xds.TypedMatchingPolicies{
api.MeshAccessLogType: {
Type: api.MeshAccessLogType,
ToRules: given.toRules,
},
},
},
}
plugin := plugin.NewPlugin().(core_plugins.PolicyPlugin)

Expect(plugin.Apply(resourceSet, context, &proxy)).To(Succeed())

for i, r := range resourceSet.ListOf(envoy_resource.ListenerType) {
actual, err := util_proto.ToYAML(r.Resource)
Expect(err).ToNot(HaveOccurred())

Expect(actual).To(MatchYAML(given.expectedListeners[i]))
}
},
Entry("basic gateway", gatewayTestCase{
resources: []core_xds.Resource{{
Name: "gateway",
Origin: gateway.OriginGateway,
Resource: NewListenerBuilder(envoy_common.APIV3).
Configure(
InboundListener(
envoy_names.GetGatewayListenerName("gateway", "HTTP", 8080), "127.0.0.1", 8080, xds.SocketAddressProtocolTCP,
),
EnableReusePort(true),
TLSInspector(),
FilterChain(
NewFilterChainBuilder(envoy_common.APIV3).Configure(
HttpConnectionManager("gateway", false),
ServerHeader("Kuma Gateway"),
),
),
).MustBuild(),
}},
toRules: core_xds.ToRules{
Rules: []*core_xds.Rule{
{
Subset: core_xds.Subset{},
Conf: &api.MeshAccessLog_Conf{
Backends: []*api.MeshAccessLog_Backend{{
File: &api.MeshAccessLog_FileBackend{
Format: &api.MeshAccessLog_Format{
Plain: "",
},
Path: "/tmp/log",
},
}},
},
},
}},
expectedListeners: []string{`
address:
socketAddress:
address: 127.0.0.1
portValue: 8080
enableReusePort: true
filterChains:
- filters:
- name: envoy.filters.network.http_connection_manager
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
httpFilters:
- name: envoy.filters.http.router
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
serverName: Kuma Gateway
statPrefix: gateway
accessLog:
- name: envoy.access_loggers.file
typedConfig:
'@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
logFormat:
textFormatSource:
inlineString: |
[%START_TIME%] default "%REQ(:method)% %REQ(x-envoy-original-path?:path)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(x-envoy-upstream-service-time)% "%REQ(x-forwarded-for)%" "%REQ(user-agent)%" "%REQ(x-b3-traceid?x-datadog-traceid)%" "%REQ(x-request-id)%" "%REQ(:authority)%" "gateway" "*" "127.0.0.1" "%UPSTREAM_HOST%"
path: /tmp/log
listenerFilters:
- name: envoy.filters.listener.tls_inspector
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.listener.tls_inspector.v3.TlsInspector
name: gateway:HTTP:8080
trafficDirection: INBOUND`,
}}),
)
})
Loading

0 comments on commit 6999678

Please sign in to comment.