Skip to content

Commit

Permalink
Switch to JSON for access logs
Browse files Browse the repository at this point in the history
  • Loading branch information
@dinofizz
dinofizz committed May 8, 2024
1 parent 3bc9d49 commit 9ee3d3c
Show file tree
Hide file tree
Showing 9 changed files with 863 additions and 634 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Image URL to use all building/pushing image targets
IMG ?= ${ACC}.dkr.ecr.eu-west-1.amazonaws.com/monzo/egress-operator:manager-$(shell git rev-parse --short head)
# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
CRD_OPTIONS ?= "crd:trivialVersions=false"
CRD_OPTIONS ?= "crd:trivialVersions=true"

# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))
Expand Down
62 changes: 47 additions & 15 deletions controllers/configmap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package controllers
import (
"context"
"fmt"
//structpb "github.com/golang/protobuf/ptypes/struct"
"google.golang.org/protobuf/types/known/anypb"
"google.golang.org/protobuf/types/known/durationpb"
"hash/fnv"
"strconv"
Expand All @@ -13,16 +15,16 @@ import (
envoycorev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
envoyendpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
envoylistener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
filev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3"
streamv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/stream/v3"
aggregatev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
tcpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
udpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/v3"
"github.com/golang/protobuf/ptypes/wrappers"
"google.golang.org/protobuf/types/known/wrapperspb"

"github.com/golang/protobuf/jsonpb"
"github.com/golang/protobuf/ptypes"
"github.com/golang/protobuf/ptypes/duration"
"github.com/golang/protobuf/ptypes/wrappers"
"google.golang.org/protobuf/types/known/structpb"
"google.golang.org/protobuf/types/known/wrapperspb"
corev1 "k8s.io/api/core/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -35,6 +37,34 @@ import (

// +kubebuilder:rbac:namespace=egress-operator-system,groups=core,resources=configmaps,verbs=get;list;watch;create;patch

var (
logFields = &structpb.Struct{
Fields: map[string]*structpb.Value{
"authority": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:AUTHORITY)%"}},
"bytes_received": {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_RECEIVED%"}},
"bytes_sent": {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_SENT%"}},
"connection_termination_details": {Kind: &structpb.Value_StringValue{StringValue: "%CONNECTION_TERMINATION_DETAILS%"}},
"downstream_local_address": {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_LOCAL_ADDRESS%"}},
"downstream_remote_address": {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_REMOTE_ADDRESS%"}},
"duration": {Kind: &structpb.Value_StringValue{StringValue: "%DURATION%"}},
"method": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:METHOD)%"}},
"path": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"}},
"protocol": {Kind: &structpb.Value_StringValue{StringValue: "%PROTOCOL%"}},
"requested_server_name": {Kind: &structpb.Value_StringValue{StringValue: "%REQUESTED_SERVER_NAME%"}},
"response_code": {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE%"}},
"response_code_details": {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE_DETAILS%"}},
"response_flags": {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_FLAGS%"}},
"start_time": {Kind: &structpb.Value_StringValue{StringValue: "%START_TIME%"}},
"upstream_cluster": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_CLUSTER%"}},
"upstream_host": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_HOST%"}},
"upstream_local_address": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_LOCAL_ADDRESS%"}},
"upstream_service_time": {Kind: &structpb.Value_StringValue{StringValue: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%"}},
"upstream_transport_failure_reason": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_TRANSPORT_FAILURE_REASON%"}},
"user_agent": {Kind: &structpb.Value_StringValue{StringValue: "%REQ(USER-AGENT)%"}},
},
}
)

func (r *ExternalServiceReconciler) reconcileConfigMap(ctx context.Context, req ctrl.Request, es *egressv1.ExternalService, desired *corev1.ConfigMap) error {
if err := ctrl.SetControllerReference(es, desired, r.Scheme); err != nil {
return err
Expand Down Expand Up @@ -86,9 +116,6 @@ func adminPort(es *egressv1.ExternalService) int32 {
panic("couldn't find a port for admin listener")
}

const accessLogFormat = `[%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
`

func envoyConfig(es *egressv1.ExternalService) (string, error) {
config := bootstrap.Bootstrap{
Node: &envoycorev3.Node{
Expand Down Expand Up @@ -197,16 +224,21 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
var listener *envoylistener.Listener
switch protocol {
case envoycorev3.SocketAddress_TCP:
accessConfig, err := ptypes.MarshalAny(&filev3.FileAccessLog{
AccessLogFormat: &filev3.FileAccessLog_Format{
Format: accessLogFormat,
accessConfig, err := anypb.New(&streamv3.StdoutAccessLog{
AccessLogFormat: &streamv3.StdoutAccessLog_LogFormat{
LogFormat: &envoycorev3.SubstitutionFormatString{
Format: &envoycorev3.SubstitutionFormatString_JsonFormat{
JsonFormat: logFields,
},
OmitEmptyValues: true,
ContentType: "application/json; charset=UTF-8",
JsonFormatOptions: nil,

Check failure on line 235 in controllers/configmap.go

View workflow job for this annotation

GitHub Actions / Build 

unknown field JsonFormatOptions in struct literal of type corev3.SubstitutionFormatString
},
},
Path: "/dev/stdout",
})

filterConfig, err := ptypes.MarshalAny(&tcpproxyv3.TcpProxy{
filterConfig, err := anypb.New(&tcpproxyv3.TcpProxy{
AccessLog: []*accesslogfilterv3.AccessLog{{
Name: "envoy.file_access_log",
Name: "envoy.stdout_access_log",
ConfigType: &accesslogfilterv3.AccessLog_TypedConfig{TypedConfig: accessConfig},
}},
StatPrefix: "tcp_proxy",
Expand Down Expand Up @@ -236,7 +268,7 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
}}}}},
}
case envoycorev3.SocketAddress_UDP:
filterConfig, err := ptypes.MarshalAny(&udpproxyv3.UdpProxyConfig{
filterConfig, err := anypb.New(&udpproxyv3.UdpProxyConfig{
StatPrefix: "udp_proxy",
RouteSpecifier: &udpproxyv3.UdpProxyConfig_Cluster{
Cluster: name,
Expand Down
76 changes: 65 additions & 11 deletions controllers/configmap_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package controllers

import (
bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
"github.com/google/go-cmp/cmp"
"sigs.k8s.io/yaml"
"testing"

"github.com/google/go-cmp/cmp"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

Expand Down Expand Up @@ -97,12 +99,34 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
accessLog:
- name: envoy.file_access_log
- name: envoy.stdout_access_log
typedConfig:
'@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
format: |
[%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
path: /dev/stdout
'@type': type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
logFormat:
contentType: application/json; charset=UTF-8
jsonFormat:
authority: '%REQ(:AUTHORITY)%'
bytes_received: '%BYTES_RECEIVED%'
bytes_sent: '%BYTES_SENT%'
connection_termination_details: '%CONNECTION_TERMINATION_DETAILS%'
downstream_local_address: '%DOWNSTREAM_LOCAL_ADDRESS%'
downstream_remote_address: '%DOWNSTREAM_REMOTE_ADDRESS%'
duration: '%DURATION%'
method: '%REQ(:METHOD)%'
path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%'
protocol: '%PROTOCOL%'
requested_server_name: '%REQUESTED_SERVER_NAME%'
response_code: '%RESPONSE_CODE%'
response_code_details: '%RESPONSE_CODE_DETAILS%'
response_flags: '%RESPONSE_FLAGS%'
start_time: '%START_TIME%'
upstream_cluster: '%UPSTREAM_CLUSTER%'
upstream_host: '%UPSTREAM_HOST%'
upstream_local_address: '%UPSTREAM_LOCAL_ADDRESS%'
upstream_service_time: '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%'
upstream_transport_failure_reason: '%UPSTREAM_TRANSPORT_FAILURE_REASON%'
user_agent: '%REQ(USER-AGENT)%'
omitEmptyValues: true
cluster: foo_TCP_101
statPrefix: tcp_proxy
name: foo_TCP_101
Expand Down Expand Up @@ -209,12 +233,34 @@ staticResources:
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
accessLog:
- name: envoy.file_access_log
- name: envoy.stdout_access_log
typedConfig:
'@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
format: |
[%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
path: /dev/stdout
'@type': type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
logFormat:
contentType: application/json; charset=UTF-8
jsonFormat:
authority: '%REQ(:AUTHORITY)%'
bytes_received: '%BYTES_RECEIVED%'
bytes_sent: '%BYTES_SENT%'
connection_termination_details: '%CONNECTION_TERMINATION_DETAILS%'
downstream_local_address: '%DOWNSTREAM_LOCAL_ADDRESS%'
downstream_remote_address: '%DOWNSTREAM_REMOTE_ADDRESS%'
duration: '%DURATION%'
method: '%REQ(:METHOD)%'
path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%'
protocol: '%PROTOCOL%'
requested_server_name: '%REQUESTED_SERVER_NAME%'
response_code: '%RESPONSE_CODE%'
response_code_details: '%RESPONSE_CODE_DETAILS%'
response_flags: '%RESPONSE_FLAGS%'
start_time: '%START_TIME%'
upstream_cluster: '%UPSTREAM_CLUSTER%'
upstream_host: '%UPSTREAM_HOST%'
upstream_local_address: '%UPSTREAM_LOCAL_ADDRESS%'
upstream_service_time: '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%'
upstream_transport_failure_reason: '%UPSTREAM_TRANSPORT_FAILURE_REASON%'
user_agent: '%REQ(USER-AGENT)%'
omitEmptyValues: true
cluster: foo_TCP_101
statPrefix: tcp_proxy
name: foo_TCP_101
Expand Down Expand Up @@ -246,7 +292,15 @@ staticResources:
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.args.es.Spec.EnvoyClusterMaxConnections = tt.args.maxConns
got, _ := envoyConfig(tt.args.es)

var x bootstrap.Bootstrap
err := yaml.Unmarshal([]byte(got), &x)
if err != nil {
t.Error()
}
if got, _ := envoyConfig(tt.args.es); got != tt.want {

t.Errorf("envoyConfig() = %v, want %v", got, tt.want)
t.Error(cmp.Diff(got, tt.want))
}
Expand Down
102 changes: 102 additions & 0 deletions controllers/testdata/expected_udp_tcp.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
admin:
accessLogPath: /dev/stdout
address:
socketAddress:
address: 0.0.0.0
portValue: 11000
node:
cluster: foo
staticResources:
clusters:
- connectTimeout: 1s
dnsLookupFamily: V4_ONLY
loadAssignment:
clusterName: foo_UDP_100
endpoints:
- lbEndpoints:
- endpoint:
address:
socketAddress:
address: google.com
portValue: 100
protocol: UDP
name: foo_UDP_100
type: LOGICAL_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 5
keepaliveProbes: 3
keepaliveTime: 30
- connectTimeout: 1s
dnsLookupFamily: V4_ONLY
loadAssignment:
clusterName: foo_TCP_101
endpoints:
- lbEndpoints:
- endpoint:
address:
socketAddress:
address: google.com
portValue: 101
name: foo_TCP_101
type: LOGICAL_DNS
upstreamConnectionOptions:
tcpKeepalive:
keepaliveInterval: 5
keepaliveProbes: 3
keepaliveTime: 30
listeners:
- address:
socketAddress:
address: 0.0.0.0
portValue: 100
protocol: UDP
filterChains:
- filters:
- name: envoy.filters.udp_listener.udp_proxy
typedConfig:
"@type": type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig
cluster: foo_UDP_100
statPrefix: udp_proxy
name: foo_UDP_100
- address:
socketAddress:
address: 0.0.0.0
portValue: 101
filterChains:
- filters:
- name: envoy.tcp_proxy
typedConfig:
'@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
accessLog:
- name: envoy.stdout_access_log
typedConfig:
'@type': type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
logFormat:
contentType: "application/json; charset=UTF-8"
jsonFormat:
authority: '%REQ(:AUTHORITY)%'
bytes_received: '%BYTES_RECEIVED%'
bytes_sent: '%BYTES_SENT%'
connection_termination_details: '%CONNECTION_TERMINATION_DETAILS%'
downstream_local_address: '%DOWNSTREAM_LOCAL_ADDRESS%'
downstream_remote_address: '%DOWNSTREAM_REMOTE_ADDRESS%'
duration: '%DURATION%'
method: '%REQ(:METHOD)%'
path: '%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%'
protocol: '%PROTOCOL%'
requested_server_name: '%REQUESTED_SERVER_NAME%'
response_code: '%RESPONSE_CODE%'
response_code_details: '%RESPONSE_CODE_DETAILS%'
response_flags: '%RESPONSE_FLAGS%'
start_time: '%START__TIME%'
upstream_cluster: '%UPSTREAM_CLUSTER%'
upstream_host: '%UPSTREAM_HOST%'
upstream_local_address: '%UPSTREAM_LOCAL_ADDRESS%'
upstream_service_time: '%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%'
upstream_transport_failure_reason: '%UPSTREAM_TRANSPORT_FAILURE_REASON%'
user_agent: '%REQ(USER-AGENT)%'
omitEmptyValues: True
cluster: "foo_TCP_101"
statPrefix: "tcp_proxy"
name: foo_TCP_101
Loading

0 comments on commit 9ee3d3c

Please sign in to comment.