Allow to exclude inbound and outbound traffic when using tproxy via a…

…nnotations (#506)

We allow the exclusion of the following:
* Exclude inbound ports
* Exclude outbound ports
* Exclude outbound CIDRs
* Exclude UIDs

connect-inject/annotations.go

@@ -88,6 +88,18 @@ const (
 	// This annotation takes a boolean value (true/false).
 	annotationTransparentProxy = "consul.hashicorp.com/transparent-proxy"
 
+	// annotationTProxyExcludeInboundPorts is a comma-separated list of inbound ports to exclude from traffic redirection.
+	annotationTProxyExcludeInboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-inbound-ports"
+
+	// annotationTProxyExcludeOutboundPorts is a comma-separated list of outbound ports to exclude from traffic redirection.
+	annotationTProxyExcludeOutboundPorts = "consul.hashicorp.com/transparent-proxy-exclude-outbound-ports"
+
+	// annotationTProxyExcludeOutboundCIDRs is a comma-separated list of outbound CIDRs to exclude from traffic redirection.
+	annotationTProxyExcludeOutboundCIDRs = "consul.hashicorp.com/transparent-proxy-exclude-outbound-cidrs"
+
+	// annotationTProxyExcludeUIDs is a comma-separated list of additional user IDs to exclude from traffic redirection.
+	annotationTProxyExcludeUIDs = "consul.hashicorp.com/transparent-proxy-exclude-uids"
+
 	// injected is used as the annotation value for annotationInjected.
 	injected = "injected"
 )

connect-inject/container_init.go

@@ -46,6 +46,22 @@ type initContainerCommandData struct {
 	// i.e. run consul connect redirect-traffic command and add the required privileges to the
 	// container to do that.
 	EnableTransparentProxy bool
+
+	// TProxyExcludeInboundPorts is a list of inbound ports to exclude from traffic redirection via
+	// the consul connect redirect-traffic command.
+	TProxyExcludeInboundPorts []string
+
+	// TProxyExcludeOutboundPorts is a list of outbound ports to exclude from traffic redirection via
+	// the consul connect redirect-traffic command.
+	TProxyExcludeOutboundPorts []string
+
+	// TProxyExcludeOutboundCIDRs is a list of outbound CIDRs to exclude from traffic redirection via
+	// the consul connect redirect-traffic command.
+	TProxyExcludeOutboundCIDRs []string
+
+	// TProxyExcludeUIDs is a list of additional user IDs to exclude from traffic redirection via
+	// the consul connect redirect-traffic command.
+	TProxyExcludeUIDs []string
 }
 
 // containerInitCopyContainer returns the init container spec for the copy container which places
@@ -84,12 +100,16 @@ func (h *Handler) containerInit(pod corev1.Pod, k8sNamespace string) (corev1.Con
 	}
 
 	data := initContainerCommandData{
-		AuthMethod:                h.AuthMethod,
-		ConsulNamespace:           h.consulNamespace(k8sNamespace),
-		NamespaceMirroringEnabled: h.EnableK8SNSMirroring,
-		ConsulCACert:              h.ConsulCACert,
-		EnableTransparentProxy:    tproxyEnabled,
-		EnvoyUID:                  envoyUserAndGroupID,
+		AuthMethod:                 h.AuthMethod,
+		ConsulNamespace:            h.consulNamespace(k8sNamespace),
+		NamespaceMirroringEnabled:  h.EnableK8SNSMirroring,
+		ConsulCACert:               h.ConsulCACert,
+		EnableTransparentProxy:     tproxyEnabled,
+		TProxyExcludeInboundPorts:  splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeInboundPorts, pod),
+		TProxyExcludeOutboundPorts: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundPorts, pod),
+		TProxyExcludeOutboundCIDRs: splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeOutboundCIDRs, pod),
+		TProxyExcludeUIDs:          splitCommaSeparatedItemsFromAnnotation(annotationTProxyExcludeUIDs, pod),
+		EnvoyUID:                   envoyUserAndGroupID,
 	}
 
 	if data.AuthMethod != "" {
@@ -213,6 +233,19 @@ func pointerToBool(b bool) *bool {
 	return &b
 }
 
+// splitCommaSeparatedItemsFromAnnotation takes an annotation and a pod
+// and returns the comma-separated value of the annotation as a list of strings.
+func splitCommaSeparatedItemsFromAnnotation(annotation string, pod corev1.Pod) []string {
+	var items []string
+	if raw, ok := pod.Annotations[annotation]; ok {
+		for _, item := range strings.Split(raw, ",") {
+			items = append(items, item)
+		}
+	}
+
+	return items
+}
+
 // initContainerCommandTpl is the template for the command executed by
 // the init container.
 const initContainerCommandTpl = `
@@ -272,6 +305,18 @@ consul-k8s connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \
   {{- if .ConsulNamespace }}
   -namespace="{{ .ConsulNamespace }}" \
   {{- end }}
+  {{- range .TProxyExcludeInboundPorts }}
+  -exclude-inbound-port="{{ . }}" \
+  {{- end }}
+  {{- range .TProxyExcludeOutboundPorts }}
+  -exclude-outbound-port="{{ . }}" \
+  {{- end }}
+  {{- range .TProxyExcludeOutboundCIDRs }}
+  -exclude-outbound-cidr="{{ . }}" \
+  {{- end }}
+  {{- range .TProxyExcludeUIDs }}
+  -exclude-uid="{{ . }}" \
+  {{- end }}
   -proxy-id="$(cat /consul/connect-inject/proxyid)" \
   -proxy-uid={{ .EnvoyUID }}
 {{- end }}

connect-inject/container_init_test.go

@@ -1,7 +1,6 @@
 package connectinject
 
 import (
-	"strconv"
 	"strings"
 	"testing"
 
@@ -143,48 +142,125 @@ consul-k8s connect-init -pod-name=${POD_NAME} -pod-namespace=${POD_NAMESPACE} \
 
 func TestHandlerContainerInit_transparentProxy(t *testing.T) {
 	cases := map[string]struct {
-		globalEnabled     bool
-		annotationEnabled *bool
-		expectEnabled     bool
+		globalEnabled          bool
+		annotations            map[string]string
+		expectedContainsCmd    string
+		expectedNotContainsCmd string
 	}{
 		"enabled globally, annotation not provided": {
 			true,
 			nil,
-			true,
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
+			"",
 		},
 		"enabled globally, annotation is false": {
 			true,
-			pointerToBool(false),
-			false,
+			map[string]string{
+				annotationTransparentProxy: "false",
+			},
+			"",
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
 		},
 		"enabled globally, annotation is true": {
 			true,
-			pointerToBool(true),
-			true,
+			map[string]string{
+				annotationTransparentProxy: "true",
+			},
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
+			"",
 		},
 		"disabled globally, annotation not provided": {
 			false,
 			nil,
-			false,
+			"",
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
 		},
 		"disabled globally, annotation is false": {
 			false,
-			pointerToBool(false),
-			false,
+			map[string]string{
+				annotationTransparentProxy: "false",
+			},
+			"",
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
 		},
 		"disabled globally, annotation is true": {
 			false,
-			pointerToBool(true),
+			map[string]string{
+				annotationTransparentProxy: "true",
+			},
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
+			"",
+		},
+		"exclude-inbound-ports annotation is provided": {
+			true,
+			map[string]string{
+				annotationTransparentProxy:          "true",
+				annotationTProxyExcludeInboundPorts: "9090,9091",
+			},
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -exclude-inbound-port="9090" \
+  -exclude-inbound-port="9091" \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
+			"",
+		},
+		"exclude-outbound-ports annotation is provided": {
+			true,
+			map[string]string{
+				annotationTransparentProxy:           "true",
+				annotationTProxyExcludeOutboundPorts: "9090,9091",
+			},
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -exclude-outbound-port="9090" \
+  -exclude-outbound-port="9091" \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
+			"",
+		},
+		"exclude-outbound-cidrs annotation is provided": {
 			true,
+			map[string]string{
+				annotationTransparentProxy:           "true",
+				annotationTProxyExcludeOutboundCIDRs: "1.1.1.1,2.2.2.2/24",
+			},
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -exclude-outbound-cidr="1.1.1.1" \
+  -exclude-outbound-cidr="2.2.2.2/24" \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
+			"",
+		},
+		"exclude-uids annotation is provided": {
+			true,
+			map[string]string{
+				annotationTransparentProxy:  "true",
+				annotationTProxyExcludeUIDs: "6000,7000",
+			},
+			`/consul/connect-inject/consul connect redirect-traffic \
+  -exclude-uid="6000" \
+  -exclude-uid="7000" \
+  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
+  -proxy-uid=5995`,
+			"",
 		},
 	}
 	for name, c := range cases {
 		t.Run(name, func(t *testing.T) {
 			h := Handler{EnableTransparentProxy: c.globalEnabled}
 			pod := minimal()
-			if c.annotationEnabled != nil {
-				pod.Annotations[annotationTransparentProxy] = strconv.FormatBool(*c.annotationEnabled)
-			}
+			pod.Annotations = c.annotations
 
 			expectedSecurityContext := &corev1.SecurityContext{
 				RunAsUser:  pointerToInt64(0),
@@ -194,19 +270,16 @@ func TestHandlerContainerInit_transparentProxy(t *testing.T) {
 				},
 				RunAsNonRoot: pointerToBool(false),
 			}
-			expectedCmd := `/consul/connect-inject/consul connect redirect-traffic \
-  -proxy-id="$(cat /consul/connect-inject/proxyid)" \
-  -proxy-uid=5995`
 			container, err := h.containerInit(*pod, k8sNamespace)
 			require.NoError(t, err)
 			actualCmd := strings.Join(container.Command, " ")
 
-			if c.expectEnabled {
+			if c.expectedContainsCmd != "" {
 				require.Equal(t, expectedSecurityContext, container.SecurityContext)
-				require.Contains(t, actualCmd, expectedCmd)
+				require.Contains(t, actualCmd, c.expectedContainsCmd)
 			} else {
 				require.Nil(t, container.SecurityContext)
-				require.NotContains(t, actualCmd, expectedCmd)
+				require.NotContains(t, actualCmd, c.expectedNotContainsCmd)
 			}
 		})
 	}

connect-inject/handler_test.go

@@ -126,7 +126,6 @@ func TestHandlerHandle(t *testing.T) {
 			},
 		},
 
-		// todo: why is upstreams different then basic
 		{
 			"pod with upstreams specified",
 			Handler{