Overwrite k8s HTTP probes when transparent proxy is enabled. (#517)

When this is enabled, the mutating webhook will also overwrite the HTTP probes
of the application container to point them to Envoy, while saving the original
ports of the probes as annotations. The endpoints controller will set Expose
configuration for the proxy, setting the LocalPathPort to the port saved
in the annotation.

- Add a new flag -transparent-proxy-default-overwrite-probes (default to true)
  to allow overwriting k8s probes.
- Allow setting this on a per-pod basis via the "consul.hashicorp.com/transparent-proxy-overwrite-probes"
  annotation
- Add new annotations to allow for choosing custom ports for Envoy listeners to expose
  readiness and liveness probes

CHANGELOG.md

@@ -1,7 +1,11 @@
 ## UNRELEASED
 
+IMPROVEMENTS:
+* Connect: Overwrite Kubernetes HTTP readiness and/or liveness probes to point to Envoy proxy when
+  transparent proxy is enabled. [[GH-517](https://github.com/hashicorp/consul-k8s/pull/517)]
+
 BUG FIXES:
-* Connect: Process every Address in and Endpoints object before returning an error. This ensures an address that isn't reconciled successfully doesn't prevent the remaining addresses from getting reconciled. [[GH-519](https://github.com/hashicorp/consul-k8s/pull/519)]
+* Connect: Process every Address in an Endpoints object before returning an error. This ensures an address that isn't reconciled successfully doesn't prevent the remaining addresses from getting reconciled. [[GH-519](https://github.com/hashicorp/consul-k8s/pull/519)]
 
 ## 0.26.0-beta2 (May 06, 2021)
 

connect-inject/annotations.go

@@ -93,7 +93,7 @@ const (
 	// keyTransparentProxy enables or disables transparent proxy for a given pod. It can also be set as a label
 	// on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting
 	// with their own annotation.
-	// This annotation takes a boolean value (true/false).
+	// This annotation/label takes a boolean value (true/false).
 	keyTransparentProxy = "consul.hashicorp.com/transparent-proxy"
 
 	// annotationTProxyExcludeInboundPorts is a comma-separated list of inbound ports to exclude from traffic redirection.
@@ -108,6 +108,26 @@ const (
 	// annotationTProxyExcludeUIDs is a comma-separated list of additional user IDs to exclude from traffic redirection.
 	annotationTProxyExcludeUIDs = "consul.hashicorp.com/transparent-proxy-exclude-uids"
 
+	// annotationTransparentProxyOverwriteProbes controls whether the Kubernetes probes should be overwritten
+	// to point to the Envoy proxy when running in Transparent Proxy mode.
+	annotationTransparentProxyOverwriteProbes = "consul.hashicorp.com/transparent-proxy-overwrite-probes"
+
+	// annotationTransparentProxyReadinessListenerPort is the port for the readiness probe
+	// that we will expose through Envoy when overwrite probes is enabled.
+	annotationTransparentProxyReadinessListenerPort = "consul.hashicorp.com/transparent-proxy-readiness-listener-port"
+
+	// annotationTransparentProxyLivenessListenerPort is the port for the liveness probe
+	// that we will expose through Envoy when overwrite probes is enabled.
+	annotationTransparentProxyLivenessListenerPort = "consul.hashicorp.com/transparent-proxy-liveness-listener-port"
+
+	// annotationOriginalLivenessProbePort is the value of the port originally defined on the liveness probe
+	// of the pod before we overwrote it.
+	annotationOriginalLivenessProbePort = "consul.hashicorp.com/original-liveness-probe-port"
+
+	// annotationOriginalReadinessProbePort is the value of the port originally defined on the readiness probe
+	// of the pod before we overwrote it.
+	annotationOriginalReadinessProbePort = "consul.hashicorp.com/original-readiness-probe-port"
+
 	// injected is used as the annotation value for annotationInjected.
 	injected = "injected"
 

connect-inject/endpoints_controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 
 	"github.com/deckarep/golang-set"
@@ -40,6 +41,14 @@ const (
 	// in Consul. Note: This value should not be changed without a corresponding change in Consul.
 	// TODO: change this to a constant shared with Consul to avoid accidentally changing this.
 	clusterIPTaggedAddressName = "virtual"
+
+	// defaultExposedPathsListenerPortLiveness is the default port that we will use as the ListenerPort
+	// for the Expose configuration of the proxy registration for a liveness probe.
+	defaultExposedPathsListenerPortLiveness = 20300
+
+	// defaultExposedPathsListenerPortReadiness is the default port that we will use as the ListenerPort
+	// for the Expose configuration of the proxy registration for a readiness probe.
+	defaultExposedPathsListenerPortReadiness = 20301
 )
 
 type EndpointsController struct {
@@ -83,11 +92,14 @@ type EndpointsController struct {
 	// EnableTransparentProxy controls whether transparent proxy should be enabled
 	// for all proxy service registrations.
 	EnableTransparentProxy bool
+	// TProxyOverwriteProbes controls whether the endpoints controller should expose pod's HTTP probes
+	// via Envoy proxy.
+	TProxyOverwriteProbes bool
 
 	MetricsConfig MetricsConfig
 	Log           logr.Logger
-	Scheme        *runtime.Scheme
 
+	Scheme *runtime.Scheme
 	context.Context
 }
 
@@ -530,6 +542,45 @@ func (r *EndpointsController) createServiceRegistrations(pod corev1.Pod, service
 		} else {
 			r.Log.Info("skipping syncing service cluster IP to Consul", "name", k8sService.Name, "ns", k8sService.Namespace, "ip", k8sService.Spec.ClusterIP)
 		}
+
+		// Expose k8s probes as Envoy listeners if needed.
+		overwriteProbes, err := shouldOverwriteProbes(pod, r.TProxyOverwriteProbes)
+		if err != nil {
+			return nil, nil, err
+		}
+		if overwriteProbes {
+			if cs := pod.Spec.Containers; len(cs) > 0 {
+				appContainer := cs[0]
+				if appContainer.LivenessProbe != nil && appContainer.LivenessProbe.HTTPGet != nil {
+					if raw, ok := pod.Annotations[annotationOriginalLivenessProbePort]; ok {
+						originalLivenessPort, err := strconv.Atoi(raw)
+						if err != nil {
+							return nil, nil, err
+						}
+
+						proxyConfig.Expose.Paths = append(proxyConfig.Expose.Paths, api.ExposePath{
+							ListenerPort:  appContainer.LivenessProbe.HTTPGet.Port.IntValue(),
+							LocalPathPort: originalLivenessPort,
+							Path:          appContainer.LivenessProbe.HTTPGet.Path,
+						})
+					}
+				}
+				if appContainer.ReadinessProbe != nil && appContainer.ReadinessProbe.HTTPGet != nil {
+					if raw, ok := pod.Annotations[annotationOriginalReadinessProbePort]; ok {
+						originalReadinessPort, err := strconv.Atoi(raw)
+						if err != nil {
+							return nil, nil, err
+						}
+
+						proxyConfig.Expose.Paths = append(proxyConfig.Expose.Paths, api.ExposePath{
+							ListenerPort:  appContainer.ReadinessProbe.HTTPGet.Port.IntValue(),
+							LocalPathPort: originalReadinessPort,
+							Path:          appContainer.ReadinessProbe.HTTPGet.Path,
+						})
+					}
+				}
+			}
+		}
 	}
 
 	return service, proxyService, nil