Overwrite k8s HTTP probes when transparent proxy is enabled.

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## UNRELEASED
 
+IMPROVEMENTS:
+* Connect: Overwrite Kubernetes HTTP readiness and/or liveness probes to point to Envoy proxy when
+  transparent proxy is enabled. [[GH-517](https://github.com/hashicorp/consul-k8s/pull/517)]
+
 ## 0.26.0-beta2 (May 06, 2021)
 
 BREAKING CHANGES:

connect-inject/annotations.go

@@ -93,7 +93,7 @@ const (
 	// keyTransparentProxy enables or disables transparent proxy for a given pod. It can also be set as a label
 	// on a namespace to define the default behaviour for connect-injected pods which do not otherwise override this setting
 	// with their own annotation.
-	// This annotation takes a boolean value (true/false).
+	// This annotation/label takes a boolean value (true/false).
 	keyTransparentProxy = "consul.hashicorp.com/transparent-proxy"
 
 	// annotationTProxyExcludeInboundPorts is a comma-separated list of inbound ports to exclude from traffic redirection.
@@ -108,6 +108,26 @@ const (
 	// annotationTProxyExcludeUIDs is a comma-separated list of additional user IDs to exclude from traffic redirection.
 	annotationTProxyExcludeUIDs = "consul.hashicorp.com/transparent-proxy-exclude-uids"
 
+	// annotationTransparentProxyOverwriteProbes controls whether the Kubernetes probes should be overwritten
+	// to point to the Envoy proxy when running in Transparent Proxy mode.
+	annotationTransparentProxyOverwriteProbes = "consul.hashicorp.com/transparent-proxy-overwrite-probes"
+
+	// annotationTransparentProxyReadinessListenerPort is the port for the readiness probe
+	// that we will expose through Envoy when overwrite probes is enabled.
+	annotationTransparentProxyReadinessListenerPort = "consul.hashicorp.com/transparent-proxy-readiness-listener-port"
+
+	// annotationTransparentProxyLivenessListenerPort is the port for the liveness probe
+	// that we will expose through Envoy when overwrite probes is enabled.
+	annotationTransparentProxyLivenessListenerPort = "consul.hashicorp.com/transparent-proxy-liveness-listener-port"
+
+	// annotationOriginalLivenessProbePort is the value of the port originally defined on the liveness probe
+	// of the pod before we overwrote it.
+	annotationOriginalLivenessProbePort = "consul.hashicorp.com/original-liveness-probe-port"
+
+	// annotationOriginalReadinessProbePort is the value of the port originally defined on the readiness probe
+	// of the pod before we overwrote it.
+	annotationOriginalReadinessProbePort = "consul.hashicorp.com/original-readiness-probe-port"
+
 	// injected is used as the annotation value for annotationInjected.
 	injected = "injected"
 

connect-inject/endpoints_controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 
 	"github.com/deckarep/golang-set"
@@ -39,6 +40,14 @@ const (
 	// in Consul. Note: This value should not be changed without a corresponding change in Consul.
 	// TODO: change this to a constant shared with Consul to avoid accidentally changing this.
 	clusterIPTaggedAddressName = "virtual"
+
+	// defaultExposedPathsListenerPortLiveness is the default port that we will use as the ListenerPort
+	// for the Expose configuration of the proxy registration for a liveness probe.
+	defaultExposedPathsListenerPortLiveness = 20300
+
+	// defaultExposedPathsListenerPortReadiness is the default port that we will use as the ListenerPort
+	// for the Expose configuration of the proxy registration for a readiness probe.
+	defaultExposedPathsListenerPortReadiness = 20301
 )
 
 type EndpointsController struct {
@@ -82,11 +91,14 @@ type EndpointsController struct {
 	// EnableTransparentProxy controls whether transparent proxy should be enabled
 	// for all proxy service registrations.
 	EnableTransparentProxy bool
+	// TProxyOverwriteProbes controls whether the endpoints controller should expose pod's HTTP probes
+	// via Envoy proxy.
+	TProxyOverwriteProbes bool
 
 	MetricsConfig MetricsConfig
 	Log           logr.Logger
-	Scheme        *runtime.Scheme
 
+	Scheme *runtime.Scheme
 	context.Context
 }
 
@@ -527,6 +539,45 @@ func (r *EndpointsController) createServiceRegistrations(pod corev1.Pod, service
 		} else {
 			r.Log.Info("skipping syncing service cluster IP to Consul", "name", k8sService.Name, "ns", k8sService.Namespace, "ip", k8sService.Spec.ClusterIP)
 		}
+
+		// Expose k8s probes as Envoy listeners if needed.
+		overwriteProbes, err := shouldOverwriteProbes(pod, r.TProxyOverwriteProbes)
+		if err != nil {
+			return nil, nil, err
+		}
+		if overwriteProbes {
+			if cs := pod.Spec.Containers; len(cs) > 0 {
+				appContainer := cs[0]
+				if appContainer.LivenessProbe != nil && appContainer.LivenessProbe.HTTPGet != nil {
+					if raw, ok := pod.Annotations[annotationOriginalLivenessProbePort]; ok {
+						originalLivenessPort, err := strconv.Atoi(raw)
+						if err != nil {
+							return nil, nil, err
+						}
+
+						proxyConfig.Expose.Paths = append(proxyConfig.Expose.Paths, api.ExposePath{
+							ListenerPort:  appContainer.LivenessProbe.HTTPGet.Port.IntValue(),
+							LocalPathPort: originalLivenessPort,
+							Path:          appContainer.LivenessProbe.HTTPGet.Path,
+						})
+					}
+				}
+				if appContainer.ReadinessProbe != nil && appContainer.ReadinessProbe.HTTPGet != nil {
+					if raw, ok := pod.Annotations[annotationOriginalReadinessProbePort]; ok {
+						originalReadinessPort, err := strconv.Atoi(raw)
+						if err != nil {
+							return nil, nil, err
+						}
+
+						proxyConfig.Expose.Paths = append(proxyConfig.Expose.Paths, api.ExposePath{
+							ListenerPort:  appContainer.ReadinessProbe.HTTPGet.Port.IntValue(),
+							LocalPathPort: originalReadinessPort,
+							Path:          appContainer.ReadinessProbe.HTTPGet.Path,
+						})
+					}
+				}
+			}
+		}
 	}
 
 	return service, proxyService, nil