TProxy integration test (#17103)

* TProxy integration test
* Fix GHA compatibility integration test command

Previously, when test splitting allocated multiple test directories to a
runner, the workflow ran `go tests "./test/dir1 ./test/dir2"` which
results in a directory not found error. This fixes that.

.github/workflows/test-integrations.yml

@@ -309,7 +309,7 @@ jobs:
             docker run --rm ${{ env.CONSUL_LATEST_IMAGE_NAME }}:local consul version
             echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests"
             # shellcheck disable=SC2001
-            sed 's,|,\n,g' <<< "${{ matrix.test-cases }}"
+            sed 's, ,\n,g' <<< "${{ matrix.test-cases }}"
             go run gotest.tools/gotestsum@v${{env.GOTESTSUM_VERSION}} \
               --raw-command \
               --format=short-verbose \
@@ -321,7 +321,7 @@ jobs:
               -tags "${{ env.GOTAGS }}" \
               -timeout=30m \
               -json \
-              "${{ matrix.test-cases }}" \
+              ${{ matrix.test-cases }} \
               --target-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \
               --target-version local \
               --latest-image ${{ env.CONSUL_LATEST_IMAGE_NAME }} \

test/integration/consul-container/libs/service/assets/Dockerfile-consul-envoy

@@ -5,4 +5,17 @@ ARG ENVOY_VERSION
 FROM ${CONSUL_IMAGE} as consul
 
 FROM docker.mirror.hashicorp.services/envoyproxy/envoy:v${ENVOY_VERSION}
+
+# Install iptables and sudo, needed for tproxy.
+RUN apt update -y \
+	&& apt install -y iptables sudo curl dnsutils \
+	&& adduser envoy sudo \
+	&& echo 'envoy ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+COPY tproxy-startup.sh /bin/tproxy-startup.sh
+RUN chmod +x /bin/tproxy-startup.sh \
+	&& chown envoy:envoy /bin/tproxy-startup.sh
+
 COPY --from=consul /bin/consul /bin/consul
+
+USER envoy

test/integration/consul-container/libs/service/assets/tproxy-startup.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+
+set -ex
+
+# HACK: UID of consul in the consul-client container
+# This is conveniently also the UID of apt in the envoy container
+CONSUL_UID=100
+ENVOY_UID=$(id -u)
+
+sudo consul connect redirect-traffic \
+    -proxy-uid $ENVOY_UID \
+    -exclude-uid $CONSUL_UID \
+    $REDIRECT_TRAFFIC_ARGS
+
+exec "$@"

test/integration/consul-container/libs/service/common.go

@@ -21,26 +21,40 @@ const (
 )
 
 //go:embed assets/Dockerfile-consul-envoy
-var consulEnvoyDockerfile string
+var consulEnvoyDockerfile []byte
+
+//go:embed assets/tproxy-startup.sh
+var tproxyStartupScript []byte
 
 // getDevContainerDockerfile returns the necessary context to build a combined consul and
 // envoy image for running "consul connect envoy ..."
 func getDevContainerDockerfile() (testcontainers.FromDockerfile, error) {
 	var buf bytes.Buffer
 	tw := tar.NewWriter(&buf)
 
-	dockerfileBytes := []byte(consulEnvoyDockerfile)
-
 	hdr := &tar.Header{
 		Name: "Dockerfile",
 		Mode: 0600,
-		Size: int64(len(dockerfileBytes)),
+		Size: int64(len(consulEnvoyDockerfile)),
+	}
+	if err := tw.WriteHeader(hdr); err != nil {
+		return testcontainers.FromDockerfile{}, err
+	}
+
+	if _, err := tw.Write(consulEnvoyDockerfile); err != nil {
+		return testcontainers.FromDockerfile{}, err
+	}
+
+	hdr = &tar.Header{
+		Name: "tproxy-startup.sh",
+		Mode: 0600,
+		Size: int64(len(tproxyStartupScript)),
 	}
 	if err := tw.WriteHeader(hdr); err != nil {
 		return testcontainers.FromDockerfile{}, err
 	}
 
-	if _, err := tw.Write(dockerfileBytes); err != nil {
+	if _, err := tw.Write(tproxyStartupScript); err != nil {
 		return testcontainers.FromDockerfile{}, err
 	}
 

test/integration/consul-container/libs/service/connect.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/testcontainers/testcontainers-go"
@@ -146,9 +147,10 @@ func (g ConnectContainer) GetStatus() (string, error) {
 }
 
 type SidecarConfig struct {
-	Name      string
-	ServiceID string
-	Namespace string
+	Name         string
+	ServiceID    string
+	Namespace    string
+	EnableTProxy bool
 }
 
 // NewConnectService returns a container that runs envoy sidecar, launched by
@@ -199,6 +201,26 @@ func NewConnectService(ctx context.Context, sidecarCfg SidecarConfig, serviceBin
 		Env: make(map[string]string),
 	}
 
+	if sidecarCfg.EnableTProxy {
+		req.Entrypoint = []string{"/bin/tproxy-startup.sh"}
+		req.Env["REDIRECT_TRAFFIC_ARGS"] = strings.Join(
+			[]string{
+				"-exclude-inbound-port", fmt.Sprint(internalAdminPort),
+				"-exclude-inbound-port", "8300",
+				"-exclude-inbound-port", "8301",
+				"-exclude-inbound-port", "8302",
+				"-exclude-inbound-port", "8500",
+				"-exclude-inbound-port", "8502",
+				"-exclude-inbound-port", "8600",
+				"-consul-dns-ip", "127.0.0.1",
+				"-consul-dns-port", "8600",
+				"-proxy-id", fmt.Sprintf("%s-sidecar-proxy", sidecarCfg.ServiceID),
+			},
+			" ",
+		)
+		req.CapAdd = append(req.CapAdd, "NET_ADMIN")
+	}
+
 	nodeInfo := node.GetInfo()
 	if nodeInfo.UseTLSForAPI || nodeInfo.UseTLSForGRPC {
 		req.Mounts = append(req.Mounts, testcontainers.ContainerMount{

test/integration/consul-container/libs/service/helpers.go

@@ -27,7 +27,12 @@ type Checks struct {
 }
 
 type SidecarService struct {
-	Port int
+	Port  int
+	Proxy ConnectProxy
+}
+
+type ConnectProxy struct {
+	Mode string
 }
 
 type ServiceOpts struct {
@@ -63,9 +68,10 @@ func createAndRegisterStaticServerAndSidecar(node libcluster.Agent, httpPort int
 		_ = serverService.Terminate()
 	})
 	sidecarCfg := SidecarConfig{
-		Name:      fmt.Sprintf("%s-sidecar", svc.ID),
-		ServiceID: svc.ID,
-		Namespace: svc.Namespace,
+		Name:         fmt.Sprintf("%s-sidecar", svc.ID),
+		ServiceID:    svc.ID,
+		Namespace:    svc.Namespace,
+		EnableTProxy: svc.Proxy != nil && svc.Proxy.Mode == "transparent",
 	}
 	serverConnectProxy, err := NewConnectService(context.Background(), sidecarCfg, []int{svc.Port}, node) // bindPort not used
 	if err != nil {
@@ -102,7 +108,9 @@ func CreateAndRegisterStaticServerAndSidecar(node libcluster.Agent, serviceOpts
 		Port: p,
 		Connect: &api.AgentServiceConnect{
 			SidecarService: &api.AgentServiceRegistration{
-				Proxy: &api.AgentServiceConnectProxyConfig{},
+				Proxy: &api.AgentServiceConnectProxyConfig{
+					Mode: api.ProxyMode(serviceOpts.Connect.Proxy.Mode),
+				},
 			},
 		},
 		Namespace: serviceOpts.Namespace,
@@ -141,15 +149,34 @@ func CreateAndRegisterStaticClientSidecar(
 	node libcluster.Agent,
 	peerName string,
 	localMeshGateway bool,
+	enableTProxy bool,
 ) (*ConnectContainer, error) {
 	// Do some trickery to ensure that partial completion is correctly torn
 	// down, but successful execution is not.
 	var deferClean utils.ResettableDefer
 	defer deferClean.Execute()
 
-	mgwMode := api.MeshGatewayModeRemote
-	if localMeshGateway {
-		mgwMode = api.MeshGatewayModeLocal
+	var proxy *api.AgentServiceConnectProxyConfig
+	if enableTProxy {
+		proxy = &api.AgentServiceConnectProxyConfig{
+			Mode: "transparent",
+		}
+	} else {
+		mgwMode := api.MeshGatewayModeRemote
+		if localMeshGateway {
+			mgwMode = api.MeshGatewayModeLocal
+		}
+		proxy = &api.AgentServiceConnectProxyConfig{
+			Upstreams: []api.Upstream{{
+				DestinationName:  StaticServerServiceName,
+				DestinationPeer:  peerName,
+				LocalBindAddress: "0.0.0.0",
+				LocalBindPort:    libcluster.ServiceUpstreamLocalBindPort,
+				MeshGateway: api.MeshGatewayConfig{
+					Mode: mgwMode,
+				},
+			}},
+		}
 	}
 
 	// Register the static-client service and sidecar first to prevent race with sidecar
@@ -159,17 +186,7 @@ func CreateAndRegisterStaticClientSidecar(
 		Port: 8080,
 		Connect: &api.AgentServiceConnect{
 			SidecarService: &api.AgentServiceRegistration{
-				Proxy: &api.AgentServiceConnectProxyConfig{
-					Upstreams: []api.Upstream{{
-						DestinationName:  StaticServerServiceName,
-						DestinationPeer:  peerName,
-						LocalBindAddress: "0.0.0.0",
-						LocalBindPort:    libcluster.ServiceUpstreamLocalBindPort,
-						MeshGateway: api.MeshGatewayConfig{
-							Mode: mgwMode,
-						},
-					}},
-				},
+				Proxy: proxy,
 			},
 		},
 	}
@@ -180,8 +197,9 @@ func CreateAndRegisterStaticClientSidecar(
 
 	// Create a service and proxy instance
 	sidecarCfg := SidecarConfig{
-		Name:      fmt.Sprintf("%s-sidecar", StaticClientServiceName),
-		ServiceID: StaticClientServiceName,
+		Name:         fmt.Sprintf("%s-sidecar", StaticClientServiceName),
+		ServiceID:    StaticClientServiceName,
+		EnableTProxy: enableTProxy,
 	}
 
 	clientConnectProxy, err := NewConnectService(context.Background(), sidecarCfg, []int{libcluster.ServiceUpstreamLocalBindPort}, node)

test/integration/consul-container/libs/topology/peering_topology.go

@@ -140,7 +140,7 @@ func BasicPeeringTwoClustersSetup(
 
 		// Create a service and proxy instance
 		var err error
-		clientSidecarService, err = libservice.CreateAndRegisterStaticClientSidecar(clientNode, DialingPeerName, true)
+		clientSidecarService, err = libservice.CreateAndRegisterStaticClientSidecar(clientNode, DialingPeerName, true, false)
 		require.NoError(t, err)
 
 		libassert.CatalogServiceExists(t, dialingClient, "static-client-sidecar-proxy", nil)

test/integration/consul-container/libs/topology/service_topology.go

@@ -45,7 +45,7 @@ func CreateServices(t *testing.T, cluster *libcluster.Cluster) (libservice.Servi
 	libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil)
 
 	// Create a client proxy instance with the server as an upstream
-	clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false)
+	clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false, false)
 	require.NoError(t, err)
 
 	libassert.CatalogServiceExists(t, client, fmt.Sprintf("%s-sidecar-proxy", libservice.StaticClientServiceName), nil)

test/integration/consul-container/test/basic/connect_service_test.go

@@ -71,7 +71,7 @@ func createServices(t *testing.T, cluster *libcluster.Cluster) libservice.Servic
 	libassert.CatalogServiceExists(t, client, libservice.StaticServerServiceName, nil)
 
 	// Create a client proxy instance with the server as an upstream
-	clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false)
+	clientConnectProxy, err := libservice.CreateAndRegisterStaticClientSidecar(node, "", false, false)
 	require.NoError(t, err)
 
 	libassert.CatalogServiceExists(t, client, "static-client-sidecar-proxy", nil)