[backport] unix socket for gRPC

Makefile.cli

@@ -12,10 +12,9 @@ cli-release:
 		--rm \
 		--workdir /tetragon \
 		--volume `pwd`:/tetragon docker.io/library/golang:1.19.0-alpine3.16 \
-		sh -c "apk add --no-cache make git && \
-			addgroup -g $(RELEASE_GID) release && \
-			adduser -u $(RELEASE_UID) -D -G release release && \
-			su release -c 'make cli-local-release VERSION=${VERSION}'"
+		sh -c "apk add --no-cache make git setpriv && \
+		        /usr/bin/setpriv --reuid=$(RELEASE_UID) --regid=$(RELEASE_GID) --clear-groups \
+			make GOCACHE=/tmp/cache cli-local-release VERSION=${VERSION}"
 
 cli-local-release: cli-clean
 	set -o errexit; \

cmd/tetragon/addr.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+// splitListenAddr splits the user-provided address a to a proto and an address field to be used
+// with net.Listen.
+//
+// addresses can be:
+//  unix://absolute_path for unix sockets
+//  <host>:<port> for TCP (more specifically, an address that can be passed to net.Listen)
+//
+// Note that the client (tetra) uses https://github.com/grpc/grpc-go/blob/v1.51.0/clientconn.go#L135
+// With the syntax is documented in https://github.com/grpc/grpc/blob/master/doc/naming.md. The
+// server uses net.Listen. And so the two are not compatible because the client expects "ipv4" or
+// "ipv6" for tcp connections.
+// Hence, because we want the same string to work the same way both on the client and the server, we
+// only support the two addresses above.
+func splitListenAddr(arg string) (string, string, error) {
+
+	if strings.HasPrefix(arg, "unix://") {
+		path := strings.TrimPrefix(arg, "unix://")
+		if !filepath.IsAbs(path) {
+			return "", "", fmt.Errorf("path %s (%s) is not absolute", path, arg)
+		}
+		return "unix", path, nil
+	}
+
+	// assume everything else is TCP to support strings such as "localhost:51234" and let
+	// net.Listen figure things out.
+	return "tcp", arg, nil
+}

cmd/tetragon/addr_test.go

@@ -0,0 +1,52 @@
+package main
+
+import "testing"
+
+func TestSplitListenAddr(t *testing.T) {
+	type testCase struct {
+		arg string
+
+		expectedErr bool
+		proto, addr string
+	}
+	testCases := []testCase{
+		{
+			arg:   "unix:///var/run/tetragon/tetragon.sock",
+			proto: "unix",
+			addr:  "/var/run/tetragon/tetragon.sock",
+		}, {
+			arg:   "localhost:51234",
+			proto: "tcp",
+			addr:  "localhost:51234",
+		}, {
+			// NB: expect error on relative paths
+			arg:         "unix://var/run/tetragon/tetragon.sock",
+			expectedErr: true,
+		},
+	}
+
+	for _, c := range testCases {
+		proto, addr, err := splitListenAddr(c.arg)
+		if c.expectedErr {
+			if err == nil {
+				t.Fatalf("expected error for %s", c.arg)
+			}
+			continue
+		}
+
+		if err != nil {
+			t.Fatalf("unexpected error for %s: %s", c.arg, err)
+		}
+
+		if proto != c.proto {
+			t.Fatalf("Proto (%s) did not match expected value (%s) for %s", proto, c.proto, c.arg)
+		}
+
+		if addr != c.addr {
+			t.Fatalf("Addr (%s) did not match expected value (%s) for %s", addr, c.addr, c.arg)
+		}
+
+		t.Logf("case %+v is OK!", c)
+	}
+
+}

cmd/tetragon/main.go

@@ -36,6 +36,7 @@ import (
 	"github.com/cilium/tetragon/pkg/sensors/base"
 	"github.com/cilium/tetragon/pkg/sensors/program"
 	"github.com/cilium/tetragon/pkg/server"
+	"github.com/cilium/tetragon/pkg/unixlisten"
 	"github.com/cilium/tetragon/pkg/version"
 	"github.com/cilium/tetragon/pkg/watcher"
 	"github.com/cilium/tetragon/pkg/watcher/crd"
@@ -359,19 +360,29 @@ func startExporter(ctx context.Context, server *server.Server) error {
 	return nil
 }
 
-func Serve(ctx context.Context, address string, server *server.Server) error {
+func Serve(ctx context.Context, listenAddr string, server *server.Server) error {
 	grpcServer := grpc.NewServer()
 	tetragon.RegisterFineGuidanceSensorsServer(grpcServer, server)
-	go func(address string) {
-		listener, err := net.Listen("tcp", address)
+	proto, addr, err := splitListenAddr(listenAddr)
+	if err != nil {
+		return fmt.Errorf("failed to parse listen address: %w", err)
+	}
+	go func(proto, addr string) {
+		var listener net.Listener
+		var err error
+		if proto == "unix" {
+			listener, err = unixlisten.ListenWithRename(addr, 0660)
+		} else {
+			listener, err = net.Listen(proto, addr)
+		}
 		if err != nil {
-			log.WithError(err).WithField("address", address).Fatal("Failed to start gRPC server")
+			log.WithError(err).WithField("protocol", proto).WithField("address", addr).Fatal("Failed to start gRPC server")
 		}
-		log.WithField("address", address).Info("Starting gRPC server")
+		log.WithField("address", addr).WithField("protocol", proto).Info("Starting gRPC server")
 		if err = grpcServer.Serve(listener); err != nil {
 			log.WithError(err).Error("Failed to close gRPC server")
 		}
-	}(address)
+	}(proto, addr)
 	go func() {
 		<-ctx.Done()
 		grpcServer.Stop()

install/kubernetes/README.md

@@ -64,9 +64,8 @@ Helm chart for Tetragon
 | tetragon.fieldFilters | string | `"{}"` |  |
 | tetragon.gops.address | string | `"localhost"` | The address at which to expose gops. |
 | tetragon.gops.port | int | `8118` | The port at which to expose gops. |
-| tetragon.grpc.address | string | `"localhost"` | The address at which to expose gRPC. Set it to "" to listen on all available interfaces. |
+| tetragon.grpc.address | string | `"localhost:54321"` | The address at which to expose gRPC. Examples: localhost:54321, unix:///var/run/tetragon/tetragon.sock |
 | tetragon.grpc.enabled | bool | `true` | Whether to enable exposing Tetragon gRPC. |
-| tetragon.grpc.port | int | `54321` | The port at which to expose gRPC. |
 | tetragon.image.override | string | `nil` |  |
 | tetragon.image.repository | string | `"quay.io/cilium/tetragon"` |  |
 | tetragon.image.tag | string | `"v0.8.3"` |  |

install/kubernetes/templates/_container_tetragon.tpl

@@ -68,11 +68,15 @@
   resources:
     {{- toYaml . | nindent 4 }}
 {{- end }}
+{{- if .Values.tetragon.grpc.enabled }}
   livenessProbe:
-    exec:
-      command:
-      - tetra
-      - status
+     exec:
+       command:
+       - tetra
+       - status
+       - --server-address
+       - {{ .Values.tetragon.grpc.address }}
+{{- end -}}
 {{- end -}}
 
 {{- define "container.tetragon.init-operator" -}}

install/kubernetes/templates/tetragon_configmap.yaml

@@ -39,7 +39,7 @@ data:
   metrics-server: ""
 {{- end }}
 {{- if .Values.tetragon.grpc.enabled }}
-  server-address: {{ .Values.tetragon.grpc.address }}:{{ .Values.tetragon.grpc.port }}
+  server-address: {{ .Values.tetragon.grpc.address }}
 {{- else }}
 {{- end }}
 {{- if .Values.tetragon.tcpStatsSampleSegs }}

install/kubernetes/values.yaml

@@ -134,10 +134,8 @@ tetragon:
   grpc:
     # -- Whether to enable exposing Tetragon gRPC.
     enabled: true
-    # -- The address at which to expose gRPC. Set it to "" to listen on all available interfaces.
-    address: "localhost"
-    # -- The port at which to expose gRPC.
-    port: 54321
+    # -- The address at which to expose gRPC. Examples: localhost:54321, unix:///var/run/tetragon/tetragon.sock
+    address: "localhost:54321"
   gops:
     # -- The address at which to expose gops.
     address: "localhost"

pkg/unixlisten/unixlisten.go

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Tetragon
+package unixlisten
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+)
+
+// ListenWithRename creates a "unix" listener for the given path and the given mode
+//
+// Go's net.Listen() performs three system calls at once:
+//  - socket, where the file descriptor is created
+//  - bind, where the unix socket file is created
+//  - listen, where the socket can now accept connections
+//
+// Hence, doing a chmod(2) after Listen is racy because a client can connect
+// between the listen(2) and the chmod(2) calls. One solution would be to use
+// umask(2), but this is tricky to do in a multi-threaded program because it
+// affects other files being created from different threads. Also, other
+// threads may change the umask.
+//
+// This function, instead, creates the socket file in a private directory,
+// performs the appropriate chmod and only then  moves the file to its original
+// location. Not sure about other systems, but at least on Linux renaming a
+// unix socket file after the listen seems to work without issues.
+func ListenWithRename(path string, mode os.FileMode) (net.Listener, error) {
+	os.Remove(path)
+
+	baseName := filepath.Base(path)
+	dirName := filepath.Dir(path)
+
+	// Create a temporary directory: MkdirTemp creates the directory with 0700
+	tmpDir, err := os.MkdirTemp(dirName, fmt.Sprintf("%s-dir-*", baseName))
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	tmpPath := filepath.Join(tmpDir, baseName)
+	l, err := net.Listen("unix", tmpPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := os.Chmod(tmpPath, mode); err != nil {
+		return nil, err
+	}
+
+	err = os.Rename(tmpPath, path)
+	if err != nil {
+		return nil, err
+	}
+	return l, nil
+}