Add USM debugger

pkg/network/usm/debugger/README.md

@@ -0,0 +1,30 @@
+# USM debugger
+
+A minimal, self-contained build of USM for faster iterations of debugging eBPF
+code on remote machines.
+
+Prepare the build for the target architectures you need (the `foo` KMT stack
+does not need to exist):
+
+```
+inv -e kmt.prepare system-probe --compile-only --stack=foo --arch=x86_64
+inv -e kmt.prepare system-probe --compile-only --stack=foo --arch=arm64
+```
+
+Build the binary with one of the following commands for the architecture of
+your target host:
+
+```
+inv -e system-probe.build-usm-debugger --arch=x86_64
+inv -e system-probe.build-usm-debugger --arch=arm64
+```
+
+Copy the `bin/usm-debugger` to the `system-probe` container in the
+`datadog-agent` pod on your target host.  Open a shell in that container and
+execute the binary.
+
+The eBPF programs are always built with debug logs enabled so you can view them
+with `/sys/kernel/tracing/trace_pipe`.
+
+If you need to change the system-probe config, edit `cmd/usm_debugger.go` and
+rebuild.

pkg/network/usm/debugger/cmd/.gitignore

@@ -0,0 +1 @@
+*.o

pkg/network/usm/debugger/cmd/ebpf_bytecode.go

@@ -0,0 +1,62 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2024-present Datadog, Inc.
+
+//go:build usm_debugger
+
+package main
+
+import (
+	_ "embed"
+	"os"
+	"path"
+
+	"github.com/DataDog/datadog-agent/pkg/util/log"
+)
+
+// The code beloe is essentially responsible for embedding the CO-RE artifacts
+// during compilation time and writing them to a temporary folder during
+// runtime, so they can be loaded by the `usm.Monitor` as regular compilation
+// assets.
+
+//go:embed usm-debug.o
+var usmProgram []byte
+
+//go:embed shared-libraries-debug.o
+var sharedLibrariesProgram []byte
+
+func setupBytecode() func() {
+	type program struct {
+		filePath string
+		bytecode []byte
+	}
+
+	var (
+		bytecodeDir = os.TempDir()
+		coreDir     = path.Join(bytecodeDir, "co-re")
+	)
+
+	os.Setenv("DD_SYSTEM_PROBE_BPF_DIR", bytecodeDir)
+	err := os.MkdirAll(coreDir, os.ModePerm)
+	checkError(err)
+
+	programs := []program{
+		{path.Join(coreDir, "usm-debug.o"), usmProgram},
+		{path.Join(coreDir, "shared-libraries-debug.o"), sharedLibrariesProgram},
+	}
+
+	for _, p := range programs {
+		f, err := os.Create(p.filePath)
+		checkError(err)
+		_, err = f.Write(p.bytecode)
+		checkError(err)
+		log.Debugf("writing ebpf bytecode to %s", p.filePath)
+	}
+
+	return func() {
+		for _, p := range programs {
+			os.Remove(p.filePath)
+		}
+	}
+}

pkg/network/usm/debugger/cmd/usm_debugger.go

@@ -0,0 +1,89 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2024-present Datadog, Inc.
+
+//go:build usm_debugger
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/DataDog/datadog-agent/cmd/system-probe/config"
+	pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
+	networkconfig "github.com/DataDog/datadog-agent/pkg/network/config"
+	"github.com/DataDog/datadog-agent/pkg/network/usm"
+	pkglogsetup "github.com/DataDog/datadog-agent/pkg/util/log/setup"
+)
+
+func main() {
+	err := pkglogsetup.SetupLogger(
+		"usm-debugger",
+		"debug",
+		"",
+		pkglogsetup.GetSyslogURI(pkgconfigsetup.Datadog()),
+		false,
+		true,
+		false,
+		pkgconfigsetup.Datadog(),
+	)
+	checkError(err)
+
+	cleanupFn := setupBytecode()
+	defer cleanupFn()
+
+	monitor, err := usm.NewMonitor(getConfiguration(), nil)
+	checkError(err)
+
+	err = monitor.Start()
+	checkError(err)
+
+	go func() {
+		t := time.NewTicker(10 * time.Second)
+		for range t.C {
+			_ = monitor.GetProtocolStats()
+		}
+	}()
+
+	defer monitor.Stop()
+	done := make(chan os.Signal, 1)
+	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
+	<-done
+}
+
+func checkError(err error) {
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s", err)
+		os.Exit(-1)
+	}
+}
+
+func getConfiguration() *networkconfig.Config {
+	// done for the purposes of initializing the configuration values
+	_, err := config.New("", "")
+	checkError(err)
+
+	c := networkconfig.New()
+
+	// run debug version of the eBPF program
+	c.BPFDebug = true
+	c.EnableUSMEventStream = false
+
+	// don't buffer data in userspace
+	// this is to ensure that we won't inadvertently trigger an OOM kill
+	// by enabling the debugger inside a system-probe container.
+	c.MaxHTTPStatsBuffered = 0
+	c.MaxKafkaStatsBuffered = 0
+
+	// make sure we use the CO-RE compilation artifact embedded
+	// in this build (see `ebpf_bytecode.go`)
+	c.EnableCORE = true
+	c.EnableRuntimeCompiler = false
+
+	return c
+}

tasks/kmt.py

@@ -738,17 +738,18 @@ def prepare(
         )
 
     if ci:
-        domains = None
         stack = "ci"
         return _prepare(ctx, stack, component, arch_obj, packages, verbose, ci, compile_only)
 
-    if alien_vms is not None:
-        err_msg = f"no alient VMs discovered from provided profile {alien_vms}."
-    else:
-        err_msg = f"no vms found from list {vms}. Run `inv -e kmt.status` to see all VMs in current stack"
-    stack = get_kmt_or_alien_stack(ctx, stack, vms, alien_vms)
-    domains = get_target_domains(ctx, stack, ssh_key, arch_obj, vms, alien_vms)
-    assert len(domains) > 0, err_msg
+    domains = None
+    if not compile_only:
+        if alien_vms is not None:
+            err_msg = f"no alient VMs discovered from provided profile {alien_vms}."
+        else:
+            err_msg = f"no vms found from list {vms}. Run `inv -e kmt.status` to see all VMs in current stack"
+        stack = get_kmt_or_alien_stack(ctx, stack, vms, alien_vms)
+        domains = get_target_domains(ctx, stack, ssh_key, arch_obj, vms, alien_vms)
+        assert len(domains) > 0, err_msg
 
     _prepare(ctx, stack, component, arch_obj, packages, verbose, ci, compile_only, domains=domains)
 

tasks/system_probe.py

@@ -2024,3 +2024,40 @@ def copy_ebpf_and_related_files(ctx: Context, target: Path | str, arch: Arch | N
     ctx.run(f"chmod 0444 {target}/*.o {target}/*.c {target}/co-re/*.o")
     ctx.run(f"cp /opt/datadog-agent/embedded/bin/clang-bpf {target}")
     ctx.run(f"cp /opt/datadog-agent/embedded/bin/llc-bpf {target}")
+
+
+@task
+def build_usm_debugger(
+    ctx,
+    arch: str = CURRENT_ARCH,
+    strip_binary=False,
+):
+    build_object_files(ctx)
+
+    build_dir = os.path.join("pkg", "ebpf", "bytecode", "build", arch)
+
+    # copy compilation artifacts to the debugger root directory for the purposes of embedding
+    usm_programs = [
+        os.path.join(build_dir, "co-re", "usm-debug.o"),
+        os.path.join(build_dir, "co-re", "shared-libraries-debug.o"),
+    ]
+
+    embedded_dir = os.path.join(".", "pkg", "network", "usm", "debugger", "cmd")
+
+    for p in usm_programs:
+        print(p)
+        shutil.copy(p, embedded_dir)
+
+    arch_obj = Arch.from_str(arch)
+    ldflags, gcflags, env = get_build_flags(ctx, arch=arch_obj)
+
+    cmd = 'go build -tags="linux_bpf,usm_debugger" -o bin/usm-debugger -ldflags="{ldflags}" ./pkg/network/usm/debugger/cmd/'
+
+    if strip_binary:
+        ldflags += ' -s -w'
+
+    args = {
+        "ldflags": ldflags,
+    }
+
+    ctx.run(cmd.format(**args), env=env)