[NPM-3663] Include BTF availability in agent flare

cmd/system-probe/api/server.go

@@ -21,6 +21,7 @@ import (
 	"github.com/DataDog/datadog-agent/comp/core/settings"
 	"github.com/DataDog/datadog-agent/comp/core/telemetry"
 	workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
+	"github.com/DataDog/datadog-agent/pkg/ebpf"
 	"github.com/DataDog/datadog-agent/pkg/process/net"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 )
@@ -53,6 +54,8 @@ func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmet
 	mux.Handle("/debug/vars", http.DefaultServeMux)
 	mux.Handle("/telemetry", telemetry.Handler())
 
+	mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo)
+
 	go func() {
 		err = http.Serve(conn.GetListener(), mux)
 		if err != nil && !errors.Is(err, http.ErrServerClosed) {

pkg/ebpf/btf.go

@@ -94,11 +94,34 @@ type returnBTF struct {
 	moduleLoadFunc kernelModuleBTFLoadFunc
 }
 
+type BTFResultMetadata struct {
+	// numLoadAttempts is how many times the loader has been invoked (doesn't include cached requests)
+	numLoadAttempts int
+	// loaderUsed the name of the loader that was used to get the BTF data
+	loaderUsed string
+	// filepathUsed is the filepath it last tried to load BTF files from (only for loadUser and loadEmbedded)
+	filepathUsed string
+	// tarballUsed is the filepath for the tarball it tried to extract BTF from (only for loadEmbedded)
+	tarballUsed string
+}
+
+func (d BTFResultMetadata) String() string {
+	res := fmt.Sprintf("numLoadAttempts: %d\nloaderUsed: %s", d.numLoadAttempts, d.loaderUsed)
+	if d.filepathUsed != "" {
+		res += fmt.Sprintf("\nfilepathUsed: %s", d.filepathUsed)
+	}
+	if d.tarballUsed != "" {
+		res += fmt.Sprintf("\ntarballUsed: %s", d.tarballUsed)
+	}
+	return res
+}
+
 type orderedBTFLoader struct {
 	userBTFPath string
 	embeddedDir string
 
 	result         ebpftelemetry.BTFResult
+	resultMetadata BTFResultMetadata
 	loadFunc       funcs.CachedFunc[returnBTF]
 	delayedFlusher *time.Timer
 }
@@ -132,6 +155,8 @@ func (b *orderedBTFLoader) Flush() {
 }
 
 func (b *orderedBTFLoader) get() (*returnBTF, error) {
+	b.resultMetadata.numLoadAttempts++
+
 	loaders := []struct {
 		result ebpftelemetry.BTFResult
 		loader btfLoaderFunc
@@ -157,6 +182,7 @@ func (b *orderedBTFLoader) get() (*returnBTF, error) {
 		if ret != nil {
 			log.Debugf("successfully loaded BTF from %s", l.desc)
 			b.result = l.result
+			b.resultMetadata.loaderUsed = l.desc
 			return ret, nil
 		}
 	}
@@ -168,7 +194,11 @@ func (b *orderedBTFLoader) loadKernel() (*returnBTF, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &returnBTF{vmlinux: spec, moduleLoadFunc: nil}, nil
+	b.resultMetadata.filepathUsed = "<unknown, internal to cilium ebpf>"
+	return &returnBTF{
+		vmlinux:        spec,
+		moduleLoadFunc: nil,
+	}, nil
 }
 
 func (b *orderedBTFLoader) loadUser() (*returnBTF, error) {
@@ -179,7 +209,11 @@ func (b *orderedBTFLoader) loadUser() (*returnBTF, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &returnBTF{vmlinux: spec, moduleLoadFunc: nil}, nil
+	b.resultMetadata.filepathUsed = b.userBTFPath
+	return &returnBTF{
+		vmlinux:        spec,
+		moduleLoadFunc: nil,
+	}, nil
 }
 
 func (b *orderedBTFLoader) checkForMinimizedBTF(extractDir string) (*returnBTF, error) {
@@ -191,8 +225,11 @@ func (b *orderedBTFLoader) checkForMinimizedBTF(extractDir string) (*returnBTF,
 		if err != nil {
 			return nil, err
 		}
-		// no module load function for minimized single file BTF
-		return &returnBTF{vmlinux: spec, moduleLoadFunc: nil}, nil
+		b.resultMetadata.filepathUsed = btfRelativePath
+		return &returnBTF{
+			vmlinux:        spec,
+			moduleLoadFunc: nil,
+		}, nil
 	}
 	return nil, nil
 }
@@ -210,7 +247,11 @@ func (b *orderedBTFLoader) checkForUnminimizedBTF(extractDir string) (*returnBTF
 		if err != nil {
 			return nil, err
 		}
-		return &returnBTF{vmlinux: spec, moduleLoadFunc: modLoadFunc}, nil
+		b.resultMetadata.filepathUsed = extractedBtfPath
+		return &returnBTF{
+			vmlinux:        spec,
+			moduleLoadFunc: modLoadFunc,
+		}, nil
 	}
 	return nil, nil
 }
@@ -229,6 +270,7 @@ func (b *orderedBTFLoader) checkforBTF(extractDir string) (*returnBTF, error) {
 
 func (b *orderedBTFLoader) loadEmbedded() (*returnBTF, error) {
 	btfRelativeTarballFilename, err := b.embeddedPath()
+	b.resultMetadata.filepathUsed = btfRelativeTarballFilename
 	if err != nil {
 		return nil, err
 	}
@@ -251,7 +293,7 @@ func (b *orderedBTFLoader) loadEmbedded() (*returnBTF, error) {
 	// of BTFs as a whole is also compressed.
 	// This means that we'll need to first extract the specific BTF which  we're looking for from the collection
 	// tarball, and then unarchive it.
-	btfTarball := filepath.Join(b.embeddedDir, btfRelativeTarballFilename)
+	btfTarball := filepath.Join(b.embeddedDir)
 	if _, err := os.Stat(btfTarball); errors.Is(err, fs.ErrNotExist) {
 		collectionTarball := filepath.Join(b.embeddedDir, btfArchiveName)
 		if err := archive.TarXZExtractFile(collectionTarball, btfRelativeTarballFilename, b.embeddedDir); err != nil {

pkg/ebpf/co_re.go

@@ -42,6 +42,18 @@ func LoadCOREAsset(filename string, startFn func(bytecode.AssetReader, manager.O
 	return loader.loadCOREAsset(filename, startFn)
 }
 
+// GetBTFLoaderInfo Returns where the ebpf BTF files were sourced from
+func GetBTFLoaderInfo() (string, error) {
+	loader, err := coreLoader(NewConfig())
+	if err != nil {
+		return "", err
+	}
+
+	metadataStr := loader.btfLoader.resultMetadata.String()
+	infoStr := fmt.Sprintf("btfLoader result: %d\n%s", loader.btfLoader.result, metadataStr)
+	return infoStr, nil
+}
+
 func (c *coreAssetLoader) loadCOREAsset(filename string, startFn func(bytecode.AssetReader, manager.Options) error) error {
 	var result ebpftelemetry.COREResult
 	base := strings.TrimSuffix(filename, path.Ext(filename))

pkg/ebpf/co_re_unsupported.go

@@ -0,0 +1,15 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2024-present Datadog, Inc.
+
+//go:build !linux_bpf
+
+package ebpf
+
+import "errors"
+
+// GetBTFLoaderInfo is not supported without linux_bpf
+func GetBTFLoaderInfo() (string, error) {
+	return "", errors.New("BTF is not supported")
+}

pkg/ebpf/debug_handlers.go

@@ -0,0 +1,26 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2024-present Datadog, Inc.
+
+package ebpf
+
+import (
+	"io"
+	"net/http"
+
+	"github.com/DataDog/datadog-agent/pkg/util/log"
+)
+
+// HandleBTFLoaderInfo responds with where the system-probe found BTF data (and
+// if it was in a pre-bundled tarball, where within that tarball it came from)
+func HandleBTFLoaderInfo(w http.ResponseWriter, _ *http.Request) {
+	info, err := GetBTFLoaderInfo()
+	if err != nil {
+		log.Errorf("unable to get ebpf_btf_loader info: %s", err)
+		w.WriteHeader(500)
+		return
+	}
+
+	io.WriteString(w, info)
+}

pkg/flare/archive.go

@@ -152,6 +152,7 @@ func provideSystemProbe(fb flaretypes.FlareBuilder) error {
 		fb.AddFileFromFunc(filepath.Join("system-probe", "system_probe_telemetry.log"), getSystemProbeTelemetry) // nolint:errcheck
 		fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_cached.log"), getSystemProbeConntrackCached) // nolint:errcheck
 		fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_host.log"), getSystemProbeConntrackHost)     // nolint:errcheck
+		fb.AddFileFromFunc(filepath.Join("system-probe", "ebpf_btf_loader.log"), getSystemProbeBTFLoaderInfo)    // nolint:errcheck
 	}
 	return nil
 }
@@ -266,6 +267,9 @@ func getSystemProbeConntrackCached() ([]byte, error) {
 func getSystemProbeConntrackHost() ([]byte, error) {
 	return sysprobe.GetSystemProbeConntrackHost(getSystemProbeSocketPath())
 }
+func getSystemProbeBTFLoaderInfo() ([]byte, error) {
+	return sysprobe.GetSystemProbeBTFLoaderInfo(getSystemProbeSocketPath())
+}
 
 // getProcessAgentFullConfig fetches process-agent runtime config as YAML and returns it to be added to  process_agent_runtime_config_dump.yaml
 func getProcessAgentFullConfig() ([]byte, error) {

pkg/flare/sysprobe/archive_linux.go

@@ -27,3 +27,12 @@ func GetSystemProbeConntrackHost(socketPath string) ([]byte, error) {
 	}
 	return probeUtil.GetConnTrackHost()
 }
+
+// GetSystemProbeBTFLoaderInfo queries ebpf_btf_loader_info which gets where the BTF data came from
+func GetSystemProbeBTFLoaderInfo(socketPath string) ([]byte, error) {
+	probeUtil, err := net.GetRemoteSystemProbeUtil(socketPath)
+	if err != nil {
+		return nil, err
+	}
+	return probeUtil.GetBTFLoaderInfo()
+}

pkg/flare/sysprobe/archive_nolinux.go

@@ -9,12 +9,17 @@ package sysprobe
 
 import "errors"
 
-// GetSystemProbeConntrackCached is not supported without the process agent on linux
+// GetSystemProbeConntrackCached is a stub designed to prevent builds without the process agent from importing pkg/process/net
 func GetSystemProbeConntrackCached(_ string) ([]byte, error) {
 	return nil, errors.New("GetSystemProbeConntrackCached is not supported without the process agent on linux")
 }
 
-// GetSystemProbeConntrackHost is not supported without the process agent on linux
+// GetSystemProbeConntrackHost is a stub designed to prevent builds without the process agent from importing pkg/process/net
 func GetSystemProbeConntrackHost(_ string) ([]byte, error) {
 	return nil, errors.New("GetSystemProbeConntrackHost is not supported without the process agent on linux")
 }
+
+// GetSystemProbeBTFLoaderInfo is a stub designed to prevent builds without the process agent from importing pkg/process/net
+func GetSystemProbeBTFLoaderInfo(_ string) ([]byte, error) {
+	return nil, errors.New("GetSystemProbeBTFLoaderInfo is not supported without the process agent on linux")
+}

pkg/process/net/common_linux.go

@@ -10,6 +10,7 @@ package net
 import (
 	"context"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"os"
@@ -32,6 +33,7 @@ const (
 	telemetryURL         = "http://unix/telemetry"
 	conntrackCachedURL   = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/cached"
 	conntrackHostURL     = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host"
+	ebpfBTFLoaderURL     = "http://unix/ebpf_btf_loader_info"
 	netType              = "unix"
 )
 
@@ -83,3 +85,28 @@ func newSystemProbe(path string) *RemoteSysProbeUtil {
 		},
 	}
 }
+
+// GetBTFLoaderInfo queries ebpf_btf_loader_info to get information about where btf files came from
+func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) {
+	req, err := http.NewRequest(http.MethodGet, ebpfBTFLoaderURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := r.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf(`GetEbpfBtfInfo got non-success status code: path %s, url: %s, status_code: %d, response: "%s"`, r.path, req.URL, resp.StatusCode, data)
+	}
+
+	return data, nil
+}

pkg/process/net/common_unsupported.go

@@ -89,6 +89,9 @@ func (r *RemoteSysProbeUtil) GetConnTrackCached() ([]byte, error) { return nil,
 // GetConnTrackHost is not supported
 func (r *RemoteSysProbeUtil) GetConnTrackHost() ([]byte, error) { return nil, ErrNotImplemented }
 
+// GetBTFLoaderInfo is not supported
+func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { return nil, ErrNotImplemented }
+
 func (r *RemoteSysProbeUtil) GetDiscoveryServices() (*discoverymodel.ServicesResponse, error) {
 	return nil, ErrNotImplemented
 }
@@ -104,3 +107,8 @@ func (r *RemoteSysProbeUtil) GetPing(clientID string, host string, count int, in
 func (r *RemoteSysProbeUtil) GetTraceroute(clientID string, host string, port uint16, protocol nppayload.Protocol, maxTTL uint8, timeout time.Duration) ([]byte, error) {
 	return nil, ErrNotImplemented
 }
+
+// GetBTFLoaderInfo is not supported
+func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) {
+	return nil, ErrNotImplemented
+}

pkg/process/net/common_windows.go

@@ -38,6 +38,8 @@ const (
 	conntrackCachedURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/cached"
 	// conntrackHostURL is not used on Windows, the value is added to avoid a compilation error
 	conntrackHostURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host"
+	// ebpfBtfLoaderURL is not used on Windows, the value is added to avoid a compilation error
+	ebpfBTFLoaderURL = ""
 
 	// SystemProbePipeName is the production named pipe for system probe
 	SystemProbePipeName = `\\.\pipe\dd_system_probe`
@@ -103,3 +105,8 @@ func newSystemProbe(path string) *RemoteSysProbeUtil {
 		},
 	}
 }
+
+// GetBTFLoaderInfo is not implemented on windows
+func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) {
+	return nil, errors.New("GetBTFLoaderInfo is not supported on windows")
+}