From 195792e1195d1cd9028069f1001799a4147d453b Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Tue, 22 Oct 2024 13:12:27 -0700 Subject: [PATCH 01/19] add btf info --- pkg/ebpf/btf.go | 57 +++++++++++++++++++---------- pkg/ebpf/co_re.go | 14 +++++++ pkg/flare/sysprobe/archive_linux.go | 2 + 3 files changed, 54 insertions(+), 19 deletions(-) diff --git a/pkg/ebpf/btf.go b/pkg/ebpf/btf.go index 0173cd5d91d3a..7cf9b57416990 100644 --- a/pkg/ebpf/btf.go +++ b/pkg/ebpf/btf.go @@ -89,10 +89,21 @@ func FlushBTF() { type kernelModuleBTFLoadFunc func(string) (*btf.Spec, error) -type returnBTF struct { +type btfSpecData struct { + // sourceFilepath documents where we got this BTF data from + sourceFilepath string vmlinux *btf.Spec moduleLoadFunc kernelModuleBTFLoadFunc } +type returnBTF struct { + // loaderDesc the name of the loader that was used to get the BTF data + loaderDesc string + btfSpecData +} + +func (r *returnBTF) String() string { + return fmt.Sprintf("loader: %s\nsource file: %s\n", r.loaderDesc, r.sourceFilepath) +} type orderedBTFLoader struct { userBTFPath string @@ -114,7 +125,7 @@ func initBTFLoader(cfg *Config) *orderedBTFLoader { return btfLoader } -type btfLoaderFunc func() (*returnBTF, error) +type btfLoaderFunc func() (*btfSpecData, error) // Get returns BTF for the running kernel func (b *orderedBTFLoader) Get() (*returnBTF, ebpftelemetry.COREResult, error) { @@ -142,7 +153,7 @@ func (b *orderedBTFLoader) get() (*returnBTF, error) { {ebpftelemetry.SuccessEmbeddedBTF, b.loadEmbedded, "embedded collection"}, } var err error - var ret *returnBTF + var ret *btfSpecData for _, l := range loaders { log.Debugf("attempting BTF load from %s", l.desc) ret, err = l.loader() @@ -157,21 +168,26 @@ func (b *orderedBTFLoader) get() (*returnBTF, error) { if ret != nil { log.Debugf("successfully loaded BTF from %s", l.desc) b.result = l.result - return ret, nil + return &returnBTF{ + loaderDesc: l.desc, + btfSpecData: *ret, + }, nil } } return nil, err } -func (b *orderedBTFLoader) loadKernel() (*returnBTF, error) { +func (b *orderedBTFLoader) loadKernel() (*btfSpecData, error) { spec, err := GetKernelSpec() if err != nil { return nil, err } - return &returnBTF{vmlinux: spec, moduleLoadFunc: nil}, nil + return &btfSpecData{ + sourceFilepath: "", + vmlinux: spec, moduleLoadFunc: nil}, nil } -func (b *orderedBTFLoader) loadUser() (*returnBTF, error) { +func (b *orderedBTFLoader) loadUser() (*btfSpecData, error) { if b.userBTFPath == "" { return nil, nil } @@ -179,10 +195,14 @@ func (b *orderedBTFLoader) loadUser() (*returnBTF, error) { if err != nil { return nil, err } - return &returnBTF{vmlinux: spec, moduleLoadFunc: nil}, nil + return &btfSpecData{ + sourceFilepath: b.userBTFPath, + vmlinux: spec, + moduleLoadFunc: nil, + }, nil } -func (b *orderedBTFLoader) checkForMinimizedBTF(extractDir string) (*returnBTF, error) { +func (b *orderedBTFLoader) checkForMinimizedBTF(extractDir, sourceTarball string) (*btfSpecData, error) { // //.btf btfRelativePath := filepath.Join(extractDir, filepath.Base(extractDir)+".btf") extractedBtfPath := filepath.Join(b.embeddedDir, btfRelativePath) @@ -191,13 +211,12 @@ func (b *orderedBTFLoader) checkForMinimizedBTF(extractDir string) (*returnBTF, if err != nil { return nil, err } - // no module load function for minimized single file BTF - return &returnBTF{vmlinux: spec, moduleLoadFunc: nil}, nil + return &btfSpecData{sourceFilepath: sourceTarball, vmlinux: spec, moduleLoadFunc: nil}, nil } return nil, nil } -func (b *orderedBTFLoader) checkForUnminimizedBTF(extractDir string) (*returnBTF, error) { +func (b *orderedBTFLoader) checkForUnminimizedBTF(extractDir, sourceTarball string) (*btfSpecData, error) { absExtractDir := filepath.Join(b.embeddedDir, extractDir) modLoadFunc := func(mod string) (*btf.Spec, error) { b.delayedFlusher.Reset(btfFlushDelay) @@ -210,24 +229,24 @@ func (b *orderedBTFLoader) checkForUnminimizedBTF(extractDir string) (*returnBTF if err != nil { return nil, err } - return &returnBTF{vmlinux: spec, moduleLoadFunc: modLoadFunc}, nil + return &btfSpecData{sourceFilepath: sourceTarball, vmlinux: spec, moduleLoadFunc: modLoadFunc}, nil } return nil, nil } -func (b *orderedBTFLoader) checkforBTF(extractDir string) (*returnBTF, error) { - ret, err := b.checkForMinimizedBTF(extractDir) +func (b *orderedBTFLoader) checkforBTF(extractDir, sourceTarball string) (*btfSpecData, error) { + ret, err := b.checkForMinimizedBTF(extractDir, sourceTarball) if err != nil || ret != nil { return ret, err } - ret, err = b.checkForUnminimizedBTF(extractDir) + ret, err = b.checkForUnminimizedBTF(extractDir, sourceTarball) if err != nil || ret != nil { return ret, err } return nil, nil } -func (b *orderedBTFLoader) loadEmbedded() (*returnBTF, error) { +func (b *orderedBTFLoader) loadEmbedded() (*btfSpecData, error) { btfRelativeTarballFilename, err := b.embeddedPath() if err != nil { return nil, err @@ -241,7 +260,7 @@ func (b *orderedBTFLoader) loadEmbedded() (*returnBTF, error) { absExtractDir := filepath.Join(b.embeddedDir, extractDir) // If we've previously extracted the BTF file in question, we can just load it - ret, err := b.checkforBTF(extractDir) + ret, err := b.checkforBTF(extractDir, btfRelativeTarballFilename) if err != nil || ret != nil { return ret, err } @@ -262,7 +281,7 @@ func (b *orderedBTFLoader) loadEmbedded() (*returnBTF, error) { if err := archive.TarXZExtractAll(btfTarball, absExtractDir); err != nil { return nil, fmt.Errorf("extract kernel BTF from tarball: %w", err) } - ret, err = b.checkforBTF(extractDir) + ret, err = b.checkforBTF(extractDir, btfRelativeTarballFilename) if err != nil || ret != nil { return ret, err } diff --git a/pkg/ebpf/co_re.go b/pkg/ebpf/co_re.go index 9742aaf7e149a..d725d263a2862 100644 --- a/pkg/ebpf/co_re.go +++ b/pkg/ebpf/co_re.go @@ -41,6 +41,20 @@ func LoadCOREAsset(filename string, startFn func(bytecode.AssetReader, manager.O } return loader.loadCOREAsset(filename, startFn) } +func GetBtfLoaderInfo() (string, error) { + loader, err := coreLoader(NewConfig()) + if err != nil { + return "", err + } + res, _, err := loader.btfLoader.Get() + if err != nil { + return "", err + } + if res == nil { + return "", errors.New("GetBtfLoaderInfo expected result from btfLoader") + } + return res.String(), nil +} func (c *coreAssetLoader) loadCOREAsset(filename string, startFn func(bytecode.AssetReader, manager.Options) error) error { var result ebpftelemetry.COREResult diff --git a/pkg/flare/sysprobe/archive_linux.go b/pkg/flare/sysprobe/archive_linux.go index 72f33f11f253e..20f7143a55380 100644 --- a/pkg/flare/sysprobe/archive_linux.go +++ b/pkg/flare/sysprobe/archive_linux.go @@ -27,3 +27,5 @@ func GetSystemProbeConntrackHost(socketPath string) ([]byte, error) { } return probeUtil.GetConnTrackHost() } + +func GetSystemProbeBtfInfo(socketPath string) ([]byte, error) {} From 9e7274d73a533a4485f8d3d5e083867777124676 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Wed, 23 Oct 2024 10:51:57 -0700 Subject: [PATCH 02/19] add queries for btf info --- cmd/system-probe/modules/network_tracer.go | 11 ++++++++ .../modules/network_tracer_linux_bpf.go | 15 +++++++++++ .../modules/network_tracer_nolinux_bpf.go | 15 +++++++++++ pkg/ebpf/co_re.go | 6 +++-- pkg/flare/archive.go | 4 +++ pkg/flare/sysprobe/archive_linux.go | 9 ++++++- pkg/process/net/common.go | 25 +++++++++++++++++++ pkg/process/net/common_linux.go | 1 + pkg/process/net/common_unsupported.go | 3 +++ pkg/process/net/common_windows.go | 2 ++ pkg/process/net/shared.go | 1 + 11 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 cmd/system-probe/modules/network_tracer_linux_bpf.go create mode 100644 cmd/system-probe/modules/network_tracer_nolinux_bpf.go diff --git a/cmd/system-probe/modules/network_tracer.go b/cmd/system-probe/modules/network_tracer.go index b605a3f0e99c3..bbd7de378bfa8 100644 --- a/cmd/system-probe/modules/network_tracer.go +++ b/cmd/system-probe/modules/network_tracer.go @@ -292,6 +292,17 @@ func (nt *networkTracer) Register(httpMux *module.Router) error { utils.WriteAsJSON(w, cache) }) + httpMux.HandleFunc("/debug/ebpf_btf_loader", func(w http.ResponseWriter, _ *http.Request) { + info, err := GetBTFLoaderInfo() + if err != nil { + log.Errorf("unable to get ebpf_btf_loader info: %s", err) + w.WriteHeader(500) + return + } + + io.WriteString(w, info) + }) + httpMux.HandleFunc("/debug/usm_telemetry", telemetry.Handler) httpMux.HandleFunc("/debug/usm/traced_programs", usm.TracedProgramsEndpoint) httpMux.HandleFunc("/debug/usm/blocked_processes", usm.BlockedPathIDEndpoint) diff --git a/cmd/system-probe/modules/network_tracer_linux_bpf.go b/cmd/system-probe/modules/network_tracer_linux_bpf.go new file mode 100644 index 0000000000000..8a46118856eee --- /dev/null +++ b/cmd/system-probe/modules/network_tracer_linux_bpf.go @@ -0,0 +1,15 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux_bpf + +package modules + +import "github.com/DataDog/datadog-agent/pkg/ebpf" + +// GetBTFLoaderInfo Returns where the ebpf BTF files were sourced from, only on linux +func GetBTFLoaderInfo() (string, error) { + return ebpf.GetBTFLoaderInfo() +} diff --git a/cmd/system-probe/modules/network_tracer_nolinux_bpf.go b/cmd/system-probe/modules/network_tracer_nolinux_bpf.go new file mode 100644 index 0000000000000..d225770473ebb --- /dev/null +++ b/cmd/system-probe/modules/network_tracer_nolinux_bpf.go @@ -0,0 +1,15 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build !linux_bpf + +package modules + +import "errors" + +// GetBTFLoaderInfo is not supported on windows +func GetBTFLoaderInfo() (string, error) { + return "", errors.New("GetBTFLoaderInfo not supported without linux_bpf") +} diff --git a/pkg/ebpf/co_re.go b/pkg/ebpf/co_re.go index d725d263a2862..0e77242c2b1fe 100644 --- a/pkg/ebpf/co_re.go +++ b/pkg/ebpf/co_re.go @@ -41,7 +41,9 @@ func LoadCOREAsset(filename string, startFn func(bytecode.AssetReader, manager.O } return loader.loadCOREAsset(filename, startFn) } -func GetBtfLoaderInfo() (string, error) { + +// Returns where the ebpf BTF files were sourced from +func GetBTFLoaderInfo() (string, error) { loader, err := coreLoader(NewConfig()) if err != nil { return "", err @@ -51,7 +53,7 @@ func GetBtfLoaderInfo() (string, error) { return "", err } if res == nil { - return "", errors.New("GetBtfLoaderInfo expected result from btfLoader") + return "", errors.New("GetBTFLoaderInfo expected result from btfLoader") } return res.String(), nil } diff --git a/pkg/flare/archive.go b/pkg/flare/archive.go index 5e1807a0aa3c2..76f5d3d6bc000 100644 --- a/pkg/flare/archive.go +++ b/pkg/flare/archive.go @@ -152,6 +152,7 @@ func provideSystemProbe(fb flaretypes.FlareBuilder) error { fb.AddFileFromFunc(filepath.Join("system-probe", "system_probe_telemetry.log"), getSystemProbeTelemetry) // nolint:errcheck fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_cached.log"), getSystemProbeConntrackCached) // nolint:errcheck fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_host.log"), getSystemProbeConntrackHost) // nolint:errcheck + fb.AddFileFromFunc(filepath.Join("system-probe", "ebpf_btf_loader.log"), getSystemProbeBTFLoaderInfo) // nolint:errcheck } return nil } @@ -266,6 +267,9 @@ func getSystemProbeConntrackCached() ([]byte, error) { func getSystemProbeConntrackHost() ([]byte, error) { return sysprobe.GetSystemProbeConntrackHost(getSystemProbeSocketPath()) } +func getSystemProbeBTFLoaderInfo() ([]byte, error) { + return sysprobe.GetSystemProbeBTFLoaderInfo(getSystemProbeSocketPath()) +} // getProcessAgentFullConfig fetches process-agent runtime config as YAML and returns it to be added to process_agent_runtime_config_dump.yaml func getProcessAgentFullConfig() ([]byte, error) { diff --git a/pkg/flare/sysprobe/archive_linux.go b/pkg/flare/sysprobe/archive_linux.go index 20f7143a55380..635f9d9c4a854 100644 --- a/pkg/flare/sysprobe/archive_linux.go +++ b/pkg/flare/sysprobe/archive_linux.go @@ -28,4 +28,11 @@ func GetSystemProbeConntrackHost(socketPath string) ([]byte, error) { return probeUtil.GetConnTrackHost() } -func GetSystemProbeBtfInfo(socketPath string) ([]byte, error) {} +// GetSystemProbeBTFLoaderInfo queries ebpf_btf_loader which gets where the BTF data came from +func GetSystemProbeBTFLoaderInfo(socketPath string) ([]byte, error) { + probeUtil, err := net.GetRemoteSystemProbeUtil(socketPath) + if err != nil { + return nil, err + } + return probeUtil.GetBTFLoaderInfo() +} diff --git a/pkg/process/net/common.go b/pkg/process/net/common.go index 1e5e695154c18..7cfabf7947cee 100644 --- a/pkg/process/net/common.go +++ b/pkg/process/net/common.go @@ -476,6 +476,31 @@ func (r *RemoteSysProbeUtil) GetConnTrackHost() ([]byte, error) { return data, nil } +// GetBTFLoaderInfo queries ebpf_btf_loader to get information about where btf files came from +func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, ebpfBTFLoaderURL, nil) + if err != nil { + return nil, err + } + + resp, err := r.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf(`GetEbpfBtfInfo got non-success status code: path %s, url: %s, status_code: %d, response: "%s"`, r.path, req.URL, resp.StatusCode, data) + } + + return data, nil +} + func (r *RemoteSysProbeUtil) init() error { resp, err := r.httpClient.Get(statsURL) if err != nil { diff --git a/pkg/process/net/common_linux.go b/pkg/process/net/common_linux.go index 5f7e4365fe3fb..e3e77fdda0f8c 100644 --- a/pkg/process/net/common_linux.go +++ b/pkg/process/net/common_linux.go @@ -32,6 +32,7 @@ const ( telemetryURL = "http://unix/telemetry" conntrackCachedURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/cached" conntrackHostURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host" + ebpfBTFLoaderURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/ebpf_btf_loader" netType = "unix" ) diff --git a/pkg/process/net/common_unsupported.go b/pkg/process/net/common_unsupported.go index 7604578d4b9cd..49bdc772284bb 100644 --- a/pkg/process/net/common_unsupported.go +++ b/pkg/process/net/common_unsupported.go @@ -89,6 +89,9 @@ func (r *RemoteSysProbeUtil) GetConnTrackCached() ([]byte, error) { return nil, // GetConnTrackHost is not supported func (r *RemoteSysProbeUtil) GetConnTrackHost() ([]byte, error) { return nil, ErrNotImplemented } +// GetBTFLoaderInfo is not supported +func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { return nil, ErrNotImplemented } + func (r *RemoteSysProbeUtil) GetDiscoveryServices() (*discoverymodel.ServicesResponse, error) { return nil, ErrNotImplemented } diff --git a/pkg/process/net/common_windows.go b/pkg/process/net/common_windows.go index afa0073ea9486..daca6405f68a3 100644 --- a/pkg/process/net/common_windows.go +++ b/pkg/process/net/common_windows.go @@ -38,6 +38,8 @@ const ( conntrackCachedURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/cached" // conntrackHostURL is not used on Windows, the value is added to avoid a compilation error conntrackHostURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host" + // ebpfBtfLoaderURL is not used on Windows, the value is added to avoid a compilation error + ebpfBTFLoaderURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/ebpf_btf_loader" // SystemProbePipeName is the production named pipe for system probe SystemProbePipeName = `\\.\pipe\dd_system_probe` diff --git a/pkg/process/net/shared.go b/pkg/process/net/shared.go index 34a3dcf5e4b85..3d6bd59f892fc 100644 --- a/pkg/process/net/shared.go +++ b/pkg/process/net/shared.go @@ -30,6 +30,7 @@ type SysProbeUtil interface { GetTelemetry() ([]byte, error) GetConnTrackCached() ([]byte, error) GetConnTrackHost() ([]byte, error) + GetBTFLoaderInfo() ([]byte, error) DetectLanguage(pids []int32) ([]languagemodels.Language, error) GetPprof(path string) ([]byte, error) GetDiscoveryServices() (*discoverymodel.ServicesResponse, error) From 3ec9c30a6ca8e9c58d1d91b2b9da294819d81dfc Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Wed, 23 Oct 2024 11:59:00 -0700 Subject: [PATCH 03/19] fix build --- pkg/flare/sysprobe/archive_nolinux.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/flare/sysprobe/archive_nolinux.go b/pkg/flare/sysprobe/archive_nolinux.go index 3506c78e47ec2..37776b8f6247a 100644 --- a/pkg/flare/sysprobe/archive_nolinux.go +++ b/pkg/flare/sysprobe/archive_nolinux.go @@ -18,3 +18,8 @@ func GetSystemProbeConntrackCached(_ string) ([]byte, error) { func GetSystemProbeConntrackHost(_ string) ([]byte, error) { return nil, errors.New("GetSystemProbeConntrackHost is not supported without the process agent on linux") } + +// GetSystemProbeBTFLoaderInfo is not supported without the process agent on linux +func GetSystemProbeBTFLoaderInfo(socketPath string) ([]byte, error) { + return nil, errors.New("GetSystemProbeBTFLoaderInfo is not supported without the process agent on linux") +} From ad40d0f7d31ff009128efbfb0a4c3a5106dede54 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 24 Oct 2024 06:41:15 -0700 Subject: [PATCH 04/19] mockery --- pkg/process/net/mocks/sys_probe_util.go | 57 +++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/pkg/process/net/mocks/sys_probe_util.go b/pkg/process/net/mocks/sys_probe_util.go index 413f115e47169..d0a26d64392b7 100644 --- a/pkg/process/net/mocks/sys_probe_util.go +++ b/pkg/process/net/mocks/sys_probe_util.go @@ -88,6 +88,63 @@ func (_c *SysProbeUtil_DetectLanguage_Call) RunAndReturn(run func([]int32) ([]la return _c } +// GetBTFLoaderInfo provides a mock function with given fields: +func (_m *SysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetBTFLoaderInfo") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func() ([]byte, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() []byte); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SysProbeUtil_GetBTFLoaderInfo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetBTFLoaderInfo' +type SysProbeUtil_GetBTFLoaderInfo_Call struct { + *mock.Call +} + +// GetBTFLoaderInfo is a helper method to define mock.On call +func (_e *SysProbeUtil_Expecter) GetBTFLoaderInfo() *SysProbeUtil_GetBTFLoaderInfo_Call { + return &SysProbeUtil_GetBTFLoaderInfo_Call{Call: _e.mock.On("GetBTFLoaderInfo")} +} + +func (_c *SysProbeUtil_GetBTFLoaderInfo_Call) Run(run func()) *SysProbeUtil_GetBTFLoaderInfo_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *SysProbeUtil_GetBTFLoaderInfo_Call) Return(_a0 []byte, _a1 error) *SysProbeUtil_GetBTFLoaderInfo_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetBTFLoaderInfo_Call) RunAndReturn(run func() ([]byte, error)) *SysProbeUtil_GetBTFLoaderInfo_Call { + _c.Call.Return(run) + return _c +} + // GetCheck provides a mock function with given fields: module func (_m *SysProbeUtil) GetCheck(module types.ModuleName) (interface{}, error) { ret := _m.Called(module) From 5173c18646003f41f2e6d312c4ce63d88ae239f3 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 24 Oct 2024 07:30:35 -0700 Subject: [PATCH 05/19] fix lint --- pkg/flare/sysprobe/archive_nolinux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/flare/sysprobe/archive_nolinux.go b/pkg/flare/sysprobe/archive_nolinux.go index 37776b8f6247a..a5fdffc3689d3 100644 --- a/pkg/flare/sysprobe/archive_nolinux.go +++ b/pkg/flare/sysprobe/archive_nolinux.go @@ -20,6 +20,6 @@ func GetSystemProbeConntrackHost(_ string) ([]byte, error) { } // GetSystemProbeBTFLoaderInfo is not supported without the process agent on linux -func GetSystemProbeBTFLoaderInfo(socketPath string) ([]byte, error) { +func GetSystemProbeBTFLoaderInfo(_ string) ([]byte, error) { return nil, errors.New("GetSystemProbeBTFLoaderInfo is not supported without the process agent on linux") } From 36889f2c84a7936ea2f296159cf3795db8daa243 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 24 Oct 2024 08:56:48 -0700 Subject: [PATCH 06/19] move BTF info query to ebpf module --- cmd/system-probe/modules/ebpf.go | 12 ++++++++++++ cmd/system-probe/modules/network_tracer.go | 11 ----------- .../modules/network_tracer_linux_bpf.go | 15 --------------- pkg/ebpf/co_re.go | 2 +- .../ebpf/co_re_unsupported.go | 4 ++-- pkg/flare/sysprobe/archive_linux.go | 2 +- pkg/process/net/common.go | 2 +- pkg/process/net/common_linux.go | 2 +- pkg/process/net/common_windows.go | 2 +- 9 files changed, 19 insertions(+), 33 deletions(-) delete mode 100644 cmd/system-probe/modules/network_tracer_linux_bpf.go rename cmd/system-probe/modules/network_tracer_nolinux_bpf.go => pkg/ebpf/co_re_unsupported.go (85%) diff --git a/cmd/system-probe/modules/ebpf.go b/cmd/system-probe/modules/ebpf.go index 6ca7a5b77f4f3..36eadfb6bb715 100644 --- a/cmd/system-probe/modules/ebpf.go +++ b/cmd/system-probe/modules/ebpf.go @@ -9,6 +9,7 @@ package modules import ( "fmt" + "io" "net/http" "time" @@ -58,6 +59,17 @@ func (o *ebpfModule) Register(httpMux *module.Router) error { utils.WriteAsJSON(w, stats) })) + httpMux.HandleFunc("/btf_loader_info", func(w http.ResponseWriter, _ *http.Request) { + info, err := ebpf.GetBTFLoaderInfo() + if err != nil { + log.Errorf("unable to get ebpf_btf_loader info: %s", err) + w.WriteHeader(500) + return + } + + io.WriteString(w, info) + }) + return nil } diff --git a/cmd/system-probe/modules/network_tracer.go b/cmd/system-probe/modules/network_tracer.go index bbd7de378bfa8..b605a3f0e99c3 100644 --- a/cmd/system-probe/modules/network_tracer.go +++ b/cmd/system-probe/modules/network_tracer.go @@ -292,17 +292,6 @@ func (nt *networkTracer) Register(httpMux *module.Router) error { utils.WriteAsJSON(w, cache) }) - httpMux.HandleFunc("/debug/ebpf_btf_loader", func(w http.ResponseWriter, _ *http.Request) { - info, err := GetBTFLoaderInfo() - if err != nil { - log.Errorf("unable to get ebpf_btf_loader info: %s", err) - w.WriteHeader(500) - return - } - - io.WriteString(w, info) - }) - httpMux.HandleFunc("/debug/usm_telemetry", telemetry.Handler) httpMux.HandleFunc("/debug/usm/traced_programs", usm.TracedProgramsEndpoint) httpMux.HandleFunc("/debug/usm/blocked_processes", usm.BlockedPathIDEndpoint) diff --git a/cmd/system-probe/modules/network_tracer_linux_bpf.go b/cmd/system-probe/modules/network_tracer_linux_bpf.go deleted file mode 100644 index 8a46118856eee..0000000000000 --- a/cmd/system-probe/modules/network_tracer_linux_bpf.go +++ /dev/null @@ -1,15 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024-present Datadog, Inc. - -//go:build linux_bpf - -package modules - -import "github.com/DataDog/datadog-agent/pkg/ebpf" - -// GetBTFLoaderInfo Returns where the ebpf BTF files were sourced from, only on linux -func GetBTFLoaderInfo() (string, error) { - return ebpf.GetBTFLoaderInfo() -} diff --git a/pkg/ebpf/co_re.go b/pkg/ebpf/co_re.go index 0e77242c2b1fe..5a929827040f7 100644 --- a/pkg/ebpf/co_re.go +++ b/pkg/ebpf/co_re.go @@ -42,7 +42,7 @@ func LoadCOREAsset(filename string, startFn func(bytecode.AssetReader, manager.O return loader.loadCOREAsset(filename, startFn) } -// Returns where the ebpf BTF files were sourced from +// GetBTFLoaderInfo Returns where the ebpf BTF files were sourced from func GetBTFLoaderInfo() (string, error) { loader, err := coreLoader(NewConfig()) if err != nil { diff --git a/cmd/system-probe/modules/network_tracer_nolinux_bpf.go b/pkg/ebpf/co_re_unsupported.go similarity index 85% rename from cmd/system-probe/modules/network_tracer_nolinux_bpf.go rename to pkg/ebpf/co_re_unsupported.go index d225770473ebb..b8ff1baa5970b 100644 --- a/cmd/system-probe/modules/network_tracer_nolinux_bpf.go +++ b/pkg/ebpf/co_re_unsupported.go @@ -5,11 +5,11 @@ //go:build !linux_bpf -package modules +package ebpf import "errors" -// GetBTFLoaderInfo is not supported on windows +// GetBTFLoaderInfo is not supported without linux_bpf func GetBTFLoaderInfo() (string, error) { return "", errors.New("GetBTFLoaderInfo not supported without linux_bpf") } diff --git a/pkg/flare/sysprobe/archive_linux.go b/pkg/flare/sysprobe/archive_linux.go index 635f9d9c4a854..477a899daadd5 100644 --- a/pkg/flare/sysprobe/archive_linux.go +++ b/pkg/flare/sysprobe/archive_linux.go @@ -28,7 +28,7 @@ func GetSystemProbeConntrackHost(socketPath string) ([]byte, error) { return probeUtil.GetConnTrackHost() } -// GetSystemProbeBTFLoaderInfo queries ebpf_btf_loader which gets where the BTF data came from +// GetSystemProbeBTFLoaderInfo queries btf_loader_info which gets where the BTF data came from func GetSystemProbeBTFLoaderInfo(socketPath string) ([]byte, error) { probeUtil, err := net.GetRemoteSystemProbeUtil(socketPath) if err != nil { diff --git a/pkg/process/net/common.go b/pkg/process/net/common.go index 7cfabf7947cee..db82ddf8073c3 100644 --- a/pkg/process/net/common.go +++ b/pkg/process/net/common.go @@ -476,7 +476,7 @@ func (r *RemoteSysProbeUtil) GetConnTrackHost() ([]byte, error) { return data, nil } -// GetBTFLoaderInfo queries ebpf_btf_loader to get information about where btf files came from +// GetBTFLoaderInfo queries btf_loader_info to get information about where btf files came from func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { req, err := http.NewRequest(http.MethodGet, ebpfBTFLoaderURL, nil) if err != nil { diff --git a/pkg/process/net/common_linux.go b/pkg/process/net/common_linux.go index e3e77fdda0f8c..ea23a4c5a4b1f 100644 --- a/pkg/process/net/common_linux.go +++ b/pkg/process/net/common_linux.go @@ -32,7 +32,7 @@ const ( telemetryURL = "http://unix/telemetry" conntrackCachedURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/cached" conntrackHostURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host" - ebpfBTFLoaderURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/ebpf_btf_loader" + ebpfBTFLoaderURL = "http://unix/" + string(sysconfig.EBPFModule) + "/btf_loader_info" netType = "unix" ) diff --git a/pkg/process/net/common_windows.go b/pkg/process/net/common_windows.go index daca6405f68a3..fe2dc3b0f67ab 100644 --- a/pkg/process/net/common_windows.go +++ b/pkg/process/net/common_windows.go @@ -39,7 +39,7 @@ const ( // conntrackHostURL is not used on Windows, the value is added to avoid a compilation error conntrackHostURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host" // ebpfBtfLoaderURL is not used on Windows, the value is added to avoid a compilation error - ebpfBTFLoaderURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/ebpf_btf_loader" + ebpfBTFLoaderURL = "http://localhost:3333/" + string(sysconfig.EBPFModule) + "/btf_loader_info" // SystemProbePipeName is the production named pipe for system probe SystemProbePipeName = `\\.\pipe\dd_system_probe` From fafa3dbe0258918c0da493138db713a4d06b313e Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 24 Oct 2024 11:01:57 -0700 Subject: [PATCH 07/19] add release note --- releasenotes/notes/btf-in-flare-92c4735d6507a27a.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 releasenotes/notes/btf-in-flare-92c4735d6507a27a.yaml diff --git a/releasenotes/notes/btf-in-flare-92c4735d6507a27a.yaml b/releasenotes/notes/btf-in-flare-92c4735d6507a27a.yaml new file mode 100644 index 0000000000000..1ae714ed62f38 --- /dev/null +++ b/releasenotes/notes/btf-in-flare-92c4735d6507a27a.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Added to the Agent flare information about where the Agent sourced BTF data for eBPF. When applicable, this will appear in ``system-probe/ebpf_btf_loader.log``. From 212b198cb36e5c0893e8ecb00f77843b190fdb9a Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 24 Oct 2024 11:18:50 -0700 Subject: [PATCH 08/19] add more info in the failure case --- pkg/ebpf/co_re.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/ebpf/co_re.go b/pkg/ebpf/co_re.go index 5a929827040f7..8086d3e6bab3d 100644 --- a/pkg/ebpf/co_re.go +++ b/pkg/ebpf/co_re.go @@ -52,8 +52,12 @@ func GetBTFLoaderInfo() (string, error) { if err != nil { return "", err } + // no error but also no data: means it tried all the loaders and got nothing back if res == nil { - return "", errors.New("GetBTFLoaderInfo expected result from btfLoader") + embeddedPath, err := loader.btfLoader.embeddedPath() + message := "BTF loaders did not find BTF files. btfLoader.embeddedPath() returned:" + result := fmt.Sprintf("%s\npath: %s\nerr: %v\n", message, embeddedPath, err) + return result, nil } return res.String(), nil } From 7e7385a45883e447d0f979d0112d2c1f33c3c7c3 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 24 Oct 2024 14:33:41 -0700 Subject: [PATCH 09/19] switch to storing metadata --- pkg/ebpf/btf.go | 89 +++++++++++++++++++++++++++++------------------ pkg/ebpf/co_re.go | 16 +++------ 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/pkg/ebpf/btf.go b/pkg/ebpf/btf.go index 7cf9b57416990..ac24013e6c1ef 100644 --- a/pkg/ebpf/btf.go +++ b/pkg/ebpf/btf.go @@ -89,20 +89,31 @@ func FlushBTF() { type kernelModuleBTFLoadFunc func(string) (*btf.Spec, error) -type btfSpecData struct { - // sourceFilepath documents where we got this BTF data from - sourceFilepath string +type returnBTF struct { vmlinux *btf.Spec moduleLoadFunc kernelModuleBTFLoadFunc } -type returnBTF struct { - // loaderDesc the name of the loader that was used to get the BTF data - loaderDesc string - btfSpecData + +type BTFResultMetadata struct { + // numLoadAttempts is how many times the loader has been invoked (doesn't include cached requests) + numLoadAttempts int + // loaderUsed the name of the loader that was used to get the BTF data + loaderUsed string + // filepathUsed is the filepath it last tried to load BTF files from (only for loadUser and loadEmbedded) + filepathUsed string + // tarballUsed is the filepath for the tarball it tried to extract BTF from (only for loadEmbedded) + tarballUsed string } -func (r *returnBTF) String() string { - return fmt.Sprintf("loader: %s\nsource file: %s\n", r.loaderDesc, r.sourceFilepath) +func (d BTFResultMetadata) String() string { + res := fmt.Sprintf("numLoadAttempts: %d\nloaderUsed: %s", d.numLoadAttempts, d.loaderUsed) + if d.filepathUsed != "" { + res += fmt.Sprintf("\nfilepathUsed: %s", d.filepathUsed) + } + if d.tarballUsed != "" { + res += fmt.Sprintf("\ntarballUsed: %s", d.tarballUsed) + } + return res } type orderedBTFLoader struct { @@ -110,6 +121,7 @@ type orderedBTFLoader struct { embeddedDir string result ebpftelemetry.BTFResult + resultMetadata BTFResultMetadata loadFunc funcs.CachedFunc[returnBTF] delayedFlusher *time.Timer } @@ -125,7 +137,7 @@ func initBTFLoader(cfg *Config) *orderedBTFLoader { return btfLoader } -type btfLoaderFunc func() (*btfSpecData, error) +type btfLoaderFunc func() (*returnBTF, error) // Get returns BTF for the running kernel func (b *orderedBTFLoader) Get() (*returnBTF, ebpftelemetry.COREResult, error) { @@ -143,6 +155,8 @@ func (b *orderedBTFLoader) Flush() { } func (b *orderedBTFLoader) get() (*returnBTF, error) { + b.resultMetadata.numLoadAttempts++ + loaders := []struct { result ebpftelemetry.BTFResult loader btfLoaderFunc @@ -153,7 +167,7 @@ func (b *orderedBTFLoader) get() (*returnBTF, error) { {ebpftelemetry.SuccessEmbeddedBTF, b.loadEmbedded, "embedded collection"}, } var err error - var ret *btfSpecData + var ret *returnBTF for _, l := range loaders { log.Debugf("attempting BTF load from %s", l.desc) ret, err = l.loader() @@ -168,26 +182,26 @@ func (b *orderedBTFLoader) get() (*returnBTF, error) { if ret != nil { log.Debugf("successfully loaded BTF from %s", l.desc) b.result = l.result - return &returnBTF{ - loaderDesc: l.desc, - btfSpecData: *ret, - }, nil + b.resultMetadata.loaderUsed = l.desc + return ret, nil } } return nil, err } -func (b *orderedBTFLoader) loadKernel() (*btfSpecData, error) { +func (b *orderedBTFLoader) loadKernel() (*returnBTF, error) { spec, err := GetKernelSpec() if err != nil { return nil, err } - return &btfSpecData{ - sourceFilepath: "", - vmlinux: spec, moduleLoadFunc: nil}, nil + b.resultMetadata.filepathUsed = "" + return &returnBTF{ + vmlinux: spec, + moduleLoadFunc: nil, + }, nil } -func (b *orderedBTFLoader) loadUser() (*btfSpecData, error) { +func (b *orderedBTFLoader) loadUser() (*returnBTF, error) { if b.userBTFPath == "" { return nil, nil } @@ -195,14 +209,14 @@ func (b *orderedBTFLoader) loadUser() (*btfSpecData, error) { if err != nil { return nil, err } - return &btfSpecData{ - sourceFilepath: b.userBTFPath, + b.resultMetadata.filepathUsed = b.userBTFPath + return &returnBTF{ vmlinux: spec, moduleLoadFunc: nil, }, nil } -func (b *orderedBTFLoader) checkForMinimizedBTF(extractDir, sourceTarball string) (*btfSpecData, error) { +func (b *orderedBTFLoader) checkForMinimizedBTF(extractDir string) (*returnBTF, error) { // //.btf btfRelativePath := filepath.Join(extractDir, filepath.Base(extractDir)+".btf") extractedBtfPath := filepath.Join(b.embeddedDir, btfRelativePath) @@ -211,12 +225,16 @@ func (b *orderedBTFLoader) checkForMinimizedBTF(extractDir, sourceTarball string if err != nil { return nil, err } - return &btfSpecData{sourceFilepath: sourceTarball, vmlinux: spec, moduleLoadFunc: nil}, nil + b.resultMetadata.filepathUsed = btfRelativePath + return &returnBTF{ + vmlinux: spec, + moduleLoadFunc: nil, + }, nil } return nil, nil } -func (b *orderedBTFLoader) checkForUnminimizedBTF(extractDir, sourceTarball string) (*btfSpecData, error) { +func (b *orderedBTFLoader) checkForUnminimizedBTF(extractDir string) (*returnBTF, error) { absExtractDir := filepath.Join(b.embeddedDir, extractDir) modLoadFunc := func(mod string) (*btf.Spec, error) { b.delayedFlusher.Reset(btfFlushDelay) @@ -229,25 +247,30 @@ func (b *orderedBTFLoader) checkForUnminimizedBTF(extractDir, sourceTarball stri if err != nil { return nil, err } - return &btfSpecData{sourceFilepath: sourceTarball, vmlinux: spec, moduleLoadFunc: modLoadFunc}, nil + b.resultMetadata.filepathUsed = extractedBtfPath + return &returnBTF{ + vmlinux: spec, + moduleLoadFunc: modLoadFunc, + }, nil } return nil, nil } -func (b *orderedBTFLoader) checkforBTF(extractDir, sourceTarball string) (*btfSpecData, error) { - ret, err := b.checkForMinimizedBTF(extractDir, sourceTarball) +func (b *orderedBTFLoader) checkforBTF(extractDir string) (*returnBTF, error) { + ret, err := b.checkForMinimizedBTF(extractDir) if err != nil || ret != nil { return ret, err } - ret, err = b.checkForUnminimizedBTF(extractDir, sourceTarball) + ret, err = b.checkForUnminimizedBTF(extractDir) if err != nil || ret != nil { return ret, err } return nil, nil } -func (b *orderedBTFLoader) loadEmbedded() (*btfSpecData, error) { +func (b *orderedBTFLoader) loadEmbedded() (*returnBTF, error) { btfRelativeTarballFilename, err := b.embeddedPath() + b.resultMetadata.filepathUsed = btfRelativeTarballFilename if err != nil { return nil, err } @@ -260,7 +283,7 @@ func (b *orderedBTFLoader) loadEmbedded() (*btfSpecData, error) { absExtractDir := filepath.Join(b.embeddedDir, extractDir) // If we've previously extracted the BTF file in question, we can just load it - ret, err := b.checkforBTF(extractDir, btfRelativeTarballFilename) + ret, err := b.checkforBTF(extractDir) if err != nil || ret != nil { return ret, err } @@ -270,7 +293,7 @@ func (b *orderedBTFLoader) loadEmbedded() (*btfSpecData, error) { // of BTFs as a whole is also compressed. // This means that we'll need to first extract the specific BTF which we're looking for from the collection // tarball, and then unarchive it. - btfTarball := filepath.Join(b.embeddedDir, btfRelativeTarballFilename) + btfTarball := filepath.Join(b.embeddedDir) if _, err := os.Stat(btfTarball); errors.Is(err, fs.ErrNotExist) { collectionTarball := filepath.Join(b.embeddedDir, btfArchiveName) if err := archive.TarXZExtractFile(collectionTarball, btfRelativeTarballFilename, b.embeddedDir); err != nil { @@ -281,7 +304,7 @@ func (b *orderedBTFLoader) loadEmbedded() (*btfSpecData, error) { if err := archive.TarXZExtractAll(btfTarball, absExtractDir); err != nil { return nil, fmt.Errorf("extract kernel BTF from tarball: %w", err) } - ret, err = b.checkforBTF(extractDir, btfRelativeTarballFilename) + ret, err = b.checkforBTF(extractDir) if err != nil || ret != nil { return ret, err } diff --git a/pkg/ebpf/co_re.go b/pkg/ebpf/co_re.go index 8086d3e6bab3d..4a56dfa7719c8 100644 --- a/pkg/ebpf/co_re.go +++ b/pkg/ebpf/co_re.go @@ -48,18 +48,10 @@ func GetBTFLoaderInfo() (string, error) { if err != nil { return "", err } - res, _, err := loader.btfLoader.Get() - if err != nil { - return "", err - } - // no error but also no data: means it tried all the loaders and got nothing back - if res == nil { - embeddedPath, err := loader.btfLoader.embeddedPath() - message := "BTF loaders did not find BTF files. btfLoader.embeddedPath() returned:" - result := fmt.Sprintf("%s\npath: %s\nerr: %v\n", message, embeddedPath, err) - return result, nil - } - return res.String(), nil + + metadataStr := loader.btfLoader.resultMetadata.String() + infoStr := fmt.Sprintf("btfLoader result: %s\n%s", loader.btfLoader.result, metadataStr) + return infoStr, nil } func (c *coreAssetLoader) loadCOREAsset(filename string, startFn func(bytecode.AssetReader, manager.Options) error) error { From d6ed1c9e09afeb35a4079ca3006ee50b43a30753 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Fri, 25 Oct 2024 11:25:37 -0700 Subject: [PATCH 10/19] implement feedback --- pkg/ebpf/co_re_unsupported.go | 2 +- pkg/flare/sysprobe/archive_nolinux.go | 6 ++--- pkg/process/net/common.go | 25 ------------------ pkg/process/net/common_linux.go | 26 +++++++++++++++++++ pkg/process/net/common_unsupported.go | 5 ++++ pkg/process/net/common_windows.go | 7 ++++- .../notes/btf-in-flare-92c4735d6507a27a.yaml | 2 +- 7 files changed, 42 insertions(+), 31 deletions(-) diff --git a/pkg/ebpf/co_re_unsupported.go b/pkg/ebpf/co_re_unsupported.go index b8ff1baa5970b..9e5cc6f7ad538 100644 --- a/pkg/ebpf/co_re_unsupported.go +++ b/pkg/ebpf/co_re_unsupported.go @@ -11,5 +11,5 @@ import "errors" // GetBTFLoaderInfo is not supported without linux_bpf func GetBTFLoaderInfo() (string, error) { - return "", errors.New("GetBTFLoaderInfo not supported without linux_bpf") + return "", errors.New("BTF is not supported") } diff --git a/pkg/flare/sysprobe/archive_nolinux.go b/pkg/flare/sysprobe/archive_nolinux.go index a5fdffc3689d3..23e7d128be875 100644 --- a/pkg/flare/sysprobe/archive_nolinux.go +++ b/pkg/flare/sysprobe/archive_nolinux.go @@ -9,17 +9,17 @@ package sysprobe import "errors" -// GetSystemProbeConntrackCached is not supported without the process agent on linux +// GetSystemProbeConntrackCached is a stub designed to prevent builds without the process agent from importing pkg/process/net func GetSystemProbeConntrackCached(_ string) ([]byte, error) { return nil, errors.New("GetSystemProbeConntrackCached is not supported without the process agent on linux") } -// GetSystemProbeConntrackHost is not supported without the process agent on linux +// GetSystemProbeConntrackHost is a stub designed to prevent builds without the process agent from importing pkg/process/net func GetSystemProbeConntrackHost(_ string) ([]byte, error) { return nil, errors.New("GetSystemProbeConntrackHost is not supported without the process agent on linux") } -// GetSystemProbeBTFLoaderInfo is not supported without the process agent on linux +// GetSystemProbeBTFLoaderInfo is a stub designed to prevent builds without the process agent from importing pkg/process/net func GetSystemProbeBTFLoaderInfo(_ string) ([]byte, error) { return nil, errors.New("GetSystemProbeBTFLoaderInfo is not supported without the process agent on linux") } diff --git a/pkg/process/net/common.go b/pkg/process/net/common.go index db82ddf8073c3..1e5e695154c18 100644 --- a/pkg/process/net/common.go +++ b/pkg/process/net/common.go @@ -476,31 +476,6 @@ func (r *RemoteSysProbeUtil) GetConnTrackHost() ([]byte, error) { return data, nil } -// GetBTFLoaderInfo queries btf_loader_info to get information about where btf files came from -func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { - req, err := http.NewRequest(http.MethodGet, ebpfBTFLoaderURL, nil) - if err != nil { - return nil, err - } - - resp, err := r.httpClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(`GetEbpfBtfInfo got non-success status code: path %s, url: %s, status_code: %d, response: "%s"`, r.path, req.URL, resp.StatusCode, data) - } - - return data, nil -} - func (r *RemoteSysProbeUtil) init() error { resp, err := r.httpClient.Get(statsURL) if err != nil { diff --git a/pkg/process/net/common_linux.go b/pkg/process/net/common_linux.go index ea23a4c5a4b1f..6910960fee158 100644 --- a/pkg/process/net/common_linux.go +++ b/pkg/process/net/common_linux.go @@ -10,6 +10,7 @@ package net import ( "context" "fmt" + "io" "net" "net/http" "os" @@ -84,3 +85,28 @@ func newSystemProbe(path string) *RemoteSysProbeUtil { }, } } + +// GetBTFLoaderInfo queries btf_loader_info to get information about where btf files came from +func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, ebpfBTFLoaderURL, nil) + if err != nil { + return nil, err + } + + resp, err := r.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf(`GetEbpfBtfInfo got non-success status code: path %s, url: %s, status_code: %d, response: "%s"`, r.path, req.URL, resp.StatusCode, data) + } + + return data, nil +} diff --git a/pkg/process/net/common_unsupported.go b/pkg/process/net/common_unsupported.go index 49bdc772284bb..572d5e78afb02 100644 --- a/pkg/process/net/common_unsupported.go +++ b/pkg/process/net/common_unsupported.go @@ -107,3 +107,8 @@ func (r *RemoteSysProbeUtil) GetPing(clientID string, host string, count int, in func (r *RemoteSysProbeUtil) GetTraceroute(clientID string, host string, port uint16, protocol nppayload.Protocol, maxTTL uint8, timeout time.Duration) ([]byte, error) { return nil, ErrNotImplemented } + +// GetBTFLoaderInfo is not supported +func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { + return nil, ErrNotImplemented +} diff --git a/pkg/process/net/common_windows.go b/pkg/process/net/common_windows.go index fe2dc3b0f67ab..c48de996b79ac 100644 --- a/pkg/process/net/common_windows.go +++ b/pkg/process/net/common_windows.go @@ -39,7 +39,7 @@ const ( // conntrackHostURL is not used on Windows, the value is added to avoid a compilation error conntrackHostURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host" // ebpfBtfLoaderURL is not used on Windows, the value is added to avoid a compilation error - ebpfBTFLoaderURL = "http://localhost:3333/" + string(sysconfig.EBPFModule) + "/btf_loader_info" + ebpfBTFLoaderURL = "" // SystemProbePipeName is the production named pipe for system probe SystemProbePipeName = `\\.\pipe\dd_system_probe` @@ -105,3 +105,8 @@ func newSystemProbe(path string) *RemoteSysProbeUtil { }, } } + +// GetBTFLoaderInfo is not implemented on windows +func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { + return nil, errors.New("GetBTFLoaderInfo is not supported on windows") +} diff --git a/releasenotes/notes/btf-in-flare-92c4735d6507a27a.yaml b/releasenotes/notes/btf-in-flare-92c4735d6507a27a.yaml index 1ae714ed62f38..9e2f2ba0e3fcf 100644 --- a/releasenotes/notes/btf-in-flare-92c4735d6507a27a.yaml +++ b/releasenotes/notes/btf-in-flare-92c4735d6507a27a.yaml @@ -8,4 +8,4 @@ --- enhancements: - | - Added to the Agent flare information about where the Agent sourced BTF data for eBPF. When applicable, this will appear in ``system-probe/ebpf_btf_loader.log``. + Added information about where the Agent sourced BTF data for eBPF to the Agent flare. When applicable, this will appear in ``system-probe/ebpf_btf_loader.log``. From 5ae5c672fc29c64f86a6bbf6342f9834f56b436e Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Fri, 25 Oct 2024 11:48:42 -0700 Subject: [PATCH 11/19] fix printf --- pkg/ebpf/co_re.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ebpf/co_re.go b/pkg/ebpf/co_re.go index 4a56dfa7719c8..34f13e5c4ef4d 100644 --- a/pkg/ebpf/co_re.go +++ b/pkg/ebpf/co_re.go @@ -50,7 +50,7 @@ func GetBTFLoaderInfo() (string, error) { } metadataStr := loader.btfLoader.resultMetadata.String() - infoStr := fmt.Sprintf("btfLoader result: %s\n%s", loader.btfLoader.result, metadataStr) + infoStr := fmt.Sprintf("btfLoader result: %d\n%s", loader.btfLoader.result, metadataStr) return infoStr, nil } From 77066efeac01ff1108818d7b734e0cbb9a3784a5 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Wed, 30 Oct 2024 10:50:50 -0700 Subject: [PATCH 12/19] feedback --- cmd/system-probe/api/server.go | 3 +++ cmd/system-probe/modules/ebpf.go | 12 ------------ pkg/ebpf/debug_handlers.go | 26 ++++++++++++++++++++++++++ pkg/flare/sysprobe/archive_linux.go | 2 +- pkg/process/net/common_linux.go | 4 ++-- 5 files changed, 32 insertions(+), 15 deletions(-) create mode 100644 pkg/ebpf/debug_handlers.go diff --git a/cmd/system-probe/api/server.go b/cmd/system-probe/api/server.go index a2d20995b9804..4de2f1c426871 100644 --- a/cmd/system-probe/api/server.go +++ b/cmd/system-probe/api/server.go @@ -21,6 +21,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/settings" "github.com/DataDog/datadog-agent/comp/core/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/ebpf" "github.com/DataDog/datadog-agent/pkg/process/net" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -53,6 +54,8 @@ func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmet mux.Handle("/debug/vars", http.DefaultServeMux) mux.Handle("/telemetry", telemetry.Handler()) + mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo) + go func() { err = http.Serve(conn.GetListener(), mux) if err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/cmd/system-probe/modules/ebpf.go b/cmd/system-probe/modules/ebpf.go index 36eadfb6bb715..6ca7a5b77f4f3 100644 --- a/cmd/system-probe/modules/ebpf.go +++ b/cmd/system-probe/modules/ebpf.go @@ -9,7 +9,6 @@ package modules import ( "fmt" - "io" "net/http" "time" @@ -59,17 +58,6 @@ func (o *ebpfModule) Register(httpMux *module.Router) error { utils.WriteAsJSON(w, stats) })) - httpMux.HandleFunc("/btf_loader_info", func(w http.ResponseWriter, _ *http.Request) { - info, err := ebpf.GetBTFLoaderInfo() - if err != nil { - log.Errorf("unable to get ebpf_btf_loader info: %s", err) - w.WriteHeader(500) - return - } - - io.WriteString(w, info) - }) - return nil } diff --git a/pkg/ebpf/debug_handlers.go b/pkg/ebpf/debug_handlers.go new file mode 100644 index 0000000000000..ea10d22a844c2 --- /dev/null +++ b/pkg/ebpf/debug_handlers.go @@ -0,0 +1,26 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package ebpf + +import ( + "io" + "net/http" + + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// HandleBTFLoaderInfo responds with where the system-probe found BTF data (and +// if it was in a pre-bundled tarball, where within that tarball it came from) +func HandleBTFLoaderInfo(w http.ResponseWriter, _ *http.Request) { + info, err := GetBTFLoaderInfo() + if err != nil { + log.Errorf("unable to get ebpf_btf_loader info: %s", err) + w.WriteHeader(500) + return + } + + io.WriteString(w, info) +} diff --git a/pkg/flare/sysprobe/archive_linux.go b/pkg/flare/sysprobe/archive_linux.go index 477a899daadd5..d6312d5cc62bf 100644 --- a/pkg/flare/sysprobe/archive_linux.go +++ b/pkg/flare/sysprobe/archive_linux.go @@ -28,7 +28,7 @@ func GetSystemProbeConntrackHost(socketPath string) ([]byte, error) { return probeUtil.GetConnTrackHost() } -// GetSystemProbeBTFLoaderInfo queries btf_loader_info which gets where the BTF data came from +// GetSystemProbeBTFLoaderInfo queries ebpf_btf_loader_info which gets where the BTF data came from func GetSystemProbeBTFLoaderInfo(socketPath string) ([]byte, error) { probeUtil, err := net.GetRemoteSystemProbeUtil(socketPath) if err != nil { diff --git a/pkg/process/net/common_linux.go b/pkg/process/net/common_linux.go index 6910960fee158..d326bc3d4ac38 100644 --- a/pkg/process/net/common_linux.go +++ b/pkg/process/net/common_linux.go @@ -33,7 +33,7 @@ const ( telemetryURL = "http://unix/telemetry" conntrackCachedURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/cached" conntrackHostURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host" - ebpfBTFLoaderURL = "http://unix/" + string(sysconfig.EBPFModule) + "/btf_loader_info" + ebpfBTFLoaderURL = "http://unix/ebpf_btf_loader_info" netType = "unix" ) @@ -86,7 +86,7 @@ func newSystemProbe(path string) *RemoteSysProbeUtil { } } -// GetBTFLoaderInfo queries btf_loader_info to get information about where btf files came from +// GetBTFLoaderInfo queries ebpf_btf_loader_info to get information about where btf files came from func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { req, err := http.NewRequest(http.MethodGet, ebpfBTFLoaderURL, nil) if err != nil { From eb2ead028a44d95191e3d0ac9bf933362728b3ec Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Wed, 30 Oct 2024 12:54:54 -0700 Subject: [PATCH 13/19] use absolute path, fix tarball path --- pkg/ebpf/btf.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ebpf/btf.go b/pkg/ebpf/btf.go index ac24013e6c1ef..ec4e656318162 100644 --- a/pkg/ebpf/btf.go +++ b/pkg/ebpf/btf.go @@ -225,7 +225,7 @@ func (b *orderedBTFLoader) checkForMinimizedBTF(extractDir string) (*returnBTF, if err != nil { return nil, err } - b.resultMetadata.filepathUsed = btfRelativePath + b.resultMetadata.filepathUsed = extractedBtfPath return &returnBTF{ vmlinux: spec, moduleLoadFunc: nil, @@ -270,7 +270,7 @@ func (b *orderedBTFLoader) checkforBTF(extractDir string) (*returnBTF, error) { func (b *orderedBTFLoader) loadEmbedded() (*returnBTF, error) { btfRelativeTarballFilename, err := b.embeddedPath() - b.resultMetadata.filepathUsed = btfRelativeTarballFilename + b.resultMetadata.tarballUsed = btfRelativeTarballFilename if err != nil { return nil, err } From 9333079560b902386572e48f0b490bd8002a4626 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Wed, 30 Oct 2024 13:31:38 -0700 Subject: [PATCH 14/19] fix tarball path, use absolute, add route only on linux --- cmd/system-probe/api/server.go | 5 ++++- pkg/ebpf/btf.go | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/system-probe/api/server.go b/cmd/system-probe/api/server.go index 4de2f1c426871..5c6c31df8beab 100644 --- a/cmd/system-probe/api/server.go +++ b/cmd/system-probe/api/server.go @@ -11,6 +11,7 @@ import ( "expvar" "fmt" "net/http" + "runtime" gorilla "github.com/gorilla/mux" @@ -54,7 +55,9 @@ func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmet mux.Handle("/debug/vars", http.DefaultServeMux) mux.Handle("/telemetry", telemetry.Handler()) - mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo) + if runtime.GOOS == "linux" { + mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo) + } go func() { err = http.Serve(conn.GetListener(), mux) diff --git a/pkg/ebpf/btf.go b/pkg/ebpf/btf.go index ec4e656318162..94bd67cfbb7f9 100644 --- a/pkg/ebpf/btf.go +++ b/pkg/ebpf/btf.go @@ -270,7 +270,6 @@ func (b *orderedBTFLoader) checkforBTF(extractDir string) (*returnBTF, error) { func (b *orderedBTFLoader) loadEmbedded() (*returnBTF, error) { btfRelativeTarballFilename, err := b.embeddedPath() - b.resultMetadata.tarballUsed = btfRelativeTarballFilename if err != nil { return nil, err } @@ -293,7 +292,8 @@ func (b *orderedBTFLoader) loadEmbedded() (*returnBTF, error) { // of BTFs as a whole is also compressed. // This means that we'll need to first extract the specific BTF which we're looking for from the collection // tarball, and then unarchive it. - btfTarball := filepath.Join(b.embeddedDir) + btfTarball := filepath.Join(b.embeddedDir, btfRelativeTarballFilename) + b.resultMetadata.tarballUsed = btfTarball if _, err := os.Stat(btfTarball); errors.Is(err, fs.ErrNotExist) { collectionTarball := filepath.Join(b.embeddedDir, btfArchiveName) if err := archive.TarXZExtractFile(collectionTarball, btfRelativeTarballFilename, b.embeddedDir); err != nil { From 1ea6d0fcd60710be4fd1b5795c6522069b587daa Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 31 Oct 2024 06:58:10 -0700 Subject: [PATCH 15/19] smaller error message --- pkg/flare/sysprobe/archive_nolinux.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/flare/sysprobe/archive_nolinux.go b/pkg/flare/sysprobe/archive_nolinux.go index 23e7d128be875..c9c441ff7752f 100644 --- a/pkg/flare/sysprobe/archive_nolinux.go +++ b/pkg/flare/sysprobe/archive_nolinux.go @@ -11,15 +11,15 @@ import "errors" // GetSystemProbeConntrackCached is a stub designed to prevent builds without the process agent from importing pkg/process/net func GetSystemProbeConntrackCached(_ string) ([]byte, error) { - return nil, errors.New("GetSystemProbeConntrackCached is not supported without the process agent on linux") + return nil, errors.New("GetSystemProbeConntrackCached is not supported") } // GetSystemProbeConntrackHost is a stub designed to prevent builds without the process agent from importing pkg/process/net func GetSystemProbeConntrackHost(_ string) ([]byte, error) { - return nil, errors.New("GetSystemProbeConntrackHost is not supported without the process agent on linux") + return nil, errors.New("GetSystemProbeConntrackHost is not supported") } // GetSystemProbeBTFLoaderInfo is a stub designed to prevent builds without the process agent from importing pkg/process/net func GetSystemProbeBTFLoaderInfo(_ string) ([]byte, error) { - return nil, errors.New("GetSystemProbeBTFLoaderInfo is not supported without the process agent on linux") + return nil, errors.New("GetSystemProbeBTFLoaderInfo is not supported") } From bb94c7f770aeeda882a357fbd1d992a6dd6897fd Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 31 Oct 2024 07:18:09 -0700 Subject: [PATCH 16/19] fix build --- pkg/process/net/common_unsupported.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/process/net/common_unsupported.go b/pkg/process/net/common_unsupported.go index 572d5e78afb02..49bdc772284bb 100644 --- a/pkg/process/net/common_unsupported.go +++ b/pkg/process/net/common_unsupported.go @@ -107,8 +107,3 @@ func (r *RemoteSysProbeUtil) GetPing(clientID string, host string, count int, in func (r *RemoteSysProbeUtil) GetTraceroute(clientID string, host string, port uint16, protocol nppayload.Protocol, maxTTL uint8, timeout time.Duration) ([]byte, error) { return nil, ErrNotImplemented } - -// GetBTFLoaderInfo is not supported -func (r *RemoteSysProbeUtil) GetBTFLoaderInfo() ([]byte, error) { - return nil, ErrNotImplemented -} From 8f6989ad2dc8757eb33395d1639d6426dab2c032 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 31 Oct 2024 07:52:18 -0700 Subject: [PATCH 17/19] fix windows build --- pkg/process/net/common_windows.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/process/net/common_windows.go b/pkg/process/net/common_windows.go index c48de996b79ac..9fcc552eb84b3 100644 --- a/pkg/process/net/common_windows.go +++ b/pkg/process/net/common_windows.go @@ -9,6 +9,7 @@ package net import ( "context" + "errors" "fmt" "net" "net/http" From 3bd69f4f385615b97881595fbec66c664d135509 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 31 Oct 2024 08:03:19 -0700 Subject: [PATCH 18/19] remove constant --- pkg/process/net/common_windows.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/process/net/common_windows.go b/pkg/process/net/common_windows.go index 9fcc552eb84b3..ccf70d591b1ac 100644 --- a/pkg/process/net/common_windows.go +++ b/pkg/process/net/common_windows.go @@ -39,8 +39,6 @@ const ( conntrackCachedURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/cached" // conntrackHostURL is not used on Windows, the value is added to avoid a compilation error conntrackHostURL = "http://localhost:3333/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host" - // ebpfBtfLoaderURL is not used on Windows, the value is added to avoid a compilation error - ebpfBTFLoaderURL = "" // SystemProbePipeName is the production named pipe for system probe SystemProbePipeName = `\\.\pipe\dd_system_probe` From cf5c3b3956e2d73e1b8d533ccf90dca0a7dee598 Mon Sep 17 00:00:00 2001 From: Stuart Geipel Date: Thu, 31 Oct 2024 10:34:42 -0700 Subject: [PATCH 19/19] fix path --- pkg/process/net/common_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/process/net/common_linux.go b/pkg/process/net/common_linux.go index d326bc3d4ac38..330010c6bcfb7 100644 --- a/pkg/process/net/common_linux.go +++ b/pkg/process/net/common_linux.go @@ -33,7 +33,7 @@ const ( telemetryURL = "http://unix/telemetry" conntrackCachedURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/cached" conntrackHostURL = "http://unix/" + string(sysconfig.NetworkTracerModule) + "/debug/conntrack/host" - ebpfBTFLoaderURL = "http://unix/ebpf_btf_loader_info" + ebpfBTFLoaderURL = "http://unix/debug/ebpf_btf_loader_info" netType = "unix" )