From d734cbe9985e7c587a55381b24bf1c46d538d7e7 Mon Sep 17 00:00:00 2001 From: Alex Lopez Date: Thu, 12 Dec 2024 17:54:31 +0100 Subject: [PATCH] Revert "unbundle agent and remove associated code (#31228)" (#31796) Co-authored-by: Pythyu <45374460+Pythyu@users.noreply.github.com> (cherry picked from commit 906f61a453542d379ddfd379b9c53d401a9b696b) --- cmd/agent/launcher/launcher.c | 32 +++++++++++ cmd/agent/main.go | 51 +++++++++++++++++- cmd/agent/main_common.go | 13 +++++ cmd/agent/main_linux_cgo.go | 42 +++++++++++++++ cmd/agent/main_linux_no_cgo.go | 22 ++++++++ cmd/agent/process_agent.go | 26 +++++++++ cmd/agent/security_agent.go | 23 ++++++++ cmd/agent/system_probe.go | 23 ++++++++ cmd/agent/trace_agent.go | 23 ++++++++ docs/dev/agent_build.md | 24 +++++++++ omnibus/config/software/datadog-agent.rb | 22 +++++--- tasks/agent.py | 68 ++++++++++++++++++++---- 12 files changed, 352 insertions(+), 17 deletions(-) create mode 100644 cmd/agent/launcher/launcher.c create mode 100644 cmd/agent/main_common.go create mode 100644 cmd/agent/main_linux_cgo.go create mode 100644 cmd/agent/main_linux_no_cgo.go create mode 100644 cmd/agent/process_agent.go create mode 100644 cmd/agent/security_agent.go create mode 100644 cmd/agent/system_probe.go create mode 100644 cmd/agent/trace_agent.go diff --git a/cmd/agent/launcher/launcher.c b/cmd/agent/launcher/launcher.c new file mode 100644 index 0000000000000..0be5b6ca184e3 --- /dev/null +++ b/cmd/agent/launcher/launcher.c @@ -0,0 +1,32 @@ +#include +#include +#include +#include + +#ifndef DD_AGENT_PATH +#error DD_AGENT_PATH must be defined +#endif + +#ifndef DD_AGENT +#define DD_AGENT "agent" +#endif + +int main(int argc, char **argv) { + if (argc > 1) { + argv[0] = DD_AGENT; + } else { + argv = malloc(sizeof(char *) * 2); + argv[0] = DD_AGENT; + argv[1] = NULL; + } + + if (strlen(DD_AGENT_PATH) == 0) { + fprintf(stderr, "Cannot determine agent location\n"); + exit(1); + } + + setenv("DD_BUNDLED_AGENT", DD_AGENT, 0); + + execvp(DD_AGENT_PATH, argv); + return 1; +} diff --git a/cmd/agent/main.go b/cmd/agent/main.go index f2b5343e787ba..5a8c90176bee5 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -9,13 +9,62 @@ package main import ( + "fmt" "os" + "path" + "strings" "github.com/DataDog/datadog-agent/cmd/agent/command" "github.com/DataDog/datadog-agent/cmd/agent/subcommands" "github.com/DataDog/datadog-agent/cmd/internal/runcmd" + "github.com/spf13/cobra" ) +var agents = map[string]func() *cobra.Command{} + +func registerAgent(names []string, getCommand func() *cobra.Command) { + for _, name := range names { + agents[name] = getCommand + } +} + +func coreAgentMain() *cobra.Command { + return command.MakeCommand(subcommands.AgentSubcommands()) +} + +func init() { + registerAgent([]string{"agent", "datadog-agent", "dd-agent"}, coreAgentMain) +} + func main() { - os.Exit(runcmd.Run(command.MakeCommand(subcommands.AgentSubcommands()))) + process := strings.TrimSpace(os.Getenv("DD_BUNDLED_AGENT")) + + if process == "" { + if len(os.Args) > 0 { + process = strings.TrimSpace(path.Base(os.Args[0])) + } + + if process == "" { + executable, err := os.Executable() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to determine the Agent process name: %s\n", err.Error()) + os.Exit(1) + } + process = executable + } + + process = strings.TrimSuffix(process, path.Ext(process)) + } + + agentCmdBuilder := agents[process] + if agentCmdBuilder == nil { + fmt.Fprintf(os.Stderr, "Invoked as '%s', acting as main Agent.\n", process) + agentCmdBuilder = coreAgentMain + } + + rootCmd := agentCmdBuilder() + if err := setProcessName(process); err != nil { + fmt.Fprintf(os.Stderr, "Failed to set process name as '%s': %s\n", process, err) + } + os.Exit(runcmd.Run(rootCmd)) } diff --git a/cmd/agent/main_common.go b/cmd/agent/main_common.go new file mode 100644 index 0000000000000..2159c28f80cbf --- /dev/null +++ b/cmd/agent/main_common.go @@ -0,0 +1,13 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !linux + +package main + +// nolint: deadcode, unused +func setProcessName(_ string) error { + return nil +} diff --git a/cmd/agent/main_linux_cgo.go b/cmd/agent/main_linux_cgo.go new file mode 100644 index 0000000000000..421565bf6d0c1 --- /dev/null +++ b/cmd/agent/main_linux_cgo.go @@ -0,0 +1,42 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build linux && cgo + +package main + +/* +#include +#include +#include + +int prctl_err = 0; + +int set_process_name () __attribute__((constructor)); + +int set_process_name() +{ + const char *name = getenv("DD_BUNDLED_AGENT"); + if (name != NULL) { + int ret = prctl(PR_SET_NAME, name, 0, 0); + if (!ret) { + prctl_err = errno; + } + return ret; + } + return 0; +} +*/ +import ( + "C" +) +import "syscall" + +func setProcessName(_ string) error { + if C.prctl_err == 0 { + return nil + } + return syscall.Errno(C.prctl_err) +} diff --git a/cmd/agent/main_linux_no_cgo.go b/cmd/agent/main_linux_no_cgo.go new file mode 100644 index 0000000000000..5259fc616d3d0 --- /dev/null +++ b/cmd/agent/main_linux_no_cgo.go @@ -0,0 +1,22 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build linux && !cgo + +package main + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/unix" +) + +func setProcessName(process string) error { + processName := make([]byte, len(process)+1) + copy(processName, process) + _, _, err := syscall.AllThreadsSyscall(unix.SYS_PRCTL, unix.PR_SET_NAME, uintptr(unsafe.Pointer(&processName[0])), 0) + return err +} diff --git a/cmd/agent/process_agent.go b/cmd/agent/process_agent.go new file mode 100644 index 0000000000000..e9cf9867eeacb --- /dev/null +++ b/cmd/agent/process_agent.go @@ -0,0 +1,26 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !windows && bundle_process_agent + +// Main package for the agent binary +package main + +import ( + "os" + + processcommand "github.com/DataDog/datadog-agent/cmd/process-agent/command" + processsubcommands "github.com/DataDog/datadog-agent/cmd/process-agent/subcommands" + "github.com/DataDog/datadog-agent/pkg/util/flavor" + "github.com/spf13/cobra" +) + +func init() { + registerAgent([]string{"process-agent"}, func() *cobra.Command { + flavor.SetFlavor(flavor.ProcessAgent) + os.Args = processcommand.FixDeprecatedFlags(os.Args, os.Stdout) + return processcommand.MakeCommand(processsubcommands.ProcessAgentSubcommands(), processcommand.UseWinParams, processcommand.RootCmdRun) + }) +} diff --git a/cmd/agent/security_agent.go b/cmd/agent/security_agent.go new file mode 100644 index 0000000000000..134f04647b3d5 --- /dev/null +++ b/cmd/agent/security_agent.go @@ -0,0 +1,23 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !windows && bundle_security_agent + +// Main package for the agent binary +package main + +import ( + seccommand "github.com/DataDog/datadog-agent/cmd/security-agent/command" + secsubcommands "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands" + "github.com/DataDog/datadog-agent/pkg/util/flavor" + "github.com/spf13/cobra" +) + +func init() { + registerAgent([]string{"security-agent"}, func() *cobra.Command { + flavor.SetFlavor(flavor.SecurityAgent) + return seccommand.MakeCommand(secsubcommands.SecurityAgentSubcommands()) + }) +} diff --git a/cmd/agent/system_probe.go b/cmd/agent/system_probe.go new file mode 100644 index 0000000000000..4248bf5a7286c --- /dev/null +++ b/cmd/agent/system_probe.go @@ -0,0 +1,23 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !windows && bundle_system_probe + +// Main package for the agent binary +package main + +import ( + sysprobecommand "github.com/DataDog/datadog-agent/cmd/system-probe/command" + sysprobesubcommands "github.com/DataDog/datadog-agent/cmd/system-probe/subcommands" + "github.com/spf13/cobra" +) + +func init() { + registerAgent([]string{"system-probe"}, func() *cobra.Command { + rootCmd := sysprobecommand.MakeCommand(sysprobesubcommands.SysprobeSubcommands()) + sysprobecommand.SetDefaultCommandIfNonePresent(rootCmd) + return rootCmd + }) +} diff --git a/cmd/agent/trace_agent.go b/cmd/agent/trace_agent.go new file mode 100644 index 0000000000000..2ac852e6e2b5f --- /dev/null +++ b/cmd/agent/trace_agent.go @@ -0,0 +1,23 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !windows && bundle_trace_agent + +// Main package for the agent binary +package main + +import ( + "os" + + tracecommand "github.com/DataDog/datadog-agent/cmd/trace-agent/command" + "github.com/spf13/cobra" +) + +func init() { + registerAgent([]string{"trace-agent"}, func() *cobra.Command { + os.Args = tracecommand.FixDeprecatedFlags(os.Args, os.Stdout) + return tracecommand.MakeRootCommand() + }) +} diff --git a/docs/dev/agent_build.md b/docs/dev/agent_build.md index 26f0126ee1911..1896724f75403 100644 --- a/docs/dev/agent_build.md +++ b/docs/dev/agent_build.md @@ -52,6 +52,30 @@ Also note that the trace agent needs to be built and run separately. For more in We use `pkg-config` to make compilers and linkers aware of Python. The required .pc files are provided automatically when building python through omnibus. +As an option, the Agent can combine multiple functionalities into a single binary to reduce +the space used on disk. The `DD_BUNDLED_AGENT` environment variable is used to select +which functionality to enable. For instance, if set to `process-agent`, it will act as the process Agent. +If the environment variable is not defined, the process name is used as a fallback. +As the last resort meaning, the executable will behave as the 'main' Agent. + +Different combinations can be obtained through the usage of build tags. As an example, +building the Agent with the `bundle_process_agent` and `bundle_security_agent` will produce +a binary that has the process Agent and security Agent capabilities. + +The `--bundle` argument can be used to override the default set of functionalities bundled +into the Agent binary. For instance, to override the defaults and bundle only the process and +and the security Agents: + +``` +deva agent.build --bundle process-agent --bundle security-agent +``` + +To disable bundling entirely: + +``` +deva agent.build --bundle agent +``` + ## Testing Agent changes in containerized environments Building an Agent Docker image from scratch through an embedded build is a slow process. diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb index cabf4557c735c..9346aeb6641d0 100644 --- a/omnibus/config/software/datadog-agent.rb +++ b/omnibus/config/software/datadog-agent.rb @@ -31,6 +31,11 @@ build do license :project_license + bundled_agents = [] + if heroku_target? + bundled_agents = ["process-agent"] + end + # set GOPATH on the omnibus source dir for this software gopath = Pathname.new(project_dir) + '../../../..' msgoroot = "/usr/local/msgo" @@ -83,15 +88,16 @@ command "inv -e rtloader.clean" command "inv -e rtloader.make --install-prefix \"#{install_dir}/embedded\" --cmake-options '-DCMAKE_CXX_FLAGS:=\"-D_GLIBCXX_USE_CXX11_ABI=0\" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_FIND_FRAMEWORK:STRING=NEVER -DPython3_EXECUTABLE=#{install_dir}/embedded/bin/python3'", :env => env command "inv -e rtloader.install" + bundle_arg = bundled_agents ? bundled_agents.map { |k| "--bundle #{k}" }.join(" ") : "--bundle agent" include_sds = "" if linux_target? include_sds = "--include-sds" # we only support SDS on Linux targets for now end - command "inv -e agent.build --exclude-rtloader #{include_sds} --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded --flavor #{flavor_arg}", env: env + command "inv -e agent.build --exclude-rtloader #{include_sds} --major-version #{major_version_arg} --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded --flavor #{flavor_arg} #{bundle_arg}", env: env if heroku_target? - command "inv -e agent.build --exclude-rtloader --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded --flavor #{flavor_arg} --agent-bin=bin/agent/core-agent", env: env + command "inv -e agent.build --exclude-rtloader --major-version #{major_version_arg} --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded --flavor #{flavor_arg} --agent-bin=bin/agent/core-agent --bundle agent", env: env end end @@ -120,8 +126,10 @@ mkdir Omnibus::Config.package_dir() unless Dir.exists?(Omnibus::Config.package_dir()) end - platform = windows_arch_i386? ? "x86" : "x64" - command "invoke trace-agent.build --install-path=#{install_dir} --major-version #{major_version_arg} --flavor #{flavor_arg}", :env => env + if not bundled_agents.include? "trace-agent" + platform = windows_arch_i386? ? "x86" : "x64" + command "invoke trace-agent.build --install-path=#{install_dir} --major-version #{major_version_arg} --flavor #{flavor_arg}", :env => env + end if windows_target? copy 'bin/trace-agent/trace-agent.exe', "#{install_dir}/bin/agent" @@ -130,7 +138,9 @@ end # Process agent - command "invoke -e process-agent.build --install-path=#{install_dir} --major-version #{major_version_arg} --flavor #{flavor_arg}", :env => env + if not bundled_agents.include? "process-agent" + command "invoke -e process-agent.build --install-path=#{install_dir} --major-version #{major_version_arg} --flavor #{flavor_arg}", :env => env + end if windows_target? copy 'bin/process-agent/process-agent.exe', "#{install_dir}/bin/agent" @@ -180,7 +190,7 @@ copy 'bin/cws-instrumentation/cws-instrumentation', "#{install_dir}/embedded/bin" end - # OTel agent + # OTel agent - can never be bundled if ot_target? unless windows_target? command "invoke -e otel-agent.build", :env => env diff --git a/tasks/agent.py b/tasks/agent.py index a3eee098dd2a9..b0220fcb60037 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -21,6 +21,8 @@ REPO_PATH, bin_name, get_build_flags, + get_embedded_path, + get_goenv, get_version, gitlab_section, ) @@ -113,7 +115,7 @@ LAST_DIRECTORY_COMMIT_PATTERN = "git -C {integrations_dir} rev-list -1 HEAD {integration}" -@task +@task(iterable=['bundle']) @run_on_devcontainer def build( ctx, @@ -134,6 +136,8 @@ def build( go_mod="mod", windows_sysprobe=False, cmake_options='', + bundle=None, + bundle_ebpf=False, agent_bin=None, run_on=None, # noqa: U100, F841. Used by the run_on_devcontainer decorator ): @@ -167,12 +171,12 @@ def build( major_version=major_version, ) + bundled_agents = ["agent"] if sys.platform == 'win32' or os.getenv("GOOS") == "windows": # Important for x-compiling env["CGO_ENABLED"] = "1" build_messagetable(ctx) - # Do not call build_rc when cross-compiling on Linux as the intend is more # to streamline the development process that producing a working executable / installer if sys.platform == 'win32': @@ -183,20 +187,31 @@ def build( vars=vars, out="cmd/agent/rsrc.syso", ) + else: + bundled_agents += bundle or [] if flavor.is_iot(): # Iot mode overrides whatever passed through `--build-exclude` and `--build-include` build_tags = get_default_build_tags(build="agent", flavor=flavor) else: - include_tags = ( - get_default_build_tags(build="agent", flavor=flavor) - if build_include is None - else filter_incompatible_tags(build_include.split(",")) - ) + all_tags = set() + if bundle_ebpf and "system-probe" in bundled_agents: + all_tags.add("ebpf_bindata") + + for build in bundled_agents: + all_tags.add("bundle_" + build.replace("-", "_")) + include_tags = ( + get_default_build_tags(build=build, flavor=flavor) + if build_include is None + else filter_incompatible_tags(build_include.split(",")) + ) - exclude_tags = [] if build_exclude is None else build_exclude.split(",") - build_tags = get_build_tags(include_tags, exclude_tags) - build_tags = add_fips_tags(build_tags, fips_mode) + exclude_tags = [] if build_exclude is None else build_exclude.split(",") + build_tags = get_build_tags(include_tags, exclude_tags) + build_tags = add_fips_tags(build_tags, fips_mode) + + all_tags |= set(build_tags) + build_tags = list(all_tags) cmd = "go build -mod={go_mod} {race_opt} {build_type} -tags \"{go_build_tags}\" " @@ -221,6 +236,23 @@ def build( with gitlab_section("Build agent", collapsed=True): ctx.run(cmd.format(**args), env=env) + if embedded_path is None: + embedded_path = get_embedded_path(ctx) + assert embedded_path, "Failed to find embedded path" + + for build in bundled_agents: + if build == "agent": + continue + + bundled_agent_dir = os.path.join(BIN_DIR, build) + bundled_agent_bin = os.path.join(bundled_agent_dir, bin_name(build)) + agent_fullpath = os.path.normpath(os.path.join(embedded_path, "..", "bin", "agent", bin_name("agent"))) + + if not os.path.exists(os.path.dirname(bundled_agent_bin)): + os.mkdir(os.path.dirname(bundled_agent_bin)) + + create_launcher(ctx, build, agent_fullpath, bundled_agent_bin) + with gitlab_section("Generate configuration files", collapsed=True): render_config( ctx, @@ -233,6 +265,22 @@ def build( ) +def create_launcher(ctx, agent, src, dst): + cc = get_goenv(ctx, "CC") + if not cc: + print("Failed to find C compiler") + raise Exit(code=1) + + cmd = "{cc} -DDD_AGENT_PATH='\"{agent_bin}\"' -DDD_AGENT='\"{agent}\"' -o {launcher_bin} ./cmd/agent/launcher/launcher.c" + args = { + "cc": cc, + "agent": agent, + "agent_bin": src, + "launcher_bin": dst, + } + ctx.run(cmd.format(**args)) + + def render_config(ctx, env, flavor, skip_assets, build_tags, development, windows_sysprobe): # Remove cross-compiling bits to render config env.update({"GOOS": "", "GOARCH": ""})