diff --git a/cmd/installer/main.go b/cmd/installer/main.go index 15fcd2a8eb91c..1cd198631d420 100644 --- a/cmd/installer/main.go +++ b/cmd/installer/main.go @@ -9,13 +9,33 @@ package main import ( + "fmt" "os" "github.com/DataDog/datadog-agent/cmd/installer/command" "github.com/DataDog/datadog-agent/cmd/installer/subcommands" - "github.com/DataDog/datadog-agent/cmd/internal/runcmd" + "github.com/spf13/cobra" + "go.uber.org/dig" + + installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors" ) func main() { - os.Exit(runcmd.Run(command.MakeCommand(subcommands.InstallerSubcommands()))) + os.Exit(runCmd(command.MakeCommand(subcommands.InstallerSubcommands()))) +} + +func runCmd(cmd *cobra.Command) int { + // always silence errors, since they are handled here + cmd.SilenceErrors = true + + err := cmd.Execute() + if err != nil { + if rootCauseErr := dig.RootCause(err); rootCauseErr != err { + fmt.Fprintln(cmd.ErrOrStderr(), installerErrors.FromErr(rootCauseErr).ToJSON()) + } else { + fmt.Fprintln(cmd.ErrOrStderr(), installerErrors.FromErr(err).ToJSON()) + } + return -1 + } + return 0 } diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index f4b25dd1a164e..ebbe269aa7ebd 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -17,8 +17,8 @@ import ( "github.com/DataDog/datadog-agent/cmd/installer/command" "github.com/DataDog/datadog-agent/pkg/fleet/bootstrapper" - "github.com/DataDog/datadog-agent/pkg/fleet/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" "github.com/DataDog/datadog-agent/pkg/version" "github.com/spf13/cobra" @@ -226,7 +226,7 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry { if site == "" { site = config.Site } - t, err := telemetry.NewTelemetry(apiKey, site, "datadog-installer") // No sampling rules for commands + t, err := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands if err != nil { fmt.Printf("failed to initialize telemetry: %v\n", err) return nil diff --git a/comp/updater/telemetry/telemetryimpl/telemetry.go b/comp/updater/telemetry/telemetryimpl/telemetry.go index caf3b90e73bcc..cf19e681c5708 100644 --- a/comp/updater/telemetry/telemetryimpl/telemetry.go +++ b/comp/updater/telemetry/telemetryimpl/telemetry.go @@ -7,14 +7,17 @@ package telemetryimpl import ( + "net/http" + "go.uber.org/fx" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/updater/telemetry" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/config/utils" fleettelemetry "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" "github.com/DataDog/datadog-agent/pkg/util/fxutil" + httputils "github.com/DataDog/datadog-agent/pkg/util/http" ) type dependencies struct { @@ -32,12 +35,14 @@ func Module() fxutil.Module { } func newTelemetry(deps dependencies) (telemetry.Component, error) { - env := env.FromConfig(deps.Config) - telemetry, err := fleettelemetry.NewTelemetry(env.APIKey, env.Site, "datadog-installer", + client := &http.Client{ + Transport: httputils.CreateHTTPTransport(deps.Config), + } + telemetry, err := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon", fleettelemetry.WithSamplingRules( - tracer.NameServiceRule("cdn.*", "datadog-installer", 0.1), - tracer.NameServiceRule("*garbage_collect*", "datadog-installer", 0.05), - tracer.NameServiceRule("HTTPClient.*", "datadog-installer", 0.05), + tracer.NameServiceRule("cdn.*", "datadog-installer-daemon", 0.1), + tracer.NameServiceRule("*garbage_collect*", "datadog-installer-daemon", 0.05), + tracer.NameServiceRule("HTTPClient.*", "datadog-installer-daemon", 0.05), ), ) if err != nil { diff --git a/comp/updater/updater/updaterimpl/updater.go b/comp/updater/updater/updaterimpl/updater.go index 117ea9c937653..5ed1cbd252375 100644 --- a/comp/updater/updater/updaterimpl/updater.go +++ b/comp/updater/updater/updaterimpl/updater.go @@ -7,12 +7,14 @@ package updaterimpl import ( + "context" "errors" "fmt" "go.uber.org/fx" "github.com/DataDog/datadog-agent/comp/core/config" + "github.com/DataDog/datadog-agent/comp/core/hostname" log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/remote-config/rcservice" updatercomp "github.com/DataDog/datadog-agent/comp/updater/updater" @@ -36,6 +38,7 @@ func Module() fxutil.Module { type dependencies struct { fx.In + Hostname hostname.Component Log log.Component Config config.Component RemoteConfig optional.Option[rcservice.Component] @@ -46,7 +49,11 @@ func newUpdaterComponent(lc fx.Lifecycle, dependencies dependencies) (updatercom if !ok { return nil, errRemoteConfigRequired } - daemon, err := daemon.NewDaemon(remoteConfig, dependencies.Config) + hostname, err := dependencies.Hostname.Get(context.Background()) + if err != nil { + return nil, fmt.Errorf("could not get hostname: %w", err) + } + daemon, err := daemon.NewDaemon(hostname, remoteConfig, dependencies.Config) if err != nil { return nil, fmt.Errorf("could not create updater: %w", err) } diff --git a/pkg/fleet/bootstrapper/bootstrapper.go b/pkg/fleet/bootstrapper/bootstrapper.go index a21960a096527..e935a6cadc649 100644 --- a/pkg/fleet/bootstrapper/bootstrapper.go +++ b/pkg/fleet/bootstrapper/bootstrapper.go @@ -10,7 +10,7 @@ import ( "context" "fmt" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/internal/bootstrap" "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" diff --git a/pkg/fleet/daemon/daemon.go b/pkg/fleet/daemon/daemon.go index d4d99ff7fd68e..566275521126a 100644 --- a/pkg/fleet/daemon/daemon.go +++ b/pkg/fleet/daemon/daemon.go @@ -16,6 +16,7 @@ import ( osexec "os/exec" "path/filepath" "runtime" + "strings" "sync" "time" @@ -24,8 +25,9 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/pkg/config/remote/client" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/config/utils" "github.com/DataDog/datadog-agent/pkg/fleet/installer" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" "github.com/DataDog/datadog-agent/pkg/fleet/internal/bootstrap" @@ -83,7 +85,7 @@ func newInstaller(env *env.Env, installerBin string) installer.Installer { } // NewDaemon returns a new daemon. -func NewDaemon(rcFetcher client.ConfigFetcher, config config.Reader) (Daemon, error) { +func NewDaemon(hostname string, rcFetcher client.ConfigFetcher, config config.Reader) (Daemon, error) { installerBin, err := os.Executable() if err != nil { return nil, fmt.Errorf("could not get installer executable path: %w", err) @@ -96,7 +98,22 @@ func NewDaemon(rcFetcher client.ConfigFetcher, config config.Reader) (Daemon, er if err != nil { return nil, fmt.Errorf("could not create remote config client: %w", err) } - env := env.FromConfig(config) + env := &env.Env{ + APIKey: utils.SanitizeAPIKey(config.GetString("api_key")), + Site: config.GetString("site"), + RemoteUpdates: config.GetBool("remote_updates"), + RemotePolicies: config.GetBool("remote_policies"), + Mirror: config.GetString("installer.mirror"), + RegistryOverride: config.GetString("installer.registry.url"), + RegistryAuthOverride: config.GetString("installer.registry.auth"), + RegistryUsername: config.GetString("installer.registry.username"), + RegistryPassword: config.GetString("installer.registry.password"), + Tags: utils.GetConfiguredTags(config, false), + Hostname: hostname, + HTTPProxy: config.GetString("proxy.http"), + HTTPSProxy: config.GetString("proxy.https"), + NoProxy: strings.Join(config.GetStringSlice("proxy.no_proxy"), ","), + } installer := newInstaller(env, installerBin) cdn, err := cdn.New(env, filepath.Join(paths.RunPath, "rc_daemon")) if err != nil { @@ -552,7 +569,7 @@ func setRequestDone(ctx context.Context, err error) { state.State = pbgo.TaskState_DONE if err != nil { state.State = pbgo.TaskState_ERROR - state.Err = installerErrors.From(err) + state.Err = installerErrors.FromErr(err) } } diff --git a/pkg/fleet/daemon/daemon_test.go b/pkg/fleet/daemon/daemon_test.go index 6a5db6fcf957c..228b37089845f 100644 --- a/pkg/fleet/daemon/daemon_test.go +++ b/pkg/fleet/daemon/daemon_test.go @@ -20,7 +20,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" "github.com/DataDog/datadog-agent/pkg/fleet/internal/cdn" pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" diff --git a/pkg/fleet/env/http_client.go b/pkg/fleet/env/http_client.go deleted file mode 100644 index 7d9184e944849..0000000000000 --- a/pkg/fleet/env/http_client.go +++ /dev/null @@ -1,22 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package env - -import ( - "net/http" - - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" - httputils "github.com/DataDog/datadog-agent/pkg/util/http" -) - -// GetHTTPClient returns an HTTP client with the proxy settings loaded from the environment. -func GetHTTPClient() *http.Client { - // Load proxy settings before creating any HTTP transport - pkgconfigsetup.LoadProxyFromEnv(pkgconfigsetup.Datadog()) - httpClient := http.DefaultClient - httpClient.Transport = httputils.CreateHTTPTransport(pkgconfigsetup.Datadog()) - return httpClient -} diff --git a/pkg/fleet/env/install_script.go b/pkg/fleet/env/install_script.go deleted file mode 100644 index eb89f36b74d44..0000000000000 --- a/pkg/fleet/env/install_script.go +++ /dev/null @@ -1,33 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package env - -const ( - envApmInstrumentationEnabled = "DD_APM_INSTRUMENTATION_ENABLED" -) - -const ( - // APMInstrumentationEnabledAll enables APM instrumentation for all containers. - APMInstrumentationEnabledAll = "all" - // APMInstrumentationEnabledDocker enables APM instrumentation for Docker containers. - APMInstrumentationEnabledDocker = "docker" - // APMInstrumentationEnabledHost enables APM instrumentation for the host. - APMInstrumentationEnabledHost = "host" - // APMInstrumentationNotSet is the default value when the environment variable is not set. - APMInstrumentationNotSet = "not_set" -) - -// InstallScriptEnv contains the environment variables for the install script. -type InstallScriptEnv struct { - APMInstrumentationEnabled string -} - -func installScriptEnvFromEnv() InstallScriptEnv { - return InstallScriptEnv{ - // defaults to all if not set - APMInstrumentationEnabled: getEnvOrDefault(envApmInstrumentationEnabled, APMInstrumentationNotSet), - } -} diff --git a/pkg/fleet/installer/default_packages.go b/pkg/fleet/installer/default_packages.go index ff2e87cdb9e5f..66c7017f42bac 100644 --- a/pkg/fleet/installer/default_packages.go +++ b/pkg/fleet/installer/default_packages.go @@ -10,7 +10,7 @@ import ( "slices" "strings" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" ) diff --git a/pkg/fleet/installer/default_packages_test.go b/pkg/fleet/installer/default_packages_test.go index 0e28847f0158e..e17645ec7cefb 100644 --- a/pkg/fleet/installer/default_packages_test.go +++ b/pkg/fleet/installer/default_packages_test.go @@ -8,7 +8,7 @@ package installer import ( "testing" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" "github.com/stretchr/testify/assert" ) diff --git a/pkg/fleet/env/env.go b/pkg/fleet/installer/env/env.go similarity index 85% rename from pkg/fleet/env/env.go rename to pkg/fleet/installer/env/env.go index a1415ea1780a3..094003754ddb3 100644 --- a/pkg/fleet/env/env.go +++ b/pkg/fleet/installer/env/env.go @@ -7,16 +7,16 @@ package env import ( - "context" "fmt" + "net" + "net/http" + "net/url" "os" "slices" "strings" "time" - "github.com/DataDog/datadog-agent/pkg/config/model" - "github.com/DataDog/datadog-agent/pkg/config/utils" - "github.com/DataDog/datadog-agent/pkg/util/hostname" + "golang.org/x/net/http/httpproxy" ) const ( @@ -49,6 +49,9 @@ const ( envHTTPSProxy = "HTTPS_PROXY" envDDNoProxy = "DD_PROXY_NO_PROXY" envNoProxy = "NO_PROXY" + + // install script + envApmInstrumentationEnabled = "DD_APM_INSTRUMENTATION_ENABLED" ) var defaultEnv = Env{ @@ -80,6 +83,22 @@ type ApmLibLanguage string // ApmLibVersion is the version of the library defined in DD_APM_INSTRUMENTATION_LIBRARIES env var type ApmLibVersion string +const ( + // APMInstrumentationEnabledAll enables APM instrumentation for all containers. + APMInstrumentationEnabledAll = "all" + // APMInstrumentationEnabledDocker enables APM instrumentation for Docker containers. + APMInstrumentationEnabledDocker = "docker" + // APMInstrumentationEnabledHost enables APM instrumentation for the host. + APMInstrumentationEnabledHost = "host" + // APMInstrumentationNotSet is the default value when the environment variable is not set. + APMInstrumentationNotSet = "not_set" +) + +// InstallScriptEnv contains the environment variables for the install script. +type InstallScriptEnv struct { + APMInstrumentationEnabled string +} + // Env contains the configuration for the installer. type Env struct { APIKey string @@ -119,6 +138,32 @@ type Env struct { NoProxy string } +// HTTPClient returns an HTTP client with the proxy settings from the environment. +func (e *Env) HTTPClient() *http.Client { + proxyConfig := &httpproxy.Config{ + HTTPProxy: e.HTTPProxy, + HTTPSProxy: e.HTTPSProxy, + NoProxy: e.NoProxy, + } + proxyFunc := func(r *http.Request) (*url.URL, error) { + return proxyConfig.ProxyFunc()(r.URL) + } + client := &http.Client{ + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + Proxy: proxyFunc, + }, + } + return client +} + // FromEnv returns an Env struct with values from the environment. func FromEnv() *Env { splitFunc := func(c rune) bool { @@ -150,7 +195,9 @@ func FromEnv() *Env { AgentMinorVersion: os.Getenv(envAgentMinorVersion), AgentUserName: getEnvOrDefault(envAgentUserName, os.Getenv(envAgentUserNameCompat)), - InstallScript: installScriptEnvFromEnv(), + InstallScript: InstallScriptEnv{ + APMInstrumentationEnabled: getEnvOrDefault(envApmInstrumentationEnabled, APMInstrumentationNotSet), + }, CDNEnabled: strings.ToLower(os.Getenv(envCDNEnabled)) == "true", CDNLocalDirPath: getEnvOrDefault(envCDNLocalDirPath, ""), @@ -167,32 +214,6 @@ func FromEnv() *Env { } } -// FromConfig returns an Env struct with values from the configuration. -func FromConfig(config model.Reader) *Env { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - hostname, err := hostname.Get(ctx) - if err != nil { - hostname = "unknown" - } - return &Env{ - APIKey: utils.SanitizeAPIKey(config.GetString("api_key")), - Site: config.GetString("site"), - RemoteUpdates: config.GetBool("remote_updates"), - RemotePolicies: config.GetBool("remote_policies"), - Mirror: config.GetString("installer.mirror"), - RegistryOverride: config.GetString("installer.registry.url"), - RegistryAuthOverride: config.GetString("installer.registry.auth"), - RegistryUsername: config.GetString("installer.registry.username"), - RegistryPassword: config.GetString("installer.registry.password"), - Tags: utils.GetConfiguredTags(config, false), - Hostname: hostname, - HTTPProxy: config.GetString("proxy.http"), - HTTPSProxy: config.GetString("proxy.https"), - NoProxy: strings.Join(config.GetStringSlice("proxy.no_proxy"), ","), - } -} - // ToEnv returns a slice of environment variables from the Env struct. func (e *Env) ToEnv() []string { var env []string diff --git a/pkg/fleet/env/env_test.go b/pkg/fleet/installer/env/env_test.go similarity index 100% rename from pkg/fleet/env/env_test.go rename to pkg/fleet/installer/env/env_test.go diff --git a/pkg/fleet/installer/errors/errors.go b/pkg/fleet/installer/errors/errors.go index 589d8179a2fa8..736c609af033c 100644 --- a/pkg/fleet/installer/errors/errors.go +++ b/pkg/fleet/installer/errors/errors.go @@ -7,6 +7,7 @@ package errors import ( + "encoding/json" "errors" ) @@ -14,19 +15,15 @@ import ( type InstallerErrorCode uint64 const ( - errUnknown InstallerErrorCode = iota // This error code is purposefully not exported - // ErrInstallFailed is the code for an install failure. - ErrInstallFailed + errUnknown InstallerErrorCode = 0 // This error code is purposefully not exported // ErrDownloadFailed is the code for a download failure. - ErrDownloadFailed - // ErrInvalidHash is the code for an invalid hash. - ErrInvalidHash - // ErrInvalidState is the code for an invalid state. - ErrInvalidState + ErrDownloadFailed InstallerErrorCode = 1 + // ErrNotEnoughDiskSpace is the code for not enough disk space. + ErrNotEnoughDiskSpace InstallerErrorCode = 2 // ErrPackageNotFound is the code for a package not found. - ErrPackageNotFound - // ErrUpdateExperimentFailed is the code for an update experiment failure. - ErrUpdateExperimentFailed + ErrPackageNotFound InstallerErrorCode = 3 + // ErrFilesystemIssue is the code for a filesystem issue (e.g. permission issue). + ErrFilesystemIssue InstallerErrorCode = 4 ) // InstallerError is an error type used by the installer. @@ -35,6 +32,11 @@ type InstallerError struct { code InstallerErrorCode } +type installerErrorJSON struct { + Error string `json:"error"` + Code int `json:"code"` +} + // Error returns the error message. func (e InstallerError) Error() string { return e.err.Error() @@ -60,7 +62,7 @@ func (e InstallerError) Code() InstallerErrorCode { // If the given error is already an installer error, it is not wrapped and // left as it is. Only the deepest InstallerError remains. func Wrap(errCode InstallerErrorCode, err error) error { - if errors.Is(err, &InstallerError{}) { + if FromErr(err).code != errUnknown { return err } return &InstallerError{ @@ -69,18 +71,51 @@ func Wrap(errCode InstallerErrorCode, err error) error { } } -// From returns a new InstallerError from the given error. -func From(err error) *InstallerError { +// FromErr returns a new InstallerError from the given error. +// Unwraps the error until it finds an InstallerError and return unknown error code if not found. +func FromErr(err error) *InstallerError { if err == nil { return nil } - e, ok := err.(*InstallerError) if !ok { - return &InstallerError{ - err: err, - code: errUnknown, + unwrappedErr := errors.Unwrap(err) + if unwrappedErr == nil { + return &InstallerError{ + err: err, + code: errUnknown, + } } + return FromErr(unwrappedErr) } return e } + +// ToJSON returns the error as a JSON string. +func (e InstallerError) ToJSON() string { + tmp := installerErrorJSON{ + Error: e.err.Error(), + Code: int(e.code), + } + jsonErr, err := json.Marshal(tmp) + if err != nil { + return e.err.Error() + } + return string(jsonErr) +} + +// FromJSON returns an InstallerError from a JSON string. +func FromJSON(errStr string) InstallerError { + var jsonError installerErrorJSON + err := json.Unmarshal([]byte(errStr), &jsonError) + if err != nil { + return InstallerError{ + err: errors.New(errStr), + code: errUnknown, + } + } + return InstallerError{ + err: errors.New(jsonError.Error), + code: InstallerErrorCode(jsonError.Code), + } +} diff --git a/pkg/fleet/installer/errors/errors_test.go b/pkg/fleet/installer/errors/errors_test.go index 44a5cdc584b95..5115faaaf8a26 100644 --- a/pkg/fleet/installer/errors/errors_test.go +++ b/pkg/fleet/installer/errors/errors_test.go @@ -12,18 +12,34 @@ import ( "github.com/stretchr/testify/assert" ) -func TestFrom(t *testing.T) { +func TestFromErr(t *testing.T) { var err error = &InstallerError{ err: fmt.Errorf("test: test"), code: ErrDownloadFailed, } - taskErr := From(err) + taskErr := FromErr(err) assert.Equal(t, taskErr, &InstallerError{ err: fmt.Errorf("test: test"), code: ErrDownloadFailed, }) - assert.Nil(t, From(nil)) + assert.Nil(t, FromErr(nil)) +} + +func TestFromErrWithWrap(t *testing.T) { + err := fmt.Errorf("test: %w", &InstallerError{ + err: fmt.Errorf("test: test"), + code: ErrDownloadFailed, + }) + taskErr := FromErr(err) + assert.Equal(t, taskErr, &InstallerError{ + err: fmt.Errorf("test: test"), + code: ErrDownloadFailed, + }) + + taskErr2 := fmt.Errorf("Wrap 2: %w", fmt.Errorf("Wrap 1: %w", taskErr)) + assert.Equal(t, FromErr(taskErr2).Code(), ErrDownloadFailed) + assert.Nil(t, FromErr(nil)) } func TestWrap(t *testing.T) { @@ -36,9 +52,12 @@ func TestWrap(t *testing.T) { // Check that Wrap doesn't change anything if the error // is already an InstallerError - taskErr2 := Wrap(ErrInstallFailed, taskErr) + taskErr2 := Wrap(ErrNotEnoughDiskSpace, taskErr) assert.Equal(t, taskErr2, &InstallerError{ err: err, code: ErrDownloadFailed, }) + + taskErr3 := Wrap(ErrFilesystemIssue, fmt.Errorf("Wrap 2: %w", fmt.Errorf("Wrap 1: %w", taskErr2))) + assert.Equal(t, FromErr(taskErr3).Code(), ErrDownloadFailed) } diff --git a/pkg/fleet/installer/installer.go b/pkg/fleet/installer/installer.go index 8db7bc07954a5..8000be3c5a261 100644 --- a/pkg/fleet/installer/installer.go +++ b/pkg/fleet/installer/installer.go @@ -20,7 +20,8 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/internal/cdn" "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" - fleetEnv "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors" "github.com/DataDog/datadog-agent/pkg/fleet/installer/packages" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" "github.com/DataDog/datadog-agent/pkg/fleet/internal/db" @@ -71,7 +72,7 @@ type Installer interface { type installerImpl struct { m sync.Mutex - env *fleetEnv.Env + env *env.Env cdn *cdn.CDN db *db.PackagesDB downloader *oci.Downloader @@ -83,7 +84,7 @@ type installerImpl struct { } // NewInstaller returns a new Package Manager. -func NewInstaller(env *fleetEnv.Env) (Installer, error) { +func NewInstaller(env *env.Env) (Installer, error) { err := ensureRepositoriesExist() if err != nil { return nil, fmt.Errorf("could not ensure packages and config directory exists: %w", err) @@ -100,7 +101,7 @@ func NewInstaller(env *fleetEnv.Env) (Installer, error) { env: env, cdn: cdn, db: db, - downloader: oci.NewDownloader(env, fleetEnv.GetHTTPClient()), + downloader: oci.NewDownloader(env, env.HTTPClient()), packages: repository.NewRepositories(paths.PackagesPath, paths.LocksPath), configs: repository.NewRepositories(paths.ConfigsPath, paths.LocksPath), @@ -227,30 +228,48 @@ func (i *installerImpl) InstallExperiment(ctx context.Context, url string) error defer i.m.Unlock() pkg, err := i.downloader.Download(ctx, url) if err != nil { - return fmt.Errorf("could not download package: %w", err) + return installerErrors.Wrap( + installerErrors.ErrDownloadFailed, + fmt.Errorf("could not download package: %w", err), + ) } err = checkAvailableDiskSpace(i.packages, pkg) if err != nil { - return fmt.Errorf("not enough disk space: %w", err) + return installerErrors.Wrap( + installerErrors.ErrNotEnoughDiskSpace, + fmt.Errorf("not enough disk space: %w", err), + ) } tmpDir, err := i.packages.MkdirTemp() if err != nil { - return fmt.Errorf("could not create temporary directory: %w", err) + return installerErrors.Wrap( + installerErrors.ErrFilesystemIssue, + fmt.Errorf("could create temporary directory: %w", err), + ) } defer os.RemoveAll(tmpDir) configDir := filepath.Join(i.userConfigsDir, pkg.Name) err = pkg.ExtractLayers(oci.DatadogPackageLayerMediaType, tmpDir) if err != nil { - return fmt.Errorf("could not extract package layers: %w", err) + return installerErrors.Wrap( + installerErrors.ErrDownloadFailed, + fmt.Errorf("could not extract package layer: %w", err), + ) } err = pkg.ExtractLayers(oci.DatadogPackageConfigLayerMediaType, configDir) if err != nil { - return fmt.Errorf("could not extract package config layer: %w", err) + return installerErrors.Wrap( + installerErrors.ErrDownloadFailed, + fmt.Errorf("could not extract package config layer: %w", err), + ) } repository := i.packages.Get(pkg.Name) err = repository.SetExperiment(pkg.Version, tmpDir) if err != nil { - return fmt.Errorf("could not set experiment: %w", err) + return installerErrors.Wrap( + installerErrors.ErrFilesystemIssue, + fmt.Errorf("could not set experiment: %w", err), + ) } return i.startExperiment(ctx, pkg.Name) @@ -267,7 +286,10 @@ func (i *installerImpl) RemoveExperiment(ctx context.Context, pkg string) error // will kill the current process, delete the experiment first. err := repository.DeleteExperiment() if err != nil { - return fmt.Errorf("could not delete experiment: %w", err) + return installerErrors.Wrap( + installerErrors.ErrFilesystemIssue, + fmt.Errorf("could not delete experiment: %w", err), + ) } err = i.stopExperiment(ctx, pkg) if err != nil { @@ -280,7 +302,10 @@ func (i *installerImpl) RemoveExperiment(ctx context.Context, pkg string) error } err = repository.DeleteExperiment() if err != nil { - return fmt.Errorf("could not delete experiment: %w", err) + return installerErrors.Wrap( + installerErrors.ErrFilesystemIssue, + fmt.Errorf("could not delete experiment: %w", err), + ) } } return nil @@ -306,27 +331,42 @@ func (i *installerImpl) InstallConfigExperiment(ctx context.Context, pkg string, config, err := i.cdn.Get(ctx, pkg) if err != nil { - return fmt.Errorf("could not get cdn config: %w", err) + return installerErrors.Wrap( + installerErrors.ErrDownloadFailed, + fmt.Errorf("could not get cdn config: %w", err), + ) } if config.State().GetVersion() != version { - return fmt.Errorf("version mismatch: expected %s, got %s", config.State().GetVersion(), version) + return installerErrors.Wrap( + installerErrors.ErrDownloadFailed, + fmt.Errorf("version mismatch: expected %s, got %s", config.State().GetVersion(), version), + ) } tmpDir, err := i.packages.MkdirTemp() if err != nil { - return fmt.Errorf("could not create temporary directory: %w", err) + return installerErrors.Wrap( + installerErrors.ErrFilesystemIssue, + fmt.Errorf("could not create temporary directory: %w", err), + ) } defer os.RemoveAll(tmpDir) err = config.Write(tmpDir) if err != nil { - return fmt.Errorf("could not write agent config: %w", err) + return installerErrors.Wrap( + installerErrors.ErrFilesystemIssue, + fmt.Errorf("could not write agent config: %w", err), + ) } configRepo := i.configs.Get(pkg) err = configRepo.SetExperiment(version, tmpDir) if err != nil { - return fmt.Errorf("could not set experiment: %w", err) + return installerErrors.Wrap( + installerErrors.ErrFilesystemIssue, + fmt.Errorf("could not set experiment: %w", err), + ) } switch runtime.GOOS { @@ -349,7 +389,10 @@ func (i *installerImpl) RemoveConfigExperiment(ctx context.Context, pkg string) repository := i.configs.Get(pkg) err = repository.DeleteExperiment() if err != nil { - return fmt.Errorf("could not delete experiment: %w", err) + return installerErrors.Wrap( + installerErrors.ErrFilesystemIssue, + fmt.Errorf("could not delete experiment: %w", err), + ) } return nil } @@ -362,7 +405,10 @@ func (i *installerImpl) PromoteConfigExperiment(ctx context.Context, pkg string) repository := i.configs.Get(pkg) err := repository.PromoteExperiment() if err != nil { - return fmt.Errorf("could not promote experiment: %w", err) + return installerErrors.Wrap( + installerErrors.ErrFilesystemIssue, + fmt.Errorf("could not promote experiment: %w", err), + ) } return i.promoteExperiment(ctx, pkg) } diff --git a/pkg/fleet/installer/installer_test.go b/pkg/fleet/installer/installer_test.go index a86562cc32609..9ab772b32dc0f 100644 --- a/pkg/fleet/installer/installer_test.go +++ b/pkg/fleet/installer/installer_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" "github.com/DataDog/datadog-agent/pkg/fleet/internal/db" "github.com/DataDog/datadog-agent/pkg/fleet/internal/fixtures" diff --git a/pkg/fleet/installer/packages/apm_inject.go b/pkg/fleet/installer/packages/apm_inject.go index cd7c08504deb6..febd93ecf8dc7 100644 --- a/pkg/fleet/installer/packages/apm_inject.go +++ b/pkg/fleet/installer/packages/apm_inject.go @@ -17,7 +17,7 @@ import ( "path/filepath" "strings" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer/packages/embedded" "github.com/DataDog/datadog-agent/pkg/util/log" "go.uber.org/multierr" diff --git a/pkg/fleet/installer/packages/apm_inject_test.go b/pkg/fleet/installer/packages/apm_inject_test.go index 736077ea7b934..254feab2b0075 100644 --- a/pkg/fleet/installer/packages/apm_inject_test.go +++ b/pkg/fleet/installer/packages/apm_inject_test.go @@ -11,7 +11,7 @@ import ( "context" "testing" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/stretchr/testify/assert" ) diff --git a/pkg/fleet/installer/setup.go b/pkg/fleet/installer/setup.go index 230bc1b6a1bd1..6673e44e2b081 100644 --- a/pkg/fleet/installer/setup.go +++ b/pkg/fleet/installer/setup.go @@ -10,7 +10,7 @@ import ( "context" "fmt" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" ) diff --git a/pkg/fleet/internal/bootstrap/bootstrap.go b/pkg/fleet/internal/bootstrap/bootstrap.go index e16f0cd9d603e..5f15ec79167d5 100644 --- a/pkg/fleet/internal/bootstrap/bootstrap.go +++ b/pkg/fleet/internal/bootstrap/bootstrap.go @@ -8,7 +8,8 @@ package bootstrap import ( "context" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" ) const ( diff --git a/pkg/fleet/internal/bootstrap/bootstrap_nix.go b/pkg/fleet/internal/bootstrap/bootstrap_nix.go index 6a3a7dd36e6f2..fbc9817f0783c 100644 --- a/pkg/fleet/internal/bootstrap/bootstrap_nix.go +++ b/pkg/fleet/internal/bootstrap/bootstrap_nix.go @@ -16,12 +16,12 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" - fleetEnv "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" ) -func install(ctx context.Context, env *fleetEnv.Env, url string, experiment bool) error { +func install(ctx context.Context, env *env.Env, url string, experiment bool) error { err := os.MkdirAll(paths.RootTmpDir, 0755) if err != nil { return fmt.Errorf("failed to create temporary directory: %w", err) @@ -48,9 +48,9 @@ func install(ctx context.Context, env *fleetEnv.Env, url string, experiment bool // 2. Export the installer image as an OCI layout on the disk. // 3. Extract the installer image layers on the disk. // 4. Create an installer executor from the extract layer. -func downloadInstaller(ctx context.Context, env *fleetEnv.Env, url string, tmpDir string) (*exec.InstallerExec, error) { +func downloadInstaller(ctx context.Context, env *env.Env, url string, tmpDir string) (*exec.InstallerExec, error) { // 1. Download the installer package from the registry. - downloader := oci.NewDownloader(env, fleetEnv.GetHTTPClient()) + downloader := oci.NewDownloader(env, env.HTTPClient()) downloadedPackage, err := downloader.Download(ctx, url) if err != nil { return nil, fmt.Errorf("failed to download installer package: %w", err) diff --git a/pkg/fleet/internal/bootstrap/bootstrap_windows.go b/pkg/fleet/internal/bootstrap/bootstrap_windows.go index 9e4d260e73897..c3d867f959db2 100644 --- a/pkg/fleet/internal/bootstrap/bootstrap_windows.go +++ b/pkg/fleet/internal/bootstrap/bootstrap_windows.go @@ -15,14 +15,14 @@ import ( "os/exec" "path/filepath" - fleetEnv "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" iexec "github.com/DataDog/datadog-agent/pkg/fleet/internal/exec" "github.com/DataDog/datadog-agent/pkg/fleet/internal/oci" ) -func install(ctx context.Context, env *fleetEnv.Env, url string, experiment bool) error { +func install(ctx context.Context, env *env.Env, url string, experiment bool) error { err := paths.CreateInstallerDataDir() if err != nil { return fmt.Errorf("failed to create installer data directory: %w", err) @@ -47,8 +47,8 @@ func install(ctx context.Context, env *fleetEnv.Env, url string, experiment bool } // downloadInstaller downloads the installer package from the registry and returns the path to the executable. -func downloadInstaller(ctx context.Context, env *fleetEnv.Env, url string, tmpDir string) (*iexec.InstallerExec, error) { - downloader := oci.NewDownloader(env, fleetEnv.GetHTTPClient()) +func downloadInstaller(ctx context.Context, env *env.Env, url string, tmpDir string) (*iexec.InstallerExec, error) { + downloader := oci.NewDownloader(env, env.HTTPClient()) downloadedPackage, err := downloader.Download(ctx, url) if err != nil { return nil, fmt.Errorf("failed to download installer package: %w", err) diff --git a/pkg/fleet/internal/cdn/cdn.go b/pkg/fleet/internal/cdn/cdn.go index 5f1ae7865a1d0..ab74ce9bfe0a7 100644 --- a/pkg/fleet/internal/cdn/cdn.go +++ b/pkg/fleet/internal/cdn/cdn.go @@ -15,7 +15,7 @@ import ( "path/filepath" "runtime" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) diff --git a/pkg/fleet/internal/cdn/cdn_http.go b/pkg/fleet/internal/cdn/cdn_http.go index 53d8f379d124e..1e0c90b36d69a 100644 --- a/pkg/fleet/internal/cdn/cdn_http.go +++ b/pkg/fleet/internal/cdn/cdn_http.go @@ -11,7 +11,7 @@ import ( "encoding/json" remoteconfig "github.com/DataDog/datadog-agent/pkg/config/remote/service" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" diff --git a/pkg/fleet/internal/cdn/cdn_local.go b/pkg/fleet/internal/cdn/cdn_local.go index 6cfefe31732b3..f657d870ab7d3 100644 --- a/pkg/fleet/internal/cdn/cdn_local.go +++ b/pkg/fleet/internal/cdn/cdn_local.go @@ -11,7 +11,7 @@ import ( "os" "path/filepath" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" ) type fetcherLocal struct { diff --git a/pkg/fleet/internal/cdn/cdn_rc.go b/pkg/fleet/internal/cdn/cdn_rc.go index e3db586530dd2..b0713047b586d 100644 --- a/pkg/fleet/internal/cdn/cdn_rc.go +++ b/pkg/fleet/internal/cdn/cdn_rc.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl" remoteconfig "github.com/DataDog/datadog-agent/pkg/config/remote/service" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" pkghostname "github.com/DataDog/datadog-agent/pkg/util/hostname" "github.com/DataDog/datadog-agent/pkg/util/log" diff --git a/pkg/fleet/internal/cdn/scope_expression.go b/pkg/fleet/internal/cdn/scope_expression.go index a67e701942848..659d6798c0670 100644 --- a/pkg/fleet/internal/cdn/scope_expression.go +++ b/pkg/fleet/internal/cdn/scope_expression.go @@ -10,7 +10,7 @@ import ( "fmt" "regexp" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/version" "github.com/expr-lang/expr" ) diff --git a/pkg/fleet/internal/cdn/tags.go b/pkg/fleet/internal/cdn/tags.go index 53c20841143bb..101c3afec8ce3 100644 --- a/pkg/fleet/internal/cdn/tags.go +++ b/pkg/fleet/internal/cdn/tags.go @@ -13,7 +13,7 @@ import ( detectenv "github.com/DataDog/datadog-agent/pkg/config/env" "github.com/DataDog/datadog-agent/pkg/config/model" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" ) type hostTagsGetter struct { diff --git a/pkg/fleet/internal/exec/installer_exec.go b/pkg/fleet/internal/exec/installer_exec.go index 1841de484b841..29ca01999dcb9 100644 --- a/pkg/fleet/internal/exec/installer_exec.go +++ b/pkg/fleet/internal/exec/installer_exec.go @@ -18,7 +18,8 @@ import ( "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" "github.com/DataDog/datadog-agent/pkg/util/log" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" @@ -247,8 +248,9 @@ func (iCmd *installerCmd) Run() error { } if len(errBuf.Bytes()) == 0 { - return fmt.Errorf("run failed: %s", err.Error()) + return fmt.Errorf("run failed: %w", err) } - return fmt.Errorf("run failed: %s \n%s", strings.TrimSpace(errBuf.String()), err.Error()) + installerError := installerErrors.FromJSON(strings.TrimSpace(errBuf.String())) + return fmt.Errorf("run failed: %v \n%s", installerError, err.Error()) } diff --git a/pkg/fleet/internal/oci/download.go b/pkg/fleet/internal/oci/download.go index f4a19fe29e59c..a03f7917d27df 100644 --- a/pkg/fleet/internal/oci/download.go +++ b/pkg/fleet/internal/oci/download.go @@ -33,7 +33,8 @@ import ( "golang.org/x/net/http2" httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" + installerErrors "github.com/DataDog/datadog-agent/pkg/fleet/installer/errors" "github.com/DataDog/datadog-agent/pkg/fleet/internal/tar" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -304,7 +305,10 @@ func (d *Downloader) downloadIndex(index oci.ImageIndex) (oci.Image, error) { } return image, nil } - return nil, fmt.Errorf("no matching image found in the index") + return nil, installerErrors.Wrap( + installerErrors.ErrPackageNotFound, + fmt.Errorf("no matching image found in the index"), + ) } // ExtractLayers extracts the layers of the downloaded package with the given media type to the given directory. diff --git a/pkg/fleet/internal/oci/download_test.go b/pkg/fleet/internal/oci/download_test.go index c3fb215373c01..e3c9b61916145 100644 --- a/pkg/fleet/internal/oci/download_test.go +++ b/pkg/fleet/internal/oci/download_test.go @@ -18,7 +18,7 @@ import ( "github.com/stretchr/testify/assert" "golang.org/x/net/http2" - "github.com/DataDog/datadog-agent/pkg/fleet/env" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/env" "github.com/DataDog/datadog-agent/pkg/fleet/internal/fixtures" "github.com/google/go-containerregistry/pkg/authn" oci "github.com/google/go-containerregistry/pkg/v1" diff --git a/pkg/fleet/telemetry/telemetry.go b/pkg/fleet/telemetry/telemetry.go index 00372e69ce760..842ea1b678cbd 100644 --- a/pkg/fleet/telemetry/telemetry.go +++ b/pkg/fleet/telemetry/telemetry.go @@ -22,7 +22,6 @@ import ( "github.com/gorilla/mux" - "github.com/DataDog/datadog-agent/pkg/fleet/env" "github.com/DataDog/datadog-agent/pkg/internaltelemetry" pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" traceconfig "github.com/DataDog/datadog-agent/pkg/trace/config" @@ -60,14 +59,14 @@ type Telemetry struct { type Option func(*Telemetry) // NewTelemetry creates a new telemetry instance -func NewTelemetry(apiKey string, site string, service string, opts ...Option) (*Telemetry, error) { +func NewTelemetry(client *http.Client, apiKey string, site string, service string, opts ...Option) (*Telemetry, error) { endpoint := &traceconfig.Endpoint{ Host: fmt.Sprintf("https://%s.%s", telemetrySubdomain, strings.TrimSpace(site)), APIKey: apiKey, } listener := newTelemetryListener() t := &Telemetry{ - telemetryClient: internaltelemetry.NewClient(env.GetHTTPClient(), []*traceconfig.Endpoint{endpoint}, service, site == "datad0g.com"), + telemetryClient: internaltelemetry.NewClient(client, []*traceconfig.Endpoint{endpoint}, service, site == "datad0g.com"), site: site, service: service, listener: listener, diff --git a/pkg/network/protocols/amqp/server.go b/pkg/network/protocols/amqp/server.go index ffd049cb36667..fd343ab5a29ae 100644 --- a/pkg/network/protocols/amqp/server.go +++ b/pkg/network/protocols/amqp/server.go @@ -7,7 +7,6 @@ package amqp import ( "fmt" - globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" "os" "path/filepath" "regexp" @@ -17,6 +16,7 @@ import ( httpUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) diff --git a/pkg/network/protocols/events/consumer_test.go b/pkg/network/protocols/events/consumer_test.go index 2b1bba26b97ad..732f5cc614964 100644 --- a/pkg/network/protocols/events/consumer_test.go +++ b/pkg/network/protocols/events/consumer_test.go @@ -8,9 +8,12 @@ package events import ( + "bytes" + "encoding/binary" "math" "os" "path/filepath" + "runtime" "sync" "testing" "time" @@ -61,14 +64,14 @@ func TestConsumer(t *testing.T) { // generate test events generator := newEventGenerator(program, t) for i := 0; i < numEvents; i++ { - generator.Generate(uint64(i)) + require.NoError(t, generator.Generate(uint64(i))) } generator.Stop() time.Sleep(100 * time.Millisecond) // this ensures that any incomplete batch left in eBPF is fully processed consumer.Sync() - program.Stop(manager.CleanAll) + require.NoError(t, program.Stop(manager.CleanAll)) consumer.Stop() // ensure that we have received each event exactly once @@ -119,8 +122,8 @@ func (e *eventGenerator) Generate(eventID uint64) error { return err } - e.testFile.Write([]byte("whatever")) - return nil + _, err = e.testFile.Write([]byte("whatever")) + return err } func (e *eventGenerator) Stop() { @@ -197,3 +200,42 @@ func TestInvalidBatchCountMetric(t *testing.T) { require.Equalf(t, int(consumer.invalidBatchCount.Get()), 1, "invalidBatchCount should be greater than 0") } + +// BenchmarkConsumer benchmarks the consumer with a large number of events to measure the performance. +func BenchmarkConsumer(b *testing.B) { + const numEvents = 3500 + program, err := newEBPFProgram(config.New()) + require.NoError(b, err) + + consumer, err := NewConsumer("test", program, func([]uint64) {}) + require.NoError(b, err) + + require.NoError(b, program.Start()) + mockBatch := batch{ + Len: uint16(numEvents), + Cap: uint16(len((batch{}).Data)), + } + + // Reset the events count as the benchmark will run multiple times + consumer.eventsCount.Reset() + + var buf bytes.Buffer + require.NoError(b, binary.Write(&buf, binary.LittleEndian, &mockBatch)) + + consumer.handler.(*ddebpf.RingBufferHandler).RecordHandler(&ringbuf.Record{ + RawSample: buf.Bytes(), + }, nil, nil) + + b.ReportAllocs() + runtime.GC() + b.ResetTimer() + for i := 0; i < b.N; i++ { + consumer.Start() + require.Eventually(b, func() bool { + program.Stop(manager.CleanAll) + return true + }, 100*time.Millisecond, 10*time.Millisecond) + require.Equalf(b, numEvents, int(consumer.eventsCount.Get()), + "Not all events were processed correctly, expected %d, got %d, iteration: %d", numEvents, consumer.eventsCount.Get(), i) + } +} diff --git a/pkg/network/protocols/kafka/server.go b/pkg/network/protocols/kafka/server.go index 95c66ef23b8a1..eeb5a91c58a5d 100644 --- a/pkg/network/protocols/kafka/server.go +++ b/pkg/network/protocols/kafka/server.go @@ -8,14 +8,15 @@ package kafka import ( - globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" - "github.com/stretchr/testify/require" "os" "path/filepath" "regexp" "testing" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" + globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) diff --git a/pkg/network/protocols/mongo/server.go b/pkg/network/protocols/mongo/server.go index 59717d099a9ae..cc9bf63b55545 100644 --- a/pkg/network/protocols/mongo/server.go +++ b/pkg/network/protocols/mongo/server.go @@ -7,13 +7,14 @@ package mongo import ( "fmt" - globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" - "github.com/stretchr/testify/require" "path/filepath" "regexp" "testing" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" + globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) diff --git a/pkg/network/protocols/mysql/server.go b/pkg/network/protocols/mysql/server.go index 1ea9ebb1ac1ec..423ca963943e7 100644 --- a/pkg/network/protocols/mysql/server.go +++ b/pkg/network/protocols/mysql/server.go @@ -7,7 +7,6 @@ package mysql import ( "fmt" - globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" "path/filepath" "regexp" "testing" @@ -15,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" + globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) diff --git a/pkg/network/protocols/postgres/server.go b/pkg/network/protocols/postgres/server.go index 2b4bd55bde9c6..556c918a1bd34 100644 --- a/pkg/network/protocols/postgres/server.go +++ b/pkg/network/protocols/postgres/server.go @@ -10,7 +10,6 @@ package postgres import ( "fmt" - globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" "io" "os" "path/filepath" @@ -20,6 +19,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" + globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) diff --git a/pkg/network/protocols/redis/server.go b/pkg/network/protocols/redis/server.go index 56d36fec8855c..58e645a42b3fd 100644 --- a/pkg/network/protocols/redis/server.go +++ b/pkg/network/protocols/redis/server.go @@ -11,7 +11,6 @@ package redis import ( "fmt" - globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" "path/filepath" "regexp" "testing" @@ -19,6 +18,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" + globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) diff --git a/pkg/network/protocols/telemetry/metric.go b/pkg/network/protocols/telemetry/metric.go index 46aa226351278..45792aa02d164 100644 --- a/pkg/network/protocols/telemetry/metric.go +++ b/pkg/network/protocols/telemetry/metric.go @@ -96,6 +96,11 @@ func (m *metricBase) Get() int64 { return m.value.Load() } +// Reset value atomically +func (m *metricBase) Reset() { + m.value.Store(0) +} + // metric is the private interface shared by `Counter` and `Gauge` // the base() method simply returns the embedded `*metricBase` struct // which is all we need in the internal code that has to deal with both types diff --git a/pkg/network/protocols/tls/gotls/testutil/server.go b/pkg/network/protocols/tls/gotls/testutil/server.go index 36a58b003c0e6..ec2e56f054625 100644 --- a/pkg/network/protocols/tls/gotls/testutil/server.go +++ b/pkg/network/protocols/tls/gotls/testutil/server.go @@ -6,12 +6,13 @@ package testutil import ( - globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" - "github.com/stretchr/testify/require" "regexp" "testing" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" + globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) diff --git a/pkg/network/protocols/tls/nodejs/nodejs.go b/pkg/network/protocols/tls/nodejs/nodejs.go index fb7dc18f60a9a..54a21acf99dd8 100644 --- a/pkg/network/protocols/tls/nodejs/nodejs.go +++ b/pkg/network/protocols/tls/nodejs/nodejs.go @@ -9,15 +9,16 @@ package nodejs import ( - globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" - "github.com/stretchr/testify/require" "io" "os" "path" "regexp" "testing" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" + globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" ) diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index 24f0f7b3a5d2f..c7553370b7c7b 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -12,7 +12,6 @@ import ( "bytes" "crypto/tls" "fmt" - globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" "io" "math/rand" nethttp "net/http" @@ -46,6 +45,7 @@ import ( usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" procmontestutil "github.com/DataDog/datadog-agent/pkg/process/monitor/testutil" + globalutils "github.com/DataDog/datadog-agent/pkg/util/testutil" dockerutils "github.com/DataDog/datadog-agent/pkg/util/testutil/docker" )