From 2d123da9bf8f7e3545f8d712c2bbecc4f5b1f881 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 30 Dec 2021 19:08:48 +0100 Subject: [PATCH 01/24] wip --- dev-tools/mage/build.go | 8 +++---- dev-tools/mage/crossbuild.go | 4 ++-- libbeat/common/backoff/equal_jitter.go | 4 ++-- libbeat/logp/configure/logging.go | 2 +- libbeat/logp/core.go | 2 +- x-pack/elastic-agent/magefile.go | 2 +- .../gateway/fleet/fleet_gateway.go | 2 +- .../agent/application/pipeline/pipeline.go | 2 +- .../pkg/agent/application/reexec/manager.go | 2 +- x-pack/elastic-agent/pkg/agent/cmd/run.go | 15 ++++++++++-- .../pkg/agent/operation/operator.go | 15 ++++++++++-- .../pkg/core/plugin/process/app.go | 23 +++++++++++++++++-- .../pkg/core/plugin/service/app.go | 17 +++++++++++--- .../elastic-agent/pkg/core/server/server.go | 5 ++-- .../elastic-agent/pkg/core/status/reporter.go | 2 +- 15 files changed, 79 insertions(+), 26 deletions(-) diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 5c5156adf44..0a1159a8607 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -48,10 +48,10 @@ type BuildArgs struct { // DefaultBuildArgs returns the default BuildArgs for use in builds. func DefaultBuildArgs() BuildArgs { args := BuildArgs{ - Name: BeatName, - CGO: build.Default.CgoEnabled, + Name: BeatName, + CGO: build.Default.CgoEnabled, LDFlags: []string{ - "-s", // Strip all debug symbols from binary (does not affect Go stack traces). + // "-s", // Strip all debug symbols from binary (does not affect Go stack traces). }, Vars: map[string]string{ elasticBeatsModulePath + "/libbeat/version.buildTime": "{{ date }}", @@ -66,7 +66,7 @@ func DefaultBuildArgs() BuildArgs { if positionIndependendCodeSupported() { args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie") } - + args.ExtraFlags = append(args.ExtraFlags, `-gcflags`, `"all=-N -l"`) return args } diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index b2349310abc..24b9b0ad1d6 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -43,8 +43,8 @@ const defaultCrossBuildTarget = "golangCrossBuild" // See NewPlatformList for details about platform filtering expressions. var Platforms = BuildPlatforms.Defaults() -// Types is the list of package types -var SelectedPackageTypes []PackageType +// SelectedPackageTypes is the list of package types +var SelectedPackageTypes []PackageType = []PackageType{TarGz} func init() { // Allow overriding via PLATFORMS. diff --git a/libbeat/common/backoff/equal_jitter.go b/libbeat/common/backoff/equal_jitter.go index ff5c86f156f..d5b5c7d250c 100644 --- a/libbeat/common/backoff/equal_jitter.go +++ b/libbeat/common/backoff/equal_jitter.go @@ -47,11 +47,11 @@ func NewEqualJitterBackoff(done <-chan struct{}, init, max time.Duration) Backof // Reset resets the duration of the backoff. func (b *EqualJitterBackoff) Reset() { - // Allow to sleep at least the init period on the first wait. + // Allow sleeping at least the init period on the first wait. b.duration = b.init * 2 } -// Wait block until either the timer is completed or channel is done. +// Wait blocks until either the timer is completed or channel is done. func (b *EqualJitterBackoff) Wait() bool { // Make sure we have always some minimal back off and jitter. temp := int64(b.duration / 2) diff --git a/libbeat/logp/configure/logging.go b/libbeat/logp/configure/logging.go index 43a32dd7f2f..9a041bc2559 100644 --- a/libbeat/logp/configure/logging.go +++ b/libbeat/logp/configure/logging.go @@ -60,7 +60,7 @@ func Logging(beatName string, cfg *common.Config) error { return logp.Configure(config) } -// Logging builds a logp.Config based on the given common.Config and the specified +// LoggingWithOutputs builds a logp.Config based on the given common.Config and the specified // CLI flags along with the given outputs. func LoggingWithOutputs(beatName string, cfg *common.Config, outputs ...zapcore.Core) error { config := logp.DefaultConfig(environment) diff --git a/libbeat/logp/core.go b/libbeat/logp/core.go index 552c81e9201..262515ac8d2 100644 --- a/libbeat/logp/core.go +++ b/libbeat/logp/core.go @@ -67,7 +67,7 @@ func Configure(cfg Config) error { return ConfigureWithOutputs(cfg) } -// XXX: ConfigureWithOutputs is used by elastic-agent only (See file: x-pack/elastic-agent/pkg/core/logger/logger.go). +// ConfigureWithOutputs XXX: is used by elastic-agent only (See file: x-pack/elastic-agent/pkg/core/logger/logger.go). // The agent requires that the output specified in the config object is configured and merged with the // logging outputs given. func ConfigureWithOutputs(cfg Config, outputs ...zapcore.Core) error { diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index ccceaf06935..113f84e7221 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -508,7 +508,7 @@ func runAgent(env map[string]string) error { return err } - // docker does not exists for this commit, build it + // docker does not exist for this commit, build it if !strings.Contains(dockerImageOut, tag) { // produce docker package packageAgent([]string{ diff --git a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go index 5eaf4c47984..0c071ad779f 100644 --- a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go +++ b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go @@ -127,7 +127,7 @@ func newFleetGatewayWithScheduler( stateStore stateStore, ) (gateway.FleetGateway, error) { - // Backoff implementation doesn't support the using context as the shutdown mechanism. + // Backoff implementation doesn't support the using of a context as the shutdown mechanism. // So we keep a done channel that will be closed when the current context is shutdown. done := make(chan struct{}) diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go index c8cac5b3216..f2f02efb258 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go @@ -29,7 +29,7 @@ var DefaultRK = "default" // RoutingKey is used for routing as pipeline id. type RoutingKey = string -// Router is an interace routes programs to correspongind stream +// Router is an interface routing programs to the corresponding stream. type Router interface { Routes() *sorted.Set Route(id string, grpProg map[RoutingKey][]program.Program) error diff --git a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go index 5ccc870d948..d251926d357 100644 --- a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go +++ b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go @@ -54,7 +54,7 @@ func (m *manager) ReExec(shutdownCallback ShutdownCallbackFn, argOverrides ...st if shutdownCallback != nil { if err := shutdownCallback(); err != nil { // panic; because there is no going back, everything is shutdown - panic(errors.New(errors.TypeUnexpected, err, "failure occured during shutdown cleanup")) + panic(errors.New(errors.TypeUnexpected, err, "failure occurred during shutdown cleanup")) } } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 7f10f9faa31..615d2cd8324 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -18,6 +18,7 @@ import ( "github.com/elastic/beats/v7/libbeat/api" "github.com/elastic/beats/v7/libbeat/cmd/instance/metrics" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/service" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application" @@ -90,6 +91,7 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { return err } + cfg.Settings.LoggingConfig.Level = logp.DebugLevel logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, true) if err != nil { return err @@ -150,7 +152,11 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { control.SetRouteFn(app.Routes) control.SetMonitoringCfg(cfg.Settings.MonitoringConfig) - serverStopFn, err := setupMetrics(agentInfo, logger, cfg.Settings.DownloadConfig.OS(), cfg.Settings.MonitoringConfig, app) + serverStopFn, err := setupMetrics( + agentInfo, + logger, cfg.Settings.DownloadConfig.OS(), + cfg.Settings.MonitoringConfig, + app) if err != nil { return err } @@ -326,7 +332,12 @@ func isProcessStatsEnabled(cfg *monitoringCfg.MonitoringHTTPConfig) bool { return cfg != nil && cfg.Enabled } -func tryDelayEnroll(ctx context.Context, logger *logger.Logger, cfg *configuration.Configuration, override cfgOverrider) (*configuration.Configuration, error) { +func tryDelayEnroll( + ctx context.Context, + logger *logger.Logger, + cfg *configuration.Configuration, + override cfgOverrider) (*configuration.Configuration, error) { + enrollPath := paths.AgentEnrollFile() if _, err := os.Stat(enrollPath); err != nil { // no enrollment file exists or failed to stat it; nothing to do diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index 8e83efbd94b..e08a3436483 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -218,9 +218,20 @@ func (o *Operator) Shutdown() { o.logger.Debugf("pipeline installer '%s' done", o.pipelineID) } - for _, app := range o.apps { - app.Shutdown() + wg := sync.WaitGroup{} + started := time.Now() + for _, a := range o.apps { + // shutdown apps concurrently. + // TODO(Anderson): it's fine, right? + wg.Add(1) + go func(a Application) { + a.Shutdown() + wg.Done() + }(a) } + wg.Wait() + o.logger.Debugf("took %s to shutdown %d apps", + time.Now().Sub(started), len(o.apps)) } // Start starts a new process based on a configuration diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 37568828cac..e975a19f733 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -14,6 +14,7 @@ import ( "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" @@ -151,7 +152,16 @@ func (a *Application) Stop() { if srvState != nil { // signal stop through GRPC, wait and kill is performed later in gracefulKill - srvState.Stop(a.processConfig.StopTimeout) + if err := srvState.Stop(a.processConfig.StopTimeout); err != nil { + err := fmt.Errorf("failed to stop after %s: %w", a.processConfig.StopTimeout, err) + a.setState( + state.Failed, + err.Error(), + nil) + + a.logger.Error(err) + } + } a.appLock.Lock() @@ -279,8 +289,13 @@ func (a *Application) gracefulKill(proc *process.Info) { return } + alog := logp.NewLogger("anderson") + alog.Infof("[gracefulKill][%s] invoked", a.Name()) + // send stop signal to request stop - proc.Stop() + if err := proc.Stop(); err != nil { + a.logger.Error(fmt.Errorf("failed to stop %s: %w", a.Name(), err)) + } var wg sync.WaitGroup doneChan := make(chan struct{}) @@ -290,6 +305,7 @@ func (a *Application) gracefulKill(proc *process.Info) { if _, err := proc.Process.Wait(); err != nil { // process is not a child - some OSs requires process to be child + alog.Infof("[gracefulKill][externalProcess][%s] closing it", a.Name()) a.externalProcess(proc.Process) } close(doneChan) @@ -304,6 +320,9 @@ func (a *Application) gracefulKill(proc *process.Info) { select { case <-doneChan: case <-t.C: + alog.Infof("[gracefulKill][%s] killing it", a.Name()) + a.logger.Infof("gracefulKill timed out after %d, killing %s", + procExitTimeout, a.Name()) _ = proc.Process.Kill() } } diff --git a/x-pack/elastic-agent/pkg/core/plugin/service/app.go b/x-pack/elastic-agent/pkg/core/plugin/service/app.go index 19ef5aca60c..0ed370eae81 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/service/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/service/app.go @@ -259,7 +259,10 @@ func (a *Application) Stop() { if err := srvState.Stop(a.processConfig.StopTimeout); err != nil { a.appLock.Lock() - a.setState(state.Failed, errors.New(err, "Failed to stopped").Error(), nil) + a.setState( + state.Failed, + fmt.Errorf("failed to stop after %s: %w", a.processConfig.StopTimeout, err).Error(), + nil) } else { a.appLock.Lock() a.setState(state.Stopped, "Stopped", nil) @@ -275,6 +278,7 @@ func (a *Application) Stop() { func (a *Application) Shutdown() { a.appLock.Lock() defer a.appLock.Unlock() + a.logger.Infof("Signaling service to stop because of shutdown: %s", a.id) if a.srvState == nil { return @@ -293,7 +297,11 @@ func (a *Application) Shutdown() { // OnStatusChange is the handler called by the GRPC server code. // // It updates the status of the application and handles restarting the application is needed. -func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.StateObserved_Status, msg string, payload map[string]interface{}) { +func (a *Application) OnStatusChange( + s *server.ApplicationState, + status proto.StateObserved_Status, + msg string, payload map[string]interface{}) { + a.appLock.Lock() defer a.appLock.Unlock() @@ -307,7 +315,10 @@ func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.St } func (a *Application) setState(s state.Status, msg string, payload map[string]interface{}) { - if a.state.Status != s || a.state.Message != msg || !reflect.DeepEqual(a.state.Payload, payload) { + if a.state.Status != s || + a.state.Message != msg || + !reflect.DeepEqual(a.state.Payload, payload) { + if state.IsStateFiltered(msg, payload) { return } diff --git a/x-pack/elastic-agent/pkg/core/server/server.go b/x-pack/elastic-agent/pkg/core/server/server.go index 390283b4e15..fa7d7ba666f 100644 --- a/x-pack/elastic-agent/pkg/core/server/server.go +++ b/x-pack/elastic-agent/pkg/core/server/server.go @@ -580,8 +580,9 @@ func (as *ApplicationState) Stop(timeout time.Duration) error { s := as.status doneChan := as.checkinDone as.checkinLock.RUnlock() - if (wasConn && doneChan == nil) || (!wasConn && s == proto.StateObserved_STOPPING && doneChan == nil) { - // either occurred + if (wasConn && doneChan == nil) || + (!wasConn && s == proto.StateObserved_STOPPING && doneChan == nil) { + // either occurred: // * client was connected then disconnected on stop // * client was not connected; connected; received stopping; then disconnected as.Destroy() diff --git a/x-pack/elastic-agent/pkg/core/status/reporter.go b/x-pack/elastic-agent/pkg/core/status/reporter.go index 3add6b188c8..2e34bb15cdc 100644 --- a/x-pack/elastic-agent/pkg/core/status/reporter.go +++ b/x-pack/elastic-agent/pkg/core/status/reporter.go @@ -279,7 +279,7 @@ func (r *reporter) Update(s state.Status, message string, payload map[string]int } } -// Unregister unregister status from reporter. Reporter will no longer be taken into consideration +// Unregister unregisters status from reporter. Reporter will no longer be taken into consideration // for overall status computation. func (r *reporter) Unregister() { r.mx.Lock() From 82c3cdba2f104c9443e3f54027accd8fc3b9e147 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 17 Jan 2022 17:01:43 +0100 Subject: [PATCH 02/24] remove debug stuff --- dev-tools/mage/build.go | 2 ++ x-pack/elastic-agent/pkg/core/plugin/process/app.go | 6 ------ x-pack/elastic-agent/pkg/core/plugin/service/app.go | 9 ++------- x-pack/elastic-agent/pkg/core/server/server.go | 3 +-- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 0a1159a8607..92d95cf960f 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -51,6 +51,7 @@ func DefaultBuildArgs() BuildArgs { Name: BeatName, CGO: build.Default.CgoEnabled, LDFlags: []string{ + // TODO: comment in before final review/merge // "-s", // Strip all debug symbols from binary (does not affect Go stack traces). }, Vars: map[string]string{ @@ -66,6 +67,7 @@ func DefaultBuildArgs() BuildArgs { if positionIndependendCodeSupported() { args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie") } + // TODO: remove before final review/merge args.ExtraFlags = append(args.ExtraFlags, `-gcflags`, `"all=-N -l"`) return args } diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index e975a19f733..256c856a830 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -14,7 +14,6 @@ import ( "github.com/elastic/elastic-agent-client/v7/pkg/proto" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" @@ -289,9 +288,6 @@ func (a *Application) gracefulKill(proc *process.Info) { return } - alog := logp.NewLogger("anderson") - alog.Infof("[gracefulKill][%s] invoked", a.Name()) - // send stop signal to request stop if err := proc.Stop(); err != nil { a.logger.Error(fmt.Errorf("failed to stop %s: %w", a.Name(), err)) @@ -305,7 +301,6 @@ func (a *Application) gracefulKill(proc *process.Info) { if _, err := proc.Process.Wait(); err != nil { // process is not a child - some OSs requires process to be child - alog.Infof("[gracefulKill][externalProcess][%s] closing it", a.Name()) a.externalProcess(proc.Process) } close(doneChan) @@ -320,7 +315,6 @@ func (a *Application) gracefulKill(proc *process.Info) { select { case <-doneChan: case <-t.C: - alog.Infof("[gracefulKill][%s] killing it", a.Name()) a.logger.Infof("gracefulKill timed out after %d, killing %s", procExitTimeout, a.Name()) _ = proc.Process.Kill() diff --git a/x-pack/elastic-agent/pkg/core/plugin/service/app.go b/x-pack/elastic-agent/pkg/core/plugin/service/app.go index 0ed370eae81..bd36f8f5e4e 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/service/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/service/app.go @@ -297,10 +297,7 @@ func (a *Application) Shutdown() { // OnStatusChange is the handler called by the GRPC server code. // // It updates the status of the application and handles restarting the application is needed. -func (a *Application) OnStatusChange( - s *server.ApplicationState, - status proto.StateObserved_Status, - msg string, payload map[string]interface{}) { +func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.StateObserved_Status, msg string, payload map[string]interface{}) { a.appLock.Lock() defer a.appLock.Unlock() @@ -315,9 +312,7 @@ func (a *Application) OnStatusChange( } func (a *Application) setState(s state.Status, msg string, payload map[string]interface{}) { - if a.state.Status != s || - a.state.Message != msg || - !reflect.DeepEqual(a.state.Payload, payload) { + if a.state.Status != s || a.state.Message != msg || !reflect.DeepEqual(a.state.Payload, payload) { if state.IsStateFiltered(msg, payload) { return diff --git a/x-pack/elastic-agent/pkg/core/server/server.go b/x-pack/elastic-agent/pkg/core/server/server.go index fa7d7ba666f..f584e70bad2 100644 --- a/x-pack/elastic-agent/pkg/core/server/server.go +++ b/x-pack/elastic-agent/pkg/core/server/server.go @@ -580,8 +580,7 @@ func (as *ApplicationState) Stop(timeout time.Duration) error { s := as.status doneChan := as.checkinDone as.checkinLock.RUnlock() - if (wasConn && doneChan == nil) || - (!wasConn && s == proto.StateObserved_STOPPING && doneChan == nil) { + if (wasConn && doneChan == nil) || (!wasConn && s == proto.StateObserved_STOPPING && doneChan == nil) { // either occurred: // * client was connected then disconnected on stop // * client was not connected; connected; received stopping; then disconnected From 65af2f6847a4ed406449b10590acea584c19db4a Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 17 Jan 2022 17:39:19 +0100 Subject: [PATCH 03/24] add TODO and restore style --- x-pack/elastic-agent/pkg/agent/cmd/run.go | 1 + x-pack/elastic-agent/pkg/core/plugin/process/app.go | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 615d2cd8324..86d16fd43cb 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -91,6 +91,7 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { return err } + // TODO: remove before review cfg.Settings.LoggingConfig.Level = logp.DebugLevel logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, true) if err != nil { diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 256c856a830..7e08624efad 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -153,10 +153,7 @@ func (a *Application) Stop() { // signal stop through GRPC, wait and kill is performed later in gracefulKill if err := srvState.Stop(a.processConfig.StopTimeout); err != nil { err := fmt.Errorf("failed to stop after %s: %w", a.processConfig.StopTimeout, err) - a.setState( - state.Failed, - err.Error(), - nil) + a.setState(state.Failed, err.Error(), nil) a.logger.Error(err) } From 5c5f418d98dd4e4df7cfd6f3c30a1d9282ac8b42 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 17 Jan 2022 18:01:03 +0100 Subject: [PATCH 04/24] undo typo fixes, they will be fixed on another PR --- dev-tools/mage/crossbuild.go | 8 ++++---- libbeat/common/backoff/equal_jitter.go | 4 ++-- libbeat/logp/configure/logging.go | 2 +- libbeat/logp/core.go | 2 +- x-pack/elastic-agent/magefile.go | 2 +- .../pkg/agent/application/gateway/fleet/fleet_gateway.go | 2 +- .../pkg/agent/application/pipeline/pipeline.go | 2 +- .../elastic-agent/pkg/agent/application/reexec/manager.go | 2 +- x-pack/elastic-agent/pkg/core/server/server.go | 2 +- x-pack/elastic-agent/pkg/core/status/reporter.go | 2 +- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 24b9b0ad1d6..9f56fd125c3 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -43,8 +43,8 @@ const defaultCrossBuildTarget = "golangCrossBuild" // See NewPlatformList for details about platform filtering expressions. var Platforms = BuildPlatforms.Defaults() -// SelectedPackageTypes is the list of package types -var SelectedPackageTypes []PackageType = []PackageType{TarGz} +// Types is the list of package types +var SelectedPackageTypes []PackageType func init() { // Allow overriding via PLATFORMS. @@ -242,8 +242,8 @@ func CrossBuildImage(platform string) (string, error) { tagSuffix = "s390x" case strings.HasPrefix(platform, "linux"): // Use an older version of libc to gain greater OS compatibility. - // Debian 7 uses glibc 2.13. - tagSuffix = "main-debian7" + // Debian 8 uses glibc 2.19. + tagSuffix = "main-debian8" } goVersion, err := GoVersion() diff --git a/libbeat/common/backoff/equal_jitter.go b/libbeat/common/backoff/equal_jitter.go index d5b5c7d250c..ff5c86f156f 100644 --- a/libbeat/common/backoff/equal_jitter.go +++ b/libbeat/common/backoff/equal_jitter.go @@ -47,11 +47,11 @@ func NewEqualJitterBackoff(done <-chan struct{}, init, max time.Duration) Backof // Reset resets the duration of the backoff. func (b *EqualJitterBackoff) Reset() { - // Allow sleeping at least the init period on the first wait. + // Allow to sleep at least the init period on the first wait. b.duration = b.init * 2 } -// Wait blocks until either the timer is completed or channel is done. +// Wait block until either the timer is completed or channel is done. func (b *EqualJitterBackoff) Wait() bool { // Make sure we have always some minimal back off and jitter. temp := int64(b.duration / 2) diff --git a/libbeat/logp/configure/logging.go b/libbeat/logp/configure/logging.go index 9a041bc2559..43a32dd7f2f 100644 --- a/libbeat/logp/configure/logging.go +++ b/libbeat/logp/configure/logging.go @@ -60,7 +60,7 @@ func Logging(beatName string, cfg *common.Config) error { return logp.Configure(config) } -// LoggingWithOutputs builds a logp.Config based on the given common.Config and the specified +// Logging builds a logp.Config based on the given common.Config and the specified // CLI flags along with the given outputs. func LoggingWithOutputs(beatName string, cfg *common.Config, outputs ...zapcore.Core) error { config := logp.DefaultConfig(environment) diff --git a/libbeat/logp/core.go b/libbeat/logp/core.go index 262515ac8d2..552c81e9201 100644 --- a/libbeat/logp/core.go +++ b/libbeat/logp/core.go @@ -67,7 +67,7 @@ func Configure(cfg Config) error { return ConfigureWithOutputs(cfg) } -// ConfigureWithOutputs XXX: is used by elastic-agent only (See file: x-pack/elastic-agent/pkg/core/logger/logger.go). +// XXX: ConfigureWithOutputs is used by elastic-agent only (See file: x-pack/elastic-agent/pkg/core/logger/logger.go). // The agent requires that the output specified in the config object is configured and merged with the // logging outputs given. func ConfigureWithOutputs(cfg Config, outputs ...zapcore.Core) error { diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 113f84e7221..ccceaf06935 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -508,7 +508,7 @@ func runAgent(env map[string]string) error { return err } - // docker does not exist for this commit, build it + // docker does not exists for this commit, build it if !strings.Contains(dockerImageOut, tag) { // produce docker package packageAgent([]string{ diff --git a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go index 0c071ad779f..5eaf4c47984 100644 --- a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go +++ b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go @@ -127,7 +127,7 @@ func newFleetGatewayWithScheduler( stateStore stateStore, ) (gateway.FleetGateway, error) { - // Backoff implementation doesn't support the using of a context as the shutdown mechanism. + // Backoff implementation doesn't support the using context as the shutdown mechanism. // So we keep a done channel that will be closed when the current context is shutdown. done := make(chan struct{}) diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go index f2f02efb258..c8cac5b3216 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go @@ -29,7 +29,7 @@ var DefaultRK = "default" // RoutingKey is used for routing as pipeline id. type RoutingKey = string -// Router is an interface routing programs to the corresponding stream. +// Router is an interace routes programs to correspongind stream type Router interface { Routes() *sorted.Set Route(id string, grpProg map[RoutingKey][]program.Program) error diff --git a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go index d251926d357..5ccc870d948 100644 --- a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go +++ b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go @@ -54,7 +54,7 @@ func (m *manager) ReExec(shutdownCallback ShutdownCallbackFn, argOverrides ...st if shutdownCallback != nil { if err := shutdownCallback(); err != nil { // panic; because there is no going back, everything is shutdown - panic(errors.New(errors.TypeUnexpected, err, "failure occurred during shutdown cleanup")) + panic(errors.New(errors.TypeUnexpected, err, "failure occured during shutdown cleanup")) } } diff --git a/x-pack/elastic-agent/pkg/core/server/server.go b/x-pack/elastic-agent/pkg/core/server/server.go index f584e70bad2..390283b4e15 100644 --- a/x-pack/elastic-agent/pkg/core/server/server.go +++ b/x-pack/elastic-agent/pkg/core/server/server.go @@ -581,7 +581,7 @@ func (as *ApplicationState) Stop(timeout time.Duration) error { doneChan := as.checkinDone as.checkinLock.RUnlock() if (wasConn && doneChan == nil) || (!wasConn && s == proto.StateObserved_STOPPING && doneChan == nil) { - // either occurred: + // either occurred // * client was connected then disconnected on stop // * client was not connected; connected; received stopping; then disconnected as.Destroy() diff --git a/x-pack/elastic-agent/pkg/core/status/reporter.go b/x-pack/elastic-agent/pkg/core/status/reporter.go index 2e34bb15cdc..3add6b188c8 100644 --- a/x-pack/elastic-agent/pkg/core/status/reporter.go +++ b/x-pack/elastic-agent/pkg/core/status/reporter.go @@ -279,7 +279,7 @@ func (r *reporter) Update(s state.Status, message string, payload map[string]int } } -// Unregister unregisters status from reporter. Reporter will no longer be taken into consideration +// Unregister unregister status from reporter. Reporter will no longer be taken into consideration // for overall status computation. func (r *reporter) Unregister() { r.mx.Lock() From 091fc257acf9c56635d27ab043483b0db1922093 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 17 Jan 2022 18:08:48 +0100 Subject: [PATCH 05/24] undo --- dev-tools/mage/crossbuild.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 9f56fd125c3..b2349310abc 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -242,8 +242,8 @@ func CrossBuildImage(platform string) (string, error) { tagSuffix = "s390x" case strings.HasPrefix(platform, "linux"): // Use an older version of libc to gain greater OS compatibility. - // Debian 8 uses glibc 2.19. - tagSuffix = "main-debian8" + // Debian 7 uses glibc 2.13. + tagSuffix = "main-debian7" } goVersion, err := GoVersion() From 1191569dd209fc3c58a0e38f0e21f5360fa94a45 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 18 Jan 2022 15:39:05 +0100 Subject: [PATCH 06/24] PR changes --- x-pack/elastic-agent/pkg/agent/cmd/run.go | 3 --- x-pack/elastic-agent/pkg/agent/operation/operator.go | 8 +++++--- x-pack/elastic-agent/pkg/core/plugin/process/app.go | 2 +- x-pack/elastic-agent/pkg/core/plugin/service/app.go | 4 +--- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 86d16fd43cb..707127bcfe7 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -18,7 +18,6 @@ import ( "github.com/elastic/beats/v7/libbeat/api" "github.com/elastic/beats/v7/libbeat/cmd/instance/metrics" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/service" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application" @@ -91,8 +90,6 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { return err } - // TODO: remove before review - cfg.Settings.LoggingConfig.Level = logp.DebugLevel logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, true) if err != nil { return err diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index e08a3436483..34fd795764b 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -219,11 +219,13 @@ func (o *Operator) Shutdown() { } wg := sync.WaitGroup{} + wg.Add(len(o.apps)) + started := time.Now() + + o.appsLock.Lock() + defer o.appsLock.Unlock() for _, a := range o.apps { - // shutdown apps concurrently. - // TODO(Anderson): it's fine, right? - wg.Add(1) go func(a Application) { a.Shutdown() wg.Done() diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 7e08624efad..3b50a4c653e 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -287,7 +287,7 @@ func (a *Application) gracefulKill(proc *process.Info) { // send stop signal to request stop if err := proc.Stop(); err != nil { - a.logger.Error(fmt.Errorf("failed to stop %s: %w", a.Name(), err)) + a.logger.Errorf("failed to stop %s: %v", a.Name(), err)) } var wg sync.WaitGroup diff --git a/x-pack/elastic-agent/pkg/core/plugin/service/app.go b/x-pack/elastic-agent/pkg/core/plugin/service/app.go index bd36f8f5e4e..4b0d2b8ee0a 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/service/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/service/app.go @@ -278,7 +278,7 @@ func (a *Application) Stop() { func (a *Application) Shutdown() { a.appLock.Lock() defer a.appLock.Unlock() - a.logger.Infof("Signaling service to stop because of shutdown: %s", a.id) + a.logger.Infof("signaling service to stop because of shutdown: %s", a.id) if a.srvState == nil { return @@ -298,7 +298,6 @@ func (a *Application) Shutdown() { // // It updates the status of the application and handles restarting the application is needed. func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.StateObserved_Status, msg string, payload map[string]interface{}) { - a.appLock.Lock() defer a.appLock.Unlock() @@ -313,7 +312,6 @@ func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.St func (a *Application) setState(s state.Status, msg string, payload map[string]interface{}) { if a.state.Status != s || a.state.Message != msg || !reflect.DeepEqual(a.state.Payload, payload) { - if state.IsStateFiltered(msg, payload) { return } From 8d605e7f112a126b969b5dbec6fdf158c1ee93c5 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 18 Jan 2022 15:40:01 +0100 Subject: [PATCH 07/24] keep debug symbols, no inline nor optimisations for dev builds --- dev-tools/mage/build.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 92d95cf960f..ec740d430a4 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -48,12 +48,8 @@ type BuildArgs struct { // DefaultBuildArgs returns the default BuildArgs for use in builds. func DefaultBuildArgs() BuildArgs { args := BuildArgs{ - Name: BeatName, - CGO: build.Default.CgoEnabled, - LDFlags: []string{ - // TODO: comment in before final review/merge - // "-s", // Strip all debug symbols from binary (does not affect Go stack traces). - }, + Name: BeatName, + CGO: build.Default.CgoEnabled, Vars: map[string]string{ elasticBeatsModulePath + "/libbeat/version.buildTime": "{{ date }}", elasticBeatsModulePath + "/libbeat/version.commit": "{{ commit }}", @@ -67,8 +63,15 @@ func DefaultBuildArgs() BuildArgs { if positionIndependendCodeSupported() { args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie") } - // TODO: remove before final review/merge - args.ExtraFlags = append(args.ExtraFlags, `-gcflags`, `"all=-N -l"`) + + if DevBuild { + // Disable optimizations (-N) and inlining (-l) for debugging. + args.ExtraFlags = append(args.ExtraFlags, `-gcflags`, `"all=-N -l"`) + } else { + // Strip all debug symbols from binary (does not affect Go stack traces). + args.LDFlags = append(args.LDFlags, "-s") + } + return args } From 89a5f593f39abf68d5f0b1b0aed35d617f1ceac5 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 20 Jan 2022 10:21:01 +0100 Subject: [PATCH 08/24] pr change --- x-pack/elastic-agent/pkg/agent/operation/operator.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index 34fd795764b..3dbd2be6314 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -218,13 +218,14 @@ func (o *Operator) Shutdown() { o.logger.Debugf("pipeline installer '%s' done", o.pipelineID) } + o.appsLock.Lock() + defer o.appsLock.Unlock() + wg := sync.WaitGroup{} wg.Add(len(o.apps)) started := time.Now() - o.appsLock.Lock() - defer o.appsLock.Unlock() for _, a := range o.apps { go func(a Application) { a.Shutdown() From 9dff80f639b46c4a7850d2102cbe63ba72d2e0fb Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 24 Jan 2022 13:41:47 +0100 Subject: [PATCH 09/24] revert build flags and debug symbols --- dev-tools/mage/build.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index ec740d430a4..5c5156adf44 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -50,6 +50,9 @@ func DefaultBuildArgs() BuildArgs { args := BuildArgs{ Name: BeatName, CGO: build.Default.CgoEnabled, + LDFlags: []string{ + "-s", // Strip all debug symbols from binary (does not affect Go stack traces). + }, Vars: map[string]string{ elasticBeatsModulePath + "/libbeat/version.buildTime": "{{ date }}", elasticBeatsModulePath + "/libbeat/version.commit": "{{ commit }}", @@ -64,14 +67,6 @@ func DefaultBuildArgs() BuildArgs { args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie") } - if DevBuild { - // Disable optimizations (-N) and inlining (-l) for debugging. - args.ExtraFlags = append(args.ExtraFlags, `-gcflags`, `"all=-N -l"`) - } else { - // Strip all debug symbols from binary (does not affect Go stack traces). - args.LDFlags = append(args.LDFlags, "-s") - } - return args } From 5e9b1223c3e736971e03c630c4cb2ed85ba1dec6 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 26 Jan 2022 10:25:53 +0100 Subject: [PATCH 10/24] . --- x-pack/elastic-agent/pkg/core/plugin/process/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 3b50a4c653e..64ec2c223c3 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -287,7 +287,7 @@ func (a *Application) gracefulKill(proc *process.Info) { // send stop signal to request stop if err := proc.Stop(); err != nil { - a.logger.Errorf("failed to stop %s: %v", a.Name(), err)) + a.logger.Errorf("failed to stop %s: %v", a.Name(), err) } var wg sync.WaitGroup From 2e9311a34bf00560e99384654dc8925b4f4acf47 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 30 Dec 2021 19:08:48 +0100 Subject: [PATCH 11/24] wip --- x-pack/elastic-agent/pkg/agent/cmd/run.go | 15 ++++++++++-- .../pkg/agent/operation/operator.go | 15 ++++++++++-- .../pkg/core/plugin/process/app.go | 23 +++++++++++++++++-- .../pkg/core/plugin/service/app.go | 17 +++++++++++--- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 7f10f9faa31..615d2cd8324 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -18,6 +18,7 @@ import ( "github.com/elastic/beats/v7/libbeat/api" "github.com/elastic/beats/v7/libbeat/cmd/instance/metrics" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/service" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application" @@ -90,6 +91,7 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { return err } + cfg.Settings.LoggingConfig.Level = logp.DebugLevel logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, true) if err != nil { return err @@ -150,7 +152,11 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { control.SetRouteFn(app.Routes) control.SetMonitoringCfg(cfg.Settings.MonitoringConfig) - serverStopFn, err := setupMetrics(agentInfo, logger, cfg.Settings.DownloadConfig.OS(), cfg.Settings.MonitoringConfig, app) + serverStopFn, err := setupMetrics( + agentInfo, + logger, cfg.Settings.DownloadConfig.OS(), + cfg.Settings.MonitoringConfig, + app) if err != nil { return err } @@ -326,7 +332,12 @@ func isProcessStatsEnabled(cfg *monitoringCfg.MonitoringHTTPConfig) bool { return cfg != nil && cfg.Enabled } -func tryDelayEnroll(ctx context.Context, logger *logger.Logger, cfg *configuration.Configuration, override cfgOverrider) (*configuration.Configuration, error) { +func tryDelayEnroll( + ctx context.Context, + logger *logger.Logger, + cfg *configuration.Configuration, + override cfgOverrider) (*configuration.Configuration, error) { + enrollPath := paths.AgentEnrollFile() if _, err := os.Stat(enrollPath); err != nil { // no enrollment file exists or failed to stat it; nothing to do diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index 8e83efbd94b..e08a3436483 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -218,9 +218,20 @@ func (o *Operator) Shutdown() { o.logger.Debugf("pipeline installer '%s' done", o.pipelineID) } - for _, app := range o.apps { - app.Shutdown() + wg := sync.WaitGroup{} + started := time.Now() + for _, a := range o.apps { + // shutdown apps concurrently. + // TODO(Anderson): it's fine, right? + wg.Add(1) + go func(a Application) { + a.Shutdown() + wg.Done() + }(a) } + wg.Wait() + o.logger.Debugf("took %s to shutdown %d apps", + time.Now().Sub(started), len(o.apps)) } // Start starts a new process based on a configuration diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 37568828cac..e975a19f733 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -14,6 +14,7 @@ import ( "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" @@ -151,7 +152,16 @@ func (a *Application) Stop() { if srvState != nil { // signal stop through GRPC, wait and kill is performed later in gracefulKill - srvState.Stop(a.processConfig.StopTimeout) + if err := srvState.Stop(a.processConfig.StopTimeout); err != nil { + err := fmt.Errorf("failed to stop after %s: %w", a.processConfig.StopTimeout, err) + a.setState( + state.Failed, + err.Error(), + nil) + + a.logger.Error(err) + } + } a.appLock.Lock() @@ -279,8 +289,13 @@ func (a *Application) gracefulKill(proc *process.Info) { return } + alog := logp.NewLogger("anderson") + alog.Infof("[gracefulKill][%s] invoked", a.Name()) + // send stop signal to request stop - proc.Stop() + if err := proc.Stop(); err != nil { + a.logger.Error(fmt.Errorf("failed to stop %s: %w", a.Name(), err)) + } var wg sync.WaitGroup doneChan := make(chan struct{}) @@ -290,6 +305,7 @@ func (a *Application) gracefulKill(proc *process.Info) { if _, err := proc.Process.Wait(); err != nil { // process is not a child - some OSs requires process to be child + alog.Infof("[gracefulKill][externalProcess][%s] closing it", a.Name()) a.externalProcess(proc.Process) } close(doneChan) @@ -304,6 +320,9 @@ func (a *Application) gracefulKill(proc *process.Info) { select { case <-doneChan: case <-t.C: + alog.Infof("[gracefulKill][%s] killing it", a.Name()) + a.logger.Infof("gracefulKill timed out after %d, killing %s", + procExitTimeout, a.Name()) _ = proc.Process.Kill() } } diff --git a/x-pack/elastic-agent/pkg/core/plugin/service/app.go b/x-pack/elastic-agent/pkg/core/plugin/service/app.go index 19ef5aca60c..0ed370eae81 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/service/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/service/app.go @@ -259,7 +259,10 @@ func (a *Application) Stop() { if err := srvState.Stop(a.processConfig.StopTimeout); err != nil { a.appLock.Lock() - a.setState(state.Failed, errors.New(err, "Failed to stopped").Error(), nil) + a.setState( + state.Failed, + fmt.Errorf("failed to stop after %s: %w", a.processConfig.StopTimeout, err).Error(), + nil) } else { a.appLock.Lock() a.setState(state.Stopped, "Stopped", nil) @@ -275,6 +278,7 @@ func (a *Application) Stop() { func (a *Application) Shutdown() { a.appLock.Lock() defer a.appLock.Unlock() + a.logger.Infof("Signaling service to stop because of shutdown: %s", a.id) if a.srvState == nil { return @@ -293,7 +297,11 @@ func (a *Application) Shutdown() { // OnStatusChange is the handler called by the GRPC server code. // // It updates the status of the application and handles restarting the application is needed. -func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.StateObserved_Status, msg string, payload map[string]interface{}) { +func (a *Application) OnStatusChange( + s *server.ApplicationState, + status proto.StateObserved_Status, + msg string, payload map[string]interface{}) { + a.appLock.Lock() defer a.appLock.Unlock() @@ -307,7 +315,10 @@ func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.St } func (a *Application) setState(s state.Status, msg string, payload map[string]interface{}) { - if a.state.Status != s || a.state.Message != msg || !reflect.DeepEqual(a.state.Payload, payload) { + if a.state.Status != s || + a.state.Message != msg || + !reflect.DeepEqual(a.state.Payload, payload) { + if state.IsStateFiltered(msg, payload) { return } From 8d3115c442c259fbaf3a3b074409933f4e67fa73 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 17 Jan 2022 17:01:43 +0100 Subject: [PATCH 12/24] remove debug stuff --- x-pack/elastic-agent/pkg/core/plugin/process/app.go | 6 ------ x-pack/elastic-agent/pkg/core/plugin/service/app.go | 9 ++------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index e975a19f733..256c856a830 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -14,7 +14,6 @@ import ( "github.com/elastic/elastic-agent-client/v7/pkg/proto" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" @@ -289,9 +288,6 @@ func (a *Application) gracefulKill(proc *process.Info) { return } - alog := logp.NewLogger("anderson") - alog.Infof("[gracefulKill][%s] invoked", a.Name()) - // send stop signal to request stop if err := proc.Stop(); err != nil { a.logger.Error(fmt.Errorf("failed to stop %s: %w", a.Name(), err)) @@ -305,7 +301,6 @@ func (a *Application) gracefulKill(proc *process.Info) { if _, err := proc.Process.Wait(); err != nil { // process is not a child - some OSs requires process to be child - alog.Infof("[gracefulKill][externalProcess][%s] closing it", a.Name()) a.externalProcess(proc.Process) } close(doneChan) @@ -320,7 +315,6 @@ func (a *Application) gracefulKill(proc *process.Info) { select { case <-doneChan: case <-t.C: - alog.Infof("[gracefulKill][%s] killing it", a.Name()) a.logger.Infof("gracefulKill timed out after %d, killing %s", procExitTimeout, a.Name()) _ = proc.Process.Kill() diff --git a/x-pack/elastic-agent/pkg/core/plugin/service/app.go b/x-pack/elastic-agent/pkg/core/plugin/service/app.go index 0ed370eae81..bd36f8f5e4e 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/service/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/service/app.go @@ -297,10 +297,7 @@ func (a *Application) Shutdown() { // OnStatusChange is the handler called by the GRPC server code. // // It updates the status of the application and handles restarting the application is needed. -func (a *Application) OnStatusChange( - s *server.ApplicationState, - status proto.StateObserved_Status, - msg string, payload map[string]interface{}) { +func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.StateObserved_Status, msg string, payload map[string]interface{}) { a.appLock.Lock() defer a.appLock.Unlock() @@ -315,9 +312,7 @@ func (a *Application) OnStatusChange( } func (a *Application) setState(s state.Status, msg string, payload map[string]interface{}) { - if a.state.Status != s || - a.state.Message != msg || - !reflect.DeepEqual(a.state.Payload, payload) { + if a.state.Status != s || a.state.Message != msg || !reflect.DeepEqual(a.state.Payload, payload) { if state.IsStateFiltered(msg, payload) { return From 758b57877a13558f49eb737657fac50fd8c278ad Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 17 Jan 2022 17:39:19 +0100 Subject: [PATCH 13/24] add TODO and restore style --- x-pack/elastic-agent/pkg/agent/cmd/run.go | 1 + x-pack/elastic-agent/pkg/core/plugin/process/app.go | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 615d2cd8324..86d16fd43cb 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -91,6 +91,7 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { return err } + // TODO: remove before review cfg.Settings.LoggingConfig.Level = logp.DebugLevel logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, true) if err != nil { diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 256c856a830..7e08624efad 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -153,10 +153,7 @@ func (a *Application) Stop() { // signal stop through GRPC, wait and kill is performed later in gracefulKill if err := srvState.Stop(a.processConfig.StopTimeout); err != nil { err := fmt.Errorf("failed to stop after %s: %w", a.processConfig.StopTimeout, err) - a.setState( - state.Failed, - err.Error(), - nil) + a.setState(state.Failed, err.Error(), nil) a.logger.Error(err) } From b7887e732ac1bccfb862f1b555a413872cd1d95a Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 17 Jan 2022 18:01:03 +0100 Subject: [PATCH 14/24] undo typo fixes, they will be fixed on another PR --- libbeat/common/backoff/equal_jitter.go | 4 ++-- libbeat/logp/configure/logging.go | 2 +- libbeat/logp/core.go | 2 +- x-pack/elastic-agent/magefile.go | 2 +- .../elastic-agent/pkg/agent/application/pipeline/pipeline.go | 2 +- x-pack/elastic-agent/pkg/agent/application/reexec/manager.go | 2 +- x-pack/elastic-agent/pkg/core/server/server.go | 2 +- x-pack/elastic-agent/pkg/core/status/reporter.go | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libbeat/common/backoff/equal_jitter.go b/libbeat/common/backoff/equal_jitter.go index d5b5c7d250c..ff5c86f156f 100644 --- a/libbeat/common/backoff/equal_jitter.go +++ b/libbeat/common/backoff/equal_jitter.go @@ -47,11 +47,11 @@ func NewEqualJitterBackoff(done <-chan struct{}, init, max time.Duration) Backof // Reset resets the duration of the backoff. func (b *EqualJitterBackoff) Reset() { - // Allow sleeping at least the init period on the first wait. + // Allow to sleep at least the init period on the first wait. b.duration = b.init * 2 } -// Wait blocks until either the timer is completed or channel is done. +// Wait block until either the timer is completed or channel is done. func (b *EqualJitterBackoff) Wait() bool { // Make sure we have always some minimal back off and jitter. temp := int64(b.duration / 2) diff --git a/libbeat/logp/configure/logging.go b/libbeat/logp/configure/logging.go index 9a041bc2559..43a32dd7f2f 100644 --- a/libbeat/logp/configure/logging.go +++ b/libbeat/logp/configure/logging.go @@ -60,7 +60,7 @@ func Logging(beatName string, cfg *common.Config) error { return logp.Configure(config) } -// LoggingWithOutputs builds a logp.Config based on the given common.Config and the specified +// Logging builds a logp.Config based on the given common.Config and the specified // CLI flags along with the given outputs. func LoggingWithOutputs(beatName string, cfg *common.Config, outputs ...zapcore.Core) error { config := logp.DefaultConfig(environment) diff --git a/libbeat/logp/core.go b/libbeat/logp/core.go index 262515ac8d2..552c81e9201 100644 --- a/libbeat/logp/core.go +++ b/libbeat/logp/core.go @@ -67,7 +67,7 @@ func Configure(cfg Config) error { return ConfigureWithOutputs(cfg) } -// ConfigureWithOutputs XXX: is used by elastic-agent only (See file: x-pack/elastic-agent/pkg/core/logger/logger.go). +// XXX: ConfigureWithOutputs is used by elastic-agent only (See file: x-pack/elastic-agent/pkg/core/logger/logger.go). // The agent requires that the output specified in the config object is configured and merged with the // logging outputs given. func ConfigureWithOutputs(cfg Config, outputs ...zapcore.Core) error { diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index ed84c8ca073..a04d3c974bf 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -508,7 +508,7 @@ func runAgent(env map[string]string) error { return err } - // docker does not exist for this commit, build it + // docker does not exists for this commit, build it if !strings.Contains(dockerImageOut, tag) { // produce docker package packageAgent([]string{ diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go index f2f02efb258..c8cac5b3216 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go @@ -29,7 +29,7 @@ var DefaultRK = "default" // RoutingKey is used for routing as pipeline id. type RoutingKey = string -// Router is an interface routing programs to the corresponding stream. +// Router is an interace routes programs to correspongind stream type Router interface { Routes() *sorted.Set Route(id string, grpProg map[RoutingKey][]program.Program) error diff --git a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go index d251926d357..5ccc870d948 100644 --- a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go +++ b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go @@ -54,7 +54,7 @@ func (m *manager) ReExec(shutdownCallback ShutdownCallbackFn, argOverrides ...st if shutdownCallback != nil { if err := shutdownCallback(); err != nil { // panic; because there is no going back, everything is shutdown - panic(errors.New(errors.TypeUnexpected, err, "failure occurred during shutdown cleanup")) + panic(errors.New(errors.TypeUnexpected, err, "failure occured during shutdown cleanup")) } } diff --git a/x-pack/elastic-agent/pkg/core/server/server.go b/x-pack/elastic-agent/pkg/core/server/server.go index f584e70bad2..390283b4e15 100644 --- a/x-pack/elastic-agent/pkg/core/server/server.go +++ b/x-pack/elastic-agent/pkg/core/server/server.go @@ -581,7 +581,7 @@ func (as *ApplicationState) Stop(timeout time.Duration) error { doneChan := as.checkinDone as.checkinLock.RUnlock() if (wasConn && doneChan == nil) || (!wasConn && s == proto.StateObserved_STOPPING && doneChan == nil) { - // either occurred: + // either occurred // * client was connected then disconnected on stop // * client was not connected; connected; received stopping; then disconnected as.Destroy() diff --git a/x-pack/elastic-agent/pkg/core/status/reporter.go b/x-pack/elastic-agent/pkg/core/status/reporter.go index 2e34bb15cdc..3add6b188c8 100644 --- a/x-pack/elastic-agent/pkg/core/status/reporter.go +++ b/x-pack/elastic-agent/pkg/core/status/reporter.go @@ -279,7 +279,7 @@ func (r *reporter) Update(s state.Status, message string, payload map[string]int } } -// Unregister unregisters status from reporter. Reporter will no longer be taken into consideration +// Unregister unregister status from reporter. Reporter will no longer be taken into consideration // for overall status computation. func (r *reporter) Unregister() { r.mx.Lock() From 2830da99372f6421cf629a67f50e74c4cb2bee5d Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 17 Jan 2022 18:08:48 +0100 Subject: [PATCH 15/24] undo --- dev-tools/mage/crossbuild.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 46e75234a7e..19e92b15c03 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -242,8 +242,8 @@ func CrossBuildImage(platform string) (string, error) { tagSuffix = "s390x" case strings.HasPrefix(platform, "linux"): // Use an older version of libc to gain greater OS compatibility. - // Debian 8 uses glibc 2.19. - tagSuffix = "main-debian8" + // Debian 7 uses glibc 2.13. + tagSuffix = "main-debian7" } goVersion, err := GoVersion() From 9dd99153a41a978bba26eb247b04c53f0f6879c5 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 18 Jan 2022 15:39:05 +0100 Subject: [PATCH 16/24] PR changes --- x-pack/elastic-agent/pkg/agent/cmd/run.go | 3 --- x-pack/elastic-agent/pkg/agent/operation/operator.go | 8 +++++--- x-pack/elastic-agent/pkg/core/plugin/process/app.go | 2 +- x-pack/elastic-agent/pkg/core/plugin/service/app.go | 4 +--- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 86d16fd43cb..707127bcfe7 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -18,7 +18,6 @@ import ( "github.com/elastic/beats/v7/libbeat/api" "github.com/elastic/beats/v7/libbeat/cmd/instance/metrics" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/monitoring" "github.com/elastic/beats/v7/libbeat/service" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application" @@ -91,8 +90,6 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { return err } - // TODO: remove before review - cfg.Settings.LoggingConfig.Level = logp.DebugLevel logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig, true) if err != nil { return err diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index e08a3436483..34fd795764b 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -219,11 +219,13 @@ func (o *Operator) Shutdown() { } wg := sync.WaitGroup{} + wg.Add(len(o.apps)) + started := time.Now() + + o.appsLock.Lock() + defer o.appsLock.Unlock() for _, a := range o.apps { - // shutdown apps concurrently. - // TODO(Anderson): it's fine, right? - wg.Add(1) go func(a Application) { a.Shutdown() wg.Done() diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 7e08624efad..3b50a4c653e 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -287,7 +287,7 @@ func (a *Application) gracefulKill(proc *process.Info) { // send stop signal to request stop if err := proc.Stop(); err != nil { - a.logger.Error(fmt.Errorf("failed to stop %s: %w", a.Name(), err)) + a.logger.Errorf("failed to stop %s: %v", a.Name(), err)) } var wg sync.WaitGroup diff --git a/x-pack/elastic-agent/pkg/core/plugin/service/app.go b/x-pack/elastic-agent/pkg/core/plugin/service/app.go index bd36f8f5e4e..4b0d2b8ee0a 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/service/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/service/app.go @@ -278,7 +278,7 @@ func (a *Application) Stop() { func (a *Application) Shutdown() { a.appLock.Lock() defer a.appLock.Unlock() - a.logger.Infof("Signaling service to stop because of shutdown: %s", a.id) + a.logger.Infof("signaling service to stop because of shutdown: %s", a.id) if a.srvState == nil { return @@ -298,7 +298,6 @@ func (a *Application) Shutdown() { // // It updates the status of the application and handles restarting the application is needed. func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.StateObserved_Status, msg string, payload map[string]interface{}) { - a.appLock.Lock() defer a.appLock.Unlock() @@ -313,7 +312,6 @@ func (a *Application) OnStatusChange(s *server.ApplicationState, status proto.St func (a *Application) setState(s state.Status, msg string, payload map[string]interface{}) { if a.state.Status != s || a.state.Message != msg || !reflect.DeepEqual(a.state.Payload, payload) { - if state.IsStateFiltered(msg, payload) { return } From 23b8fcfee95d9a26bd026badf7cfe87534585cea Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 20 Jan 2022 10:21:01 +0100 Subject: [PATCH 17/24] pr change --- x-pack/elastic-agent/pkg/agent/operation/operator.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index 34fd795764b..3dbd2be6314 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -218,13 +218,14 @@ func (o *Operator) Shutdown() { o.logger.Debugf("pipeline installer '%s' done", o.pipelineID) } + o.appsLock.Lock() + defer o.appsLock.Unlock() + wg := sync.WaitGroup{} wg.Add(len(o.apps)) started := time.Now() - o.appsLock.Lock() - defer o.appsLock.Unlock() for _, a := range o.apps { go func(a Application) { a.Shutdown() From a966d1d816bce5268746d16a1612a271e9aa45eb Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 24 Jan 2022 13:41:47 +0100 Subject: [PATCH 18/24] revert build flags and debug symbols --- dev-tools/mage/build.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 54d1b9f5c18..7a6bc0c631f 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -50,6 +50,9 @@ func DefaultBuildArgs() BuildArgs { args := BuildArgs{ Name: BeatName, CGO: build.Default.CgoEnabled, + LDFlags: []string{ + "-s", // Strip all debug symbols from binary (does not affect Go stack traces). + }, Vars: map[string]string{ elasticBeatsModulePath + "/libbeat/version.buildTime": "{{ date }}", elasticBeatsModulePath + "/libbeat/version.commit": "{{ commit }}", @@ -64,14 +67,6 @@ func DefaultBuildArgs() BuildArgs { args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie") } - if DevBuild { - // Disable optimizations (-N) and inlining (-l) for debugging. - args.ExtraFlags = append(args.ExtraFlags, `-gcflags`, `"all=-N -l"`) - } else { - // Strip all debug symbols from binary (does not affect Go stack traces). - args.LDFlags = append(args.LDFlags, "-s") - } - return args } From e1333dafe06c90d0450be6b599a4fc366fb7e8ff Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 26 Jan 2022 10:25:53 +0100 Subject: [PATCH 19/24] . --- x-pack/elastic-agent/pkg/core/plugin/process/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 3b50a4c653e..64ec2c223c3 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -287,7 +287,7 @@ func (a *Application) gracefulKill(proc *process.Info) { // send stop signal to request stop if err := proc.Stop(); err != nil { - a.logger.Errorf("failed to stop %s: %v", a.Name(), err)) + a.logger.Errorf("failed to stop %s: %v", a.Name(), err) } var wg sync.WaitGroup From c698890170598f5c5f9e5ef64021d466769d6104 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 26 Jan 2022 16:28:00 +0100 Subject: [PATCH 20/24] changelog --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 37f09cec0cc..e01467cf3a7 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -94,6 +94,7 @@ - Allow HTTP metrics to run in bootstrap mode. Add ability to adjust timeouts for Fleet Server. {pull}28260[28260] - Fix agent configuration overwritten by default fleet config. {pull}29297[29297] - Allow agent containers to use basic auth to create a service token. {pull}29651[29651] +- Elastic Agent takes long to shut down {pull}29650[29650] ==== New features From 5ca3cc2acc9e81b2c67c8f0d608dfe5cc40653b3 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 31 Jan 2022 15:03:49 +0100 Subject: [PATCH 21/24] reset to master some files --- dev-tools/mage/build.go | 11 ++++++++--- dev-tools/mage/crossbuild.go | 4 ++-- libbeat/common/backoff/equal_jitter.go | 4 ++-- libbeat/logp/configure/logging.go | 2 +- libbeat/logp/core.go | 2 +- x-pack/elastic-agent/magefile.go | 2 +- .../pkg/agent/application/pipeline/pipeline.go | 2 +- .../pkg/agent/application/reexec/manager.go | 2 +- x-pack/elastic-agent/pkg/core/server/server.go | 2 +- x-pack/elastic-agent/pkg/core/status/reporter.go | 2 +- 10 files changed, 19 insertions(+), 14 deletions(-) diff --git a/dev-tools/mage/build.go b/dev-tools/mage/build.go index 7a6bc0c631f..54d1b9f5c18 100644 --- a/dev-tools/mage/build.go +++ b/dev-tools/mage/build.go @@ -50,9 +50,6 @@ func DefaultBuildArgs() BuildArgs { args := BuildArgs{ Name: BeatName, CGO: build.Default.CgoEnabled, - LDFlags: []string{ - "-s", // Strip all debug symbols from binary (does not affect Go stack traces). - }, Vars: map[string]string{ elasticBeatsModulePath + "/libbeat/version.buildTime": "{{ date }}", elasticBeatsModulePath + "/libbeat/version.commit": "{{ commit }}", @@ -67,6 +64,14 @@ func DefaultBuildArgs() BuildArgs { args.ExtraFlags = append(args.ExtraFlags, "-buildmode", "pie") } + if DevBuild { + // Disable optimizations (-N) and inlining (-l) for debugging. + args.ExtraFlags = append(args.ExtraFlags, `-gcflags`, `"all=-N -l"`) + } else { + // Strip all debug symbols from binary (does not affect Go stack traces). + args.LDFlags = append(args.LDFlags, "-s") + } + return args } diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 420d9121bcb..7a08a035bc4 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -242,8 +242,8 @@ func CrossBuildImage(platform string) (string, error) { tagSuffix = "s390x" case strings.HasPrefix(platform, "linux"): // Use an older version of libc to gain greater OS compatibility. - // Debian 7 uses glibc 2.13. - tagSuffix = "main-debian7" + // Debian 8 uses glibc 2.19. + tagSuffix = "main-debian8" } goVersion, err := GoVersion() diff --git a/libbeat/common/backoff/equal_jitter.go b/libbeat/common/backoff/equal_jitter.go index ff5c86f156f..d5b5c7d250c 100644 --- a/libbeat/common/backoff/equal_jitter.go +++ b/libbeat/common/backoff/equal_jitter.go @@ -47,11 +47,11 @@ func NewEqualJitterBackoff(done <-chan struct{}, init, max time.Duration) Backof // Reset resets the duration of the backoff. func (b *EqualJitterBackoff) Reset() { - // Allow to sleep at least the init period on the first wait. + // Allow sleeping at least the init period on the first wait. b.duration = b.init * 2 } -// Wait block until either the timer is completed or channel is done. +// Wait blocks until either the timer is completed or channel is done. func (b *EqualJitterBackoff) Wait() bool { // Make sure we have always some minimal back off and jitter. temp := int64(b.duration / 2) diff --git a/libbeat/logp/configure/logging.go b/libbeat/logp/configure/logging.go index 43a32dd7f2f..9a041bc2559 100644 --- a/libbeat/logp/configure/logging.go +++ b/libbeat/logp/configure/logging.go @@ -60,7 +60,7 @@ func Logging(beatName string, cfg *common.Config) error { return logp.Configure(config) } -// Logging builds a logp.Config based on the given common.Config and the specified +// LoggingWithOutputs builds a logp.Config based on the given common.Config and the specified // CLI flags along with the given outputs. func LoggingWithOutputs(beatName string, cfg *common.Config, outputs ...zapcore.Core) error { config := logp.DefaultConfig(environment) diff --git a/libbeat/logp/core.go b/libbeat/logp/core.go index 552c81e9201..262515ac8d2 100644 --- a/libbeat/logp/core.go +++ b/libbeat/logp/core.go @@ -67,7 +67,7 @@ func Configure(cfg Config) error { return ConfigureWithOutputs(cfg) } -// XXX: ConfigureWithOutputs is used by elastic-agent only (See file: x-pack/elastic-agent/pkg/core/logger/logger.go). +// ConfigureWithOutputs XXX: is used by elastic-agent only (See file: x-pack/elastic-agent/pkg/core/logger/logger.go). // The agent requires that the output specified in the config object is configured and merged with the // logging outputs given. func ConfigureWithOutputs(cfg Config, outputs ...zapcore.Core) error { diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index ccceaf06935..113f84e7221 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -508,7 +508,7 @@ func runAgent(env map[string]string) error { return err } - // docker does not exists for this commit, build it + // docker does not exist for this commit, build it if !strings.Contains(dockerImageOut, tag) { // produce docker package packageAgent([]string{ diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go index c8cac5b3216..f2f02efb258 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go @@ -29,7 +29,7 @@ var DefaultRK = "default" // RoutingKey is used for routing as pipeline id. type RoutingKey = string -// Router is an interace routes programs to correspongind stream +// Router is an interface routing programs to the corresponding stream. type Router interface { Routes() *sorted.Set Route(id string, grpProg map[RoutingKey][]program.Program) error diff --git a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go index 5ccc870d948..d251926d357 100644 --- a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go +++ b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go @@ -54,7 +54,7 @@ func (m *manager) ReExec(shutdownCallback ShutdownCallbackFn, argOverrides ...st if shutdownCallback != nil { if err := shutdownCallback(); err != nil { // panic; because there is no going back, everything is shutdown - panic(errors.New(errors.TypeUnexpected, err, "failure occured during shutdown cleanup")) + panic(errors.New(errors.TypeUnexpected, err, "failure occurred during shutdown cleanup")) } } diff --git a/x-pack/elastic-agent/pkg/core/server/server.go b/x-pack/elastic-agent/pkg/core/server/server.go index 390283b4e15..f584e70bad2 100644 --- a/x-pack/elastic-agent/pkg/core/server/server.go +++ b/x-pack/elastic-agent/pkg/core/server/server.go @@ -581,7 +581,7 @@ func (as *ApplicationState) Stop(timeout time.Duration) error { doneChan := as.checkinDone as.checkinLock.RUnlock() if (wasConn && doneChan == nil) || (!wasConn && s == proto.StateObserved_STOPPING && doneChan == nil) { - // either occurred + // either occurred: // * client was connected then disconnected on stop // * client was not connected; connected; received stopping; then disconnected as.Destroy() diff --git a/x-pack/elastic-agent/pkg/core/status/reporter.go b/x-pack/elastic-agent/pkg/core/status/reporter.go index 3add6b188c8..2e34bb15cdc 100644 --- a/x-pack/elastic-agent/pkg/core/status/reporter.go +++ b/x-pack/elastic-agent/pkg/core/status/reporter.go @@ -279,7 +279,7 @@ func (r *reporter) Update(s state.Status, message string, payload map[string]int } } -// Unregister unregister status from reporter. Reporter will no longer be taken into consideration +// Unregister unregisters status from reporter. Reporter will no longer be taken into consideration // for overall status computation. func (r *reporter) Unregister() { r.mx.Lock() From 70d7b5401823208fe0c0978cb58fe1c22f7360d3 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 31 Jan 2022 15:05:15 +0100 Subject: [PATCH 22/24] checkout file to master --- x-pack/elastic-agent/pkg/agent/cmd/run.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 707127bcfe7..7f10f9faa31 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -150,11 +150,7 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { control.SetRouteFn(app.Routes) control.SetMonitoringCfg(cfg.Settings.MonitoringConfig) - serverStopFn, err := setupMetrics( - agentInfo, - logger, cfg.Settings.DownloadConfig.OS(), - cfg.Settings.MonitoringConfig, - app) + serverStopFn, err := setupMetrics(agentInfo, logger, cfg.Settings.DownloadConfig.OS(), cfg.Settings.MonitoringConfig, app) if err != nil { return err } @@ -330,12 +326,7 @@ func isProcessStatsEnabled(cfg *monitoringCfg.MonitoringHTTPConfig) bool { return cfg != nil && cfg.Enabled } -func tryDelayEnroll( - ctx context.Context, - logger *logger.Logger, - cfg *configuration.Configuration, - override cfgOverrider) (*configuration.Configuration, error) { - +func tryDelayEnroll(ctx context.Context, logger *logger.Logger, cfg *configuration.Configuration, override cfgOverrider) (*configuration.Configuration, error) { enrollPath := paths.AgentEnrollFile() if _, err := os.Stat(enrollPath); err != nil { // no enrollment file exists or failed to stat it; nothing to do From e2b14d850d9023f2775bbca94fd267f90e124a64 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 1 Feb 2022 17:24:37 +0100 Subject: [PATCH 23/24] pr changes --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 2 +- x-pack/elastic-agent/pkg/agent/operation/operator.go | 4 ++++ x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go | 7 +++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index e01467cf3a7..a3c7a4592b2 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -94,7 +94,7 @@ - Allow HTTP metrics to run in bootstrap mode. Add ability to adjust timeouts for Fleet Server. {pull}28260[28260] - Fix agent configuration overwritten by default fleet config. {pull}29297[29297] - Allow agent containers to use basic auth to create a service token. {pull}29651[29651] -- Elastic Agent takes long to shut down {pull}29650[29650] +- Reduce Elastic Agent shut down by stopping processes in parallel {pull}29650[29650] ==== New features diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index 3dbd2be6314..c0bd85ea444 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -228,8 +228,12 @@ func (o *Operator) Shutdown() { for _, a := range o.apps { go func(a Application) { + started := time.Now() a.Shutdown() wg.Done() + o.logger.Debugf("took %s to shutdown %s", + time.Now().Sub(started), a.Name()) + }(a) } wg.Wait() diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go index 7a14e63c00d..940ec89ac26 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go @@ -73,6 +73,7 @@ func TestIsSubpath(t *testing.T) { } func TestExecFile_Success(t *testing.T) { + t.Skip("skipping failing tests") pwd, err := os.Getwd() if err != nil { panic(err) @@ -86,6 +87,8 @@ func TestExecFile_Success(t *testing.T) { } func TestExecFile_StdErr(t *testing.T) { + t.Skip("skipping failing tests") + pwd, err := os.Getwd() if err != nil { panic(err) @@ -103,6 +106,8 @@ func TestExecFile_StdErr(t *testing.T) { } func TestExecFile_StdOut(t *testing.T) { + t.Skip("skipping failing tests") + pwd, err := os.Getwd() if err != nil { panic(err) @@ -120,6 +125,8 @@ func TestExecFile_StdOut(t *testing.T) { } func TestExecFile_NoOutput(t *testing.T) { + t.Skip("skipping failing tests") + pwd, err := os.Getwd() if err != nil { panic(err) From 079d3e211a098f2dfbe534313aa05f4dbb48358c Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 2 Feb 2022 12:01:45 +0100 Subject: [PATCH 24/24] don't skip tests --- x-pack/elastic-agent/pkg/agent/operation/operator.go | 1 - x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go | 7 ------- 2 files changed, 8 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/operation/operator.go b/x-pack/elastic-agent/pkg/agent/operation/operator.go index c0bd85ea444..9041b62136a 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operator.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operator.go @@ -233,7 +233,6 @@ func (o *Operator) Shutdown() { wg.Done() o.logger.Debugf("took %s to shutdown %s", time.Now().Sub(started), a.Name()) - }(a) } wg.Wait() diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go index 940ec89ac26..7a14e63c00d 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/steps_test.go @@ -73,7 +73,6 @@ func TestIsSubpath(t *testing.T) { } func TestExecFile_Success(t *testing.T) { - t.Skip("skipping failing tests") pwd, err := os.Getwd() if err != nil { panic(err) @@ -87,8 +86,6 @@ func TestExecFile_Success(t *testing.T) { } func TestExecFile_StdErr(t *testing.T) { - t.Skip("skipping failing tests") - pwd, err := os.Getwd() if err != nil { panic(err) @@ -106,8 +103,6 @@ func TestExecFile_StdErr(t *testing.T) { } func TestExecFile_StdOut(t *testing.T) { - t.Skip("skipping failing tests") - pwd, err := os.Getwd() if err != nil { panic(err) @@ -125,8 +120,6 @@ func TestExecFile_StdOut(t *testing.T) { } func TestExecFile_NoOutput(t *testing.T) { - t.Skip("skipping failing tests") - pwd, err := os.Getwd() if err != nil { panic(err)