diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 480b3e4deb486..d14bf23e1332c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -390,7 +390,6 @@ /pkg/process/util/address*.go @DataDog/Networks /pkg/process/util/netns*.go @DataDog/Networks /pkg/process/checks/net*.go @DataDog/Networks -/pkg/process/checks/pod*.go @DataDog/container-app /pkg/process/metadata/parser/ @DataDog/universal-service-monitoring @DataDog/processes @DataDog/Networks /pkg/process/metadata/parser/*windows* @DataDog/universal-service-monitoring @DataDog/processes @DataDog/Networks @DataDog/windows-kernel-integrations /pkg/process/monitor/ @DataDog/universal-service-monitoring @@ -401,6 +400,9 @@ /pkg/proto/pbgo/languagedetection @DataDog/agent-apm /pkg/proto/pbgo/process @DataDog/processes /pkg/proto/pbgo/core @DataDog/agent-shared-components +/pkg/proto/pbgo/core/remoteconfig.pb.go @DataDog/remote-config +/pkg/proto/pbgo/core/remoteconfig_gen.go @DataDog/remote-config +/pkg/proto/pbgo/core/remoteconfig_gen_test.go @DataDog/remote-config /pkg/proto/pbgo/mocks/core @DataDog/agent-shared-components /pkg/orchestrator/ @DataDog/container-app /pkg/network/ @DataDog/Networks diff --git a/.markdown-link-check b/.markdown-link-check index 6f73db8343342..df7a7fc62e62b 100644 --- a/.markdown-link-check +++ b/.markdown-link-check @@ -5,6 +5,9 @@ }, { "pattern": "^https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/" + }, + { + "pattern": "^/security/threats/" } ], "replacementPatterns": [ diff --git a/cmd/agent/subcommands/jmx/command.go b/cmd/agent/subcommands/jmx/command.go index fee666b63e726..0da1158dd4e57 100644 --- a/cmd/agent/subcommands/jmx/command.go +++ b/cmd/agent/subcommands/jmx/command.go @@ -26,6 +26,7 @@ import ( "github.com/DataDog/datadog-agent/comp/aggregator/diagnosesendermanager" "github.com/DataDog/datadog-agent/comp/aggregator/diagnosesendermanager/diagnosesendermanagerimpl" internalAPI "github.com/DataDog/datadog-agent/comp/api/api" + "github.com/DataDog/datadog-agent/comp/collector/collector" "github.com/DataDog/datadog-agent/comp/api/api/apiimpl" "github.com/DataDog/datadog-agent/comp/core" @@ -296,7 +297,7 @@ func runJmxCommandConsole(config config.Component, cliParams *cliParams, wmeta w // Create the CheckScheduler, but do not attach it to // AutoDiscovery. - pkgcollector.InitCheckScheduler(optional.NewNoneOption[pkgcollector.Collector](), senderManager) + pkgcollector.InitCheckScheduler(optional.NewNoneOption[collector.Component](), senderManager) // if cliSelectedChecks is empty, then we want to fetch all check configs; // otherwise, we fetch only the matching cehck configs. diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go index 9327647bc5db7..a1a9de53e600e 100644 --- a/cmd/agent/subcommands/run/command.go +++ b/cmd/agent/subcommands/run/command.go @@ -509,6 +509,9 @@ func startAgent( // Subscribe to `AGENT_TASK` product rcclient.SubscribeAgentTask() + // Subscribe to `APM_TRACING` product + rcclient.SubscribeApmTracing() + if pkgconfig.Datadog.GetBool("remote_configuration.agent_integrations.enabled") { // Spin up the config provider to schedule integrations through remote-config rcProvider := providers.NewRemoteConfigProvider() @@ -584,8 +587,7 @@ func startAgent( // Set up check collector commonchecks.RegisterChecks(wmeta) - common.AC.AddScheduler("check", pkgcollector.InitCheckScheduler(optional.NewOption[pkgcollector.Collector](collector), demultiplexer), true) - collector.Start() + common.AC.AddScheduler("check", pkgcollector.InitCheckScheduler(optional.NewOption(collector), demultiplexer), true) diagnose.Init(optional.NewOption(collector)) demultiplexer.AddAgentStartupTelemetry(version.AgentVersion) diff --git a/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go b/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go index 5c1550241860b..2f5a3162cb6fb 100644 --- a/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go +++ b/cmd/cluster-agent-cloudfoundry/subcommands/run/command.go @@ -148,8 +148,7 @@ func run(log log.Component, taggerComp tagger.Component, demultiplexer demultipl common.LoadComponents(secretResolver, wmeta, pkgconfig.Datadog.GetString("confd_path")) // Set up check collector - common.AC.AddScheduler("check", pkgcollector.InitCheckScheduler(optional.NewOption[pkgcollector.Collector](collector), demultiplexer), true) - collector.Start() + common.AC.AddScheduler("check", pkgcollector.InitCheckScheduler(optional.NewOption(collector), demultiplexer), true) diagnose.Init(optional.NewOption(collector)) // start the autoconfig, this will immediately run any configured check diff --git a/cmd/cluster-agent/subcommands/start/command.go b/cmd/cluster-agent/subcommands/start/command.go index 383eeed5d2de7..239008854a299 100644 --- a/cmd/cluster-agent/subcommands/start/command.go +++ b/cmd/cluster-agent/subcommands/start/command.go @@ -11,7 +11,7 @@ package start import ( "context" "fmt" - "github.com/DataDog/datadog-agent/pkg/util/optional" + "net/http" "os" "os/signal" @@ -19,6 +19,9 @@ import ( "syscall" "time" + "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" + "github.com/DataDog/datadog-agent/pkg/util/optional" + "github.com/DataDog/datadog-agent/cmd/agent/common" "github.com/DataDog/datadog-agent/cmd/agent/common/path" admissioncmd "github.com/DataDog/datadog-agent/cmd/cluster-agent/admission" @@ -51,13 +54,13 @@ import ( "github.com/DataDog/datadog-agent/pkg/clusteragent" admissionpkg "github.com/DataDog/datadog-agent/pkg/clusteragent/admission" "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate" + agentsidecar "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate/agent_sidecar" admissionpatch "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/patch" apidca "github.com/DataDog/datadog-agent/pkg/clusteragent/api" "github.com/DataDog/datadog-agent/pkg/clusteragent/clusterchecks" pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" pkgconfig "github.com/DataDog/datadog-agent/pkg/config" rcclient "github.com/DataDog/datadog-agent/pkg/config/remote/client" - "github.com/DataDog/datadog-agent/pkg/config/remote/data" commonsettings "github.com/DataDog/datadog-agent/pkg/config/settings" "github.com/DataDog/datadog-agent/pkg/status/health" "github.com/DataDog/datadog-agent/pkg/util" @@ -70,7 +73,6 @@ import ( pkglog "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" - "github.com/DataDog/datadog-agent/pkg/clusteragent/languagedetection" "github.com/gorilla/mux" "github.com/spf13/cobra" "go.uber.org/fx" @@ -79,6 +81,8 @@ import ( corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" + "github.com/DataDog/datadog-agent/pkg/clusteragent/languagedetection" + // Core checks corecheckLoader "github.com/DataDog/datadog-agent/pkg/collector/corechecks" @@ -306,8 +310,7 @@ func start(log log.Component, config config.Component, taggerComp tagger.Compone // Set up check collector registerChecks() - common.AC.AddScheduler("check", pkgcollector.InitCheckScheduler(optional.NewOption[pkgcollector.Collector](collector), demultiplexer), true) - collector.Start() + common.AC.AddScheduler("check", pkgcollector.InitCheckScheduler(optional.NewOption(collector), demultiplexer), true) // start the autoconfig, this will immediately run any configured check common.AC.LoadAndRun(mainCtx) @@ -396,6 +399,7 @@ func start(log log.Component, config config.Component, taggerComp tagger.Compone server.Register(pkgconfig.Datadog.GetString("admission_controller.inject_config.endpoint"), mutate.InjectConfig, apiCl.DynamicCl, apiCl.Cl) server.Register(pkgconfig.Datadog.GetString("admission_controller.inject_tags.endpoint"), mutate.InjectTags, apiCl.DynamicCl, apiCl.Cl) server.Register(pkgconfig.Datadog.GetString("admission_controller.auto_instrumentation.endpoint"), mutate.InjectAutoInstrumentation, apiCl.DynamicCl, apiCl.Cl) + server.Register(pkgconfig.Datadog.GetString("admission_controller.agent_sidecar.endpoint"), agentsidecar.InjectAgentSidecar, apiCl.DynamicCl, apiCl.Cl) // CWS Instrumentation webhooks cwsInstrumentation, err := mutate.NewCWSInstrumentation() @@ -503,7 +507,7 @@ func initializeRemoteConfigClient(ctx context.Context, rcService rccomp.Componen rcClient, err := rcclient.NewClient(rcService, rcclient.WithAgent("cluster-agent", version.AgentVersion), rcclient.WithCluster(clusterName, clusterID), - rcclient.WithProducts([]data.Product{data.ProductAPMTracing}), + rcclient.WithProducts(state.ProductAPMTracing), rcclient.WithPollInterval(5*time.Second), rcclient.WithDirectorRootOverride(pkgconfig.Datadog.GetString("remote_configuration.director_root")), ) diff --git a/cmd/trace-agent/config/remote/config.go b/cmd/trace-agent/config/remote/config.go index 9f7c37fe883fb..21444c067f20a 100644 --- a/cmd/trace-agent/config/remote/config.go +++ b/cmd/trace-agent/config/remote/config.go @@ -15,6 +15,11 @@ import ( "sync" "time" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/DataDog/datadog-go/v5/statsd" + rcclient "github.com/DataDog/datadog-agent/pkg/config/remote/client" pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" "github.com/DataDog/datadog-agent/pkg/trace/api" @@ -22,9 +27,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/trace/timing" "github.com/DataDog/datadog-agent/pkg/trace/traceutil" "github.com/DataDog/datadog-agent/pkg/util/log" - "github.com/DataDog/datadog-go/v5/statsd" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) var bufferPool = sync.Pool{ @@ -76,7 +78,7 @@ func ConfigHandler(r *api.HTTPReceiver, client rcclient.ConfigUpdater, cfg *conf } configsRequest.Client.ClientTracer.Tags = append(configsRequest.Client.ClientTracer.Tags, getContainerTags(req, cfg, cidProvider)...) } - cfg, err := client.ClientGetConfigs(req.Context(), &configsRequest) + cfgResponse, err := client.ClientGetConfigs(req.Context(), &configsRequest) if err != nil { statusCode = http.StatusInternalServerError if e, ok := status.FromError(err); ok { @@ -88,12 +90,12 @@ func ConfigHandler(r *api.HTTPReceiver, client rcclient.ConfigUpdater, cfg *conf http.Error(w, err.Error(), statusCode) return } - if cfg == nil { + if cfgResponse == nil { w.WriteHeader(http.StatusNoContent) return } - content, err := json.Marshal(cfg) + content, err := json.Marshal(cfgResponse) if err != nil { statusCode = http.StatusInternalServerError http.Error(w, err.Error(), http.StatusInternalServerError) diff --git a/comp/README.md b/comp/README.md index af8db9ca35b8f..c11963a116ae3 100644 --- a/comp/README.md +++ b/comp/README.md @@ -327,10 +327,6 @@ Package forwarders implements a component to provide forwarders used by the proc Package hostinfo wraps the hostinfo inside a component. This is useful because it is relied on by other components. -### [comp/process/podcheck](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/process/podcheck) - -Package podcheck implements a component to handle Kubernetes data collection in the Process Agent. - ### [comp/process/processcheck](https://pkg.go.dev/github.com/DataDog/datadog-agent/comp/process/processcheck) Package processcheck implements a component to handle Process data collection in the Process Agent. diff --git a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go index ee434d1fdc17f..c063bcdd338ba 100644 --- a/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go +++ b/comp/aggregator/demultiplexer/demultiplexerimpl/demultiplexer.go @@ -56,7 +56,8 @@ type provides struct { // implements diagnosesendermanager.Component). This has the nice consequence of preventing having // demultiplexerimpl.Module and diagnosesendermanagerimpl.Module in the same fx.App because there would // be two ways to create diagnosesendermanager.Component. - SenderManager diagnosesendermanager.Component + DiagnosticSenderManager diagnosesendermanager.Component + SenderManager sender.SenderManager StatusProvider status.InformationProvider AggregatorDemultiplexer aggregator.Demultiplexer } @@ -88,8 +89,9 @@ func newDemultiplexer(deps dependencies) (provides, error) { }}) return provides{ - Comp: demultiplexer, - SenderManager: demultiplexer, + Comp: demultiplexer, + DiagnosticSenderManager: demultiplexer, + SenderManager: demultiplexer, StatusProvider: status.NewInformationProvider(demultiplexerStatus{ Log: deps.Log, }), diff --git a/comp/collector/collector/collectorimpl/collector.go b/comp/collector/collector/collectorimpl/collector.go index c744887d2b5f2..efcdd710eb8c4 100644 --- a/comp/collector/collector/collectorimpl/collector.go +++ b/comp/collector/collector/collectorimpl/collector.go @@ -7,20 +7,37 @@ package collectorimpl import ( + "context" + "fmt" + "sync" + "time" + + "go.uber.org/atomic" "go.uber.org/fx" "github.com/DataDog/datadog-agent/cmd/agent/common" - "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" "github.com/DataDog/datadog-agent/comp/collector/collector" + "github.com/DataDog/datadog-agent/comp/collector/collector/collectorimpl/internal/middleware" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log" "github.com/DataDog/datadog-agent/comp/core/status" - pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" + "github.com/DataDog/datadog-agent/pkg/aggregator/sender" + pkgCollector "github.com/DataDog/datadog-agent/pkg/collector" + "github.com/DataDog/datadog-agent/pkg/collector/check" + checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" + "github.com/DataDog/datadog-agent/pkg/collector/runner" + "github.com/DataDog/datadog-agent/pkg/collector/runner/expvars" + "github.com/DataDog/datadog-agent/pkg/collector/scheduler" collectorStatus "github.com/DataDog/datadog-agent/pkg/status/collector" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" ) +const ( + stopped uint32 = iota + started +) + type dependencies struct { fx.In @@ -28,11 +45,26 @@ type dependencies struct { Config config.Component Log log.Component - Demultiplexer demultiplexer.Component + SenderManager sender.SenderManager } type collectorImpl struct { - pkgcollector.Collector + log log.Component + + senderManager sender.SenderManager + checkInstances int64 + + // state is 'started' or 'stopped' + state *atomic.Uint32 + + scheduler *scheduler.Scheduler + runner *runner.Runner + checks map[checkid.ID]*middleware.CheckWrapper + eventReceivers []collector.EventReceiver + + cancelCheckTimeout time.Duration + + m sync.RWMutex } type provides struct { @@ -46,7 +78,7 @@ type provides struct { // Module defines the fx options for this component. func Module() fxutil.Module { return fxutil.Component( - fx.Provide(newCollector), + fx.Provide(newProvides), ) } @@ -57,11 +89,8 @@ func ModuleNoneCollector() fxutil.Module { ) } -func newCollector(deps dependencies) provides { - c := &collectorImpl{ - Collector: pkgcollector.NewCollector(deps.Demultiplexer, deps.Config.GetDuration("check_cancel_timeout"), common.GetPythonPaths()...), - } - +func newProvides(deps dependencies) provides { + c := newCollector(deps) return provides{ Comp: c, OptionalComp: optional.NewOption[collector.Component](c), @@ -75,3 +104,251 @@ func newNoneCollector() provides { Provider: status.NewInformationProvider(collectorStatus.Provider{}), } } + +func newCollector(deps dependencies) *collectorImpl { + c := &collectorImpl{ + log: deps.Log, + senderManager: deps.SenderManager, + checks: make(map[checkid.ID]*middleware.CheckWrapper), + state: atomic.NewUint32(stopped), + checkInstances: int64(0), + cancelCheckTimeout: deps.Config.GetDuration("check_cancel_timeout"), + } + + pkgCollector.InitPython(common.GetPythonPaths()...) + + deps.Lc.Append(fx.Hook{ + OnStart: c.start, + OnStop: c.stop, + }) + + c.log.Debug("Collector up and running!") + return c +} + +// AddEventReceiver adds a callback to the collector to be called each time a check is added or removed. +func (c *collectorImpl) AddEventReceiver(cb collector.EventReceiver) { + c.m.Lock() + defer c.m.Unlock() + + c.eventReceivers = append(c.eventReceivers, cb) +} + +func (c *collectorImpl) notify(cid checkid.ID, e collector.EventType) { + for _, cb := range c.eventReceivers { + cb(cid, e) + } +} + +// start begins the collector's operation. The scheduler will not run any checks until this has been called. +func (c *collectorImpl) start(_ context.Context) error { + c.m.Lock() + defer c.m.Unlock() + + run := runner.NewRunner(c.senderManager) + sched := scheduler.NewScheduler(run.GetChan()) + + // let the runner some visibility into the scheduler + run.SetScheduler(sched) + sched.Run() + + c.scheduler = sched + c.runner = run + c.state.Store(started) + + return nil +} + +// stop halts any component involved in running a Check +func (c *collectorImpl) stop(_ context.Context) error { + c.m.Lock() + defer c.m.Unlock() + + if c.scheduler != nil { + c.scheduler.Stop() //nolint:errcheck + c.scheduler = nil + } + if c.runner != nil { + c.runner.Stop() + c.runner = nil + } + c.state.Store(stopped) + return nil +} + +// RunCheck sends a Check in the execution queue +func (c *collectorImpl) RunCheck(inner check.Check) (checkid.ID, error) { + c.m.Lock() + defer c.m.Unlock() + + ch := middleware.NewCheckWrapper(inner, c.senderManager) + + var emptyID checkid.ID + + if c.state.Load() != started { + return emptyID, fmt.Errorf("the collector is not running") + } + + if _, found := c.checks[ch.ID()]; found { + return emptyID, fmt.Errorf("a check with ID %s is already running", ch.ID()) + } + + if err := c.scheduler.Enter(ch); err != nil { + return emptyID, fmt.Errorf("unable to schedule the check: %s", err) + } + + // Track the total number of checks running in order to have an appropriate number of workers + c.checkInstances++ + if ch.Interval() == 0 { + // Adding a temporary runner for long running check in case the + // number of runners is lower than the number of long running + // checks. + c.log.Infof("Adding an extra runner for the '%s' long running check", ch) + c.runner.AddWorker() + } else { + c.runner.UpdateNumWorkers(c.checkInstances) + } + + c.checks[ch.ID()] = ch + c.notify(ch.ID(), collector.CheckRun) + return ch.ID(), nil +} + +// StopCheck halts a check and remove the instance +func (c *collectorImpl) StopCheck(id checkid.ID) error { + if !c.started() { + return fmt.Errorf("the collector is not running") + } + + ch, found := c.get(id) + if !found { + return fmt.Errorf("cannot find a check with ID %s", id) + } + + // unschedule the instance + if err := c.scheduler.Cancel(id); err != nil { + return fmt.Errorf("an error occurred while canceling the check schedule: %s", err) + } + + if err := c.runner.StopCheck(id); err != nil { + // still attempt to cancel the check before returning the error + _ = c.cancelCheck(ch, c.cancelCheckTimeout) + return fmt.Errorf("an error occurred while stopping the check: %s", err) + } + + if err := c.cancelCheck(ch, c.cancelCheckTimeout); err != nil { + return fmt.Errorf("an error occurred while calling check.Cancel(): %s", err) + } + + // remove the check from the stats map + expvars.RemoveCheckStats(id) + + // vaporize the check + c.delete(id) + + return nil +} + +// cancelCheck calls Cancel on the passed check, with a timeout +func (c *collectorImpl) cancelCheck(ch check.Check, timeout time.Duration) error { + done := make(chan struct{}) + + go func() { + ch.Cancel() + close(done) + }() + + select { + case <-done: + return nil + case <-time.After(timeout): + return fmt.Errorf("timeout while calling check.Cancel() on check ID %s, timeout: %s", ch.ID(), timeout) + } +} + +func (c *collectorImpl) get(id checkid.ID) (check.Check, bool) { + c.m.RLock() + defer c.m.RUnlock() + + ch, found := c.checks[id] + return ch, found +} + +// remove the check from the list +func (c *collectorImpl) delete(id checkid.ID) { + c.m.Lock() + defer c.m.Unlock() + + delete(c.checks, id) + c.notify(id, collector.CheckStop) +} + +// lightweight shortcut to see if the collector has started +func (c *collectorImpl) started() bool { + return c.state.Load() == started +} + +// MapOverChecks call the callback with the list of checks locked. +func (c *collectorImpl) MapOverChecks(cb func([]check.Info)) { + c.m.RLock() + defer c.m.RUnlock() + + cInfo := make([]check.Info, 0, len(c.checks)) + for _, c := range c.checks { + cInfo = append(cInfo, c) + } + cb(cInfo) +} + +// GetChecks copies checks +func (c *collectorImpl) GetChecks() []check.Check { + c.m.RLock() + defer c.m.RUnlock() + + chks := make([]check.Check, 0, len(c.checks)) + for _, chck := range c.checks { + chks = append(chks, chck) + } + + return chks +} + +// GetAllInstanceIDs returns the ID's of all instances of a check +func (c *collectorImpl) GetAllInstanceIDs(checkName string) []checkid.ID { + c.m.RLock() + defer c.m.RUnlock() + + instances := []checkid.ID{} + for id, check := range c.checks { + if check.String() == checkName { + instances = append(instances, id) + } + } + + return instances +} + +// ReloadAllCheckInstances completely restarts a check with a new configuration and returns a list of killed check IDs +func (c *collectorImpl) ReloadAllCheckInstances(name string, newInstances []check.Check) ([]checkid.ID, error) { + if !c.started() { + return nil, fmt.Errorf("The collector is not running") + } + + // Stop all the old instances + killed := c.GetAllInstanceIDs(name) + for _, id := range killed { + e := c.StopCheck(id) + if e != nil { + return nil, fmt.Errorf("Error stopping check %s: %s", id, e) + } + } + + // Start the new instances + for _, check := range newInstances { + id, e := c.RunCheck(check) + if e != nil { + return nil, fmt.Errorf("Error adding check %s: %s", id, e) + } + } + return killed, nil +} diff --git a/pkg/collector/collector_demux_test.go b/comp/collector/collector/collectorimpl/collector_demux_test.go similarity index 88% rename from pkg/collector/collector_demux_test.go rename to comp/collector/collector/collectorimpl/collector_demux_test.go index c1ea961025334..1f9511df251ab 100644 --- a/pkg/collector/collector_demux_test.go +++ b/comp/collector/collector/collectorimpl/collector_demux_test.go @@ -5,20 +5,26 @@ //go:build test -package collector +package collectorimpl import ( + "context" "testing" "time" - "github.com/DataDog/datadog-agent/comp/core/log/logimpl" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "go.uber.org/fx" + + "github.com/DataDog/datadog-agent/comp/core" + "github.com/DataDog/datadog-agent/comp/core/config" + "github.com/DataDog/datadog-agent/comp/core/log/logimpl" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer" "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameimpl" + "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/collector/check/stub" "github.com/DataDog/datadog-agent/pkg/metrics" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -28,18 +34,25 @@ type CollectorDemuxTestSuite struct { suite.Suite demux demultiplexer.FakeSamplerMock - c *collector + c *collectorImpl } func (suite *CollectorDemuxTestSuite) SetupTest() { suite.demux = fxutil.Test[demultiplexer.FakeSamplerMock](suite.T(), logimpl.MockModule(), demultiplexerimpl.FakeSamplerMockModule(), hostnameimpl.MockModule()) - suite.c = NewCollector(suite.demux, 500*time.Millisecond).(*collector) - - suite.c.Start() + suite.c = newCollector(fxutil.Test[dependencies](suite.T(), + core.MockBundle(), + fx.Provide(func() sender.SenderManager { + return suite.demux + }), + fx.Replace(config.MockParams{ + Overrides: map[string]interface{}{"check_cancel_timeout": 500 * time.Millisecond}, + }))) + + suite.c.start(context.TODO()) } func (suite *CollectorDemuxTestSuite) TearDownTest() { - suite.c.Stop() + suite.c.stop(context.TODO()) suite.demux.Stop(false) suite.c = nil } @@ -119,7 +132,7 @@ func (suite *CollectorDemuxTestSuite) TestCancelledCheckDestroysSender() { newSender, err := suite.demux.GetSender(ch.ID()) assert.Nil(suite.T(), err) - assert.NotEqual(suite.T(), sender, newSender) // GetSedner returns a new instance, which means the old sender was destroyed correctly. + assert.NotEqual(suite.T(), sender, newSender) // GetSender returns a new instance, which means the old sender was destroyed correctly. } func (suite *CollectorDemuxTestSuite) TestRescheduledCheckReusesSampler() { diff --git a/comp/collector/collector/collectorimpl/collector_mock.go b/comp/collector/collector/collectorimpl/collector_mock.go index aa8de22f10587..38e89f9074b60 100644 --- a/comp/collector/collector/collectorimpl/collector_mock.go +++ b/comp/collector/collector/collectorimpl/collector_mock.go @@ -12,7 +12,8 @@ import ( "go.uber.org/fx" "github.com/DataDog/datadog-agent/comp/collector/collector" - pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" + "github.com/DataDog/datadog-agent/pkg/collector/check" + checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" ) @@ -20,6 +21,7 @@ import ( // MockModule defines the fx options for the mock component. func MockModule() fxutil.Module { return fxutil.Component( + fx.Supply(MockParams{}), fx.Provide(newMock), fx.Provide(func(collector collector.Component) optional.Option[collector.Component] { return optional.NewOption(collector) @@ -27,10 +29,66 @@ func MockModule() fxutil.Module { ) } -type mock struct { - pkgcollector.Collector +// MockParams defines the parameters for the mock component. +type MockParams struct { + ChecksInfo []check.Info } -func newMock() collector.Component { - return &mock{} +type mockDependencies struct { + fx.In + + Params MockParams +} + +type mockimpl struct { + collector.Component + + checksInfo []check.Info +} + +func newMock(deps mockDependencies) collector.Component { + return &mockimpl{ + checksInfo: deps.Params.ChecksInfo, + } +} + +// Start begins the collector's operation. The scheduler will not run any checks until this has been called. +func (c *mockimpl) Start() { +} + +// Stop halts any component involved in running a Check +func (c *mockimpl) Stop() { +} + +// RunCheck sends a Check in the execution queue +func (c *mockimpl) RunCheck(_ check.Check) (checkid.ID, error) { + return checkid.ID(""), nil } + +// StopCheck halts a check and remove the instance +func (c *mockimpl) StopCheck(_ checkid.ID) error { + return nil +} + +// MapOverChecks call the callback with the list of checks locked. +func (c *mockimpl) MapOverChecks(cb func([]check.Info)) { + cb(c.checksInfo) +} + +// GetChecks copies checks +func (c *mockimpl) GetChecks() []check.Check { + return nil +} + +// GetAllInstanceIDs returns the ID's of all instances of a check +func (c *mockimpl) GetAllInstanceIDs(_ string) []checkid.ID { + return nil +} + +// ReloadAllCheckInstances completely restarts a check with a new configuration +func (c *mockimpl) ReloadAllCheckInstances(_ string, _ []check.Check) ([]checkid.ID, error) { + return []checkid.ID{checkid.ID("")}, nil +} + +// AddEventReceiver adds a callback to the collector to be called each time a check is added or removed. +func (c *mockimpl) AddEventReceiver(_ collector.EventReceiver) {} diff --git a/pkg/collector/collector_test.go b/comp/collector/collector/collectorimpl/collector_test.go similarity index 89% rename from pkg/collector/collector_test.go rename to comp/collector/collector/collectorimpl/collector_test.go index ea0c0d8addf44..d459006084b56 100644 --- a/pkg/collector/collector_test.go +++ b/comp/collector/collector/collectorimpl/collector_test.go @@ -5,27 +5,33 @@ //go:build test -package collector +package collectorimpl import ( + "context" "sort" "time" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" + tmock "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" + "github.com/DataDog/datadog-agent/comp/collector/collector/collectorimpl/internal/middleware" + "github.com/DataDog/datadog-agent/comp/core" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/collector/check" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" "github.com/DataDog/datadog-agent/pkg/collector/check/stub" - "github.com/DataDog/datadog-agent/pkg/collector/internal/middleware" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) // FIXTURE type TestCheck struct { stub.StubCheck - mock.Mock + tmock.Mock uniqueID checkid.ID name string stop chan bool @@ -81,16 +87,21 @@ func (p ChecksList) Less(i, j int) bool { return p[i] < p[j] } type CollectorTestSuite struct { suite.Suite - c *collector + c *collectorImpl } func (suite *CollectorTestSuite) SetupTest() { - suite.c = NewCollector(aggregator.NewNoOpSenderManager(), 500*time.Millisecond).(*collector) - suite.c.Start() + suite.c = newCollector(fxutil.Test[dependencies](suite.T(), + core.MockBundle(), + demultiplexerimpl.MockModule(), + fx.Replace(config.MockParams{ + Overrides: map[string]interface{}{"check_cancel_timeout": 500 * time.Millisecond}, + }))) + suite.c.start(context.TODO()) } func (suite *CollectorTestSuite) TearDownTest() { - suite.c.Stop() + suite.c.stop(context.TODO()) suite.c = nil } @@ -101,7 +112,7 @@ func (suite *CollectorTestSuite) TestNewCollector() { } func (suite *CollectorTestSuite) TestStop() { - suite.c.Stop() + suite.c.stop(context.TODO()) assert.Nil(suite.T(), suite.c.runner) assert.Nil(suite.T(), suite.c.scheduler) assert.Equal(suite.T(), stopped, suite.c.state.Load()) @@ -177,7 +188,7 @@ func (suite *CollectorTestSuite) TestDelete() { func (suite *CollectorTestSuite) TestStarted() { assert.True(suite.T(), suite.c.started()) - suite.c.Stop() + suite.c.stop(context.TODO()) assert.False(suite.T(), suite.c.started()) } diff --git a/pkg/collector/internal/middleware/check_wrapper.go b/comp/collector/collector/collectorimpl/internal/middleware/check_wrapper.go similarity index 100% rename from pkg/collector/internal/middleware/check_wrapper.go rename to comp/collector/collector/collectorimpl/internal/middleware/check_wrapper.go diff --git a/pkg/collector/internal/middleware/test_utils.go b/comp/collector/collector/collectorimpl/internal/middleware/test_utils.go similarity index 100% rename from pkg/collector/internal/middleware/test_utils.go rename to comp/collector/collector/collectorimpl/internal/middleware/test_utils.go diff --git a/comp/collector/collector/component.go b/comp/collector/collector/component.go index 085355687ce1e..bfd674e4bc440 100644 --- a/comp/collector/collector/component.go +++ b/comp/collector/collector/component.go @@ -7,19 +7,27 @@ package collector import ( - "github.com/DataDog/datadog-agent/pkg/collector" "github.com/DataDog/datadog-agent/pkg/collector/check" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" ) // team: agent-metrics-logs +// EventType represents the type of events emitted by the collector +type EventType uint32 + +const ( + // CheckRun is emitted when a check is added to the collector + CheckRun EventType = iota + // CheckStop is emitted when a check is stopped and removed from the collector + CheckStop +) + +// EventReceiver represents a function to receive notification from the collector when running or stopping checks. +type EventReceiver func(checkid.ID, EventType) + // Component is the component type. type Component interface { - // Start begins the collector's operation. The scheduler will not run any checks until this has been called. - Start() - // Stop halts any component involved in running a Check - Stop() // RunCheck sends a Check in the execution queue RunCheck(inner check.Check) (checkid.ID, error) // StopCheck halts a check and remove the instance @@ -30,8 +38,8 @@ type Component interface { GetChecks() []check.Check // GetAllInstanceIDs returns the ID's of all instances of a check GetAllInstanceIDs(checkName string) []checkid.ID - // ReloadAllCheckInstances completely restarts a check with a new configuration + // ReloadAllCheckInstances completely restarts a check with a new configuration and returns a list of killed check IDs ReloadAllCheckInstances(name string, newInstances []check.Check) ([]checkid.ID, error) // AddEventReceiver adds a callback to the collector to be called each time a check is added or removed. - AddEventReceiver(cb collector.EventReceiver) + AddEventReceiver(cb EventReceiver) } diff --git a/comp/forwarder/defaultforwarder/default_forwarder.go b/comp/forwarder/defaultforwarder/default_forwarder.go index d4b7e79b26918..ec369cc248013 100644 --- a/comp/forwarder/defaultforwarder/default_forwarder.go +++ b/comp/forwarder/defaultforwarder/default_forwarder.go @@ -34,6 +34,8 @@ const ( Stopped uint32 = iota // Started represent the internal state of an started Forwarder. Started + // Disabled represent the internal state of a disabled Forwarder. + Disabled ) const ( @@ -60,7 +62,7 @@ type Forwarder interface { Start() error Stop() SubmitV1Series(payload transaction.BytesPayloads, extra http.Header) error - SubmitV1Intake(payload transaction.BytesPayloads, extra http.Header) error + SubmitV1Intake(payload transaction.BytesPayloads, kind transaction.Kind, extra http.Header) error SubmitV1CheckRuns(payload transaction.BytesPayloads, extra http.Header) error SubmitSeries(payload transaction.BytesPayloads, extra http.Header) error SubmitSketchSeries(payload transaction.BytesPayloads, extra http.Header) error @@ -272,6 +274,16 @@ func NewDefaultForwarder(config config.Component, log log.Component, options *Op transactionContainerSort := transaction.SortByCreatedTimeAndPriority{HighPriorityFirst: false} for domain, resolver := range options.DomainResolvers { + isHA := false + if config.GetBool("ha.enabled") && config.GetString("ha.site") != "" { + log.Infof("HA is enabled, checking site: %v ", config.GetString("ha.site")) + siteURL := utils.BuildURLWithPrefix(utils.InfraURLPrefix, config.GetString("ha.site")) + if domain == siteURL { + log.Infof("HA domain '%s', configured ", domain) + isHA = true + } + + } domain, _ := utils.AddAgentVersionToDomain(domain, "app") resolver.SetBaseDomain(domain) if resolver.GetAPIKeys() == nil || len(resolver.GetAPIKeys()) == 0 { @@ -301,6 +313,7 @@ func NewDefaultForwarder(config config.Component, log log.Component, options *Op config, log, domain, + isHA, transactionContainer, options.NumberOfWorkers, options.ConnectionResetInterval, @@ -430,11 +443,11 @@ func (f *DefaultForwarder) State() uint32 { return f.internalState.Load() } -func (f *DefaultForwarder) createHTTPTransactions(endpoint transaction.Endpoint, payloads transaction.BytesPayloads, extra http.Header) []*transaction.HTTPTransaction { - return f.createAdvancedHTTPTransactions(endpoint, payloads, extra, transaction.TransactionPriorityNormal, true) +func (f *DefaultForwarder) createHTTPTransactions(endpoint transaction.Endpoint, payloads transaction.BytesPayloads, kind transaction.Kind, extra http.Header) []*transaction.HTTPTransaction { + return f.createAdvancedHTTPTransactions(endpoint, payloads, extra, transaction.TransactionPriorityNormal, kind, true) } -func (f *DefaultForwarder) createAdvancedHTTPTransactions(endpoint transaction.Endpoint, payloads transaction.BytesPayloads, extra http.Header, priority transaction.Priority, storableOnDisk bool) []*transaction.HTTPTransaction { +func (f *DefaultForwarder) createAdvancedHTTPTransactions(endpoint transaction.Endpoint, payloads transaction.BytesPayloads, extra http.Header, priority transaction.Priority, kind transaction.Kind, storableOnDisk bool) []*transaction.HTTPTransaction { transactions := make([]*transaction.HTTPTransaction, 0, len(payloads)*len(f.domainForwarders)) allowArbitraryTags := f.config.GetBool("allow_arbitrary_tags") @@ -446,6 +459,7 @@ func (f *DefaultForwarder) createAdvancedHTTPTransactions(endpoint transaction.E t.Endpoint = endpoint t.Payload = payload t.Priority = priority + t.Kind = kind t.StorableOnDisk = storableOnDisk t.Headers.Set(apiHTTPHeaderKey, apiKey) t.Headers.Set(versionHTTPHeaderKey, version.AgentVersion) @@ -484,6 +498,7 @@ func (f *DefaultForwarder) sendHTTPTransactions(transactions []*transaction.HTTP now := time.Now() for _, t := range transactions { forwarder := f.domainForwarders[t.Domain] + forwarder.sendHTTPTransactions(t) if f.queueDurationCapacity != nil { @@ -516,67 +531,68 @@ func (f *DefaultForwarder) sendHTTPTransactions(transactions []*transaction.HTTP // SubmitSketchSeries will send payloads to Datadog backend - PROTOTYPE FOR PERCENTILE func (f *DefaultForwarder) SubmitSketchSeries(payload transaction.BytesPayloads, extra http.Header) error { - transactions := f.createHTTPTransactions(endpoints.SketchSeriesEndpoint, payload, extra) + transactions := f.createHTTPTransactions(endpoints.SketchSeriesEndpoint, payload, transaction.Sketches, extra) return f.sendHTTPTransactions(transactions) } // SubmitHostMetadata will send a host_metadata tag type payload to Datadog backend. func (f *DefaultForwarder) SubmitHostMetadata(payload transaction.BytesPayloads, extra http.Header) error { - return f.submitV1IntakeWithTransactionsFactory(payload, extra, - func(endpoint transaction.Endpoint, payloads transaction.BytesPayloads, extra http.Header) []*transaction.HTTPTransaction { + return f.submitV1IntakeWithTransactionsFactory(payload, transaction.Metadata, extra, + func(endpoint transaction.Endpoint, payloads transaction.BytesPayloads, kind transaction.Kind, extra http.Header) []*transaction.HTTPTransaction { // Host metadata contains the API KEY and should not be stored on disk. storableOnDisk := false - return f.createAdvancedHTTPTransactions(endpoint, payloads, extra, transaction.TransactionPriorityHigh, storableOnDisk) + return f.createAdvancedHTTPTransactions(endpoint, payloads, extra, transaction.TransactionPriorityHigh, transaction.Metadata, storableOnDisk) }) } // SubmitAgentChecksMetadata will send a agentchecks_metadata tag type payload to Datadog backend. func (f *DefaultForwarder) SubmitAgentChecksMetadata(payload transaction.BytesPayloads, extra http.Header) error { - return f.submitV1IntakeWithTransactionsFactory(payload, extra, - func(endpoint transaction.Endpoint, payloads transaction.BytesPayloads, extra http.Header) []*transaction.HTTPTransaction { + return f.submitV1IntakeWithTransactionsFactory(payload, transaction.Metadata, extra, + func(endpoint transaction.Endpoint, payloads transaction.BytesPayloads, kind transaction.Kind, extra http.Header) []*transaction.HTTPTransaction { // Agentchecks metadata contains the API KEY and should not be stored on disk. storableOnDisk := false - return f.createAdvancedHTTPTransactions(endpoint, payloads, extra, transaction.TransactionPriorityNormal, storableOnDisk) + return f.createAdvancedHTTPTransactions(endpoint, payloads, extra, transaction.TransactionPriorityNormal, transaction.Metadata, storableOnDisk) }) } // SubmitMetadata will send a metadata type payload to Datadog backend. func (f *DefaultForwarder) SubmitMetadata(payload transaction.BytesPayloads, extra http.Header) error { - transactions := f.createHTTPTransactions(endpoints.V1MetadataEndpoint, payload, extra) + transactions := f.createHTTPTransactions(endpoints.V1MetadataEndpoint, payload, transaction.Metadata, extra) return f.sendHTTPTransactions(transactions) } // SubmitV1Series will send timeserie to v1 endpoint (this will be remove once // the backend handles v2 endpoints). func (f *DefaultForwarder) SubmitV1Series(payloads transaction.BytesPayloads, extra http.Header) error { - transactions := f.createHTTPTransactions(endpoints.V1SeriesEndpoint, payloads, extra) + transactions := f.createHTTPTransactions(endpoints.V1SeriesEndpoint, payloads, transaction.Series, extra) return f.sendHTTPTransactions(transactions) } // SubmitSeries will send timeseries to the v2 endpoint func (f *DefaultForwarder) SubmitSeries(payloads transaction.BytesPayloads, extra http.Header) error { - transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payloads, extra) + transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payloads, transaction.Series, extra) return f.sendHTTPTransactions(transactions) } // SubmitV1CheckRuns will send service checks to v1 endpoint (this will be removed once // the backend handles v2 endpoints). func (f *DefaultForwarder) SubmitV1CheckRuns(payload transaction.BytesPayloads, extra http.Header) error { - transactions := f.createHTTPTransactions(endpoints.V1CheckRunsEndpoint, payload, extra) + transactions := f.createHTTPTransactions(endpoints.V1CheckRunsEndpoint, payload, transaction.CheckRuns, extra) return f.sendHTTPTransactions(transactions) } // SubmitV1Intake will send payloads to the universal `/intake/` endpoint used by Agent v.5 -func (f *DefaultForwarder) SubmitV1Intake(payload transaction.BytesPayloads, extra http.Header) error { - return f.submitV1IntakeWithTransactionsFactory(payload, extra, f.createHTTPTransactions) +func (f *DefaultForwarder) SubmitV1Intake(payload transaction.BytesPayloads, kind transaction.Kind, extra http.Header) error { + return f.submitV1IntakeWithTransactionsFactory(payload, kind, extra, f.createHTTPTransactions) } func (f *DefaultForwarder) submitV1IntakeWithTransactionsFactory( payload transaction.BytesPayloads, + kind transaction.Kind, extra http.Header, - createHTTPTransactions func(endpoint transaction.Endpoint, payload transaction.BytesPayloads, extra http.Header) []*transaction.HTTPTransaction, + createHTTPTransactions func(endpoint transaction.Endpoint, payload transaction.BytesPayloads, kind transaction.Kind, extra http.Header) []*transaction.HTTPTransaction, ) error { - transactions := createHTTPTransactions(endpoints.V1IntakeEndpoint, payload, extra) + transactions := createHTTPTransactions(endpoints.V1IntakeEndpoint, payload, kind, extra) // the intake endpoint requires the Content-Type header to be set for _, t := range transactions { @@ -640,7 +656,7 @@ func (f *DefaultForwarder) SubmitOrchestratorManifests(payload transaction.Bytes } func (f *DefaultForwarder) submitProcessLikePayload(ep transaction.Endpoint, payload transaction.BytesPayloads, extra http.Header, retryable bool) (chan Response, error) { - transactions := f.createHTTPTransactions(ep, payload, extra) + transactions := f.createHTTPTransactions(ep, payload, transaction.Process, extra) results := make(chan Response, len(transactions)) internalResults := make(chan Response, len(transactions)) expectedResponses := len(transactions) diff --git a/comp/forwarder/defaultforwarder/domain_forwarder.go b/comp/forwarder/defaultforwarder/domain_forwarder.go index 4519020d2c3b3..29bfc2880d8d5 100644 --- a/comp/forwarder/defaultforwarder/domain_forwarder.go +++ b/comp/forwarder/defaultforwarder/domain_forwarder.go @@ -30,6 +30,7 @@ type domainForwarder struct { log log.Component isRetrying *atomic.Bool domain string + isHA bool numberOfWorkers int highPrio chan transaction.Transaction // use to receive new transactions lowPrio chan transaction.Transaction // use to retry transactions @@ -50,6 +51,7 @@ func newDomainForwarder( config config.Component, log log.Component, domain string, + ha bool, retryQueue *retry.TransactionRetryQueue, numberOfWorkers int, connectionResetInterval time.Duration, @@ -59,6 +61,7 @@ func newDomainForwarder( config: config, log: log, isRetrying: atomic.NewBool(false), + isHA: ha, domain: domain, numberOfWorkers: numberOfWorkers, retryQueue: retryQueue, @@ -263,6 +266,30 @@ func (f *domainForwarder) State() uint32 { } func (f *domainForwarder) sendHTTPTransactions(t transaction.Transaction) { + // Metadata types should always be submitted in dual-shipping fashion - no special considerations for + // Metadata transactions. + if f.isHA && t.GetKind() != transaction.Metadata { + if f.State() == Disabled { + if f.config.GetBool("ha.enabled") && f.config.GetBool("ha.failover") { + f.m.Lock() + f.internalState = Started + f.m.Unlock() + f.log.Debugf("Forwarder for domain %v has been failed over to, enabling it for HA.", t.GetTarget()) + } else { + f.log.Debugf("Forwarder for domain %v is disabled; dropping transaction for this domain.", t.GetTarget()) + return + } + } else { + if (!f.config.GetBool("ha.enabled") || !f.config.GetBool("ha.failover")) && f.State() != Disabled { + f.m.Lock() + f.internalState = Disabled + f.m.Unlock() + f.log.Infof("Forwarder for domain %v was disabled; transactions will be dropped for this domain.", t.GetTarget()) + return + } + } + } + // We don't want to block the collector if the highPrio queue is full select { case f.highPrio <- t: diff --git a/comp/forwarder/defaultforwarder/domain_forwarder_test.go b/comp/forwarder/defaultforwarder/domain_forwarder_test.go index 5f3fdd4b9a14f..7c9ec8abf6701 100644 --- a/comp/forwarder/defaultforwarder/domain_forwarder_test.go +++ b/comp/forwarder/defaultforwarder/domain_forwarder_test.go @@ -27,7 +27,7 @@ import ( func TestNewDomainForwarder(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 120*time.Second) + forwarder := newDomainForwarderForTest(mockConfig, log, 120*time.Second, false) assert.NotNil(t, forwarder) assert.Equal(t, 1, forwarder.numberOfWorkers) @@ -46,7 +46,7 @@ func TestNewDomainForwarder(t *testing.T) { func TestDomainForwarderStart(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) err := forwarder.Start() assert.Nil(t, err) @@ -67,7 +67,7 @@ func TestDomainForwarderStart(t *testing.T) { func TestDomainForwarderInit(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) forwarder.init() assert.Len(t, forwarder.workers, 0) requireLenForwarderRetryQueue(t, forwarder, 0) @@ -76,7 +76,7 @@ func TestDomainForwarderInit(t *testing.T) { func TestDomainForwarderStop(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) forwarder.Stop(false) // this should be a noop forwarder.Start() assert.Equal(t, Started, forwarder.State()) @@ -89,7 +89,7 @@ func TestDomainForwarderStop(t *testing.T) { func TestDomainForwarderStop_WithConnectionReset(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 120*time.Second) + forwarder := newDomainForwarderForTest(mockConfig, log, 120*time.Second, false) forwarder.Stop(false) // this should be a noop forwarder.Start() assert.Equal(t, Started, forwarder.State()) @@ -102,7 +102,7 @@ func TestDomainForwarderStop_WithConnectionReset(t *testing.T) { func TestDomainForwarderSendHTTPTransactions(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) tr := newTestTransactionDomainForwarder() // fw is stopped, we should get an error @@ -121,10 +121,107 @@ func TestDomainForwarderSendHTTPTransactions(t *testing.T) { forwarder.workers = nil } +func TestDomainForwarderHAPreFailover(t *testing.T) { + + datadogYaml := ` +ha: + enabled: true + failover: false + apikey: foo + site: bar.ddhq.com +` + mockConfig := pkgconfigsetup.ConfFromYAML(datadogYaml) + log := fxutil.Test[log.Component](t, logimpl.MockModule()) + // HA forwarder + forwarder := newDomainForwarderForTest(mockConfig, log, 0, true) + trSeries := newTestTransactionWithKindDomainForwarder(transaction.Series) + trMeta := newTestTransactionWithKindDomainForwarder(transaction.Metadata) + + // fw is stopped, we should get an error + forwarder.sendHTTPTransactions(trMeta) + forwarder.sendHTTPTransactions(trSeries) + + defer forwarder.Stop(false) + forwarder.Start() + + // Stopping the worker + forwarder.workers[0].Stop(false) + + trSeries.On("Process", forwarder.workers[0].Client).Return(nil).Times(1) + trMeta.On("Process", forwarder.workers[0].Client).Return(nil).Times(1) + + forwarder.sendHTTPTransactions(trMeta) + transactionToProcess := <-forwarder.highPrio + assert.Equal(t, trMeta, transactionToProcess) + + // haven't failed over, so transaction should be dropped and not queued + forwarder.sendHTTPTransactions(trSeries) + timedOut := false + select { + case <-forwarder.highPrio: + // should not happen + case <-time.After(time.Second): + timedOut = true + } + assert.Equal(t, timedOut, true) + + // Reset `forwarder.workers` otherwise `defer forwarder.Stop(false)` will timeout. + forwarder.workers = nil +} + +func TestDomainForwarderHAFailover(t *testing.T) { + + datadogYaml := ` +ha: + enabled: true + failover: true + apikey: foo + site: bar.ddhq.com +` + mockConfig := pkgconfigsetup.ConfFromYAML(datadogYaml) + log := fxutil.Test[log.Component](t, logimpl.MockModule()) + // HA forwarder + forwarder := newDomainForwarderForTest(mockConfig, log, 0, true) + trSeries := newTestTransactionWithKindDomainForwarder(transaction.Series) + trMeta := newTestTransactionWithKindDomainForwarder(transaction.Metadata) + + // fw is stopped, we should get an error + forwarder.sendHTTPTransactions(trMeta) + forwarder.sendHTTPTransactions(trSeries) + + defer forwarder.Stop(false) + forwarder.Start() + + // Stopping the worker + forwarder.workers[0].Stop(false) + + trSeries.On("Process", forwarder.workers[0].Client).Return(nil).Times(1) + trMeta.On("Process", forwarder.workers[0].Client).Return(nil).Times(1) + + forwarder.sendHTTPTransactions(trMeta) + transactionToProcess := <-forwarder.highPrio + assert.Equal(t, trMeta, transactionToProcess) + + // haven't failed over, so transaction should be dropped and not queued + forwarder.sendHTTPTransactions(trSeries) + timedOut := false + select { + case transactionToProcess = <-forwarder.highPrio: + // should not happen + case <-time.After(time.Second): + timedOut = true + } + assert.Equal(t, timedOut, false) + assert.Equal(t, trSeries, transactionToProcess) + + // Reset `forwarder.workers` otherwise `defer forwarder.Stop(false)` will timeout. + forwarder.workers = nil +} + func TestRequeueTransaction(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) tr := transaction.NewHTTPTransaction() requireLenForwarderRetryQueue(t, forwarder, 0) forwarder.requeueTransaction(tr) @@ -134,7 +231,7 @@ func TestRequeueTransaction(t *testing.T) { func TestRetryTransactions(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) forwarder.init() // Default value should be 0 @@ -169,7 +266,7 @@ func TestRetryTransactions(t *testing.T) { func TestForwarderRetry(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) forwarder.Start() defer forwarder.Stop(false) @@ -205,7 +302,7 @@ func TestForwarderRetry(t *testing.T) { func TestForwarderRetryLifo(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) forwarder.init() transaction1 := newTestTransactionDomainForwarder() @@ -236,7 +333,7 @@ func TestForwarderRetryLifo(t *testing.T) { func TestForwarderRetryLimitQueue(t *testing.T) { mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) forwarder.init() forwarder.blockedList.close("blocked") forwarder.blockedList.errorPerEndpoint["blocked"].until = time.Now().Add(1 * time.Minute) @@ -282,7 +379,7 @@ func TestDomainForwarderRetryQueueAllPayloadsMaxSize(t *testing.T) { retry.NewPointCountTelemetryMock()) mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarder(mockConfig, log, "test", transactionRetryQueue, 0, 10, transaction.SortByCreatedTimeAndPriority{HighPriorityFirst: true}, retry.NewPointCountTelemetry("domain")) + forwarder := newDomainForwarder(mockConfig, log, "test", false, transactionRetryQueue, 0, 10, transaction.SortByCreatedTimeAndPriority{HighPriorityFirst: true}, retry.NewPointCountTelemetry("domain")) forwarder.blockedList.close("blocked") forwarder.blockedList.errorPerEndpoint["blocked"].until = time.Now().Add(1 * time.Minute) @@ -310,7 +407,7 @@ func TestDomainForwarderInitConfigs(t *testing.T) { // Test default values mockConfig := pkgconfigsetup.Conf() log := fxutil.Test[log.Component](t, logimpl.MockModule()) - forwarder := newDomainForwarderForTest(mockConfig, log, 0) + forwarder := newDomainForwarderForTest(mockConfig, log, 0, false) forwarder.init() assert.Equal(t, 100, cap(forwarder.highPrio)) assert.Equal(t, 100, cap(forwarder.lowPrio)) @@ -326,14 +423,14 @@ forwarder_requeue_buffer_size: 1300 err := mockConfig.ReadConfig(bytes.NewBuffer([]byte(datadogYaml))) assert.NoError(t, err) - forwarder = newDomainForwarderForTest(mockConfig, log, 0) + forwarder = newDomainForwarderForTest(mockConfig, log, 0, false) forwarder.init() assert.Equal(t, 1100, cap(forwarder.highPrio)) assert.Equal(t, 1200, cap(forwarder.lowPrio)) assert.Equal(t, 1300, cap(forwarder.requeuedTransaction)) } -func newDomainForwarderForTest(config config.Component, log log.Component, connectionResetInterval time.Duration) *domainForwarder { +func newDomainForwarderForTest(config config.Component, log log.Component, connectionResetInterval time.Duration, ha bool) *domainForwarder { sorter := transaction.SortByCreatedTimeAndPriority{HighPriorityFirst: true} telemetry := retry.NewTransactionRetryQueueTelemetry("domain") transactionRetryQueue := retry.NewTransactionRetryQueue( @@ -344,7 +441,7 @@ func newDomainForwarderForTest(config config.Component, log log.Component, conne telemetry, retry.NewPointCountTelemetryMock()) - return newDomainForwarder(config, log, "test", transactionRetryQueue, 1, connectionResetInterval, sorter, retry.NewPointCountTelemetry("domain")) + return newDomainForwarder(config, log, "test", ha, transactionRetryQueue, 1, connectionResetInterval, sorter, retry.NewPointCountTelemetry("domain")) } func requireLenForwarderRetryQueue(t *testing.T, forwarder *domainForwarder, expectedValue int) { @@ -356,3 +453,10 @@ func newTestTransactionDomainForwarder() *testTransaction { tr.On("GetPayloadSize").Return(1) return tr } + +func newTestTransactionWithKindDomainForwarder(kind transaction.Kind) *testTransaction { + tr := newTestTransactionWithKind(kind) + tr.On("GetPayloadSize").Return(1) + tr.On("GetTarget").Return("foo.ddhq.com") + return tr +} diff --git a/comp/forwarder/defaultforwarder/forwarder_test.go b/comp/forwarder/defaultforwarder/forwarder_test.go index 73c63f8765965..03970ba976117 100644 --- a/comp/forwarder/defaultforwarder/forwarder_test.go +++ b/comp/forwarder/defaultforwarder/forwarder_test.go @@ -137,7 +137,7 @@ func TestSubmitIfStopped(t *testing.T) { assert.NotNil(t, forwarder.SubmitMetadata(nil, make(http.Header))) assert.NotNil(t, forwarder.SubmitV1Series(nil, make(http.Header))) assert.NotNil(t, forwarder.SubmitSeries(nil, make(http.Header))) - assert.NotNil(t, forwarder.SubmitV1Intake(nil, make(http.Header))) + assert.NotNil(t, forwarder.SubmitV1Intake(nil, transaction.Series, make(http.Header))) assert.NotNil(t, forwarder.SubmitV1CheckRuns(nil, make(http.Header))) } @@ -152,7 +152,7 @@ func TestCreateHTTPTransactions(t *testing.T) { headers := make(http.Header) headers.Set("HTTP-MAGIC", "foo") - transactions := forwarder.createHTTPTransactions(endpoint, payloads, headers) + transactions := forwarder.createHTTPTransactions(endpoint, payloads, transaction.Series, headers) require.Len(t, transactions, 4) assert.Equal(t, testVersionDomain, transactions[0].Domain) assert.Equal(t, testVersionDomain, transactions[1].Domain) @@ -184,7 +184,7 @@ func TestCreateHTTPTransactionsWithMultipleDomains(t *testing.T) { headers := make(http.Header) headers.Set("HTTP-MAGIC", "foo") - transactions := forwarder.createHTTPTransactions(endpoint, payloads, headers) + transactions := forwarder.createHTTPTransactions(endpoint, payloads, transaction.Series, headers) require.Len(t, transactions, 3, "should contain 3 transactions, contains %d", len(transactions)) var txNormal, txBar []*transaction.HTTPTransaction @@ -225,7 +225,7 @@ func TestCreateHTTPTransactionsWithDifferentResolvers(t *testing.T) { headers := make(http.Header) headers.Set("HTTP-MAGIC", "foo") - transactions := forwarder.createHTTPTransactions(endpoint, payloads, headers) + transactions := forwarder.createHTTPTransactions(endpoint, payloads, transaction.Series, headers) require.Len(t, transactions, 4, "should contain 4 transactions, contains %d", len(transactions)) var txNormal, txBar, txVector []*transaction.HTTPTransaction @@ -270,14 +270,14 @@ func TestCreateHTTPTransactionsWithOverrides(t *testing.T) { headers := make(http.Header) headers.Set("HTTP-MAGIC", "foo") - transactions := forwarder.createHTTPTransactions(endpoint, payloads, headers) + transactions := forwarder.createHTTPTransactions(endpoint, payloads, transaction.Series, headers) require.Len(t, transactions, 1, "should contain 1 transaction, contains %d", len(transactions)) assert.Equal(t, transactions[0].Endpoint.Route, "/api/foo") assert.Equal(t, transactions[0].Domain, testVersionDomain) endpoint.Name = "diverted" - transactions = forwarder.createHTTPTransactions(endpoint, payloads, headers) + transactions = forwarder.createHTTPTransactions(endpoint, payloads, transaction.Series, headers) require.Len(t, transactions, 1, "should contain 1 transaction, contains %d", len(transactions)) assert.Equal(t, transactions[0].Endpoint.Route, "/api/foo") @@ -294,7 +294,7 @@ func TestArbitraryTagsHTTPHeader(t *testing.T) { payload := []byte("A payload") headers := make(http.Header) - transactions := forwarder.createHTTPTransactions(endpoint, transaction.NewBytesPayloadsWithoutMetaData([]*[]byte{&payload}), headers) + transactions := forwarder.createHTTPTransactions(endpoint, transaction.NewBytesPayloadsWithoutMetaData([]*[]byte{&payload}), transaction.Series, headers) require.True(t, len(transactions) > 0) assert.Equal(t, "true", transactions[0].Headers.Get(arbitraryTagHTTPHeaderKey)) } @@ -307,7 +307,7 @@ func TestSendHTTPTransactions(t *testing.T) { p1 := []byte("A payload") payloads := transaction.NewBytesPayloadsWithoutMetaData([]*[]byte{&p1}) headers := make(http.Header) - tr := forwarder.createHTTPTransactions(endpoint, payloads, headers) + tr := forwarder.createHTTPTransactions(endpoint, payloads, transaction.Series, headers) // fw is stopped, we should get an error err := forwarder.sendHTTPTransactions(tr) @@ -336,7 +336,7 @@ func TestSubmitV1Intake(t *testing.T) { defer func() { df.highPrio = bk }() p := []byte("test") - assert.Nil(t, forwarder.SubmitV1Intake(transaction.NewBytesPayloadsWithoutMetaData([]*[]byte{&p}), make(http.Header))) + assert.Nil(t, forwarder.SubmitV1Intake(transaction.NewBytesPayloadsWithoutMetaData([]*[]byte{&p}), transaction.Metadata, make(http.Header))) select { case tr := <-df.highPrio: @@ -386,7 +386,7 @@ func TestForwarderEndtoEnd(t *testing.T) { assert.Nil(t, f.SubmitSeries(payload, headers)) numReqs += 4 - assert.Nil(t, f.SubmitV1Intake(payload, headers)) + assert.Nil(t, f.SubmitV1Intake(payload, transaction.Series, headers)) numReqs += 4 assert.Nil(t, f.SubmitV1CheckRuns(payload, headers)) @@ -431,7 +431,7 @@ func TestTransactionEventHandlers(t *testing.T) { headers := http.Header{} headers.Set("key", "value") - transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payload, headers) + transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payload, transaction.Series, headers) require.Len(t, transactions, 1) attempts := atomic.NewInt64(0) @@ -486,7 +486,7 @@ func TestTransactionEventHandlersOnRetry(t *testing.T) { headers := http.Header{} headers.Set("key", "value") - transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payload, headers) + transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payload, transaction.Series, headers) require.Len(t, transactions, 1) attempts := atomic.NewInt64(0) @@ -537,7 +537,7 @@ func TestTransactionEventHandlersNotRetryable(t *testing.T) { headers := http.Header{} headers.Set("key", "value") - transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payload, headers) + transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payload, transaction.Series, headers) require.Len(t, transactions, 1) attempts := atomic.NewInt64(0) @@ -591,7 +591,7 @@ func TestProcessLikePayloadResponseTimeout(t *testing.T) { headers := http.Header{} headers.Set("key", "value") - transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payload, headers) + transactions := f.createHTTPTransactions(endpoints.SeriesEndpoint, payload, transaction.Series, headers) require.Len(t, transactions, 1) responses, err := f.submitProcessLikePayload(endpoints.SeriesEndpoint, payload, headers, true) diff --git a/comp/forwarder/defaultforwarder/internal/retry/http_transactions_serializer_test.go b/comp/forwarder/defaultforwarder/internal/retry/http_transactions_serializer_test.go index fd2840edabc80..0fa532384dbee 100644 --- a/comp/forwarder/defaultforwarder/internal/retry/http_transactions_serializer_test.go +++ b/comp/forwarder/defaultforwarder/internal/retry/http_transactions_serializer_test.go @@ -108,7 +108,7 @@ func TestHTTPTransactionSerializerMissingAPIKey(t *testing.T) { func TestHTTPTransactionFieldsCount(t *testing.T) { tr := transaction.HTTPTransaction{} transactionType := reflect.TypeOf(tr) - assert.Equalf(t, 11, transactionType.NumField(), + assert.Equalf(t, 12, transactionType.NumField(), "A field was added or remove from HTTPTransaction. "+ "You probably need to update the implementation of "+ "HTTPTransactionsSerializer and then adjust this unit test.") diff --git a/comp/forwarder/defaultforwarder/noop_forwarder.go b/comp/forwarder/defaultforwarder/noop_forwarder.go index 7335cc431774d..47b4fba1698d3 100644 --- a/comp/forwarder/defaultforwarder/noop_forwarder.go +++ b/comp/forwarder/defaultforwarder/noop_forwarder.go @@ -26,7 +26,7 @@ func (f NoopForwarder) SubmitV1Series(_ transaction.BytesPayloads, _ http.Header } // SubmitV1Intake does nothing. -func (f NoopForwarder) SubmitV1Intake(_ transaction.BytesPayloads, _ http.Header) error { +func (f NoopForwarder) SubmitV1Intake(_ transaction.BytesPayloads, _ transaction.Kind, _ http.Header) error { return nil } diff --git a/comp/forwarder/defaultforwarder/sync_forwarder.go b/comp/forwarder/defaultforwarder/sync_forwarder.go index c89df2b8949b3..64ea25d0efa12 100644 --- a/comp/forwarder/defaultforwarder/sync_forwarder.go +++ b/comp/forwarder/defaultforwarder/sync_forwarder.go @@ -68,19 +68,20 @@ func (f *SyncForwarder) sendHTTPTransactions(transactions []*transaction.HTTPTra // SubmitV1Series will send timeserie to v1 endpoint (this will be remove once // the backend handles v2 endpoints). func (f *SyncForwarder) SubmitV1Series(payload transaction.BytesPayloads, extra http.Header) error { - transactions := f.defaultForwarder.createHTTPTransactions(endpoints.V1SeriesEndpoint, payload, extra) + transactions := f.defaultForwarder.createHTTPTransactions(endpoints.V1SeriesEndpoint, payload, transaction.Series, extra) return f.sendHTTPTransactions(transactions) } // SubmitSeries will send timeseries to the v2 endpoint func (f *SyncForwarder) SubmitSeries(payload transaction.BytesPayloads, extra http.Header) error { - transactions := f.defaultForwarder.createHTTPTransactions(endpoints.SeriesEndpoint, payload, extra) + transactions := f.defaultForwarder.createHTTPTransactions(endpoints.SeriesEndpoint, payload, transaction.Series, extra) return f.sendHTTPTransactions(transactions) } // SubmitV1Intake will send payloads to the universal `/intake/` endpoint used by Agent v.5 -func (f *SyncForwarder) SubmitV1Intake(payload transaction.BytesPayloads, extra http.Header) error { - transactions := f.defaultForwarder.createHTTPTransactions(endpoints.V1IntakeEndpoint, payload, extra) +func (f *SyncForwarder) SubmitV1Intake(payload transaction.BytesPayloads, kind transaction.Kind, extra http.Header) error { + // treat as a Series transaction + transactions := f.defaultForwarder.createHTTPTransactions(endpoints.V1IntakeEndpoint, payload, kind, extra) // the intake endpoint requires the Content-Type header to be set for _, t := range transactions { t.Headers.Set("Content-Type", "application/json") @@ -91,29 +92,29 @@ func (f *SyncForwarder) SubmitV1Intake(payload transaction.BytesPayloads, extra // SubmitV1CheckRuns will send service checks to v1 endpoint (this will be removed once // the backend handles v2 endpoints). func (f *SyncForwarder) SubmitV1CheckRuns(payload transaction.BytesPayloads, extra http.Header) error { - transactions := f.defaultForwarder.createHTTPTransactions(endpoints.V1CheckRunsEndpoint, payload, extra) + transactions := f.defaultForwarder.createHTTPTransactions(endpoints.V1CheckRunsEndpoint, payload, transaction.CheckRuns, extra) return f.sendHTTPTransactions(transactions) } // SubmitSketchSeries will send payloads to Datadog backend - PROTOTYPE FOR PERCENTILE func (f *SyncForwarder) SubmitSketchSeries(payload transaction.BytesPayloads, extra http.Header) error { - transactions := f.defaultForwarder.createHTTPTransactions(endpoints.SketchSeriesEndpoint, payload, extra) + transactions := f.defaultForwarder.createHTTPTransactions(endpoints.SketchSeriesEndpoint, payload, transaction.Sketches, extra) return f.sendHTTPTransactions(transactions) } // SubmitHostMetadata will send a host_metadata tag type payload to Datadog backend. func (f *SyncForwarder) SubmitHostMetadata(payload transaction.BytesPayloads, extra http.Header) error { - return f.SubmitV1Intake(payload, extra) + return f.SubmitV1Intake(payload, transaction.Metadata, extra) } // SubmitMetadata will send a metadata type payload to Datadog backend. func (f *SyncForwarder) SubmitMetadata(payload transaction.BytesPayloads, extra http.Header) error { - return f.SubmitV1Intake(payload, extra) + return f.SubmitV1Intake(payload, transaction.Metadata, extra) } // SubmitAgentChecksMetadata will send a agentchecks_metadata tag type payload to Datadog backend. func (f *SyncForwarder) SubmitAgentChecksMetadata(payload transaction.BytesPayloads, extra http.Header) error { - return f.SubmitV1Intake(payload, extra) + return f.SubmitV1Intake(payload, transaction.Metadata, extra) } // SubmitProcessChecks sends process checks diff --git a/comp/forwarder/defaultforwarder/test_common.go b/comp/forwarder/defaultforwarder/test_common.go index 9bd88aef3110f..1e6a4416e0cc9 100644 --- a/comp/forwarder/defaultforwarder/test_common.go +++ b/comp/forwarder/defaultforwarder/test_common.go @@ -24,6 +24,7 @@ type testTransaction struct { assertClient bool processed chan bool pointCount int + kind transaction.Kind } func newTestTransaction() *testTransaction { @@ -33,6 +34,14 @@ func newTestTransaction() *testTransaction { return t } +func newTestTransactionWithKind(kind transaction.Kind) *testTransaction { + t := new(testTransaction) + t.assertClient = true + t.kind = kind + t.processed = make(chan bool, 1) + return t +} + func newTestTransactionWithoutClientAssert() *testTransaction { t := new(testTransaction) t.assertClient = false @@ -61,6 +70,10 @@ func (t *testTransaction) GetPriority() transaction.Priority { return transaction.TransactionPriorityNormal } +func (t *testTransaction) GetKind() transaction.Kind { + return t.kind +} + func (t *testTransaction) GetEndpointName() string { return "" } @@ -106,7 +119,7 @@ func (tf *MockedForwarder) SubmitSeries(payload transaction.BytesPayloads, extra } // SubmitV1Intake updates the internal mock struct -func (tf *MockedForwarder) SubmitV1Intake(payload transaction.BytesPayloads, extra http.Header) error { +func (tf *MockedForwarder) SubmitV1Intake(payload transaction.BytesPayloads, _ transaction.Kind, extra http.Header) error { return tf.Called(payload, extra).Error(0) } diff --git a/comp/forwarder/defaultforwarder/transaction/transaction.go b/comp/forwarder/defaultforwarder/transaction/transaction.go index 8da94c5d0f7ae..b33adad27bb78 100644 --- a/comp/forwarder/defaultforwarder/transaction/transaction.go +++ b/comp/forwarder/defaultforwarder/transaction/transaction.go @@ -174,6 +174,26 @@ const ( TransactionPriorityHigh Priority = iota ) +// Kind defines de kind of transaction (metrics, metadata, process, ...) +type Kind int + +const ( + // Series is the transaction type for metrics series + Series = iota + // Sketches is the transaction type for distribution sketches + Sketches + // ServiceChecks is the transaction type for service checks + ServiceChecks + // Events is the transaction type for events + Events + // CheckRuns is the transaction type for agent check runs + CheckRuns + // Metadata is the transaction type for metadata payloads + Metadata + // Process is the transaction type for live-process monitoring payloads + Process +) + // HTTPTransaction represents one Payload for one Endpoint on one Domain. type HTTPTransaction struct { // Domain represents the domain target by the HTTPTransaction. @@ -202,6 +222,8 @@ type HTTPTransaction struct { CompletionHandler HTTPCompletionHandler Priority Priority + + Kind Kind } // TransactionsSerializer serializes Transaction instances. @@ -215,6 +237,7 @@ type Transaction interface { GetCreatedAt() time.Time GetTarget() string GetPriority() Priority + GetKind() Kind GetEndpointName() string GetPayloadSize() int GetPointCount() int @@ -261,6 +284,11 @@ func (t *HTTPTransaction) GetPriority() Priority { return t.Priority } +// GetKind returns the transaction kind +func (t *HTTPTransaction) GetKind() Kind { + return t.Kind +} + // GetEndpointName returns the name of the endpoint used by the transaction func (t *HTTPTransaction) GetEndpointName() string { return t.Endpoint.Name diff --git a/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks.go b/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks.go index 213e91c0cc7a0..6f540ea7bcd96 100644 --- a/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks.go +++ b/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks.go @@ -24,7 +24,6 @@ import ( "github.com/DataDog/datadog-agent/comp/metadata/internal/util" "github.com/DataDog/datadog-agent/comp/metadata/inventorychecks" "github.com/DataDog/datadog-agent/comp/metadata/runner/runnerimpl" - pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" "github.com/DataDog/datadog-agent/pkg/collector/check" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" "github.com/DataDog/datadog-agent/pkg/logs/sources" @@ -121,7 +120,7 @@ func newInventoryChecksProvider(deps dependencies) provides { // component we can migrate it there and remove the entire logic to emit event from the collector. if coll, isSet := ic.coll.Get(); isSet { - coll.AddEventReceiver(func(_ checkid.ID, _ pkgcollector.EventType) { ic.Refresh() }) + coll.AddEventReceiver(func(_ checkid.ID, _ collector.EventType) { ic.Refresh() }) } if logAgent, isSet := deps.LogAgent.Get(); isSet { diff --git a/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks_test.go b/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks_test.go index 026f4ffe8b77c..083c3d6f587c1 100644 --- a/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks_test.go +++ b/comp/metadata/inventorychecks/inventorychecksimpl/inventorychecks_test.go @@ -10,17 +10,16 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "go.uber.org/fx" "github.com/DataDog/datadog-agent/comp/collector/collector" + "github.com/DataDog/datadog-agent/comp/collector/collector/collectorimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log/logimpl" logagent "github.com/DataDog/datadog-agent/comp/logs/agent" logConfig "github.com/DataDog/datadog-agent/comp/logs/agent/config" "github.com/DataDog/datadog-agent/comp/metadata/inventoryagent/inventoryagentimpl" - pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" "github.com/DataDog/datadog-agent/pkg/collector/check" checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" "github.com/DataDog/datadog-agent/pkg/logs/sources" @@ -123,9 +122,13 @@ func TestGetPayload(t *testing.T) { }, } - mockColl := pkgcollector.NewMock(cInfo) - mockColl.On("AddEventReceiver", mock.AnythingOfType("EventReceiver")).Return() - mockColl.On("MapOverChecks", mock.AnythingOfType("func([]check.Info)")).Return() + mockColl := fxutil.Test[collector.Component](t, + fx.Replace(collectorimpl.MockParams{ + ChecksInfo: cInfo, + }), + collectorimpl.MockModule(), + core.MockBundle(), + ) // Setup log sources logSources := sources.NewLogSources() diff --git a/comp/process/bundle.go b/comp/process/bundle.go index dad7afd0e1e0b..8c7d8c825d1c9 100644 --- a/comp/process/bundle.go +++ b/comp/process/bundle.go @@ -18,7 +18,6 @@ import ( "github.com/DataDog/datadog-agent/comp/process/expvars/expvarsimpl" "github.com/DataDog/datadog-agent/comp/process/forwarders/forwardersimpl" "github.com/DataDog/datadog-agent/comp/process/hostinfo/hostinfoimpl" - "github.com/DataDog/datadog-agent/comp/process/podcheck/podcheckimpl" "github.com/DataDog/datadog-agent/comp/process/processcheck/processcheckimpl" "github.com/DataDog/datadog-agent/comp/process/processdiscoverycheck/processdiscoverycheckimpl" "github.com/DataDog/datadog-agent/comp/process/processeventscheck/processeventscheckimpl" @@ -41,7 +40,6 @@ func Bundle() fxutil.BundleOptions { // Checks connectionscheckimpl.Module(), containercheckimpl.Module(), - podcheckimpl.Module(), processcheckimpl.Module(), processeventscheckimpl.Module(), rtcontainercheckimpl.Module(), diff --git a/comp/process/bundle_test.go b/comp/process/bundle_test.go index 0cb9dd566e943..1d0198c043e7d 100644 --- a/comp/process/bundle_test.go +++ b/comp/process/bundle_test.go @@ -42,7 +42,7 @@ func TestBundleDependencies(t *testing.T) { func TestBundleOneShot(t *testing.T) { runCmd := func(r runner.Component) { checks := r.GetProvidedChecks() - require.Len(t, checks, 7) + require.Len(t, checks, 6) var names []string for _, c := range checks { @@ -55,7 +55,6 @@ func TestBundleOneShot(t *testing.T) { "rtcontainer", "process_events", "connections", - "pod", "process_discovery", }, names) } diff --git a/comp/process/podcheck/component.go b/comp/process/podcheck/component.go deleted file mode 100644 index 9f94ca3527c69..0000000000000 --- a/comp/process/podcheck/component.go +++ /dev/null @@ -1,18 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -// Package podcheck implements a component to handle Kubernetes data collection in the Process Agent. -package podcheck - -import ( - "github.com/DataDog/datadog-agent/comp/process/types" -) - -// team: processes - -//nolint:revive // TODO(PROC) Fix revive linter -type Component interface { - types.CheckComponent -} diff --git a/comp/process/podcheck/podcheckimpl/check.go b/comp/process/podcheck/podcheckimpl/check.go deleted file mode 100644 index df9a3da047511..0000000000000 --- a/comp/process/podcheck/podcheckimpl/check.go +++ /dev/null @@ -1,51 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -// Package podcheckimpl implements a component to handle Kubernetes data collection in the Process Agent. -package podcheckimpl - -import ( - "go.uber.org/fx" - - "github.com/DataDog/datadog-agent/comp/process/podcheck" - "github.com/DataDog/datadog-agent/comp/process/types" - "github.com/DataDog/datadog-agent/pkg/process/checks" - "github.com/DataDog/datadog-agent/pkg/util/fxutil" -) - -// Module defines the fx options for this component. -func Module() fxutil.Module { - return fxutil.Component( - fx.Provide(newCheck)) -} - -var _ types.CheckComponent = (*check)(nil) - -type check struct { - podCheck *checks.PodCheck -} - -type result struct { - fx.Out - - Check types.ProvidesCheck - Component podcheck.Component -} - -func newCheck() result { - c := &check{ - podCheck: checks.NewPodCheck(), - } - return result{ - Check: types.ProvidesCheck{ - CheckComponent: c, - }, - Component: c, - } -} - -func (c *check) Object() checks.Check { - return c.podCheck -} diff --git a/comp/remote-config/rcclient/apm_tracing.go b/comp/remote-config/rcclient/apm_tracing.go new file mode 100644 index 0000000000000..d58c364b97746 --- /dev/null +++ b/comp/remote-config/rcclient/apm_tracing.go @@ -0,0 +1,16 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !linux + +package rcclient + +import ( + pkglog "github.com/DataDog/datadog-agent/pkg/util/log" +) + +func (rc rcClient) SubscribeApmTracing() { + pkglog.Info("APM TRACING config product is not supported outside Linux currently.") +} diff --git a/comp/remote-config/rcclient/apm_tracing_linux.go b/comp/remote-config/rcclient/apm_tracing_linux.go new file mode 100644 index 0000000000000..d2fd72701de6b --- /dev/null +++ b/comp/remote-config/rcclient/apm_tracing_linux.go @@ -0,0 +1,169 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build linux + +package rcclient + +import ( + "encoding/json" + "os" + + yamlv2 "gopkg.in/yaml.v2" + + "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" + pkglog "github.com/DataDog/datadog-agent/pkg/util/log" +) + +var apmTracingFilePath = "/opt/datadog-agent/run/inject_config.yaml" + +// InvalidAPMTracingPayload indicates we received an APM_TRACING payload we were unable to decode +const InvalidAPMTracingPayload = "INVALID_APM_TRACING_PAYLOAD" + +// MissingServiceTarget indicates we were missing the service_target field +const MissingServiceTarget = "MISSING_SERVICE_TARGET" + +// FileWriteFailure indicates we were unable to write the RC Updates to a local file for use by the injector +const FileWriteFailure = "FILE_WRITE_FAILURE" + +// DuplicateHostConfig indicates received more than one InfraTarget configuration with a different env, +// this leads to inconsistent env values +const DuplicateHostConfig = "DUPLICATE_HOST_CONFIG" + +type serviceEnvConfig struct { + Service string `yaml:"service"` + Env string `yaml:"env"` + TracingEnabled bool `yaml:"tracing_enabled"` +} + +type tracingEnabledConfig struct { + TracingEnabled bool `yaml:"tracing_enabled"` + Env string `yaml:"env"` + ServiceEnvConfigs []serviceEnvConfig `yaml:"service_env_configs"` +} + +type tracingConfigUpdate struct { + ID string `json:"id"` + Revision int64 `json:"revision"` + SchemaVersion string `json:"schema_version"` + Action string `json:"action"` + LibConfig struct { + ServiceName string `json:"service_name"` + Env string `json:"env"` + TracingEnabled bool `json:"tracing_enabled"` + } `json:"lib_config"` + ServiceTarget *struct { + Service string `json:"service"` + Env string `json:"env"` + } `json:"service_target"` + InfraTarget *struct { + Tags []string `json:"tags"` + } `json:"infra_target"` +} + +func (rc rcClient) SubscribeApmTracing() { + if rc.client == nil { + pkglog.Errorf("No remote-config client") + return + } + rc.client.Subscribe(state.ProductAPMTracing, rc.onAPMTracingUpdate) +} + +func (rc rcClient) onAPMTracingUpdate(update map[string]state.RawConfig, applyStateCallback func(string, state.ApplyStatus)) { //nolint:revive + if len(update) == 0 { + // Empty update means revert to default behavior, so remove any existing config file + err := os.Remove(apmTracingFilePath) + if err == nil { + pkglog.Infof("Removed APM_TRACING remote config file, APM injection will revert to default behavior") + } else if !os.IsNotExist(err) { + // If the file already wasn't there then it wasn't an error + pkglog.Errorf("Failed to remove APM_TRACING remote config file, previous APM injection behavior will continue: %v", err) + } + return + } + + var senvConfigs []serviceEnvConfig + // Maps update IDs to their error, empty string indicates success + updateStatus := map[string]string{} + var hostTracingEnabled bool + var hostEnvTarget string + var hostConfigID string + for id, rawConfig := range update { + tcu := tracingConfigUpdate{} + err := json.Unmarshal(rawConfig.Config, &tcu) + updateStatus[id] = "" + if err != nil { + pkglog.Warnf("Skipping invalid APM_TRACING remote update %s: %v, any err: %v", id, tcu, err) + updateStatus[id] = InvalidAPMTracingPayload + continue + } + pkglog.Infof("Received APM_TRACING remote update %s: %v, any err: %v", id, tcu, err) + if tcu.InfraTarget != nil { + // This is an infra targeting payload, skip adding it to the service env config map + if hostConfigID != "" && tcu.LibConfig.Env != hostEnvTarget { + // We already saw a InfraTarget configuration and the envs are different, this is generally not desired + // To be consistent we will apply the "lowest" config ID and report a failure for the un-applied host config + pkglog.Warnf("Received more than 1 InfraTarget APM_TRACING config, the 'lowest' config will be used, but inconsistent behavior may occur. Check your Single Step Instrumentation configurations.") + if id < hostConfigID { + updateStatus[hostConfigID] = DuplicateHostConfig + // fallthrough to use this update's config values + } else { + // The previous infra target was lower, keep the current values + updateStatus[id] = DuplicateHostConfig + continue + } + } + hostTracingEnabled = tcu.LibConfig.TracingEnabled + hostEnvTarget = tcu.LibConfig.Env + hostConfigID = id + continue + } + if tcu.ServiceTarget == nil { + pkglog.Warnf("Missing service_target from APM_TRACING config update, SKIPPING: %v", tcu) + updateStatus[id] = MissingServiceTarget + continue + } + senvConfigs = append(senvConfigs, serviceEnvConfig{ + Service: tcu.ServiceTarget.Service, + Env: tcu.ServiceTarget.Env, + TracingEnabled: tcu.LibConfig.TracingEnabled, + }) + } + tec := tracingEnabledConfig{ + TracingEnabled: hostTracingEnabled, + Env: hostEnvTarget, + ServiceEnvConfigs: senvConfigs, + } + configFile, err := yamlv2.Marshal(tec) + if err != nil { + pkglog.Errorf("Failed to marshal APM_TRACING config update %v", err) + return + } + err = os.WriteFile(apmTracingFilePath, configFile, 0644) + if err != nil { + pkglog.Errorf("Failed to write single step config data file from APM_TRACING config: %v", err) + // Failed to write file, report failure for all updates + for id := range update { + applyStateCallback(id, state.ApplyStatus{ + State: state.ApplyStateError, + Error: FileWriteFailure, + }) + } + return + } + pkglog.Debugf("Successfully wrote APM_TRACING config to %s", apmTracingFilePath) + // Successfully wrote file, report success/failure per update + for id, errStatus := range updateStatus { + applyState := state.ApplyStateAcknowledged + if errStatus != "" { + applyState = state.ApplyStateError + } + applyStateCallback(id, state.ApplyStatus{ + State: applyState, + Error: errStatus, + }) + } + +} diff --git a/comp/remote-config/rcclient/apm_tracing_linux_test.go b/comp/remote-config/rcclient/apm_tracing_linux_test.go new file mode 100644 index 0000000000000..2a36284341983 --- /dev/null +++ b/comp/remote-config/rcclient/apm_tracing_linux_test.go @@ -0,0 +1,116 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +//go:build linux + +package rcclient + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" +) + +func TestOnAPMTracingUpdate(t *testing.T) { + mkTemp := func(t *testing.T) func() { + oldPath := apmTracingFilePath + f, err := os.CreateTemp("", "test") + require.NoError(t, err) + f.Close() // This is required for windows unit tests as windows will not allow this file to be deleted while we have this handle. + apmTracingFilePath = f.Name() + return func() { + apmTracingFilePath = oldPath + } + } + + t.Run("Empty update deletes file", func(t *testing.T) { + defer mkTemp(t)() + rc := rcClient{} + + rc.onAPMTracingUpdate(map[string]state.RawConfig{}, nil) + + _, err := os.Open(apmTracingFilePath) + if !os.IsNotExist(err) { + // file still exists when it shouldn't + assert.Fail(t, "Empty update did not delete existing config file") + } + }) + + t.Run("Valid update writes file", func(t *testing.T) { + defer mkTemp(t)() + rc := rcClient{} + callbackCalls := map[string]string{} + callback := func(id string, status state.ApplyStatus) { + callbackCalls[id] = status.Error + } + + hostConfig := state.RawConfig{Config: []byte(`{"infra_target": {"tags":["k:v"]},"lib_config":{"env":"someEnv","tracing_enabled":true}}`)} + senvConfig := state.RawConfig{Config: []byte(`{"service_target": {"service":"s1", "env":"e1"}}`)} + + updates := map[string]state.RawConfig{ + "host1": hostConfig, + "srv1": senvConfig, + } + rc.onAPMTracingUpdate(updates, callback) + + assert.Len(t, callbackCalls, 2) + assert.Empty(t, callbackCalls["host1"]) + assert.Empty(t, callbackCalls["srv1"]) + actualBytes, err := os.ReadFile(apmTracingFilePath) + assert.NoError(t, err) + assert.Equal(t, "tracing_enabled: true\nenv: someEnv\nservice_env_configs:\n- service: s1\n env: e1\n tracing_enabled: false\n", string(actualBytes)) + }) + + t.Run("lowest config-id wins", func(t *testing.T) { + defer mkTemp(t)() + rc := rcClient{} + callbackCalls := map[string]string{} + callback := func(id string, status state.ApplyStatus) { + callbackCalls[id] = status.Error + } + + hostConfig := state.RawConfig{Config: []byte(`{"id":"abc","infra_target": {"tags":["k:v"]},"lib_config":{"env":"someEnv","tracing_enabled":true}}`)} + hostConfig2 := state.RawConfig{Config: []byte(`{"id":"xyz","infra_target": {"tags":["k:v"]},"lib_config":{"env":"someEnv2","tracing_enabled":true}}`)} + + updates := map[string]state.RawConfig{ + "abc": hostConfig, + "xyz": hostConfig2, + } + rc.onAPMTracingUpdate(updates, callback) + + assert.Len(t, callbackCalls, 2) + assert.Empty(t, callbackCalls["abc"]) + assert.Equal(t, "DUPLICATE_HOST_CONFIG", callbackCalls["xyz"]) + actualBytes, err := os.ReadFile(apmTracingFilePath) + assert.NoError(t, err) + assert.Equal(t, "tracing_enabled: true\nenv: someEnv\nservice_env_configs: []\n", string(actualBytes)) + }) + + t.Run("bad updates report failure", func(t *testing.T) { + defer mkTemp(t)() + rc := rcClient{} + calls := map[string]string{} + callback := func(id string, status state.ApplyStatus) { + calls[id] = status.Error + } + + missingTarget := state.RawConfig{Config: []byte(`{}`)} + badPayload := state.RawConfig{Config: []byte(`{`)} + + updates := map[string]state.RawConfig{ + "missingTarget": missingTarget, + "badPayload": badPayload, + } + rc.onAPMTracingUpdate(updates, callback) + + assert.Len(t, calls, 2) + assert.Equal(t, calls["missingTarget"], MissingServiceTarget) + assert.Equal(t, calls["badPayload"], InvalidAPMTracingPayload) + }) +} diff --git a/comp/remote-config/rcclient/component.go b/comp/remote-config/rcclient/component.go index ad1111bd1e119..8ee542e14dffe 100644 --- a/comp/remote-config/rcclient/component.go +++ b/comp/remote-config/rcclient/component.go @@ -6,10 +6,11 @@ package rcclient //nolint:revive // TODO(RC) Fix revive linter import ( + "go.uber.org/fx" + "github.com/DataDog/datadog-agent/pkg/config/remote/data" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "go.uber.org/fx" ) // team: remote-config @@ -22,6 +23,8 @@ type Component interface { Start(agentName string) error // SubscribeAgentTask subscribe the remote-config client to AGENT_TASK SubscribeAgentTask() + // SubscribeApmTracing subscribes the remote-config client to APM_TRACING + SubscribeApmTracing() // Subscribe is the generic way to start listening to a specific product update // Component can also automatically subscribe to updates by returning a `ListenerProvider` struct Subscribe(product data.Product, fn func(update map[string]state.RawConfig, applyStateCallback func(string, state.ApplyStatus))) diff --git a/comp/remote-config/rcclient/rcclient_test.go b/comp/remote-config/rcclient/rcclient_test.go index db49cd99da79c..aae3c1667eaf5 100644 --- a/comp/remote-config/rcclient/rcclient_test.go +++ b/comp/remote-config/rcclient/rcclient_test.go @@ -15,7 +15,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/config/remote/client" - "github.com/DataDog/datadog-agent/pkg/config/remote/data" "github.com/DataDog/datadog-agent/pkg/config/settings" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -79,7 +78,7 @@ func TestAgentConfigCallback(t *testing.T) { structRC.client, _ = client.NewUnverifiedGRPCClient( ipcAddress, config.GetIPCPort(), func() (string, error) { return security.FetchAuthToken(config.Datadog) }, client.WithAgent("test-agent", "9.99.9"), - client.WithProducts([]data.Product{data.ProductAgentConfig}), + client.WithProducts(state.ProductAgentConfig), client.WithPollInterval(time.Hour), ) diff --git a/comp/remote-config/rcservice/component.go b/comp/remote-config/rcservice/component.go index a237f65dead0d..467bb1b6914b7 100644 --- a/comp/remote-config/rcservice/component.go +++ b/comp/remote-config/rcservice/component.go @@ -8,11 +8,19 @@ package rcservice import ( "context" + + "github.com/DataDog/datadog-agent/pkg/config/remote/service" pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" ) // team: remote-config +// Params is the set of parameters required for the remote config service. +type Params struct { + // Options is the set of options for the remote config service. + Options []service.Option +} + // Component is the component type. type Component interface { // ClientGetConfigs is the polling API called by tracers and agents to get the latest configurations diff --git a/comp/remote-config/rcservice/rcserviceimpl/rcservice.go b/comp/remote-config/rcservice/rcserviceimpl/rcservice.go index 39e864a79e885..0914a1bea5528 100644 --- a/comp/remote-config/rcservice/rcserviceimpl/rcservice.go +++ b/comp/remote-config/rcservice/rcserviceimpl/rcservice.go @@ -9,6 +9,7 @@ package rcserviceimpl import ( "context" "fmt" + "github.com/DataDog/datadog-agent/comp/core/log" "github.com/DataDog/datadog-agent/pkg/util/optional" @@ -37,6 +38,7 @@ type dependencies struct { Lc fx.Lifecycle + Params *rcservice.Params `optional:"true"` DdRcTelemetryReporter rctelemetryreporter.Component Hostname hostname.Component Cfg cfgcomp.Component @@ -70,6 +72,13 @@ func newRemoteConfigService(deps dependencies) (rcservice.Component, error) { traceAgentEnv := configUtils.GetTraceAgentDefaultEnv(config.Datadog) configuredTags := configUtils.GetConfiguredTags(config.Datadog, false) + options := []remoteconfig.Option{ + remoteconfig.WithTraceAgentEnv(traceAgentEnv), + } + if deps.Params != nil { + options = append(options, deps.Params.Options...) + } + configService, err := remoteconfig.NewService( config.Datadog, apiKey, @@ -78,7 +87,7 @@ func newRemoteConfigService(deps dependencies) (rcservice.Component, error) { configuredTags, deps.DdRcTelemetryReporter, version.AgentVersion, - remoteconfig.WithTraceAgentEnv(traceAgentEnv), + options..., ) if err != nil { return nil, fmt.Errorf("unable to create remote config service: %w", err) diff --git a/comp/trace/config/setup.go b/comp/trace/config/setup.go index 483356fce1e5b..594be8e7faaee 100644 --- a/comp/trace/config/setup.go +++ b/comp/trace/config/setup.go @@ -31,8 +31,8 @@ import ( coreconfig "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/config/model" rc "github.com/DataDog/datadog-agent/pkg/config/remote/client" - "github.com/DataDog/datadog-agent/pkg/config/remote/data" "github.com/DataDog/datadog-agent/pkg/config/utils" + "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" //nolint:revive // TODO(APM) Fix revive linter configUtils "github.com/DataDog/datadog-agent/pkg/config/utils" @@ -124,7 +124,7 @@ func prepareConfig(c corecompcfg.Component) (*config.AgentConfig, error) { coreconfig.GetIPCPort(), func() (string, error) { return security.FetchAuthToken(c) }, rc.WithAgent(rcClientName, version.AgentVersion), - rc.WithProducts([]data.Product{data.ProductAPMSampling, data.ProductAgentConfig}), + rc.WithProducts(state.ProductAPMSampling, state.ProductAgentConfig), rc.WithPollInterval(rcClientPollInterval), rc.WithDirectorRootOverride(c.GetString("remote_configuration.director_root")), ) diff --git a/docs/cloud-workload-security/backend.md b/docs/cloud-workload-security/backend.md index 28f9983724698..16433428347a3 100644 --- a/docs/cloud-workload-security/backend.md +++ b/docs/cloud-workload-security/backend.md @@ -16,7 +16,7 @@ CSM Threats logs have the following JSON schema: {{< code-block lang="json" collapsible="true" filename="BACKEND_EVENT_JSON_SCHEMA" >}} { - "$id": "https://github.com/DataDog/datadog-agent/pkg/security/serializers/event", + "$id": "https://github.com/DataDog/datadog-agent/tree/main/pkg/security/serializers", "$defs": { "AnomalyDetectionSyscallEvent": { "properties": { diff --git a/docs/cloud-workload-security/backend.schema.json b/docs/cloud-workload-security/backend.schema.json index 1db0c944c75fb..01d6cc8c2e86a 100644 --- a/docs/cloud-workload-security/backend.schema.json +++ b/docs/cloud-workload-security/backend.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/DataDog/datadog-agent/pkg/security/serializers/event", + "$id": "https://github.com/DataDog/datadog-agent/tree/main/pkg/security/serializers", "$defs": { "AnomalyDetectionSyscallEvent": { "properties": { diff --git a/omnibus/package-scripts/agent-deb/prerm b/omnibus/package-scripts/agent-deb/prerm index ce2fff5c028cb..1f5e54abb08cd 100755 --- a/omnibus/package-scripts/agent-deb/prerm +++ b/omnibus/package-scripts/agent-deb/prerm @@ -145,6 +145,14 @@ remove_remote_config_db() fi } +remove_apm_remote_config() +{ + if [ -f "$INSTALL_DIR/run/inject_config.yaml" ]; then + echo "Removing APM remote configuration file" + rm "$INSTALL_DIR/run/inject_config.yaml" || true + fi +} + stop_agent deregister_agent remove_custom_integrations diff --git a/omnibus/package-scripts/agent-rpm/prerm b/omnibus/package-scripts/agent-rpm/prerm index 3fafd3af77317..520cb64c33e01 100755 --- a/omnibus/package-scripts/agent-rpm/prerm +++ b/omnibus/package-scripts/agent-rpm/prerm @@ -129,6 +129,14 @@ remove_remote_config_db() fi } +remove_apm_remote_config() +{ + if [ -f "$INSTALL_DIR/run/inject_config.yaml" ]; then + echo "Removing APM remote configuration file" + rm "$INSTALL_DIR/run/inject_config.yaml" || true + fi +} + stop_agent deregister_agent diff --git a/pkg/cli/subcommands/check/command.go b/pkg/cli/subcommands/check/command.go index aa1e144f0a42a..9cd3c2a7b6124 100644 --- a/pkg/cli/subcommands/check/command.go +++ b/pkg/cli/subcommands/check/command.go @@ -31,6 +31,7 @@ import ( "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" internalAPI "github.com/DataDog/datadog-agent/comp/api/api" "github.com/DataDog/datadog-agent/comp/api/api/apiimpl" + collectorComp "github.com/DataDog/datadog-agent/comp/collector/collector" "github.com/DataDog/datadog-agent/comp/collector/collector/collectorimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" @@ -290,7 +291,7 @@ func run( // Create the CheckScheduler, but do not attach it to // AutoDiscovery. - pkgcollector.InitCheckScheduler(optional.NewNoneOption[pkgcollector.Collector](), demultiplexer) + pkgcollector.InitCheckScheduler(optional.NewNoneOption[collectorComp.Component](), demultiplexer) waitCtx, cancelTimeout := context.WithTimeout( context.Background(), time.Duration(cliParams.discoveryTimeout)*time.Second) diff --git a/pkg/clusteragent/admission/controllers/webhook/common.go b/pkg/clusteragent/admission/controllers/webhook/common.go index c946497b37289..3bccccbdfdfaa 100644 --- a/pkg/clusteragent/admission/controllers/webhook/common.go +++ b/pkg/clusteragent/admission/controllers/webhook/common.go @@ -8,6 +8,10 @@ package webhook import ( + "encoding/json" + agentsidecar "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate/agent_sidecar" + "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/common" "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate" "github.com/DataDog/datadog-agent/pkg/config" @@ -15,6 +19,53 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// Selector specifies an object label selector and a namespace label selector +type Selector struct { + ObjectSelector metav1.LabelSelector `yaml:"objectSelector,omitempty"` + NamespaceSelector metav1.LabelSelector `yaml:"namespaceSelector,omitempty"` +} + +// buildAgentSidecarObjectSelectors returns the mutating webhooks object selectors based on the configuration +func buildAgentSidecarObjectSelectors() (namespaceSelector, objectSelector *metav1.LabelSelector) { + // Read and parse selectors + selectorsJSON := config.Datadog.GetString("admission_controller.agent_sidecar.selectors") + + var selectors []Selector + + err := json.Unmarshal([]byte(selectorsJSON), &selectors) + if err != nil { + log.Errorf("failed to parse selectors for admission controller agent sidecar injection webhook: %s", err) + return nil, nil + } + + if len(selectors) > 1 { + log.Errorf("configuring more than 1 selector is not supported") + return nil, nil + } + + if len(selectors) == 1 { + namespaceSelector = &selectors[0].NamespaceSelector + objectSelector = &selectors[0].ObjectSelector + } else { + provider := config.Datadog.GetString("admission_controller.agent_sidecar.provider") + + if !agentsidecar.ProviderIsSupported(provider) { + log.Errorf("agent sidecar provider is not supported: %v", provider) + return nil, nil + } + + log.Infof("using default selector \"agent.datadoghq.com/sidecar\": \"%v\" for provider %v", provider, provider) + namespaceSelector = nil + objectSelector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "agent.datadoghq.com/sidecar": provider, + }, + } + } + + return namespaceSelector, objectSelector +} + // buildCWSInstrumentationLabelSelectors returns the mutating webhook object selector based on the configuration func buildCWSInstrumentationLabelSelectors(useNamespaceSelector bool) (namespaceSelector, objectSelector *metav1.LabelSelector) { var labelSelector metav1.LabelSelector diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1.go index a8bd1cbbba912..45482c549b82a 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1.go @@ -275,6 +275,26 @@ func (c *ControllerV1) generateTemplates() { } } + // Agent sidecar injection + if config.Datadog.GetBool("admission_controller.agent_sidecar.enabled") { + nsSelector, objSelector := buildAgentSidecarObjectSelectors() + + if nsSelector != nil || objSelector != nil { + webhook := c.getWebhookSkeleton( + "agent-sidecar", + config.Datadog.GetString("admission_controller.agent_sidecar.endpoint"), + []admiv1.OperationType{ + admiv1.Create, + }, + []string{"pods"}, + nsSelector, + objSelector, + ) + log.Info("registered side car injection v1") + webhooks = append(webhooks, webhook) + } + } + c.webhookTemplates = webhooks } diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go index c67e4241bfceb..2566524a44a80 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go @@ -677,6 +677,170 @@ func TestGenerateTemplatesV1(t *testing.T) { return []admiv1.MutatingWebhook{podWebhook, execWebhook} }, }, + { + name: "agent sidecar injection, no selectors specified, supported provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.provider", "fargate") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1.MutatingWebhook { + podWebhook := webhook( + "datadog.webhook.agent.sidecar", + "/agentsidecar", + &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "agent.datadoghq.com/sidecar": "fargate", + }, + }, + nil, + []admiv1.OperationType{admiv1.Create}, + []string{"pods"}, + ) + return []admiv1.MutatingWebhook{podWebhook} + }, + }, + { + name: "agent sidecar injection, no selectors specified, unsupported provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.provider", "unsupported-prov") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1.MutatingWebhook { + return []admiv1.MutatingWebhook{} + }, + }, + { + name: "agent sidecar injection, no selectors specified, no provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1.MutatingWebhook { + return []admiv1.MutatingWebhook{} + }, + }, + { + name: "agent sidecar injection, only single namespace selector, no provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1.MutatingWebhook { + podWebhook := webhook( + "datadog.webhook.agent.sidecar", + "/agentsidecar", + &metav1.LabelSelector{}, + &metav1.LabelSelector{ + MatchLabels: map[string]string{"labelKey": "labelVal"}, + }, + []admiv1.OperationType{admiv1.Create}, + []string{"pods"}, + ) + return []admiv1.MutatingWebhook{podWebhook} + }, + }, + { + name: "agent sidecar injection, only single object selector, no provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1.MutatingWebhook { + podWebhook := webhook( + "datadog.webhook.agent.sidecar", + "/agentsidecar", + &metav1.LabelSelector{MatchLabels: map[string]string{"labelKey": "labelVal"}}, + &metav1.LabelSelector{}, + []admiv1.OperationType{admiv1.Create}, + []string{"pods"}, + ) + return []admiv1.MutatingWebhook{podWebhook} + }, + }, + { + name: "agent sidecar injection, one object selector and one namespace selector, no provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey1\": \"labelVal1\"}}, \"NamespaceSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1.MutatingWebhook { + podWebhook := webhook( + "datadog.webhook.agent.sidecar", + "/agentsidecar", + &metav1.LabelSelector{MatchLabels: map[string]string{"labelKey1": "labelVal1"}}, + &metav1.LabelSelector{MatchLabels: map[string]string{"labelKey2": "labelVal2"}}, + []admiv1.OperationType{admiv1.Create}, + []string{"pods"}, + ) + return []admiv1.MutatingWebhook{podWebhook} + }, + }, + { + name: "agent sidecar injection, multiple selectors (should refuse to create webhook), provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.provider", "fargate") + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\":{\"labelKey1\": \"labelVal1\"}}} , {\"ObjectSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1.MutatingWebhook { + return []admiv1.MutatingWebhook{} + }, + }, } mockConfig.SetWithoutSource("kube_resources_namespace", "nsfoo") diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1.go index c63c6b01f15a0..515f3359fe8cb 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1.go @@ -275,6 +275,26 @@ func (c *ControllerV1beta1) generateTemplates() { } } + // Agent sidecar injection + if config.Datadog.GetBool("admission_controller.agent_sidecar.enabled") { + nsSelector, objSelector := buildAgentSidecarObjectSelectors() + + if nsSelector != nil || objSelector != nil { + webhook := c.getWebhookSkeleton( + "agent-sidecar", + config.Datadog.GetString("admission_controller.agent_sidecar.endpoint"), + []admiv1beta1.OperationType{ + admiv1beta1.Create, + }, + []string{"pods"}, + nsSelector, + objSelector, + ) + log.Info("registered side car injection v1beta1") + webhooks = append(webhooks, webhook) + } + } + c.webhookTemplates = webhooks } diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go index bf47ac5dec3c3..0ea40ae8024fe 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go @@ -673,6 +673,170 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { return []admiv1beta1.MutatingWebhook{podWebhook, execWebhook} }, }, + { + name: "agent sidecar injection, no selectors specified, supported provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.provider", "fargate") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1beta1.MutatingWebhook { + podWebhook := webhook( + "datadog.webhook.agent.sidecar", + "/agentsidecar", + &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "agent.datadoghq.com/sidecar": "fargate", + }, + }, + nil, + []admiv1beta1.OperationType{admiv1beta1.Create}, + []string{"pods"}, + ) + return []admiv1beta1.MutatingWebhook{podWebhook} + }, + }, + { + name: "agent sidecar injection, no selectors specified, unsupported provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.provider", "unsupported-prov") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1beta1.MutatingWebhook { + return []admiv1beta1.MutatingWebhook{} + }, + }, + { + name: "agent sidecar injection, no selectors specified, no provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1beta1.MutatingWebhook { + return []admiv1beta1.MutatingWebhook{} + }, + }, + { + name: "agent sidecar injection, only single namespace selector, no provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1beta1.MutatingWebhook { + podWebhook := webhook( + "datadog.webhook.agent.sidecar", + "/agentsidecar", + &metav1.LabelSelector{}, + &metav1.LabelSelector{ + MatchLabels: map[string]string{"labelKey": "labelVal"}, + }, + []admiv1beta1.OperationType{admiv1beta1.Create}, + []string{"pods"}, + ) + return []admiv1beta1.MutatingWebhook{podWebhook} + }, + }, + { + name: "agent sidecar injection, only single object selector, no provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1beta1.MutatingWebhook { + podWebhook := webhook( + "datadog.webhook.agent.sidecar", + "/agentsidecar", + &metav1.LabelSelector{MatchLabels: map[string]string{"labelKey": "labelVal"}}, + &metav1.LabelSelector{}, + []admiv1beta1.OperationType{admiv1beta1.Create}, + []string{"pods"}, + ) + return []admiv1beta1.MutatingWebhook{podWebhook} + }, + }, + { + name: "agent sidecar injection, one object selector and one namespace selector, no provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey1\": \"labelVal1\"}}, \"NamespaceSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1beta1.MutatingWebhook { + podWebhook := webhook( + "datadog.webhook.agent.sidecar", + "/agentsidecar", + &metav1.LabelSelector{MatchLabels: map[string]string{"labelKey1": "labelVal1"}}, + &metav1.LabelSelector{MatchLabels: map[string]string{"labelKey2": "labelVal2"}}, + []admiv1beta1.OperationType{admiv1beta1.Create}, + []string{"pods"}, + ) + return []admiv1beta1.MutatingWebhook{podWebhook} + }, + }, + { + name: "agent sidecar injection, multiple selectors (should refuse to create webhook), provider specified", + setupConfig: func() { + mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", false) + mockConfig.SetWithoutSource("admission_controller.inject_config.enabled", false) + mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) + mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) + mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.enabled", true) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.provider", "fargate") + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\":{\"labelKey1\": \"labelVal1\"}}} , {\"ObjectSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") + }, + configFunc: func() Config { return NewConfig(true, true) }, + want: func() []admiv1beta1.MutatingWebhook { + return []admiv1beta1.MutatingWebhook{} + }, + }, } mockConfig.SetWithoutSource("kube_resources_namespace", "nsfoo") diff --git a/pkg/clusteragent/admission/controllers/webhook/utils_test.go b/pkg/clusteragent/admission/controllers/webhook/utils_test.go index 82165c7dce687..96bf89b939ab9 100644 --- a/pkg/clusteragent/admission/controllers/webhook/utils_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/utils_test.go @@ -46,4 +46,5 @@ func resetMockConfig(c *config.MockConfig) { c.SetWithoutSource("admission_controller.namespace_selector_fallback", false) c.SetWithoutSource("admission_controller.add_aks_selectors", false) c.SetWithoutSource("admission_controller.admission_controller.cws_instrumentation.enabled", false) + c.SetWithoutSource("admission_controller.agent_sidecar.enabled", false) } diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/README.md b/pkg/clusteragent/admission/mutate/agent_sidecar/README.md new file mode 100644 index 0000000000000..a8aa55e6f5c83 --- /dev/null +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/README.md @@ -0,0 +1,87 @@ +# Agent Sidecar Auto-Injection + +## Overview + +Agent sidecar auto-injection is implemented as a webhook in the DCA admission controller. A mutation function is defined to process the pod mutation request that is forwarded to the webhook on pod creation. + +The main goal of this webhook is to facilitate the user experience when running in environments where the agent should be deployed as a sidecar. + +The agent sidecar is injected on pods that match the webhook selector(s). + +## Providers + +We support the use of providers. In this context, a provider is tied to a specific runtime environment (e.g. fargate). + +Currently, only `fargate` provider is supported. + +A provider serves to auto-configure the injected agent sidecar to target the specified provider environment by setting some extra environment variables for example. + +## Profiles + +A profile defines a set of overrides that the user would like to apply to the agent sidecar such as environment variables and/or resource limits. + +## Configuration Modes + +The configuration of the webhook depends on the user needs and can go from simple configuration to complex and advanced configuration. + +### Simplest Configuration + +The minimum requirement to activate this feature includes the following: +- Enabling the feature +- Setting the provider +- Creating datadog secrets in every namespace where you wish to inject the agent sidecar + +With this configuration, all pods having the label `agent.datadoghq.com/sidecar: ` will be injected with an agent sidecar. + +The injected sidecar will automatically have all the configuration needed for the specified provider. + +### Custom Selectors/Profiles Without Provider + +A more complex setup can include the following: +- Enabling the feature +- Setting custom selectors +- Setting custom profiles +- Creating datadog secrets in every namespace where you wish to inject the agent sidecar + +This allows the user to customize the matching criteria for the webhook. It allows specifying which pods will be injected with the agent sidecar. + +With this configuration, the default agent sidecar will be injected, in addition to any overrides set by the user in the specified profiles. + +### Custom Selectors/Profiles Without Provider + +This configuration includes the following: +- Enabling the feature +- Setting custom selectors +- Setting custom profiles +- Setting a provider +- Creating datadog secrets in every namespace where you wish to inject the agent sidecar + +This allows the user to customize the matching criteria for the webhook. It allows specifying which pods will be injected with the agent sidecar. + +With this configuration, the default agent sidecar will be injected, in addition to any overrides set by the user. + +Having set a provider, the agent sidecar will also get automatically the necessary configurations for the targeted provider. + +## Expected Behaviour + +The table below shows the expected behaviour when the feature is enabled. + +Note that currently we only support creating 1 selector and 1 profile (config override). +Creating multiple selectors and/or overrides will result in not registering any webhook. + +| Custom Selectors / Profiles Set | Provider Set | Provider Supported | Expected Behaviour | +|---------------------------------|--------------------|--------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| +| :x: | :heavy_check_mark: | :heavy_check_mark: | Agent sidecar should be injected on pods having the provider label key set (`agent.datadoghq.com/sidecar: `) | +| :x: | :heavy_check_mark: | :x: | No agent sidecar should be injected, and an error message will be logged in the cluster agent: "agent sidecar provider is not supported: foo-provider" | +| :x: | :x: | :x: | No agent sidecar should be injected, and an error message will be logged in the cluster agent: "agent sidecar provider is not supported" | +| :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The agent sidecar container should be injected only on pods matching the selector, and the `DD_EKS_FARGATE` label should be set to `true` | +| :heavy_check_mark: | :heavy_check_mark: | :x: | Agent sidecar should be injected, and you must find an error message in the cluster agent logs "unsupported provider: foo" | +| :heavy_check_mark: | :x: | :x: | The agent sidecar container should be injected only on pods matching the selector | + + + + +## Notes +- For now, we only support configuring 1 custom selector and 1 custom profile. +- For now, only `fargate` provider is supported +- For now, only 1 selector and 1 profile (config override) can be configured. diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go new file mode 100644 index 0000000000000..5311796b68ef8 --- /dev/null +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go @@ -0,0 +1,44 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build kubeapiserver + +// Package agentsidecar defines the mutation logic for the agentsidecar webhook +package agentsidecar + +import ( + "errors" + dca_ac "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate" + "github.com/DataDog/datadog-agent/pkg/util/log" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/dynamic" + k8s "k8s.io/client-go/kubernetes" +) + +// InjectAgentSidecar handles mutating pod requests for the agentsidecat webhook +func InjectAgentSidecar(rawPod []byte, _ string, ns string, _ *authenticationv1.UserInfo, dc dynamic.Interface, _ k8s.Interface) ([]byte, error) { + return dca_ac.Mutate(rawPod, ns, injectAgentSidecar, dc) +} + +func injectAgentSidecar(pod *corev1.Pod, _ string, _ dynamic.Interface) error { + if pod == nil { + return errors.New("can't inject agent sidecar into nil pod") + } + + for _, container := range pod.Spec.Containers { + if container.Name == agentSidecarContainerName { + log.Info("skipping agent sidecar injection: agent sidecar already exists") + return nil + } + } + + agentSidecarContainer := getDefaultSidecarTemplate() + + applyProviderOverrides(agentSidecarContainer) + + pod.Spec.Containers = append(pod.Spec.Containers, *agentSidecarContainer) + return nil +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go new file mode 100644 index 0000000000000..03fbaa14f6dbe --- /dev/null +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go @@ -0,0 +1,154 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build kubeapiserver + +package agentsidecar + +import ( + "github.com/DataDog/datadog-agent/pkg/config" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "reflect" + "testing" +) + +func TestInjectAgentSidecar(t *testing.T) { + mockConfig := config.Mock(t) + + tests := []struct { + Name string + Pod *corev1.Pod + provider string + ExpectError bool + ExpectedPodAfterInjection *corev1.Pod + }{ + { + Name: "should return error for nil pod", + Pod: nil, + provider: "", + ExpectError: true, + ExpectedPodAfterInjection: nil, + }, + { + Name: "should inject sidecar if no sidecar present, no provider set", + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + }, + }, + }, + provider: "", + ExpectError: false, + ExpectedPodAfterInjection: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + *getDefaultSidecarTemplate(), + }, + }, + }, + }, + { + Name: "should skip injecting sidecar when sidecar already exists", + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + *getDefaultSidecarTemplate(), + }, + }, + }, + provider: "", + ExpectError: false, + ExpectedPodAfterInjection: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + *getDefaultSidecarTemplate(), + }, + }, + }, + }, + { + Name: "should inject sidecar if no sidecar present, with supported provider", + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + }, + }, + }, + provider: "fargate", + ExpectError: false, + ExpectedPodAfterInjection: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + *sidecarWithEnvOverrides(corev1.EnvVar{ + Name: "DD_EKS_FARGATE", + Value: "true", + }), + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.Name, func(tt *testing.T) { + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.provider", test.provider) + + err := injectAgentSidecar(test.Pod, "", nil) + + if test.ExpectError { + assert.Error(tt, err, "expected non-nil error to be returned") + } else { + assert.NoError(tt, err, "expected returned error to be nil") + } + + if test.ExpectedPodAfterInjection == nil { + assert.Nil(tt, test.Pod) + } else { + assert.NotNil(tt, test.Pod) + assert.Truef( + tt, + reflect.DeepEqual(*test.ExpectedPodAfterInjection, *test.Pod), + "expected %v, found %v", + *test.ExpectedPodAfterInjection, + *test.Pod, + ) + } + + }) + } + +} + +func sidecarWithEnvOverrides(extraEnv ...corev1.EnvVar) *corev1.Container { + sidecar := getDefaultSidecarTemplate() + sidecar.Env = append(sidecar.Env, extraEnv...) + return sidecar +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/util.go b/pkg/clusteragent/admission/mutate/agent_sidecar/util.go new file mode 100644 index 0000000000000..6a261d24d8164 --- /dev/null +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/util.go @@ -0,0 +1,118 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build kubeapiserver + +package agentsidecar + +import ( + "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/log" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "os" +) + +const ( + agentSidecarContainerName = "datadog-agent-injected" + + providerFargate = "fargate" +) + +func getDefaultSidecarTemplate() *corev1.Container { + ddSite := os.Getenv("DD_SITE") + if ddSite == "" { + ddSite = config.DefaultSite + } + + agentContainer := &corev1.Container{ + Env: []corev1.EnvVar{ + { + Name: "DD_API_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "api-key", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "datadog-secret", + }, + }, + }, + }, + { + Name: "DD_SITE", + Value: ddSite, + }, + { + Name: "DD_CLUSTER_NAME", + Value: config.Datadog.GetString("cluster_name"), + }, + { + Name: "DD_KUBERNETES_KUBELET_NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + }, + Image: "datadog/agent", + ImagePullPolicy: corev1.PullIfNotPresent, + Name: agentSidecarContainerName, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("256Mi"), + "cpu": resource.MustParse("200m"), + }, + Limits: map[corev1.ResourceName]resource.Quantity{ + "memory": resource.MustParse("256Mi"), + "cpu": resource.MustParse("200m"), + }, + }, + } + + return agentContainer +} + +//////////////////////////////// +// // +// Provider Overrides // +// // +//////////////////////////////// + +// ProviderIsSupported indicates whether the provider is supported by agent sidecar injection +func ProviderIsSupported(provider string) bool { + switch provider { + case providerFargate: + return true + default: + return false + } +} + +func applyProviderOverrides(container *corev1.Container) { + provider := config.Datadog.GetString("admission_controller.agent_sidecar.provider") + + if !ProviderIsSupported(provider) { + log.Errorf("unsupported provider: %v", provider) + return + } + + switch provider { + case providerFargate: + applyFargateOverrides(container) + } +} + +func applyFargateOverrides(container *corev1.Container) { + if container == nil { + return + } + + container.Env = append(container.Env, corev1.EnvVar{ + Name: "DD_EKS_FARGATE", + Value: "true", + }) +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/util_test.go b/pkg/clusteragent/admission/mutate/agent_sidecar/util_test.go new file mode 100644 index 0000000000000..dddb0d79af583 --- /dev/null +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/util_test.go @@ -0,0 +1,110 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build kubeapiserver + +package agentsidecar + +import ( + "github.com/DataDog/datadog-agent/pkg/config" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "reflect" + "testing" +) + +func TestProviderIsSupported(t *testing.T) { + + tests := []struct { + name string + provider string + expectIsSupported bool + }{ + { + name: "supported provider", + provider: "fargate", + expectIsSupported: true, + }, + { + name: "unsupported provider", + provider: "foo-provider", + expectIsSupported: false, + }, + { + name: "empty provider", + provider: "", + expectIsSupported: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + isSupported := ProviderIsSupported(test.provider) + if test.expectIsSupported { + assert.True(tt, isSupported) + } else { + assert.False(tt, isSupported) + } + }) + } +} + +func TestApplyProviderOverrides(t *testing.T) { + mockConfig := config.Mock(t) + + tests := []struct { + name string + provider string + baseContainer *corev1.Container + // assertions assume the order of overrides is deterministic + // changing the order will cause the tests to fail + expectedContainerAfterOverride *corev1.Container + }{ + { + name: "nil container should be skipped", + provider: "fargate", + baseContainer: nil, + expectedContainerAfterOverride: nil, + }, + { + name: "fargate provider", + provider: "fargate", + baseContainer: &corev1.Container{}, + expectedContainerAfterOverride: &corev1.Container{ + Env: []corev1.EnvVar{ + { + Name: "DD_EKS_FARGATE", + Value: "true", + }, + }, + }, + }, + { + name: "unsupported provider", + provider: "foo-provider", + baseContainer: &corev1.Container{}, + expectedContainerAfterOverride: &corev1.Container{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.provider", test.provider) + applyProviderOverrides(test.baseContainer) + + if test.expectedContainerAfterOverride == nil { + assert.Nil(tt, test.baseContainer) + } else { + assert.NotNil(tt, test.baseContainer) + assert.Truef(tt, + reflect.DeepEqual(*test.baseContainer, *test.expectedContainerAfterOverride), + "overrides not applied as expected. expected %v, but found %v", + *test.expectedContainerAfterOverride, + *test.baseContainer, + ) + } + }) + } +} diff --git a/pkg/clusteragent/admission/mutate/auto_instrumentation.go b/pkg/clusteragent/admission/mutate/auto_instrumentation.go index 05d33f63a39fd..b4d4867566e04 100644 --- a/pkg/clusteragent/admission/mutate/auto_instrumentation.go +++ b/pkg/clusteragent/admission/mutate/auto_instrumentation.go @@ -117,7 +117,7 @@ var ( // InjectAutoInstrumentation injects APM libraries into pods func InjectAutoInstrumentation(rawPod []byte, _ string, ns string, _ *authenticationv1.UserInfo, dc dynamic.Interface, _ k8s.Interface) ([]byte, error) { - return mutate(rawPod, ns, injectAutoInstrumentation, dc) + return Mutate(rawPod, ns, injectAutoInstrumentation, dc) } func initContainerName(lang language) string { diff --git a/pkg/clusteragent/admission/mutate/common.go b/pkg/clusteragent/admission/mutate/common.go index ea7dea2e8ff1d..6e3292c46f1d7 100644 --- a/pkg/clusteragent/admission/mutate/common.go +++ b/pkg/clusteragent/admission/mutate/common.go @@ -31,9 +31,9 @@ const ( type mutateFunc func(*corev1.Pod, string, dynamic.Interface) error type mutatePodExecFunc func(*corev1.PodExecOptions, string, string, *authenticationv1.UserInfo, dynamic.Interface, kubernetes.Interface) error -// mutate handles mutating pods and encoding and decoding admission +// Mutate handles mutating pods and encoding and decoding admission // requests and responses for the public mutate functions -func mutate(rawPod []byte, ns string, m mutateFunc, dc dynamic.Interface) ([]byte, error) { +func Mutate(rawPod []byte, ns string, m mutateFunc, dc dynamic.Interface) ([]byte, error) { var pod corev1.Pod if err := json.Unmarshal(rawPod, &pod); err != nil { return nil, fmt.Errorf("failed to decode raw object: %v", err) diff --git a/pkg/clusteragent/admission/mutate/common_test.go b/pkg/clusteragent/admission/mutate/common_test.go index 5eda51c0998c9..0fc8fdd77dc9d 100644 --- a/pkg/clusteragent/admission/mutate/common_test.go +++ b/pkg/clusteragent/admission/mutate/common_test.go @@ -416,7 +416,7 @@ func TestJSONPatchCorrectness(t *testing.T) { podJSON, err := json.Marshal(pod) assert.NoError(t, err) - jsonPatch, err := mutate(podJSON, "bar", injectConfig, nil) + jsonPatch, err := Mutate(podJSON, "bar", injectConfig, nil) assert.NoError(t, err) expected, err := os.ReadFile("./testdata/expected_jsonpatch.json") @@ -443,7 +443,7 @@ func BenchmarkJSONPatch(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - jsonPatch, err := mutate(podJSON, "foobar-bax", injectConfig, nil) + jsonPatch, err := Mutate(podJSON, "foobar-bax", injectConfig, nil) if err != nil { b.Fatal(err) } diff --git a/pkg/clusteragent/admission/mutate/config.go b/pkg/clusteragent/admission/mutate/config.go index 4f52672bfc9f0..921b824d4c824 100644 --- a/pkg/clusteragent/admission/mutate/config.go +++ b/pkg/clusteragent/admission/mutate/config.go @@ -80,7 +80,7 @@ var ( // InjectConfig adds the DD_AGENT_HOST and DD_ENTITY_ID env vars to the pod template if they don't exist func InjectConfig(rawPod []byte, _ string, ns string, _ *authenticationv1.UserInfo, dc dynamic.Interface, _ kubernetes.Interface) ([]byte, error) { - return mutate(rawPod, ns, injectConfig, dc) + return Mutate(rawPod, ns, injectConfig, dc) } // injectConfig injects DD_AGENT_HOST and DD_ENTITY_ID into a pod template if needed diff --git a/pkg/clusteragent/admission/mutate/cws_instrumentation.go b/pkg/clusteragent/admission/mutate/cws_instrumentation.go index fafbd06ed9304..c9884d1557534 100644 --- a/pkg/clusteragent/admission/mutate/cws_instrumentation.go +++ b/pkg/clusteragent/admission/mutate/cws_instrumentation.go @@ -208,7 +208,7 @@ func (ci *CWSInstrumentation) injectCWSCommandInstrumentation(exec *corev1.PodEx // InjectCWSPodInstrumentation injects CWS pod instrumentation func (ci *CWSInstrumentation) InjectCWSPodInstrumentation(rawPod []byte, _ string, ns string, _ *authenticationv1.UserInfo, dc dynamic.Interface, _ kubernetes.Interface) ([]byte, error) { - return mutate(rawPod, ns, ci.injectCWSPodInstrumentation, dc) + return Mutate(rawPod, ns, ci.injectCWSPodInstrumentation, dc) } func (ci *CWSInstrumentation) injectCWSPodInstrumentation(pod *corev1.Pod, ns string, _ dynamic.Interface) error { diff --git a/pkg/clusteragent/admission/mutate/tags.go b/pkg/clusteragent/admission/mutate/tags.go index 60287e5013dca..4485cdcaca7fd 100644 --- a/pkg/clusteragent/admission/mutate/tags.go +++ b/pkg/clusteragent/admission/mutate/tags.go @@ -59,7 +59,7 @@ func (o *ownerInfo) buildID(ns string) string { // InjectTags adds the DD_ENV, DD_VERSION, DD_SERVICE env vars to // the pod template from pod and higher-level resource labels func InjectTags(rawPod []byte, _ string, ns string, _ *authenticationv1.UserInfo, dc dynamic.Interface, _ k8s.Interface) ([]byte, error) { - return mutate(rawPod, ns, injectTags, dc) + return Mutate(rawPod, ns, injectTags, dc) } // injectTags injects DD_ENV, DD_VERSION, DD_SERVICE diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go deleted file mode 100644 index 7a39ccc9b1667..0000000000000 --- a/pkg/collector/collector.go +++ /dev/null @@ -1,349 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//nolint:revive // TODO(AML) Fix revive linter -package collector - -import ( - "fmt" - "sync" - "time" - - "go.uber.org/atomic" - - "github.com/DataDog/datadog-agent/pkg/aggregator/sender" - "github.com/DataDog/datadog-agent/pkg/collector/check" - checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" - "github.com/DataDog/datadog-agent/pkg/collector/internal/middleware" - "github.com/DataDog/datadog-agent/pkg/collector/runner" - "github.com/DataDog/datadog-agent/pkg/collector/runner/expvars" - "github.com/DataDog/datadog-agent/pkg/collector/scheduler" - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -const ( - stopped uint32 = iota - started -) - -// EventType represents the type of events emitted by the collector -type EventType uint32 - -const ( - // CheckRun is emitted when a check is added to the collector - CheckRun EventType = iota - // CheckStop is emitted when a check is stopped and removed from the collector - CheckStop -) - -// EventReceiver represents a function to receive notification from the collector when running or stopping checks. -type EventReceiver func(checkid.ID, EventType) - -// Collector manages a collection of checks and provides operations over them -type Collector interface { - // Start begins the collector's operation. The scheduler will not run any checks until this has been called. - Start() - // Stop halts any component involved in running a Check - Stop() - // RunCheck sends a Check in the execution queue - RunCheck(inner check.Check) (checkid.ID, error) - // StopCheck halts a check and remove the instance - StopCheck(id checkid.ID) error - // MapOverChecks call the callback with the list of checks locked. - MapOverChecks(cb func([]check.Info)) - // GetChecks copies checks - GetChecks() []check.Check - // GetAllInstanceIDs returns the ID's of all instances of a check - GetAllInstanceIDs(checkName string) []checkid.ID - // ReloadAllCheckInstances completely restarts a check with a new configuration - ReloadAllCheckInstances(name string, newInstances []check.Check) ([]checkid.ID, error) - // AddEventReceiver adds a callback to the collector to be called each time a check is added or removed. - AddEventReceiver(cb EventReceiver) -} - -type collector struct { - senderManager sender.SenderManager - checkInstances int64 - - // state is 'started' or 'stopped' - state *atomic.Uint32 - - scheduler *scheduler.Scheduler - runner *runner.Runner - checks map[checkid.ID]*middleware.CheckWrapper - eventReceivers []EventReceiver - - cancelCheckTimeout time.Duration - - m sync.RWMutex -} - -// NewCollector create a Collector instance and sets up the Python Environment -func NewCollector(senderManager sender.SenderManager, cancelCheckTimeout time.Duration, paths ...string) Collector { - c := &collector{ - senderManager: senderManager, - checks: make(map[checkid.ID]*middleware.CheckWrapper), - state: atomic.NewUint32(stopped), - checkInstances: int64(0), - cancelCheckTimeout: cancelCheckTimeout, - } - InitPython(paths...) - - log.Debug("Collector up and running!") - return c -} - -// Sets up the Python environment -func InitPython(paths ...string) { - pyVer, pyHome, pyPath := pySetup(paths...) - - // print the Python info if the interpreter was embedded - if pyVer != "" { - log.Infof("Embedding Python %s", pyVer) - log.Debugf("Python Home: %s", pyHome) - log.Debugf("Python path: %s", pyPath) - } - - // Prepare python environment if necessary - if err := pyPrepareEnv(); err != nil { - log.Errorf("Unable to perform additional configuration of the python environment: %v", err) - } -} - -// AddEventReceiver adds a callback to the collector to be called each time a check is added or removed. -func (c *collector) AddEventReceiver(cb EventReceiver) { - c.m.Lock() - defer c.m.Unlock() - - c.eventReceivers = append(c.eventReceivers, cb) -} - -func (c *collector) notify(cid checkid.ID, e EventType) { - for _, cb := range c.eventReceivers { - cb(cid, e) - } -} - -// Start begins the collector's operation. The scheduler will not run any checks until this has been called. -func (c *collector) Start() { - c.m.Lock() - defer c.m.Unlock() - - if c.state.Load() == started { - return - } - - run := runner.NewRunner(c.senderManager) - sched := scheduler.NewScheduler(run.GetChan()) - - // let the runner some visibility into the scheduler - run.SetScheduler(sched) - sched.Run() - - c.scheduler = sched - c.runner = run - c.state.Store(started) -} - -// Stop halts any component involved in running a Check -func (c *collector) Stop() { - c.m.Lock() - defer c.m.Unlock() - - if c.state.Load() == stopped { - return - } - - if c.scheduler != nil { - c.scheduler.Stop() //nolint:errcheck - c.scheduler = nil - } - if c.runner != nil { - c.runner.Stop() - c.runner = nil - } - c.state.Store(stopped) -} - -// RunCheck sends a Check in the execution queue -func (c *collector) RunCheck(inner check.Check) (checkid.ID, error) { - c.m.Lock() - defer c.m.Unlock() - - ch := middleware.NewCheckWrapper(inner, c.senderManager) - - var emptyID checkid.ID - - if c.state.Load() != started { - return emptyID, fmt.Errorf("the collector is not running") - } - - if _, found := c.checks[ch.ID()]; found { - return emptyID, fmt.Errorf("a check with ID %s is already running", ch.ID()) - } - - err := c.scheduler.Enter(ch) - if err != nil { - return emptyID, fmt.Errorf("unable to schedule the check: %s", err) - } - - // Track the total number of checks running in order to have an appropriate number of workers - c.checkInstances++ - if ch.Interval() == 0 { - // Adding a temporary runner for long running check in case the - // number of runners is lower than the number of long running - // checks. - log.Infof("Adding an extra runner for the '%s' long running check", ch) - c.runner.AddWorker() - } else { - c.runner.UpdateNumWorkers(c.checkInstances) - } - - c.checks[ch.ID()] = ch - c.notify(ch.ID(), CheckRun) - return ch.ID(), nil -} - -// StopCheck halts a check and remove the instance -func (c *collector) StopCheck(id checkid.ID) error { - if !c.started() { - return fmt.Errorf("the collector is not running") - } - - ch, found := c.get(id) - if !found { - return fmt.Errorf("cannot find a check with ID %s", id) - } - - // unschedule the instance - err := c.scheduler.Cancel(id) - if err != nil { - return fmt.Errorf("an error occurred while canceling the check schedule: %s", err) - } - - err = c.runner.StopCheck(id) - if err != nil { - // still attempt to cancel the check before returning the error - _ = c.cancelCheck(ch, c.cancelCheckTimeout) - return fmt.Errorf("an error occurred while stopping the check: %s", err) - } - - err = c.cancelCheck(ch, c.cancelCheckTimeout) - if err != nil { - return fmt.Errorf("an error occurred while calling check.Cancel(): %s", err) - } - - // remove the check from the stats map - expvars.RemoveCheckStats(id) - - // vaporize the check - c.delete(id) - - return nil -} - -// cancelCheck calls Cancel on the passed check, with a timeout -func (c *collector) cancelCheck(ch check.Check, timeout time.Duration) error { - done := make(chan struct{}) - - go func() { - ch.Cancel() - close(done) - }() - - select { - case <-done: - return nil - case <-time.After(timeout): - return fmt.Errorf("timeout while calling check.Cancel() on check ID %s, timeout: %s", ch.ID(), timeout) - } -} - -func (c *collector) get(id checkid.ID) (check.Check, bool) { - c.m.RLock() - defer c.m.RUnlock() - - ch, found := c.checks[id] - return ch, found -} - -// remove the check from the list -func (c *collector) delete(id checkid.ID) { - c.m.Lock() - defer c.m.Unlock() - - delete(c.checks, id) - c.notify(id, CheckStop) -} - -// lightweight shortcut to see if the collector has started -func (c *collector) started() bool { - return c.state.Load() == started -} - -// MapOverChecks call the callback with the list of checks locked. -func (c *collector) MapOverChecks(cb func([]check.Info)) { - c.m.RLock() - defer c.m.RUnlock() - - cInfo := []check.Info{} - for _, c := range c.checks { - cInfo = append(cInfo, c) - } - cb(cInfo) -} - -// GetChecks copies checks -func (c *collector) GetChecks() []check.Check { - c.m.RLock() - defer c.m.RUnlock() - - chks := make([]check.Check, 0, len(c.checks)) - for _, chck := range c.checks { - chks = append(chks, chck) - } - - return chks -} - -// GetAllInstanceIDs returns the ID's of all instances of a check -func (c *collector) GetAllInstanceIDs(checkName string) []checkid.ID { - c.m.RLock() - defer c.m.RUnlock() - - instances := []checkid.ID{} - for id, check := range c.checks { - if check.String() == checkName { - instances = append(instances, id) - } - } - - return instances -} - -// ReloadAllCheckInstances completely restarts a check with a new configuration -func (c *collector) ReloadAllCheckInstances(name string, newInstances []check.Check) ([]checkid.ID, error) { - if !c.started() { - return nil, fmt.Errorf("The collector is not running") - } - - // Stop all the old instances - killed := c.GetAllInstanceIDs(name) - for _, id := range killed { - e := c.StopCheck(id) - if e != nil { - return nil, fmt.Errorf("Error stopping check %s: %s", id, e) - } - } - - // Start the new instances - for _, check := range newInstances { - id, e := c.RunCheck(check) - if e != nil { - return nil, fmt.Errorf("Error adding check %s: %s", id, e) - } - } - return killed, nil -} diff --git a/pkg/collector/collector_mock.go b/pkg/collector/collector_mock.go deleted file mode 100644 index a97282e4c8d81..0000000000000 --- a/pkg/collector/collector_mock.go +++ /dev/null @@ -1,80 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build test - -package collector - -import ( - "github.com/DataDog/datadog-agent/pkg/collector/check" - checkid "github.com/DataDog/datadog-agent/pkg/collector/check/id" - "github.com/stretchr/testify/mock" -) - -// MockCollector mock the Collector interface -type MockCollector struct { - mock.Mock - checksInfo []check.Info -} - -// NewMock returns a mock collector. -// -// 'mockChecksInfo' will be used as argument for the callback when 'MapOverChecks' is called. -func NewMock(mockChecksInfo []check.Info) *MockCollector { - return &MockCollector{ - checksInfo: mockChecksInfo, - } -} - -// Start begins the collector's operation. The scheduler will not run any checks until this has been called. -func (c *MockCollector) Start() { - c.Called() -} - -// Stop halts any component involved in running a Check -func (c *MockCollector) Stop() { - c.Called() -} - -// RunCheck sends a Check in the execution queue -func (c *MockCollector) RunCheck(inner check.Check) (checkid.ID, error) { - args := c.Called(inner) - return args.Get(0).(checkid.ID), args.Error(1) -} - -// StopCheck halts a check and remove the instance -func (c *MockCollector) StopCheck(id checkid.ID) error { - args := c.Called(id) - return args.Error(0) -} - -// MapOverChecks call the callback with the list of checks locked. -func (c *MockCollector) MapOverChecks(cb func([]check.Info)) { - c.Called(cb) - cb(c.checksInfo) -} - -// GetChecks copies checks -func (c *MockCollector) GetChecks() []check.Check { - args := c.Called() - return args.Get(0).([]check.Check) -} - -// GetAllInstanceIDs returns the ID's of all instances of a check -func (c *MockCollector) GetAllInstanceIDs(checkName string) []checkid.ID { - args := c.Called(checkName) - return args.Get(0).([]checkid.ID) -} - -// ReloadAllCheckInstances completely restarts a check with a new configuration -func (c *MockCollector) ReloadAllCheckInstances(name string, newInstances []check.Check) ([]checkid.ID, error) { - args := c.Called(name, newInstances) - return args.Get(0).([]checkid.ID), args.Error(1) -} - -// AddEventReceiver adds a callback to the collector to be called each time a check is added or removed. -func (c *MockCollector) AddEventReceiver(cb EventReceiver) { - c.Called(cb) -} diff --git a/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/horizontalpodautoscaler.go b/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/horizontalpodautoscaler.go index 8747a95c19f9a..e9ad98beadf98 100644 --- a/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/horizontalpodautoscaler.go +++ b/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/horizontalpodautoscaler.go @@ -76,13 +76,7 @@ func (c *HorizontalPodAutoscalerCollector) Run(rcfg *collectors.CollectorRunConf return nil, collectors.NewListingError(err) } - ctx := &processors.ProcessorContext{ - APIClient: rcfg.APIClient, - Cfg: rcfg.Config, - ClusterID: rcfg.ClusterID, - MsgGroupID: rcfg.MsgGroupRef.Inc(), - NodeType: c.metadata.NodeType, - } + ctx := collectors.NewProcessorContext(rcfg, c.metadata) processResult, processed := c.processor.Process(ctx, list) diff --git a/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/verticalpodautoscaler.go b/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/verticalpodautoscaler.go index bbefc0691d9d9..8b883b887795f 100644 --- a/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/verticalpodautoscaler.go +++ b/pkg/collector/corechecks/cluster/orchestrator/collectors/k8s/verticalpodautoscaler.go @@ -76,13 +76,7 @@ func (c *VerticalPodAutoscalerCollector) Run(rcfg *collectors.CollectorRunConfig return nil, collectors.NewListingError(err) } - ctx := &processors.ProcessorContext{ - APIClient: rcfg.APIClient, - Cfg: rcfg.Config, - ClusterID: rcfg.ClusterID, - MsgGroupID: rcfg.MsgGroupRef.Inc(), - NodeType: c.metadata.NodeType, - } + ctx := collectors.NewProcessorContext(rcfg, c.metadata) processResult, processed := c.processor.Process(ctx, list) diff --git a/pkg/collector/corechecks/orchestrator/pod/pod.go b/pkg/collector/corechecks/orchestrator/pod/pod.go index ee5218978bb43..928416b7101a0 100644 --- a/pkg/collector/corechecks/orchestrator/pod/pod.go +++ b/pkg/collector/corechecks/orchestrator/pod/pod.go @@ -81,10 +81,6 @@ func (c *Check) Configure( if err != nil { return err } - if !c.config.CoreCheck { - log.Warn("The Node Agent version for pods is currently disabled. See the changelog.") - return nil - } if !c.config.OrchestrationCollectionEnabled { log.Warn("orchestrator pod check is configured but the feature is disabled") return nil @@ -115,11 +111,6 @@ func (c *Check) Configure( // Run executes the check func (c *Check) Run() error { - - if !c.config.CoreCheck { - return nil - } - if c.clusterID == "" { clusterID, err := clustername.GetClusterID() if err != nil { diff --git a/pkg/collector/corechecks/orchestrator/pod/pod_test.go b/pkg/collector/corechecks/orchestrator/pod/pod_test.go index e7937ce4bb673..c5afd471d5324 100644 --- a/pkg/collector/corechecks/orchestrator/pod/pod_test.go +++ b/pkg/collector/corechecks/orchestrator/pod/pod_test.go @@ -30,9 +30,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" ) -var ( - testHostName = "test-host" -) +var testHostName = "test-host" // dummyKubelet allows tests to mock a kubelet's responses type dummyKubelet struct { @@ -121,7 +119,6 @@ func (suite *PodTestSuite) SetupTest() { mockConfig.SetWithoutSource("kubernetes_https_kubelet_port", kubeletPort) mockConfig.SetWithoutSource("kubelet_tls_verify", false) mockConfig.SetWithoutSource("orchestrator_explorer.enabled", true) - mockConfig.SetWithoutSource("orchestrator_explorer.run_on_node_agent", true) mockConfig.SetWithoutSource("orchestrator_explorer.manifest_collection.enabled", true) kubeutil, _ := kubelet.GetKubeUtilWithRetrier() @@ -129,7 +126,6 @@ func (suite *PodTestSuite) SetupTest() { suite.kubeUtil = kubeutil orchConfig := oconfig.NewDefaultOrchestratorConfig() - orchConfig.CoreCheck = true require.NoError(suite.T(), err) suite.check = &Check{ sender: mockSender, diff --git a/pkg/collector/embed_nopy.go b/pkg/collector/embed_nopy.go index e04b088e4d7b3..b4195f3d8a02e 100644 --- a/pkg/collector/embed_nopy.go +++ b/pkg/collector/embed_nopy.go @@ -7,11 +7,5 @@ package collector -//nolint:revive // TODO(AML) Fix revive linter -func pySetup(paths ...string) (pythonVersion, pythonHome, pythonPath string) { - return "", "", "" -} - -func pyPrepareEnv() error { - return nil -} +// InitPython is a no-op when the build tag is not set +func InitPython(_ ...string) {} diff --git a/pkg/collector/embed_python.go b/pkg/collector/embed_python.go index 0008fe43319bd..81999e2793017 100644 --- a/pkg/collector/embed_python.go +++ b/pkg/collector/embed_python.go @@ -13,6 +13,23 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" ) +// InitPython sets up the Python environment +func InitPython(paths ...string) { + pyVer, pyHome, pyPath := pySetup(paths...) + + // print the Python info if the interpreter was embedded + if pyVer != "" { + log.Infof("Embedding Python %s", pyVer) + log.Debugf("Python Home: %s", pyHome) + log.Debugf("Python path: %s", pyPath) + } + + // Prepare python environment if necessary + if err := pyPrepareEnv(); err != nil { + log.Errorf("Unable to perform additional configuration of the python environment: %v", err) + } +} + func pySetup(paths ...string) (pythonVersion, pythonHome, pythonPath string) { if err := python.Initialize(paths...); err != nil { log.Errorf("Could not initialize Python: %s", err) diff --git a/pkg/collector/scheduler.go b/pkg/collector/scheduler.go index bee82ba644248..fdeb1c717a289 100644 --- a/pkg/collector/scheduler.go +++ b/pkg/collector/scheduler.go @@ -3,6 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. +// Package collector provides the implementation of the collector package collector import ( @@ -14,6 +15,7 @@ import ( "golang.org/x/text/cases" "golang.org/x/text/language" + "github.com/DataDog/datadog-agent/comp/collector/collector" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/collector/check" @@ -54,13 +56,13 @@ func init() { type CheckScheduler struct { configToChecks map[string][]checkid.ID // cache the ID of checks we load for each config loaders []check.Loader - collector optional.Option[Collector] + collector optional.Option[collector.Component] senderManager sender.SenderManager m sync.RWMutex } // InitCheckScheduler creates and returns a check scheduler -func InitCheckScheduler(collector optional.Option[Collector], senderManager sender.SenderManager) *CheckScheduler { +func InitCheckScheduler(collector optional.Option[collector.Component], senderManager sender.SenderManager) *CheckScheduler { checkScheduler = &CheckScheduler{ collector: collector, senderManager: senderManager, @@ -137,12 +139,8 @@ func (s *CheckScheduler) Unschedule(configs []integration.Config) { } } -// Stop handles clean stop of registered schedulers -func (s *CheckScheduler) Stop() { - if coll, ok := s.collector.Get(); ok { - coll.Stop() - } -} +// Stop is a stub to satisfy the scheduler interface +func (s *CheckScheduler) Stop() {} // AddLoader adds a new Loader that AutoConfig can use to load a check. func (s *CheckScheduler) AddLoader(loader check.Loader) { diff --git a/pkg/compliance/agent.go b/pkg/compliance/agent.go index e9b22ba34978c..61a0c48f061b2 100644 --- a/pkg/compliance/agent.go +++ b/pkg/compliance/agent.go @@ -280,7 +280,7 @@ func (a *Agent) Stop() { log.Tracef("shutting down compliance agent") a.cancel() select { - case <-time.After(10 * time.Second): + case <-time.After(20 * time.Second): case <-a.finish: } a.opts.Reporter.Stop() @@ -317,6 +317,9 @@ func (a *Agent) runRegoBenchmarks(ctx context.Context) { resolver := NewResolver(ctx, a.opts.ResolverOptions) for _, rule := range benchmark.Rules { inputs, err := resolver.ResolveInputs(ctx, rule) + if err := ctx.Err(); err != nil { + return + } if err != nil { a.reportCheckEvents(checkInterval, CheckEventFromError(RegoEvaluator, rule, benchmark, err)) } else { diff --git a/pkg/config/remote/client/client.go b/pkg/config/remote/client/client.go index 1251db4a39b12..c633e52de7c2c 100644 --- a/pkg/config/remote/client/client.go +++ b/pkg/config/remote/client/client.go @@ -20,14 +20,14 @@ import ( "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" - "github.com/DataDog/datadog-agent/pkg/config/remote/data" + "github.com/pkg/errors" + "github.com/DataDog/datadog-agent/pkg/config/remote/meta" pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/util/backoff" ddgrpc "github.com/DataDog/datadog-agent/pkg/util/grpc" "github.com/DataDog/datadog-agent/pkg/util/log" - "github.com/pkg/errors" ) // Constraints on the maximum backoff time when errors occur @@ -72,6 +72,8 @@ type Client struct { // Options describes the client options type Options struct { + isUpdater bool + updaterTags []string agentVersion string agentName string products []string @@ -155,12 +157,9 @@ func NewUnverifiedGRPCClient(ipcAddress string, cmdPort string, authTokenFetcher } // WithProducts specifies the product lists -func WithProducts(products []data.Product) func(opts *Options) { +func WithProducts(products ...string) func(opts *Options) { return func(opts *Options) { - opts.products = make([]string, len(products)) - for i, product := range products { - opts.products[i] = string(product) - } + opts.products = products } } @@ -189,6 +188,14 @@ func WithAgent(name, version string) func(opts *Options) { return func(opts *Options) { opts.agentName, opts.agentVersion = name, version } } +// WithUpdater specifies that this client is an updater +func WithUpdater(tags ...string) func(opts *Options) { + return func(opts *Options) { + opts.isUpdater = true + opts.updaterTags = tags + } +} + func newClient(updater ConfigUpdater, opts ...func(opts *Options)) (*Client, error) { var options = defaultOptions for _, opt := range opts { @@ -486,19 +493,33 @@ func (c *Client) newUpdateRequest() (*pbgo.ClientGetConfigsRequest, error) { }, Id: c.ID, Products: c.products, - IsAgent: true, - IsTracer: false, - ClientAgent: &pbgo.ClientAgent{ - Name: c.agentName, - Version: c.agentVersion, - ClusterName: c.clusterName, - ClusterId: c.clusterID, - CwsWorkloads: c.cwsWorkloads, - }, }, CachedTargetFiles: pbCachedFiles, } + switch c.Options.isUpdater { + case true: + req.Client.IsUpdater = true + req.Client.ClientUpdater = &pbgo.ClientUpdater{ + Tags: c.Options.updaterTags, + Packages: []*pbgo.PackageState{ + { + Package: "datadog-agent", + StableVersion: "7.50.0", + }, + }, + } + case false: + req.Client.IsAgent = true + req.Client.ClientAgent = &pbgo.ClientAgent{ + Name: c.agentName, + Version: c.agentVersion, + ClusterName: c.clusterName, + ClusterId: c.clusterID, + CwsWorkloads: c.cwsWorkloads, + } + } + return req, nil } diff --git a/pkg/config/remote/service/service.go b/pkg/config/remote/service/service.go index e48c51b014f17..f7737b22f236c 100644 --- a/pkg/config/remote/service/service.go +++ b/pkg/config/remote/service/service.go @@ -150,15 +150,37 @@ func init() { exportedMapStatus.Set("lastError", &exportedLastUpdateErr) } +type options struct { + traceAgentEnv string + databaseFileName string +} + +var defaultOptions = options{ + traceAgentEnv: "", + databaseFileName: "remote-config.db", +} + +// Option is a service option +type Option func(s *options) + // WithTraceAgentEnv sets the service trace-agent environment variable -func WithTraceAgentEnv(env string) func(s *Service) { - return func(s *Service) { s.traceAgentEnv = env } +func WithTraceAgentEnv(env string) func(s *options) { + return func(s *options) { s.traceAgentEnv = env } +} + +// WithDatabaseFileName sets the service database file name +func WithDatabaseFileName(fileName string) func(s *options) { + return func(s *options) { s.databaseFileName = fileName } } // NewService instantiates a new remote configuration management service -func NewService(cfg model.Reader, apiKey, baseRawURL, hostname string, tags []string, telemetryReporter RcTelemetryReporter, agentVersion string, opts ...func(s *Service)) (*Service, error) { - refreshIntervalOverrideAllowed := false // If a user provides a value we don't want to override +func NewService(cfg model.Reader, apiKey, baseRawURL, hostname string, tags []string, telemetryReporter RcTelemetryReporter, agentVersion string, opts ...Option) (*Service, error) { + options := defaultOptions + for _, opt := range opts { + opt(&options) + } + refreshIntervalOverrideAllowed := false // If a user provides a value we don't want to override var refreshInterval time.Duration if cfg.IsSet("remote_configuration.refresh_interval") { refreshInterval = cfg.GetDuration("remote_configuration.refresh_interval") @@ -216,7 +238,7 @@ func NewService(cfg model.Reader, apiKey, baseRawURL, hostname string, tags []st return nil, err } - dbPath := path.Join(cfg.GetString("run_path"), "remote-config.db") + dbPath := path.Join(cfg.GetString("run_path"), options.databaseFileName) db, err := openCacheDB(dbPath, agentVersion) if err != nil { return nil, err @@ -260,7 +282,7 @@ func NewService(cfg model.Reader, apiKey, baseRawURL, hostname string, tags []st clientsCacheBypassLimit = defaultCacheBypassLimit } - s := &Service{ + return &Service{ firstUpdate: true, defaultRefreshInterval: refreshInterval, refreshIntervalOverrideAllowed: refreshIntervalOverrideAllowed, @@ -290,13 +312,7 @@ func NewService(cfg model.Reader, apiKey, baseRawURL, hostname string, tags []st agentVersion: agentVersion, stopOrgPoller: make(chan struct{}), stopConfigPoller: make(chan struct{}), - } - - for _, opt := range opts { - opt(s) - } - - return s, nil + }, nil } func newRCBackendOrgUUIDProvider(http api.API) uptane.OrgUUIDProvider { @@ -759,12 +775,16 @@ func validateRequest(request *pbgo.ClientGetConfigsRequest) error { return status.Error(codes.InvalidArgument, "client.client_tracer is a required field for tracer client config update requests") } - if request.Client.IsTracer && request.Client.IsAgent { - return status.Error(codes.InvalidArgument, "client.is_tracer and client.is_agent cannot both be true") + if request.Client.IsUpdater && request.Client.ClientUpdater == nil { + return status.Error(codes.InvalidArgument, "client.client_updater is a required field for updater client config update requests") + } + + if (request.Client.IsTracer && request.Client.IsAgent) || (request.Client.IsTracer && request.Client.IsUpdater) || (request.Client.IsAgent && request.Client.IsUpdater) { + return status.Error(codes.InvalidArgument, "client.is_tracer, client.is_agent, and client.is_updater are mutually exclusive") } - if !request.Client.IsTracer && !request.Client.IsAgent { - return status.Error(codes.InvalidArgument, "agents only support remote config updates from tracer or agent at this time") + if !request.Client.IsTracer && !request.Client.IsAgent && !request.Client.IsUpdater { + return status.Error(codes.InvalidArgument, "agents only support remote config updates from tracer or agent or updater at this time") } if request.Client.Id == "" { diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index fe57d6a5fbdd6..e326a8bf968a8 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -1069,6 +1069,11 @@ func InitConfig(config pkgconfigmodel.Config) { config.BindEnvAndSetDefault("admission_controller.cws_instrumentation.image_tag", "latest") config.BindEnv("admission_controller.cws_instrumentation.init_resources.cpu") config.BindEnv("admission_controller.cws_instrumentation.init_resources.memory") + config.BindEnvAndSetDefault("admission_controller.agent_sidecar.enabled", false) + config.BindEnvAndSetDefault("admission_controller.agent_sidecar.provider", "") + config.BindEnvAndSetDefault("admission_controller.agent_sidecar.endpoint", "/agentsidecar") + // Should be able to parse it to a list of webhook selectors + config.BindEnvAndSetDefault("admission_controller.agent_sidecar.selectors", "[]") // Telemetry // Enable telemetry metrics on the internals of the Agent. @@ -1111,7 +1116,6 @@ func InitConfig(config pkgconfigmodel.Config) { config.BindEnvAndSetDefault("orchestrator_explorer.manifest_collection.enabled", true) config.BindEnvAndSetDefault("orchestrator_explorer.manifest_collection.buffer_manifest", true) config.BindEnvAndSetDefault("orchestrator_explorer.manifest_collection.buffer_flush_interval", 20*time.Second) - config.BindEnvAndSetDefault("orchestrator_explorer.run_on_node_agent", true) // Container lifecycle configuration config.BindEnvAndSetDefault("container_lifecycle.enabled", true) diff --git a/pkg/config/utils/endpoints.go b/pkg/config/utils/endpoints.go index 5cb71950dd9f6..1c943fed217c3 100644 --- a/pkg/config/utils/endpoints.go +++ b/pkg/config/utils/endpoints.go @@ -18,7 +18,8 @@ import ( ) const ( - infraURLPrefix = "https://app." + // InfraURLPrefix is the default infra URL prefix for datadog + InfraURLPrefix = "https://app." ) func getResolvedDDUrl(c pkgconfigmodel.Reader, urlKey string) string { @@ -97,23 +98,37 @@ func GetMultipleEndpoints(c pkgconfigmodel.Reader) (map[string][]string, error) } additionalEndpoints := c.GetStringMapStringSlice("additional_endpoints") + + // populate with HA endpoints too + if c.GetBool("ha.enabled") { + site := c.GetString("ha.site") + if site != "" { + url := BuildURLWithPrefix(InfraURLPrefix, site) + additionalEndpoints[url] = []string{c.GetString("ha.api_key")} + } + } return mergeAdditionalEndpoints(keysPerDomain, additionalEndpoints) } +// BuildURLWithPrefix will return an HTTP(s) URL for a site given a certain prefix +func BuildURLWithPrefix(prefix, site string) string { + return prefix + strings.TrimSpace(site) +} + // GetMainEndpoint returns the main DD URL defined in the config, based on `site` and the prefix, or ddURLKey func GetMainEndpoint(c pkgconfigmodel.Reader, prefix string, ddURLKey string) string { // value under ddURLKey takes precedence over 'site' if c.IsSet(ddURLKey) && c.GetString(ddURLKey) != "" { return getResolvedDDUrl(c, ddURLKey) } else if c.GetString("site") != "" { - return prefix + strings.TrimSpace(c.GetString("site")) + return BuildURLWithPrefix(prefix, c.GetString("site")) } - return prefix + pkgconfigsetup.DefaultSite + return BuildURLWithPrefix(prefix, pkgconfigsetup.DefaultSite) } // GetInfraEndpoint returns the main DD Infra URL defined in config, based on the value of `site` and `dd_url` func GetInfraEndpoint(c pkgconfigmodel.Reader) string { - return GetMainEndpoint(c, infraURLPrefix, "dd_url") + return GetMainEndpoint(c, InfraURLPrefix, "dd_url") } // ddURLRegexp determines if an URL belongs to Datadog or not. If the URL belongs to Datadog it's prefixed with the Agent diff --git a/pkg/diagnose/check.go b/pkg/diagnose/check.go index c2ebe4b382a16..498c7c6c721b9 100644 --- a/pkg/diagnose/check.go +++ b/pkg/diagnose/check.go @@ -121,7 +121,7 @@ func diagnoseChecksInCLIProcess(diagCfg diagnosis.Config, senderManager diagnose // Create the CheckScheduler, but do not attach it to // AutoDiscovery. - pkgcollector.InitCheckScheduler(optional.NewNoneOption[pkgcollector.Collector](), senderManagerInstance) + pkgcollector.InitCheckScheduler(optional.NewNoneOption[collector.Component](), senderManagerInstance) // Load matching configurations (should we use common.AC.GetAllConfigs()) waitCtx, cancelTimeout := context.WithTimeout(context.Background(), time.Duration(5*time.Second)) diff --git a/pkg/flare/archive_security_test.go b/pkg/flare/archive_security_test.go index f5bd115821564..e6a3206dee456 100644 --- a/pkg/flare/archive_security_test.go +++ b/pkg/flare/archive_security_test.go @@ -17,6 +17,7 @@ import ( // Required to initialize the "dogstatsd" expvar _ "github.com/DataDog/datadog-agent/comp/dogstatsd/server" + _ "github.com/DataDog/datadog-agent/pkg/collector/runner/expvars" ) func TestCreateSecurityAgentArchive(t *testing.T) { diff --git a/pkg/orchestrator/config/config.go b/pkg/orchestrator/config/config.go index 5b242a26e0fc8..89bf03c868466 100644 --- a/pkg/orchestrator/config/config.go +++ b/pkg/orchestrator/config/config.go @@ -35,7 +35,6 @@ const ( type OrchestratorConfig struct { CollectorDiscoveryEnabled bool OrchestrationCollectionEnabled bool - CoreCheck bool KubeClusterName string IsScrubbingEnabled bool Scrubber *redact.DataScrubber @@ -111,7 +110,7 @@ func (oc *OrchestratorConfig) Load() error { } // Orchestrator Explorer - oc.OrchestrationCollectionEnabled, oc.CoreCheck, oc.KubeClusterName = IsOrchestratorEnabled() + oc.OrchestrationCollectionEnabled, oc.KubeClusterName = IsOrchestratorEnabled() oc.CollectorDiscoveryEnabled = config.Datadog.GetBool(OrchestratorNSKey("collector_discovery.enabled")) oc.IsScrubbingEnabled = config.Datadog.GetBool(OrchestratorNSKey("container_scrubbing.enabled")) @@ -182,8 +181,8 @@ func setBoundedConfigIntValue(configKey string, upperBound int, setter func(v in setter(val) } -// IsOrchestratorEnabled checks if orchestrator explorer features are enabled, it returns the boolean, the coreCheck flag and the cluster name -func IsOrchestratorEnabled() (bool, bool, string) { +// IsOrchestratorEnabled checks if orchestrator explorer features are enabled, it returns the boolean and the cluster name +func IsOrchestratorEnabled() (bool, string) { enabled := config.Datadog.GetBool(OrchestratorNSKey("enabled")) var clusterName string if enabled { @@ -191,6 +190,5 @@ func IsOrchestratorEnabled() (bool, bool, string) { hname, _ := hostname.Get(context.TODO()) clusterName = clustername.GetRFC1123CompliantClusterName(context.TODO(), hname) } - coreCheck := config.Datadog.GetBool(OrchestratorNSKey("run_on_node_agent")) - return enabled, coreCheck, clusterName + return enabled, clusterName } diff --git a/pkg/orchestrator/config/config_test.go b/pkg/orchestrator/config/config_test.go index 60dff58cfef7f..e6c85da46e4b8 100644 --- a/pkg/orchestrator/config/config_test.go +++ b/pkg/orchestrator/config/config_test.go @@ -219,7 +219,6 @@ func (suite *YamlConfigTestSuite) TestEnvConfigSensitiveWords() { } func (suite *YamlConfigTestSuite) TestNoEnvConfigArgsScrubbing() { - orchestratorCfg := NewDefaultOrchestratorConfig() err := orchestratorCfg.Load() suite.NoError(err) @@ -241,7 +240,6 @@ func (suite *YamlConfigTestSuite) TestNoEnvConfigArgsScrubbing() { } func (suite *YamlConfigTestSuite) TestOnlyEnvConfigArgsScrubbing() { - suite.config.SetWithoutSource("orchestrator_explorer.custom_sensitive_words", `["token","consul"]`) orchestratorCfg := NewDefaultOrchestratorConfig() @@ -265,7 +263,6 @@ func (suite *YamlConfigTestSuite) TestOnlyEnvConfigArgsScrubbing() { } func (suite *YamlConfigTestSuite) TestOnlyEnvContainsConfigArgsScrubbing() { - suite.config.SetWithoutSource("orchestrator_explorer.custom_sensitive_words", `["token","consul"]`) orchestratorCfg := NewDefaultOrchestratorConfig() diff --git a/pkg/process/checks/checks.go b/pkg/process/checks/checks.go index ac196d051abfe..dce58b55ba7b7 100644 --- a/pkg/process/checks/checks.go +++ b/pkg/process/checks/checks.go @@ -21,8 +21,6 @@ const ( ContainerCheckName = "container" RTContainerCheckName = "rtcontainer" ConnectionsCheckName = "connections" - PodCheckName = "pod" - PodCheckManifestName = "pod_manifest" DiscoveryCheckName = "process_discovery" ProcessEventsCheckName = "process_events" ) @@ -111,7 +109,6 @@ func All(config, sysprobeYamlCfg ddconfig.ReaderWriter, syscfg *sysconfigtypes.C NewContainerCheck(config), NewRTContainerCheck(config), NewConnectionsCheck(config, sysprobeYamlCfg, syscfg), - NewPodCheck(), NewProcessDiscoveryCheck(config), NewProcessEventsCheck(config), } diff --git a/pkg/process/checks/enabled_checks_pod_test.go b/pkg/process/checks/enabled_checks_pod_test.go deleted file mode 100644 index f5355cdbc288a..0000000000000 --- a/pkg/process/checks/enabled_checks_pod_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build kubelet && orchestrator - -package checks - -import ( - "testing" - - "github.com/DataDog/datadog-agent/pkg/config" - "github.com/DataDog/datadog-agent/pkg/util/kubernetes/clustername" -) - -func TestPodCheck(t *testing.T) { - config.SetFeatures(t, config.Kubernetes) - - t.Run("enabled", func(t *testing.T) { - // Resets the cluster name so that it isn't cached during the call to `getEnabledChecks()` - clustername.ResetClusterName() - defer clustername.ResetClusterName() - - cfg := config.Mock(t) - cfg.SetWithoutSource("orchestrator_explorer.enabled", true) - cfg.SetWithoutSource("orchestrator_explorer.run_on_node_agent", false) - cfg.SetWithoutSource("cluster_name", "test") - - enabledChecks := getEnabledChecks(t, cfg, config.MockSystemProbe(t)) - assertContainsCheck(t, enabledChecks, PodCheckName) - }) - - t.Run("disabled", func(t *testing.T) { - clustername.ResetClusterName() - defer clustername.ResetClusterName() - - cfg := config.Mock(t) - cfg.SetWithoutSource("orchestrator_explorer.enabled", false) - - enabledChecks := getEnabledChecks(t, cfg, config.MockSystemProbe(t)) - assertNotContainsCheck(t, enabledChecks, PodCheckName) - }) -} diff --git a/pkg/process/checks/interval.go b/pkg/process/checks/interval.go index 63371e2c879d9..b1f44ad6b9cd7 100644 --- a/pkg/process/checks/interval.go +++ b/pkg/process/checks/interval.go @@ -24,8 +24,6 @@ const ( //nolint:revive // TODO(PROC) Fix revive linter ConnectionsCheckDefaultInterval = 30 * time.Second //nolint:revive // TODO(PROC) Fix revive linter - PodCheckDefaultInterval = 10 * time.Second - //nolint:revive // TODO(PROC) Fix revive linter ProcessDiscoveryCheckDefaultInterval = 4 * time.Hour discoveryMinInterval = 10 * time.Minute @@ -49,7 +47,6 @@ var ( ContainerCheckName: ContainerCheckDefaultInterval, RTContainerCheckName: RTContainerCheckDefaultInterval, ConnectionsCheckName: ConnectionsCheckDefaultInterval, - PodCheckName: PodCheckDefaultInterval, DiscoveryCheckName: ProcessDiscoveryCheckDefaultInterval, ProcessEventsCheckName: config.DefaultProcessEventsCheckInterval, } diff --git a/pkg/process/checks/interval_test.go b/pkg/process/checks/interval_test.go index 592e1402effd2..80a83ec3d9644 100644 --- a/pkg/process/checks/interval_test.go +++ b/pkg/process/checks/interval_test.go @@ -45,11 +45,6 @@ func TestLegacyIntervalDefault(t *testing.T) { checkName: ConnectionsCheckName, expectedInterval: ConnectionsCheckDefaultInterval, }, - { - name: "pod default", - checkName: PodCheckName, - expectedInterval: PodCheckDefaultInterval, - }, } { t.Run(tc.name, func(t *testing.T) { cfg := config.Mock(t) @@ -91,7 +86,6 @@ func TestLegacyIntervalOverride(t *testing.T) { setting: "process_config.intervals.connections", checkName: ConnectionsCheckName, }, - // Note: non-default overridden handling of pod check interval is in pkg/orhestrator/config } { t.Run(tc.name, func(t *testing.T) { cfg := config.Mock(t) diff --git a/pkg/process/checks/pod.go b/pkg/process/checks/pod.go deleted file mode 100644 index a3c63b67f582e..0000000000000 --- a/pkg/process/checks/pod.go +++ /dev/null @@ -1,128 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build kubelet && orchestrator - -package checks - -import ( - "context" - "fmt" - "time" - - "github.com/DataDog/datadog-agent/pkg/collector/corechecks/cluster/orchestrator/processors" - k8sProcessors "github.com/DataDog/datadog-agent/pkg/collector/corechecks/cluster/orchestrator/processors/k8s" - "github.com/DataDog/datadog-agent/pkg/orchestrator" - oconfig "github.com/DataDog/datadog-agent/pkg/orchestrator/config" - "github.com/DataDog/datadog-agent/pkg/process/util" - "github.com/DataDog/datadog-agent/pkg/util/kubernetes/clustername" - "github.com/DataDog/datadog-agent/pkg/util/kubernetes/kubelet" - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -// NewPodCheck returns an instance of the Pod check -func NewPodCheck() *PodCheck { - return &PodCheck{ - config: oconfig.NewDefaultOrchestratorConfig(), - } -} - -// PodCheck is a check that returns container metadata and stats. -type PodCheck struct { - hostInfo *HostInfo - containerFailedLogLimit *util.LogLimit - processor *processors.Processor - config *oconfig.OrchestratorConfig -} - -// Init initializes a PodCheck instance. -func (c *PodCheck) Init(_ *SysProbeConfig, hostInfo *HostInfo, _ bool) error { - c.hostInfo = hostInfo - c.containerFailedLogLimit = util.NewLogLimit(10, time.Minute*10) - c.processor = processors.NewProcessor(new(k8sProcessors.PodHandlers)) - return c.config.Load() -} - -// IsEnabled returns true if the check is enabled by configuration -func (c *PodCheck) IsEnabled() bool { - // activate the pod collection if enabled and we have the cluster name set - orchestratorEnabled, coreCheck, kubeClusterName := oconfig.IsOrchestratorEnabled() - if !orchestratorEnabled { - return false - } - - // Do not run this check if we enabled the corecheck for pods - if coreCheck { - log.Info("Skipping pod check on process agent") - return false - } - - log.Warn("This Process Agent check will be deprecated in 7.51.0 and moved to the Node Agent") - - if kubeClusterName == "" { - _ = log.Warnf("Failed to auto-detect a Kubernetes cluster name. Pod collection will not start. To fix this, set it manually via the cluster_name config option") - return false - } - return true -} - -// SupportsRunOptions returns true if the check supports RunOptions -func (c *PodCheck) SupportsRunOptions() bool { - return false -} - -// Name returns the name of the ProcessCheck. -func (c *PodCheck) Name() string { return PodCheckName } - -// Realtime indicates if this check only runs in real-time mode. -func (c *PodCheck) Realtime() bool { return false } - -// ShouldSaveLastRun indicates if the output from the last run should be saved for use in flares -func (c *PodCheck) ShouldSaveLastRun() bool { return true } - -// Run runs the PodCheck to collect a list of running pods -func (c *PodCheck) Run(nextGroupID func() int32, _ *RunOptions) (RunResult, error) { - kubeUtil, err := kubelet.GetKubeUtil() - if err != nil { - return nil, err - } - - clusterID, err := clustername.GetClusterID() - if err != nil { - return nil, err - } - - podList, err := kubeUtil.GetRawLocalPodList(context.TODO()) - if err != nil { - return nil, err - } - - groupID := nextGroupID() - ctx := &processors.ProcessorContext{ - ClusterID: clusterID, - Cfg: c.config, - HostName: c.hostInfo.HostName, - MsgGroupID: groupID, - NodeType: orchestrator.K8sPod, - ApiGroupVersionTag: fmt.Sprintf("kube_api_version:%s", "v1"), - } - - processResult, processed := c.processor.Process(ctx, podList) - - if processed == -1 { - return nil, fmt.Errorf("unable to process pods: a panic occurred") - } - - // Append manifestMessages behind metadataMessages to avoiding modifying the func signature. - // Split the messages during forwarding. - metadataMessages := append(processResult.MetadataMessages, processResult.ManifestMessages...) - - orchestrator.SetCacheStats(len(podList), processed, ctx.NodeType) - - return StandardRunResult(metadataMessages), nil -} - -// Cleanup frees any resource held by the PodCheck before the agent exits -func (c *PodCheck) Cleanup() {} diff --git a/pkg/process/checks/pod_null.go b/pkg/process/checks/pod_null.go deleted file mode 100644 index af87f3a9835e7..0000000000000 --- a/pkg/process/checks/pod_null.go +++ /dev/null @@ -1,53 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build !kubelet || !orchestrator - -package checks - -import ( - "fmt" -) - -// NewPodCheck returns an instance of the Pod check -func NewPodCheck() *PodCheck { - return &PodCheck{} -} - -// PodCheck is a check that returns container metadata and stats. -type PodCheck struct { -} - -// Init initializes a PodCheck instance. -func (c *PodCheck) Init(_ *SysProbeConfig, _ *HostInfo, _ bool) error { - return nil -} - -// IsEnabled returns true if the check is enabled by configuration -func (c *PodCheck) IsEnabled() bool { - return false -} - -// SupportsRunOptions returns true if the check supports RunOptions -func (c *PodCheck) SupportsRunOptions() bool { - return false -} - -// Name returns the name of the ProcessCheck. -func (c *PodCheck) Name() string { return "pod" } - -// Realtime indicates if this check only runs in real-time mode. -func (c *PodCheck) Realtime() bool { return false } - -// ShouldSaveLastRun indicates if the output from the last run should be saved for use in flares -func (c *PodCheck) ShouldSaveLastRun() bool { return true } - -// Run runs the PodCheck to collect a list of running pods -func (c *PodCheck) Run(_ func() int32, _ *RunOptions) (RunResult, error) { - return nil, fmt.Errorf("not implemented") -} - -// Cleanup frees any resource held by the PodCheck before the agent exits -func (c *PodCheck) Cleanup() {} diff --git a/pkg/process/runner/collector_api_test.go b/pkg/process/runner/collector_api_test.go index e007a271cd835..c7aa9fd247e88 100644 --- a/pkg/process/runner/collector_api_test.go +++ b/pkg/process/runner/collector_api_test.go @@ -58,19 +58,6 @@ func setProcessEventsEndpointsForTest(config ddconfig.Config, eps ...apicfg.Endp config.SetWithoutSource("process_config.events_additional_endpoints", additionalEps) } -func setOrchestratorEndpointsForTest(config ddconfig.Config, eps ...apicfg.Endpoint) { - additionalEps := make(map[string][]string) - for i, ep := range eps { - if i == 0 { - config.SetWithoutSource("api_key", ep.APIKey) - config.SetWithoutSource("orchestrator_explorer.orchestrator_dd_url", ep.Endpoint) - } else { - additionalEps[ep.Endpoint.String()] = append(additionalEps[ep.Endpoint.String()], ep.APIKey) - } - } - config.SetWithoutSource("orchestrator_explorer.orchestrator_additional_endpoints", additionalEps) -} - func TestSendConnectionsMessage(t *testing.T) { m := &process.CollectorConnections{ HostName: testHostName, @@ -356,102 +343,6 @@ func TestRTProcMessageNotRetried(t *testing.T) { }) } -func TestSendPodMessageSendManifestPayload(t *testing.T) { - clusterID, check := getPodCheckMessage(t) - - ddconfig.SetFeatures(t, ddconfig.Kubernetes) - - ddcfg := ddconfig.Mock(t) - ddcfg.SetWithoutSource("orchestrator_explorer.enabled", true) - ddcfg.SetWithoutSource("orchestrator_explorer.manifest_collection.enabled", true) - - runCollectorTest(t, check, &endpointConfig{}, ddconfig.Mock(t), func(c *CheckRunner, ep *mockEndpoint) { - testPodMessageMetadata(t, clusterID, c, ep) - testPodMessageManifest(t, clusterID, c, ep) - }) -} - -func TestSendPodMessageNotSendManifestPayload(t *testing.T) { - clusterID, check := getPodCheckMessage(t) - - ddconfig.SetFeatures(t, ddconfig.Kubernetes) - - ddcfg := ddconfig.Mock(t) - ddcfg.SetWithoutSource("orchestrator_explorer.enabled", true) - ddcfg.SetWithoutSource("orchestrator_explorer.manifest_collection.enabled", false) - - runCollectorTest(t, check, &endpointConfig{}, ddconfig.Mock(t), func(c *CheckRunner, ep *mockEndpoint) { - testPodMessageMetadata(t, clusterID, c, ep) - select { - case q := <-ep.Requests: - t.Fatalf("should not have received manifest payload %+v", q) - case <-time.After(1 * time.Second): - } - }) -} - -func getPodCheckMessage(t *testing.T) (string, checks.Check) { - clusterID := "d801b2b1-4811-11ea-8618-121d4d0938a3" - - t.Setenv("DD_ORCHESTRATOR_CLUSTER_ID", clusterID) - - pd := make([]process.MessageBody, 0, 2) - m := &process.CollectorPod{ - HostName: testHostName, - GroupId: 1, - } - mm := &process.CollectorManifest{ - ClusterId: clusterID, - } - pd = append(pd, m, mm) - - check := &testCheck{ - name: checks.PodCheckName, - data: [][]process.MessageBody{pd}, - } - return clusterID, check -} - -func testPodMessageMetadata(t *testing.T, clusterID string, c *CheckRunner, ep *mockEndpoint) { - req := <-ep.Requests - - assert.Equal(t, "/api/v2/orch", req.uri) - - assert.Equal(t, testHostName, req.headers.Get(headers.HostHeader)) - assert.Equal(t, c.orchestrator.OrchestratorEndpoints[0].APIKey, req.headers.Get("DD-Api-Key")) - assert.Equal(t, "0", req.headers.Get(headers.ContainerCountHeader)) - assert.Equal(t, "1", req.headers.Get("X-DD-Agent-Attempts")) - assert.NotEmpty(t, req.headers.Get(headers.TimestampHeader)) - - reqBody, err := process.DecodeMessage(req.body) - require.NoError(t, err) - - cp, ok := reqBody.Body.(*process.CollectorPod) - require.True(t, ok) - - assert.Equal(t, clusterID, req.headers.Get(headers.ClusterIDHeader)) - assert.Equal(t, testHostName, cp.HostName) -} - -func testPodMessageManifest(t *testing.T, clusterID string, c *CheckRunner, ep *mockEndpoint) { - req := <-ep.Requests - - assert.Equal(t, "/api/v2/orchmanif", req.uri) - assert.Equal(t, testHostName, req.headers.Get(headers.HostHeader)) - assert.Equal(t, c.orchestrator.OrchestratorEndpoints[0].APIKey, req.headers.Get("DD-Api-Key")) - assert.Equal(t, "0", req.headers.Get(headers.ContainerCountHeader)) - assert.Equal(t, "1", req.headers.Get("X-DD-Agent-Attempts")) - assert.NotEmpty(t, req.headers.Get(headers.TimestampHeader)) - - reqBody, err := process.DecodeMessage(req.body) - require.NoError(t, err) - - cm, ok := reqBody.Body.(*process.CollectorManifest) - require.True(t, ok) - - assert.Equal(t, clusterID, cm.ClusterId) -} - func TestQueueSpaceNotAvailable(t *testing.T) { m := &process.CollectorRealTime{ HostName: testHostName, @@ -532,7 +423,7 @@ func TestMultipleAPIKeys(t *testing.T) { apiKeys := []string{"apiKeyI", "apiKeyII", "apiKeyIII"} - runCollectorTestWithAPIKeys(t, check, &endpointConfig{}, apiKeys, nil, ddconfig.Mock(t), func(_ *CheckRunner, ep *mockEndpoint) { + runCollectorTestWithAPIKeys(t, check, &endpointConfig{}, apiKeys, ddconfig.Mock(t), func(_ *CheckRunner, ep *mockEndpoint) { for _, expectedAPIKey := range apiKeys { request := <-ep.Requests assert.Equal(t, expectedAPIKey, request.headers.Get("DD-Api-Key")) @@ -541,12 +432,12 @@ func TestMultipleAPIKeys(t *testing.T) { } func runCollectorTest(t *testing.T, check checks.Check, epConfig *endpointConfig, mockConfig ddconfig.Config, tc func(c *CheckRunner, ep *mockEndpoint)) { - runCollectorTestWithAPIKeys(t, check, epConfig, []string{"apiKey"}, []string{"orchestratorApiKey"}, mockConfig, tc) + runCollectorTestWithAPIKeys(t, check, epConfig, []string{"apiKey"}, mockConfig, tc) } -func runCollectorTestWithAPIKeys(t *testing.T, check checks.Check, epConfig *endpointConfig, apiKeys, orchAPIKeys []string, mockConfig ddconfig.Config, tc func(c *CheckRunner, ep *mockEndpoint)) { +func runCollectorTestWithAPIKeys(t *testing.T, check checks.Check, epConfig *endpointConfig, apiKeys []string, mockConfig ddconfig.Config, tc func(c *CheckRunner, ep *mockEndpoint)) { ep := newMockEndpoint(t, epConfig) - collectorAddr, eventsAddr, orchestratorAddr := ep.start() + collectorAddr, eventsAddr := ep.start() defer ep.stop() eps := make([]apicfg.Endpoint, 0, len(apiKeys)) @@ -561,12 +452,6 @@ func runCollectorTestWithAPIKeys(t *testing.T, check checks.Check, epConfig *end } setProcessEventsEndpointsForTest(mockConfig, eventsEps...) - orchestratorEndpoints := make([]apicfg.Endpoint, 0, len(orchAPIKeys)) - for _, key := range orchAPIKeys { - orchestratorEndpoints = append(orchestratorEndpoints, apicfg.Endpoint{APIKey: key, Endpoint: orchestratorAddr}) - } - setOrchestratorEndpointsForTest(mockConfig, orchestratorEndpoints...) - hostInfo := &checks.HostInfo{ HostName: testHostName, } @@ -641,15 +526,14 @@ type endpointConfig struct { } type mockEndpoint struct { - t *testing.T - collectorServer *http.Server - eventsServer *http.Server - orchestratorServer *http.Server - stopper sync.WaitGroup - Requests chan request - errorCount int - errorsSent int - closeOnce sync.Once + t *testing.T + collectorServer *http.Server + eventsServer *http.Server + stopper sync.WaitGroup + Requests chan request + errorCount int + errorsSent int + closeOnce sync.Once } func newMockEndpoint(t *testing.T, config *endpointConfig) *mockEndpoint { @@ -669,20 +553,14 @@ func newMockEndpoint(t *testing.T, config *endpointConfig) *mockEndpoint { eventsMux := http.NewServeMux() eventsMux.HandleFunc("/api/v2/proclcycle", m.handleEvents) - orchestratorMux := http.NewServeMux() - orchestratorMux.HandleFunc("/api/v1/validate", m.handleValidate) - orchestratorMux.HandleFunc("/api/v2/orch", m.handle) - orchestratorMux.HandleFunc("/api/v2/orchmanif", m.handle) - m.collectorServer = &http.Server{Addr: ":", Handler: collectorMux} m.eventsServer = &http.Server{Addr: ":", Handler: eventsMux} - m.orchestratorServer = &http.Server{Addr: ":", Handler: orchestratorMux} return m } -// start starts the http endpoints and returns (collector server url, orchestrator server url) -func (m *mockEndpoint) start() (*url.URL, *url.URL, *url.URL) { +// start starts the http endpoints and returns (collector server url) +func (m *mockEndpoint) start() (*url.URL, *url.URL) { addrC := make(chan net.Addr, 1) m.stopper.Add(1) @@ -713,20 +591,6 @@ func (m *mockEndpoint) start() (*url.URL, *url.URL, *url.URL) { eventsAddr := <-addrC - m.stopper.Add(1) - go func() { - defer m.stopper.Done() - - listener, err := net.Listen("tcp", ":") - require.NoError(m.t, err) - - addrC <- listener.Addr() - - _ = m.orchestratorServer.Serve(listener) - }() - - orchestratorAddr := <-addrC - close(addrC) collectorEndpoint, err := url.Parse(fmt.Sprintf("http://%s", collectorAddr.String())) @@ -735,10 +599,7 @@ func (m *mockEndpoint) start() (*url.URL, *url.URL, *url.URL) { eventsEndpoint, err := url.Parse(fmt.Sprintf("http://%s", eventsAddr.String())) require.NoError(m.t, err) - orchestratorEndpoint, err := url.Parse(fmt.Sprintf("http://%s", orchestratorAddr.String())) - require.NoError(m.t, err) - - return collectorEndpoint, eventsEndpoint, orchestratorEndpoint + return collectorEndpoint, eventsEndpoint } func (m *mockEndpoint) stop() { @@ -748,9 +609,6 @@ func (m *mockEndpoint) stop() { err = m.eventsServer.Close() require.NoError(m.t, err) - err = m.orchestratorServer.Close() - require.NoError(m.t, err) - m.stopper.Wait() m.closeOnce.Do(func() { close(m.Requests) diff --git a/pkg/process/runner/runner.go b/pkg/process/runner/runner.go index 7850a40de3f0e..e47561d2f1be3 100644 --- a/pkg/process/runner/runner.go +++ b/pkg/process/runner/runner.go @@ -50,8 +50,7 @@ type checkPayload struct { } //nolint:revive // TODO(PROC) Fix revive linter -type Runner interface { -} +type Runner interface{} // CheckRunner will collect metrics from the local system and ship to the backend. type CheckRunner struct { @@ -517,7 +516,7 @@ func readResponseStatuses(checkName string, responses <-chan defaultforwarder.Re func ignoreResponseBody(checkName string) bool { switch checkName { - case checks.PodCheckName, checks.PodCheckManifestName, checks.ProcessEventsCheckName: + case checks.ProcessEventsCheckName: return true default: return false diff --git a/pkg/process/runner/runner_test.go b/pkg/process/runner/runner_test.go index 84828b1550ee6..c5ad6b7dc8b99 100644 --- a/pkg/process/runner/runner_test.go +++ b/pkg/process/runner/runner_test.go @@ -148,8 +148,6 @@ func TestIgnoreResponseBody(t *testing.T) { {checkName: checks.DiscoveryCheckName, ignore: false}, {checkName: checks.ContainerCheckName, ignore: false}, {checkName: checks.RTContainerCheckName, ignore: false}, - {checkName: checks.PodCheckName, ignore: true}, - {checkName: checks.PodCheckManifestName, ignore: true}, {checkName: checks.ConnectionsCheckName, ignore: false}, {checkName: checks.ProcessEventsCheckName, ignore: true}, } { diff --git a/pkg/process/runner/submitter.go b/pkg/process/runner/submitter.go index c451608ad1cc8..48820c64f627f 100644 --- a/pkg/process/runner/submitter.go +++ b/pkg/process/runner/submitter.go @@ -18,16 +18,14 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/log" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" + //nolint:revive // TODO(PROC) Fix revive linter forwarder "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder" "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" "github.com/DataDog/datadog-agent/comp/process/forwarders" "github.com/DataDog/datadog-agent/comp/process/types" - "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/resolver" ddconfig "github.com/DataDog/datadog-agent/pkg/config" - "github.com/DataDog/datadog-agent/pkg/orchestrator" - oconfig "github.com/DataDog/datadog-agent/pkg/orchestrator/config" "github.com/DataDog/datadog-agent/pkg/process/checks" "github.com/DataDog/datadog-agent/pkg/process/runner/endpoint" "github.com/DataDog/datadog-agent/pkg/process/statsd" @@ -35,7 +33,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/process/util/api" apicfg "github.com/DataDog/datadog-agent/pkg/process/util/api/config" "github.com/DataDog/datadog-agent/pkg/process/util/api/headers" - "github.com/DataDog/datadog-agent/pkg/util/kubernetes/clustername" "github.com/DataDog/datadog-agent/pkg/version" ) @@ -56,17 +53,14 @@ type CheckSubmitter struct { rtProcessResults *api.WeightedQueue eventResults *api.WeightedQueue connectionsResults *api.WeightedQueue - podResults *api.WeightedQueue // Forwarders processForwarder defaultforwarder.Component rtProcessForwarder defaultforwarder.Component connectionsForwarder defaultforwarder.Component - podForwarder *forwarder.DefaultForwarder eventForwarder defaultforwarder.Component - orchestrator *oconfig.OrchestratorConfig - hostname string + hostname string exit chan struct{} wg *sync.WaitGroup @@ -113,13 +107,6 @@ func NewSubmitter(config config.Component, log log.Component, forwarders forward connectionsResults := api.NewWeightedQueue(queueSize, int64(queueBytes)) log.Debugf("Creating connections queue with max_size=%d and max_weight=%d", connectionsResults.MaxSize(), connectionsResults.MaxWeight()) - orchestrator := oconfig.NewDefaultOrchestratorConfig() - if err := orchestrator.Load(); err != nil { - return nil, err - } - podResults := api.NewWeightedQueue(queueSize, int64(orchestrator.PodQueueBytes)) - log.Debugf("Creating pod check queue with max_size=%d and max_weight=%d", podResults.MaxSize(), podResults.MaxWeight()) - eventResults := api.NewWeightedQueue(queueSize, int64(queueBytes)) log.Debugf("Creating event check queue with max_size=%d and max_weight=%d", eventResults.MaxSize(), eventResults.MaxWeight()) @@ -135,33 +122,25 @@ func NewSubmitter(config config.Component, log log.Component, forwarders forward return nil, err } - podForwarderOpts := forwarder.NewOptionsWithResolvers(config, log, resolver.NewSingleDomainResolvers(apicfg.KeysPerDomains(orchestrator.OrchestratorEndpoints))) - podForwarderOpts.DisableAPIKeyChecking = true - podForwarderOpts.RetryQueuePayloadsTotalMaxSize = queueBytes // Allow more in-flight requests than the default - podForwarder := forwarder.NewDefaultForwarder(config, log, podForwarderOpts) - processEventsAPIEndpoints, err := endpoint.GetEventsAPIEndpoints(config) if err != nil { return nil, err } - printStartMessage(log, hostname, processAPIEndpoints, processEventsAPIEndpoints, orchestrator.OrchestratorEndpoints) + printStartMessage(log, hostname, processAPIEndpoints, processEventsAPIEndpoints) return &CheckSubmitter{ log: log, processResults: processResults, rtProcessResults: rtProcessResults, eventResults: eventResults, connectionsResults: connectionsResults, - podResults: podResults, processForwarder: forwarders.GetProcessForwarder(), rtProcessForwarder: forwarders.GetRTProcessForwarder(), connectionsForwarder: forwarders.GetConnectionsForwarder(), - podForwarder: podForwarder, eventForwarder: forwarders.GetEventForwarder(), - orchestrator: orchestrator, - hostname: hostname, + hostname: hostname, dropCheckPayloads: dropCheckPayloads, @@ -176,34 +155,22 @@ func NewSubmitter(config config.Component, log log.Component, forwarders forward }, nil } -func printStartMessage(log log.Component, hostname string, processAPIEndpoints, processEventsAPIEndpoints, orchestratorEndpoints []apicfg.Endpoint) { +func printStartMessage(log log.Component, hostname string, processAPIEndpoints []apicfg.Endpoint, processEventsAPIEndpoints []apicfg.Endpoint) { eps := make([]string, 0, len(processAPIEndpoints)) for _, e := range processAPIEndpoints { eps = append(eps, e.Endpoint.String()) } - orchestratorEps := make([]string, 0, len(orchestratorEndpoints)) - for _, e := range orchestratorEndpoints { - orchestratorEps = append(orchestratorEps, e.Endpoint.String()) - } eventsEps := make([]string, 0, len(processEventsAPIEndpoints)) for _, e := range processEventsAPIEndpoints { eventsEps = append(eventsEps, e.Endpoint.String()) } - log.Infof("Starting CheckSubmitter for host=%s, endpoints=%s, events endpoints=%s orchestrator endpoints=%s", hostname, eps, eventsEps, orchestratorEps) + log.Infof("Starting CheckSubmitter for host=%s, endpoints=%s, events endpoints=%s", hostname, eps, eventsEps) } //nolint:revive // TODO(PROC) Fix revive linter func (s *CheckSubmitter) Submit(start time.Time, name string, messages *types.Payload) { results := s.resultsQueueForCheck(name) - if name == checks.PodCheckName { - s.messagesToResultsQueue(start, checks.PodCheckName, messages.Message[:len(messages.Message)/2], results) - if s.orchestrator.IsManifestCollectionEnabled { - s.messagesToResultsQueue(start, checks.PodCheckManifestName, messages.Message[len(messages.Message)/2:], results) - } - return - } - s.messagesToResultsQueue(start, name, messages.Message, results) } @@ -221,10 +188,6 @@ func (s *CheckSubmitter) Start() error { return fmt.Errorf("error starting connections forwarder: %s", err) } - if err := s.podForwarder.Start(); err != nil { - return fmt.Errorf("error starting pod forwarder: %s", err) - } - if err := s.eventForwarder.Start(); err != nil { return fmt.Errorf("error starting event forwarder: %s", err) } @@ -247,12 +210,6 @@ func (s *CheckSubmitter) Start() error { s.consumePayloads(s.connectionsResults, s.connectionsForwarder) }() - s.wg.Add(1) - go func() { - defer s.wg.Done() - s.consumePayloads(s.podResults, s.podForwarder) - }() - s.wg.Add(1) go func() { defer s.wg.Done() @@ -287,12 +244,10 @@ func (s *CheckSubmitter) Start() error { RtProcessQueueSize: s.rtProcessResults.Len(), ConnectionsQueueSize: s.connectionsResults.Len(), EventQueueSize: s.eventResults.Len(), - PodQueueSize: s.podResults.Len(), ProcessQueueBytes: s.processResults.Weight(), RtProcessQueueBytes: s.rtProcessResults.Weight(), ConnectionsQueueBytes: s.connectionsResults.Weight(), EventQueueBytes: s.eventResults.Weight(), - PodQueueBytes: s.podResults.Weight(), }) case <-queueLogTicker.C: s.logQueuesSize() @@ -312,7 +267,6 @@ func (s *CheckSubmitter) Stop() { s.processResults.Stop() s.rtProcessResults.Stop() s.connectionsResults.Stop() - s.podResults.Stop() s.eventResults.Stop() s.wg.Wait() @@ -320,7 +274,6 @@ func (s *CheckSubmitter) Stop() { s.processForwarder.Stop() s.rtProcessForwarder.Stop() s.connectionsForwarder.Stop() - s.podForwarder.Stop() s.eventForwarder.Stop() close(s.rtNotifierChan) @@ -366,12 +319,6 @@ func (s *CheckSubmitter) consumePayloads(results *api.WeightedQueue, fwd forward responses, err = fwd.SubmitRTContainerChecks(forwarderPayload, payload.headers) case checks.ConnectionsCheckName: responses, err = fwd.SubmitConnectionChecks(forwarderPayload, payload.headers) - // Pod check metadata - case checks.PodCheckName: - responses, err = fwd.SubmitOrchestratorChecks(forwarderPayload, payload.headers, int(orchestrator.K8sPod)) - // Pod check manifest data - case checks.PodCheckManifestName: - responses, err = fwd.SubmitOrchestratorManifests(forwarderPayload, payload.headers) case checks.DiscoveryCheckName: // A Process Discovery check does not change the RT mode responses, err = fwd.SubmitProcessDiscoveryChecks(forwarderPayload, payload.headers) @@ -397,8 +344,6 @@ func (s *CheckSubmitter) consumePayloads(results *api.WeightedQueue, fwd forward func (s *CheckSubmitter) resultsQueueForCheck(name string) *api.WeightedQueue { switch name { - case checks.PodCheckName: - return s.podResults case checks.RTProcessCheckName, checks.RTContainerCheckName: return s.rtProcessResults case checks.ConnectionsCheckName: @@ -415,24 +360,21 @@ func (s *CheckSubmitter) logQueuesSize() { rtProcessSize = s.rtProcessResults.Len() connectionsSize = s.connectionsResults.Len() eventsSize = s.eventResults.Len() - podSize = s.podResults.Len() ) if processSize == 0 && rtProcessSize == 0 && connectionsSize == 0 && - eventsSize == 0 && - podSize == 0 { + eventsSize == 0 { return } s.log.Infof( - "Delivery queues: process[size=%d, weight=%d], rtprocess[size=%d, weight=%d], connections[size=%d, weight=%d], event[size=%d, weight=%d], pod[size=%d, weight=%d]", + "Delivery queues: process[size=%d, weight=%d], rtprocess[size=%d, weight=%d], connections[size=%d, weight=%d], event[size=%d, weight=%d]", processSize, s.processResults.Weight(), rtProcessSize, s.rtProcessResults.Weight(), connectionsSize, s.connectionsResults.Weight(), eventsSize, s.eventResults.Weight(), - podSize, s.podResults.Weight(), ) } @@ -470,14 +412,6 @@ func (s *CheckSubmitter) messagesToCheckResult(start time.Time, name string, mes extraHeaders.Set(headers.ContentTypeHeader, headers.ProtobufContentType) extraHeaders.Set(headers.AgentStartTime, strconv.FormatInt(s.agentStartTime, 10)) - if s.orchestrator.OrchestrationCollectionEnabled { - if cid, err := clustername.GetClusterID(); err == nil && cid != "" { - extraHeaders.Set(headers.ClusterIDHeader, cid) - } - extraHeaders.Set(headers.EVPOriginHeader, "process-agent") - extraHeaders.Set(headers.EVPOriginVersionHeader, version.AgentVersion) - } - switch name { case checks.ProcessEventsCheckName: extraHeaders.Set(headers.EVPOriginHeader, "process-agent") diff --git a/pkg/process/runner/submitter_test.go b/pkg/process/runner/submitter_test.go index 8b9ac3ba173ab..14f5943de90dc 100644 --- a/pkg/process/runner/submitter_test.go +++ b/pkg/process/runner/submitter_test.go @@ -68,7 +68,6 @@ func TestNewCollectorQueueSize(t *testing.T) { c, err := NewSubmitter(mockConfig, deps.Log, deps.Forwarders, testHostName) assert.NoError(t, err) assert.Equal(t, tc.expectedQueueSize, c.processResults.MaxSize()) - assert.Equal(t, tc.expectedQueueSize, c.podResults.MaxSize()) }) } } diff --git a/pkg/proto/datadog/remoteconfig/remoteconfig.proto b/pkg/proto/datadog/remoteconfig/remoteconfig.proto index 95c731c2aa899..f71ce0d8e96ff 100644 --- a/pkg/proto/datadog/remoteconfig/remoteconfig.proto +++ b/pkg/proto/datadog/remoteconfig/remoteconfig.proto @@ -85,6 +85,9 @@ message Client { ClientAgent client_agent = 9; uint64 last_seen = 10; bytes capabilities = 11; + reserved 12, 13; + bool is_updater = 14; + ClientUpdater client_updater = 15; } message ClientTracer { @@ -106,6 +109,29 @@ message ClientAgent { repeated string cws_workloads = 5; } +message ClientUpdater { + repeated string tags = 1; + repeated PackageState packages = 2; +} + +enum ExperimentState { + NO_EXPERIMENT = 0; + DOWNLOADING = 1; + INSTALLING = 2; + PENDING = 3; + RUNNING = 4; + ERROR = 5; +} + +message PackageState { + string package = 1; + string stable_version = 2; + string experiment_version = 3; + ExperimentState experiment_state = 4; + uint64 experiment_error_code = 5; + string experiment_error_message = 6; +} + message ConfigState { string id = 1; uint64 version = 2; diff --git a/pkg/proto/pbgo/core/remoteconfig.pb.go b/pkg/proto/pbgo/core/remoteconfig.pb.go index 7b199f7dcbb04..b468933528725 100644 --- a/pkg/proto/pbgo/core/remoteconfig.pb.go +++ b/pkg/proto/pbgo/core/remoteconfig.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 -// protoc v3.6.1 +// protoc-gen-go v1.30.0 +// protoc v3.21.7 // source: datadog/remoteconfig/remoteconfig.proto package core @@ -20,6 +20,64 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type ExperimentState int32 + +const ( + ExperimentState_NO_EXPERIMENT ExperimentState = 0 + ExperimentState_DOWNLOADING ExperimentState = 1 + ExperimentState_INSTALLING ExperimentState = 2 + ExperimentState_PENDING ExperimentState = 3 + ExperimentState_RUNNING ExperimentState = 4 + ExperimentState_ERROR ExperimentState = 5 +) + +// Enum value maps for ExperimentState. +var ( + ExperimentState_name = map[int32]string{ + 0: "NO_EXPERIMENT", + 1: "DOWNLOADING", + 2: "INSTALLING", + 3: "PENDING", + 4: "RUNNING", + 5: "ERROR", + } + ExperimentState_value = map[string]int32{ + "NO_EXPERIMENT": 0, + "DOWNLOADING": 1, + "INSTALLING": 2, + "PENDING": 3, + "RUNNING": 4, + "ERROR": 5, + } +) + +func (x ExperimentState) Enum() *ExperimentState { + p := new(ExperimentState) + *p = x + return p +} + +func (x ExperimentState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ExperimentState) Descriptor() protoreflect.EnumDescriptor { + return file_datadog_remoteconfig_remoteconfig_proto_enumTypes[0].Descriptor() +} + +func (ExperimentState) Type() protoreflect.EnumType { + return &file_datadog_remoteconfig_remoteconfig_proto_enumTypes[0] +} + +func (x ExperimentState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ExperimentState.Descriptor instead. +func (ExperimentState) EnumDescriptor() ([]byte, []int) { + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{0} +} + type ConfigMetas struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -665,15 +723,17 @@ type Client struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - State *ClientState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` - Products []string `protobuf:"bytes,3,rep,name=products,proto3" json:"products,omitempty"` - IsTracer bool `protobuf:"varint,6,opt,name=is_tracer,json=isTracer,proto3" json:"is_tracer,omitempty"` - ClientTracer *ClientTracer `protobuf:"bytes,7,opt,name=client_tracer,json=clientTracer,proto3" json:"client_tracer,omitempty"` - IsAgent bool `protobuf:"varint,8,opt,name=is_agent,json=isAgent,proto3" json:"is_agent,omitempty"` - ClientAgent *ClientAgent `protobuf:"bytes,9,opt,name=client_agent,json=clientAgent,proto3" json:"client_agent,omitempty"` - LastSeen uint64 `protobuf:"varint,10,opt,name=last_seen,json=lastSeen,proto3" json:"last_seen,omitempty"` - Capabilities []byte `protobuf:"bytes,11,opt,name=capabilities,proto3" json:"capabilities,omitempty"` + State *ClientState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Products []string `protobuf:"bytes,3,rep,name=products,proto3" json:"products,omitempty"` + IsTracer bool `protobuf:"varint,6,opt,name=is_tracer,json=isTracer,proto3" json:"is_tracer,omitempty"` + ClientTracer *ClientTracer `protobuf:"bytes,7,opt,name=client_tracer,json=clientTracer,proto3" json:"client_tracer,omitempty"` + IsAgent bool `protobuf:"varint,8,opt,name=is_agent,json=isAgent,proto3" json:"is_agent,omitempty"` + ClientAgent *ClientAgent `protobuf:"bytes,9,opt,name=client_agent,json=clientAgent,proto3" json:"client_agent,omitempty"` + LastSeen uint64 `protobuf:"varint,10,opt,name=last_seen,json=lastSeen,proto3" json:"last_seen,omitempty"` + Capabilities []byte `protobuf:"bytes,11,opt,name=capabilities,proto3" json:"capabilities,omitempty"` + IsUpdater bool `protobuf:"varint,14,opt,name=is_updater,json=isUpdater,proto3" json:"is_updater,omitempty"` + ClientUpdater *ClientUpdater `protobuf:"bytes,15,opt,name=client_updater,json=clientUpdater,proto3" json:"client_updater,omitempty"` } func (x *Client) Reset() { @@ -771,6 +831,20 @@ func (x *Client) GetCapabilities() []byte { return nil } +func (x *Client) GetIsUpdater() bool { + if x != nil { + return x.IsUpdater + } + return false +} + +func (x *Client) GetClientUpdater() *ClientUpdater { + if x != nil { + return x.ClientUpdater + } + return nil +} + type ClientTracer struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -953,6 +1027,148 @@ func (x *ClientAgent) GetCwsWorkloads() []string { return nil } +type ClientUpdater struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Tags []string `protobuf:"bytes,1,rep,name=tags,proto3" json:"tags,omitempty"` + Packages []*PackageState `protobuf:"bytes,2,rep,name=packages,proto3" json:"packages,omitempty"` +} + +func (x *ClientUpdater) Reset() { + *x = ClientUpdater{} + if protoimpl.UnsafeEnabled { + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClientUpdater) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientUpdater) ProtoMessage() {} + +func (x *ClientUpdater) ProtoReflect() protoreflect.Message { + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientUpdater.ProtoReflect.Descriptor instead. +func (*ClientUpdater) Descriptor() ([]byte, []int) { + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{12} +} + +func (x *ClientUpdater) GetTags() []string { + if x != nil { + return x.Tags + } + return nil +} + +func (x *ClientUpdater) GetPackages() []*PackageState { + if x != nil { + return x.Packages + } + return nil +} + +type PackageState struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Package string `protobuf:"bytes,1,opt,name=package,proto3" json:"package,omitempty"` + StableVersion string `protobuf:"bytes,2,opt,name=stable_version,json=stableVersion,proto3" json:"stable_version,omitempty"` + ExperimentVersion string `protobuf:"bytes,3,opt,name=experiment_version,json=experimentVersion,proto3" json:"experiment_version,omitempty"` + ExperimentState ExperimentState `protobuf:"varint,4,opt,name=experiment_state,json=experimentState,proto3,enum=datadog.config.ExperimentState" json:"experiment_state,omitempty"` + ExperimentErrorCode uint64 `protobuf:"varint,5,opt,name=experiment_error_code,json=experimentErrorCode,proto3" json:"experiment_error_code,omitempty"` + ExperimentErrorMessage string `protobuf:"bytes,6,opt,name=experiment_error_message,json=experimentErrorMessage,proto3" json:"experiment_error_message,omitempty"` +} + +func (x *PackageState) Reset() { + *x = PackageState{} + if protoimpl.UnsafeEnabled { + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PackageState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PackageState) ProtoMessage() {} + +func (x *PackageState) ProtoReflect() protoreflect.Message { + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PackageState.ProtoReflect.Descriptor instead. +func (*PackageState) Descriptor() ([]byte, []int) { + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{13} +} + +func (x *PackageState) GetPackage() string { + if x != nil { + return x.Package + } + return "" +} + +func (x *PackageState) GetStableVersion() string { + if x != nil { + return x.StableVersion + } + return "" +} + +func (x *PackageState) GetExperimentVersion() string { + if x != nil { + return x.ExperimentVersion + } + return "" +} + +func (x *PackageState) GetExperimentState() ExperimentState { + if x != nil { + return x.ExperimentState + } + return ExperimentState_NO_EXPERIMENT +} + +func (x *PackageState) GetExperimentErrorCode() uint64 { + if x != nil { + return x.ExperimentErrorCode + } + return 0 +} + +func (x *PackageState) GetExperimentErrorMessage() string { + if x != nil { + return x.ExperimentErrorMessage + } + return "" +} + type ConfigState struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -968,7 +1184,7 @@ type ConfigState struct { func (x *ConfigState) Reset() { *x = ConfigState{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[12] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -981,7 +1197,7 @@ func (x *ConfigState) String() string { func (*ConfigState) ProtoMessage() {} func (x *ConfigState) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[12] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -994,7 +1210,7 @@ func (x *ConfigState) ProtoReflect() protoreflect.Message { // Deprecated: Use ConfigState.ProtoReflect.Descriptor instead. func (*ConfigState) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{12} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{14} } func (x *ConfigState) GetId() string { @@ -1048,7 +1264,7 @@ type ClientState struct { func (x *ClientState) Reset() { *x = ClientState{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[13] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1061,7 +1277,7 @@ func (x *ClientState) String() string { func (*ClientState) ProtoMessage() {} func (x *ClientState) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[13] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1074,7 +1290,7 @@ func (x *ClientState) ProtoReflect() protoreflect.Message { // Deprecated: Use ClientState.ProtoReflect.Descriptor instead. func (*ClientState) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{13} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{15} } func (x *ClientState) GetRootVersion() uint64 { @@ -1131,7 +1347,7 @@ type TargetFileHash struct { func (x *TargetFileHash) Reset() { *x = TargetFileHash{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[14] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1144,7 +1360,7 @@ func (x *TargetFileHash) String() string { func (*TargetFileHash) ProtoMessage() {} func (x *TargetFileHash) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[14] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1157,7 +1373,7 @@ func (x *TargetFileHash) ProtoReflect() protoreflect.Message { // Deprecated: Use TargetFileHash.ProtoReflect.Descriptor instead. func (*TargetFileHash) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{14} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{16} } func (x *TargetFileHash) GetAlgorithm() string { @@ -1187,7 +1403,7 @@ type TargetFileMeta struct { func (x *TargetFileMeta) Reset() { *x = TargetFileMeta{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[15] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1200,7 +1416,7 @@ func (x *TargetFileMeta) String() string { func (*TargetFileMeta) ProtoMessage() {} func (x *TargetFileMeta) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[15] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1213,7 +1429,7 @@ func (x *TargetFileMeta) ProtoReflect() protoreflect.Message { // Deprecated: Use TargetFileMeta.ProtoReflect.Descriptor instead. func (*TargetFileMeta) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{15} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{17} } func (x *TargetFileMeta) GetPath() string { @@ -1249,7 +1465,7 @@ type ClientGetConfigsRequest struct { func (x *ClientGetConfigsRequest) Reset() { *x = ClientGetConfigsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[16] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1262,7 +1478,7 @@ func (x *ClientGetConfigsRequest) String() string { func (*ClientGetConfigsRequest) ProtoMessage() {} func (x *ClientGetConfigsRequest) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[16] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[18] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1275,7 +1491,7 @@ func (x *ClientGetConfigsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ClientGetConfigsRequest.ProtoReflect.Descriptor instead. func (*ClientGetConfigsRequest) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{16} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{18} } func (x *ClientGetConfigsRequest) GetClient() *Client { @@ -1306,7 +1522,7 @@ type ClientGetConfigsResponse struct { func (x *ClientGetConfigsResponse) Reset() { *x = ClientGetConfigsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[17] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1319,7 +1535,7 @@ func (x *ClientGetConfigsResponse) String() string { func (*ClientGetConfigsResponse) ProtoMessage() {} func (x *ClientGetConfigsResponse) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[17] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[19] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1332,7 +1548,7 @@ func (x *ClientGetConfigsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ClientGetConfigsResponse.ProtoReflect.Descriptor instead. func (*ClientGetConfigsResponse) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{17} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{19} } func (x *ClientGetConfigsResponse) GetRoots() [][]byte { @@ -1375,7 +1591,7 @@ type FileMetaState struct { func (x *FileMetaState) Reset() { *x = FileMetaState{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[18] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1388,7 +1604,7 @@ func (x *FileMetaState) String() string { func (*FileMetaState) ProtoMessage() {} func (x *FileMetaState) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[18] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[20] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1401,7 +1617,7 @@ func (x *FileMetaState) ProtoReflect() protoreflect.Message { // Deprecated: Use FileMetaState.ProtoReflect.Descriptor instead. func (*FileMetaState) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{18} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{20} } func (x *FileMetaState) GetVersion() uint64 { @@ -1432,7 +1648,7 @@ type GetStateConfigResponse struct { func (x *GetStateConfigResponse) Reset() { *x = GetStateConfigResponse{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[19] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1445,7 +1661,7 @@ func (x *GetStateConfigResponse) String() string { func (*GetStateConfigResponse) ProtoMessage() {} func (x *GetStateConfigResponse) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[19] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[21] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1458,7 +1674,7 @@ func (x *GetStateConfigResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetStateConfigResponse.ProtoReflect.Descriptor instead. func (*GetStateConfigResponse) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{19} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{21} } func (x *GetStateConfigResponse) GetConfigState() map[string]*FileMetaState { @@ -1506,7 +1722,7 @@ type TracerPredicateV1 struct { func (x *TracerPredicateV1) Reset() { *x = TracerPredicateV1{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[20] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1519,7 +1735,7 @@ func (x *TracerPredicateV1) String() string { func (*TracerPredicateV1) ProtoMessage() {} func (x *TracerPredicateV1) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[20] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[22] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1532,7 +1748,7 @@ func (x *TracerPredicateV1) ProtoReflect() protoreflect.Message { // Deprecated: Use TracerPredicateV1.ProtoReflect.Descriptor instead. func (*TracerPredicateV1) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{20} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{22} } func (x *TracerPredicateV1) GetClientID() string { @@ -1595,7 +1811,7 @@ type TracerPredicates struct { func (x *TracerPredicates) Reset() { *x = TracerPredicates{} if protoimpl.UnsafeEnabled { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[21] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1608,7 +1824,7 @@ func (x *TracerPredicates) String() string { func (*TracerPredicates) ProtoMessage() {} func (x *TracerPredicates) ProtoReflect() protoreflect.Message { - mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[21] + mi := &file_datadog_remoteconfig_remoteconfig_proto_msgTypes[23] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1621,7 +1837,7 @@ func (x *TracerPredicates) ProtoReflect() protoreflect.Message { // Deprecated: Use TracerPredicates.ProtoReflect.Descriptor instead. func (*TracerPredicates) Descriptor() ([]byte, []int) { - return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{21} + return file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP(), []int{23} } func (x *TracerPredicates) GetTracerPredicatesV1() []*TracerPredicateV1 { @@ -1740,7 +1956,7 @@ var file_datadog_remoteconfig_remoteconfig_proto_rawDesc = []byte{ 0x18, 0x0a, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, - 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x22, 0xef, 0x02, 0x0a, 0x06, 0x43, 0x6c, + 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x22, 0xe0, 0x03, 0x0a, 0x06, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x31, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, @@ -1762,160 +1978,199 @@ var file_datadog_remoteconfig_remoteconfig_proto_rawDesc = []byte{ 0x0a, 0x09, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x65, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x4a, - 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0xf8, 0x01, 0x0a, 0x0c, - 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, - 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x09, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6c, - 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, - 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x72, 0x61, 0x63, 0x65, - 0x72, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0d, 0x74, 0x72, 0x61, 0x63, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, - 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x78, 0x74, 0x72, - 0x61, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0d, 0x65, 0x78, 0x74, 0x72, 0x61, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, - 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x76, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x65, 0x6e, - 0x76, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x70, 0x70, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x70, 0x70, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0xa2, 0x01, 0x0a, 0x0b, 0x43, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x77, 0x73, 0x5f, 0x77, 0x6f, - 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, - 0x77, 0x73, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x22, 0x93, 0x01, 0x0a, 0x0b, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x12, - 0x1f, 0x0a, 0x0b, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x22, 0x80, 0x02, 0x0a, 0x0b, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x72, 0x6f, 0x6f, 0x74, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x5f, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x40, 0x0a, - 0x0d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, - 0x1b, 0x0a, 0x09, 0x68, 0x61, 0x73, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x08, 0x68, 0x61, 0x73, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x12, 0x30, 0x0a, 0x14, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x5f, 0x63, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x12, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x22, 0x48, 0x0a, 0x0e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, - 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6c, 0x67, 0x6f, 0x72, 0x69, - 0x74, 0x68, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x6c, 0x67, 0x6f, 0x72, - 0x69, 0x74, 0x68, 0x6d, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0x74, - 0x0a, 0x0e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x36, 0x0a, 0x06, - 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, - 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x52, 0x06, 0x68, 0x61, - 0x73, 0x68, 0x65, 0x73, 0x22, 0x99, 0x01, 0x0a, 0x17, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, - 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x2e, 0x0a, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x12, 0x4e, 0x0a, 0x13, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, - 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x11, 0x63, - 0x61, 0x63, 0x68, 0x65, 0x64, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, - 0x22, 0xaa, 0x01, 0x0a, 0x18, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, - 0x05, 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x05, 0x72, 0x6f, - 0x6f, 0x74, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x37, 0x0a, - 0x0c, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x22, 0x3d, 0x0a, - 0x0d, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, - 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x22, 0x81, 0x05, 0x0a, - 0x16, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, - 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x47, - 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x60, 0x0a, 0x0e, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x64, 0x61, - 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x47, 0x65, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x66, 0x0a, 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, - 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x3b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, - 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x3d, 0x0a, - 0x0e, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x0d, 0x61, - 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0x5d, 0x0a, 0x10, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x33, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5f, 0x0a, 0x12, 0x44, + 0x0c, 0x52, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, + 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72, 0x18, 0x0e, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72, 0x12, 0x44, + 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72, + 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x72, 0x52, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x72, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, + 0x4a, 0x04, 0x08, 0x0c, 0x10, 0x0d, 0x4a, 0x04, 0x08, 0x0d, 0x10, 0x0e, 0x22, 0xf8, 0x01, 0x0a, + 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x72, 0x12, 0x1d, 0x0a, + 0x0a, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, + 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x72, 0x61, 0x63, + 0x65, 0x72, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0d, 0x74, 0x72, 0x61, 0x63, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x78, 0x74, + 0x72, 0x61, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0d, 0x65, 0x78, 0x74, 0x72, 0x61, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x76, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x65, + 0x6e, 0x76, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x70, 0x70, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, 0x70, 0x70, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0xa2, 0x01, 0x0a, 0x0b, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x77, 0x73, 0x5f, 0x77, + 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, + 0x63, 0x77, 0x73, 0x57, 0x6f, 0x72, 0x6b, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x22, 0x5d, 0x0a, 0x0d, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, + 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, + 0x73, 0x12, 0x38, 0x0a, 0x08, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x08, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x22, 0xb8, 0x02, 0x0a, 0x0c, + 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, + 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x73, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2d, 0x0a, + 0x12, 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x65, 0x78, 0x70, 0x65, 0x72, + 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4a, 0x0a, 0x10, + 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, + 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0f, 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x65, 0x78, 0x70, 0x65, + 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x13, 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, + 0x65, 0x6e, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x38, 0x0a, 0x18, + 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, + 0x65, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x64, 0x75, 0x63, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x70, + 0x70, 0x6c, 0x79, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0a, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x61, + 0x70, 0x70, 0x6c, 0x79, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0a, 0x61, 0x70, 0x70, 0x6c, 0x79, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x80, 0x02, 0x0a, + 0x0b, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x72, 0x6f, 0x6f, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0b, 0x72, 0x6f, 0x6f, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x27, 0x0a, 0x0f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x40, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x1b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0c, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x68, 0x61, + 0x73, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x68, + 0x61, 0x73, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x30, 0x0a, + 0x14, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x62, 0x61, 0x63, + 0x6b, 0x65, 0x6e, 0x64, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, + 0x48, 0x0a, 0x0e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, + 0x68, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, + 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, + 0x61, 0x73, 0x68, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0x74, 0x0a, 0x0e, 0x54, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, + 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x36, 0x0a, 0x06, 0x68, 0x61, 0x73, 0x68, 0x65, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, + 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, + 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x52, 0x06, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, 0x22, + 0x99, 0x01, 0x0a, 0x17, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x06, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x64, 0x61, + 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x52, 0x06, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x4e, 0x0a, 0x13, 0x63, + 0x61, 0x63, 0x68, 0x65, 0x64, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x66, 0x69, 0x6c, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, + 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x11, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, + 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x22, 0xaa, 0x01, 0x0a, 0x18, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x6f, 0x6f, 0x74, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x05, 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x37, 0x0a, 0x0c, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, + 0x46, 0x69, 0x6c, 0x65, 0x52, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, + 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x73, 0x22, 0x3d, 0x0a, 0x0d, 0x46, 0x69, 0x6c, 0x65, + 0x4d, 0x65, 0x74, 0x61, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x22, 0x81, 0x05, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, + 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, + 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x60, + 0x0a, 0x0e, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, + 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x33, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x42, 0x0a, 0x14, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0xeb, 0x01, 0x0a, 0x11, 0x54, 0x72, 0x61, 0x63, 0x65, 0x72, 0x50, 0x72, 0x65, 0x64, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x56, 0x31, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, - 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1e, - 0x0a, 0x0a, 0x61, 0x70, 0x70, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x61, 0x70, 0x70, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x24, - 0x0a, 0x0d, 0x74, 0x72, 0x61, 0x63, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x72, 0x61, 0x63, 0x65, 0x72, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, - 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x49, 0x44, 0x22, 0x67, - 0x0a, 0x10, 0x54, 0x72, 0x61, 0x63, 0x65, 0x72, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, - 0x65, 0x73, 0x12, 0x53, 0x0a, 0x14, 0x74, 0x72, 0x61, 0x63, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x65, - 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x31, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x21, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x2e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x72, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, - 0x65, 0x56, 0x31, 0x52, 0x12, 0x74, 0x72, 0x61, 0x63, 0x65, 0x72, 0x50, 0x72, 0x65, 0x64, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x73, 0x56, 0x31, 0x42, 0x15, 0x5a, 0x13, 0x70, 0x6b, 0x67, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x67, 0x6f, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x79, 0x52, 0x0d, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x66, 0x0a, 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x6e, + 0x61, 0x6d, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x64, 0x61, 0x74, + 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, + 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x46, + 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x3d, 0x0a, 0x0e, 0x61, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x52, 0x0d, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0x5d, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x33, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x64, + 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x46, 0x69, + 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5f, 0x0a, 0x12, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x33, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x64, 0x61, 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x42, 0x0a, 0x14, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x46, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xeb, 0x01, 0x0a, 0x11, + 0x54, 0x72, 0x61, 0x63, 0x65, 0x72, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x56, + 0x31, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x49, 0x44, 0x12, 0x18, 0x0a, + 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x61, 0x70, 0x70, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x61, + 0x70, 0x70, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x74, 0x72, 0x61, + 0x63, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0d, 0x74, 0x72, 0x61, 0x63, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1a, 0x0a, 0x08, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x72, + 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x49, 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x49, 0x44, 0x22, 0x67, 0x0a, 0x10, 0x54, 0x72, 0x61, + 0x63, 0x65, 0x72, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, 0x12, 0x53, 0x0a, + 0x14, 0x74, 0x72, 0x61, 0x63, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, + 0x65, 0x73, 0x5f, 0x76, 0x31, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x64, 0x61, + 0x74, 0x61, 0x64, 0x6f, 0x67, 0x2e, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x54, 0x72, 0x61, + 0x63, 0x65, 0x72, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x56, 0x31, 0x52, 0x12, + 0x74, 0x72, 0x61, 0x63, 0x65, 0x72, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x73, + 0x56, 0x31, 0x2a, 0x6a, 0x0a, 0x0f, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x4e, 0x4f, 0x5f, 0x45, 0x58, 0x50, 0x45, + 0x52, 0x49, 0x4d, 0x45, 0x4e, 0x54, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x4f, 0x57, 0x4e, + 0x4c, 0x4f, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x49, 0x4e, 0x53, + 0x54, 0x41, 0x4c, 0x4c, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x45, 0x4e, + 0x44, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, + 0x47, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x05, 0x42, 0x15, + 0x5a, 0x13, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x62, 0x67, 0x6f, + 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1930,68 +2185,75 @@ func file_datadog_remoteconfig_remoteconfig_proto_rawDescGZIP() []byte { return file_datadog_remoteconfig_remoteconfig_proto_rawDescData } -var file_datadog_remoteconfig_remoteconfig_proto_msgTypes = make([]protoimpl.MessageInfo, 25) +var file_datadog_remoteconfig_remoteconfig_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_datadog_remoteconfig_remoteconfig_proto_msgTypes = make([]protoimpl.MessageInfo, 27) var file_datadog_remoteconfig_remoteconfig_proto_goTypes = []interface{}{ - (*ConfigMetas)(nil), // 0: datadog.config.ConfigMetas - (*DirectorMetas)(nil), // 1: datadog.config.DirectorMetas - (*DelegatedMeta)(nil), // 2: datadog.config.DelegatedMeta - (*TopMeta)(nil), // 3: datadog.config.TopMeta - (*File)(nil), // 4: datadog.config.File - (*LatestConfigsRequest)(nil), // 5: datadog.config.LatestConfigsRequest - (*LatestConfigsResponse)(nil), // 6: datadog.config.LatestConfigsResponse - (*OrgDataResponse)(nil), // 7: datadog.config.OrgDataResponse - (*OrgStatusResponse)(nil), // 8: datadog.config.OrgStatusResponse - (*Client)(nil), // 9: datadog.config.Client - (*ClientTracer)(nil), // 10: datadog.config.ClientTracer - (*ClientAgent)(nil), // 11: datadog.config.ClientAgent - (*ConfigState)(nil), // 12: datadog.config.ConfigState - (*ClientState)(nil), // 13: datadog.config.ClientState - (*TargetFileHash)(nil), // 14: datadog.config.TargetFileHash - (*TargetFileMeta)(nil), // 15: datadog.config.TargetFileMeta - (*ClientGetConfigsRequest)(nil), // 16: datadog.config.ClientGetConfigsRequest - (*ClientGetConfigsResponse)(nil), // 17: datadog.config.ClientGetConfigsResponse - (*FileMetaState)(nil), // 18: datadog.config.FileMetaState - (*GetStateConfigResponse)(nil), // 19: datadog.config.GetStateConfigResponse - (*TracerPredicateV1)(nil), // 20: datadog.config.TracerPredicateV1 - (*TracerPredicates)(nil), // 21: datadog.config.TracerPredicates - nil, // 22: datadog.config.GetStateConfigResponse.ConfigStateEntry - nil, // 23: datadog.config.GetStateConfigResponse.DirectorStateEntry - nil, // 24: datadog.config.GetStateConfigResponse.TargetFilenamesEntry + (ExperimentState)(0), // 0: datadog.config.ExperimentState + (*ConfigMetas)(nil), // 1: datadog.config.ConfigMetas + (*DirectorMetas)(nil), // 2: datadog.config.DirectorMetas + (*DelegatedMeta)(nil), // 3: datadog.config.DelegatedMeta + (*TopMeta)(nil), // 4: datadog.config.TopMeta + (*File)(nil), // 5: datadog.config.File + (*LatestConfigsRequest)(nil), // 6: datadog.config.LatestConfigsRequest + (*LatestConfigsResponse)(nil), // 7: datadog.config.LatestConfigsResponse + (*OrgDataResponse)(nil), // 8: datadog.config.OrgDataResponse + (*OrgStatusResponse)(nil), // 9: datadog.config.OrgStatusResponse + (*Client)(nil), // 10: datadog.config.Client + (*ClientTracer)(nil), // 11: datadog.config.ClientTracer + (*ClientAgent)(nil), // 12: datadog.config.ClientAgent + (*ClientUpdater)(nil), // 13: datadog.config.ClientUpdater + (*PackageState)(nil), // 14: datadog.config.PackageState + (*ConfigState)(nil), // 15: datadog.config.ConfigState + (*ClientState)(nil), // 16: datadog.config.ClientState + (*TargetFileHash)(nil), // 17: datadog.config.TargetFileHash + (*TargetFileMeta)(nil), // 18: datadog.config.TargetFileMeta + (*ClientGetConfigsRequest)(nil), // 19: datadog.config.ClientGetConfigsRequest + (*ClientGetConfigsResponse)(nil), // 20: datadog.config.ClientGetConfigsResponse + (*FileMetaState)(nil), // 21: datadog.config.FileMetaState + (*GetStateConfigResponse)(nil), // 22: datadog.config.GetStateConfigResponse + (*TracerPredicateV1)(nil), // 23: datadog.config.TracerPredicateV1 + (*TracerPredicates)(nil), // 24: datadog.config.TracerPredicates + nil, // 25: datadog.config.GetStateConfigResponse.ConfigStateEntry + nil, // 26: datadog.config.GetStateConfigResponse.DirectorStateEntry + nil, // 27: datadog.config.GetStateConfigResponse.TargetFilenamesEntry } var file_datadog_remoteconfig_remoteconfig_proto_depIdxs = []int32{ - 3, // 0: datadog.config.ConfigMetas.roots:type_name -> datadog.config.TopMeta - 3, // 1: datadog.config.ConfigMetas.timestamp:type_name -> datadog.config.TopMeta - 3, // 2: datadog.config.ConfigMetas.snapshot:type_name -> datadog.config.TopMeta - 3, // 3: datadog.config.ConfigMetas.topTargets:type_name -> datadog.config.TopMeta - 2, // 4: datadog.config.ConfigMetas.delegatedTargets:type_name -> datadog.config.DelegatedMeta - 3, // 5: datadog.config.DirectorMetas.roots:type_name -> datadog.config.TopMeta - 3, // 6: datadog.config.DirectorMetas.timestamp:type_name -> datadog.config.TopMeta - 3, // 7: datadog.config.DirectorMetas.snapshot:type_name -> datadog.config.TopMeta - 3, // 8: datadog.config.DirectorMetas.targets:type_name -> datadog.config.TopMeta - 9, // 9: datadog.config.LatestConfigsRequest.active_clients:type_name -> datadog.config.Client - 0, // 10: datadog.config.LatestConfigsResponse.config_metas:type_name -> datadog.config.ConfigMetas - 1, // 11: datadog.config.LatestConfigsResponse.director_metas:type_name -> datadog.config.DirectorMetas - 4, // 12: datadog.config.LatestConfigsResponse.target_files:type_name -> datadog.config.File - 13, // 13: datadog.config.Client.state:type_name -> datadog.config.ClientState - 10, // 14: datadog.config.Client.client_tracer:type_name -> datadog.config.ClientTracer - 11, // 15: datadog.config.Client.client_agent:type_name -> datadog.config.ClientAgent - 12, // 16: datadog.config.ClientState.config_states:type_name -> datadog.config.ConfigState - 14, // 17: datadog.config.TargetFileMeta.hashes:type_name -> datadog.config.TargetFileHash - 9, // 18: datadog.config.ClientGetConfigsRequest.client:type_name -> datadog.config.Client - 15, // 19: datadog.config.ClientGetConfigsRequest.cached_target_files:type_name -> datadog.config.TargetFileMeta - 4, // 20: datadog.config.ClientGetConfigsResponse.target_files:type_name -> datadog.config.File - 22, // 21: datadog.config.GetStateConfigResponse.config_state:type_name -> datadog.config.GetStateConfigResponse.ConfigStateEntry - 23, // 22: datadog.config.GetStateConfigResponse.director_state:type_name -> datadog.config.GetStateConfigResponse.DirectorStateEntry - 24, // 23: datadog.config.GetStateConfigResponse.target_filenames:type_name -> datadog.config.GetStateConfigResponse.TargetFilenamesEntry - 9, // 24: datadog.config.GetStateConfigResponse.active_clients:type_name -> datadog.config.Client - 20, // 25: datadog.config.TracerPredicates.tracer_predicates_v1:type_name -> datadog.config.TracerPredicateV1 - 18, // 26: datadog.config.GetStateConfigResponse.ConfigStateEntry.value:type_name -> datadog.config.FileMetaState - 18, // 27: datadog.config.GetStateConfigResponse.DirectorStateEntry.value:type_name -> datadog.config.FileMetaState - 28, // [28:28] is the sub-list for method output_type - 28, // [28:28] is the sub-list for method input_type - 28, // [28:28] is the sub-list for extension type_name - 28, // [28:28] is the sub-list for extension extendee - 0, // [0:28] is the sub-list for field type_name + 4, // 0: datadog.config.ConfigMetas.roots:type_name -> datadog.config.TopMeta + 4, // 1: datadog.config.ConfigMetas.timestamp:type_name -> datadog.config.TopMeta + 4, // 2: datadog.config.ConfigMetas.snapshot:type_name -> datadog.config.TopMeta + 4, // 3: datadog.config.ConfigMetas.topTargets:type_name -> datadog.config.TopMeta + 3, // 4: datadog.config.ConfigMetas.delegatedTargets:type_name -> datadog.config.DelegatedMeta + 4, // 5: datadog.config.DirectorMetas.roots:type_name -> datadog.config.TopMeta + 4, // 6: datadog.config.DirectorMetas.timestamp:type_name -> datadog.config.TopMeta + 4, // 7: datadog.config.DirectorMetas.snapshot:type_name -> datadog.config.TopMeta + 4, // 8: datadog.config.DirectorMetas.targets:type_name -> datadog.config.TopMeta + 10, // 9: datadog.config.LatestConfigsRequest.active_clients:type_name -> datadog.config.Client + 1, // 10: datadog.config.LatestConfigsResponse.config_metas:type_name -> datadog.config.ConfigMetas + 2, // 11: datadog.config.LatestConfigsResponse.director_metas:type_name -> datadog.config.DirectorMetas + 5, // 12: datadog.config.LatestConfigsResponse.target_files:type_name -> datadog.config.File + 16, // 13: datadog.config.Client.state:type_name -> datadog.config.ClientState + 11, // 14: datadog.config.Client.client_tracer:type_name -> datadog.config.ClientTracer + 12, // 15: datadog.config.Client.client_agent:type_name -> datadog.config.ClientAgent + 13, // 16: datadog.config.Client.client_updater:type_name -> datadog.config.ClientUpdater + 14, // 17: datadog.config.ClientUpdater.packages:type_name -> datadog.config.PackageState + 0, // 18: datadog.config.PackageState.experiment_state:type_name -> datadog.config.ExperimentState + 15, // 19: datadog.config.ClientState.config_states:type_name -> datadog.config.ConfigState + 17, // 20: datadog.config.TargetFileMeta.hashes:type_name -> datadog.config.TargetFileHash + 10, // 21: datadog.config.ClientGetConfigsRequest.client:type_name -> datadog.config.Client + 18, // 22: datadog.config.ClientGetConfigsRequest.cached_target_files:type_name -> datadog.config.TargetFileMeta + 5, // 23: datadog.config.ClientGetConfigsResponse.target_files:type_name -> datadog.config.File + 25, // 24: datadog.config.GetStateConfigResponse.config_state:type_name -> datadog.config.GetStateConfigResponse.ConfigStateEntry + 26, // 25: datadog.config.GetStateConfigResponse.director_state:type_name -> datadog.config.GetStateConfigResponse.DirectorStateEntry + 27, // 26: datadog.config.GetStateConfigResponse.target_filenames:type_name -> datadog.config.GetStateConfigResponse.TargetFilenamesEntry + 10, // 27: datadog.config.GetStateConfigResponse.active_clients:type_name -> datadog.config.Client + 23, // 28: datadog.config.TracerPredicates.tracer_predicates_v1:type_name -> datadog.config.TracerPredicateV1 + 21, // 29: datadog.config.GetStateConfigResponse.ConfigStateEntry.value:type_name -> datadog.config.FileMetaState + 21, // 30: datadog.config.GetStateConfigResponse.DirectorStateEntry.value:type_name -> datadog.config.FileMetaState + 31, // [31:31] is the sub-list for method output_type + 31, // [31:31] is the sub-list for method input_type + 31, // [31:31] is the sub-list for extension type_name + 31, // [31:31] is the sub-list for extension extendee + 0, // [0:31] is the sub-list for field type_name } func init() { file_datadog_remoteconfig_remoteconfig_proto_init() } @@ -2145,7 +2407,7 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ConfigState); i { + switch v := v.(*ClientUpdater); i { case 0: return &v.state case 1: @@ -2157,7 +2419,7 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ClientState); i { + switch v := v.(*PackageState); i { case 0: return &v.state case 1: @@ -2169,7 +2431,7 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TargetFileHash); i { + switch v := v.(*ConfigState); i { case 0: return &v.state case 1: @@ -2181,7 +2443,7 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TargetFileMeta); i { + switch v := v.(*ClientState); i { case 0: return &v.state case 1: @@ -2193,7 +2455,7 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ClientGetConfigsRequest); i { + switch v := v.(*TargetFileHash); i { case 0: return &v.state case 1: @@ -2205,7 +2467,7 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ClientGetConfigsResponse); i { + switch v := v.(*TargetFileMeta); i { case 0: return &v.state case 1: @@ -2217,7 +2479,7 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FileMetaState); i { + switch v := v.(*ClientGetConfigsRequest); i { case 0: return &v.state case 1: @@ -2229,7 +2491,7 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetStateConfigResponse); i { + switch v := v.(*ClientGetConfigsResponse); i { case 0: return &v.state case 1: @@ -2241,7 +2503,7 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TracerPredicateV1); i { + switch v := v.(*FileMetaState); i { case 0: return &v.state case 1: @@ -2253,6 +2515,30 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { } } file_datadog_remoteconfig_remoteconfig_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetStateConfigResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_datadog_remoteconfig_remoteconfig_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TracerPredicateV1); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_datadog_remoteconfig_remoteconfig_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*TracerPredicates); i { case 0: return &v.state @@ -2270,13 +2556,14 @@ func file_datadog_remoteconfig_remoteconfig_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_datadog_remoteconfig_remoteconfig_proto_rawDesc, - NumEnums: 0, - NumMessages: 25, + NumEnums: 1, + NumMessages: 27, NumExtensions: 0, NumServices: 0, }, GoTypes: file_datadog_remoteconfig_remoteconfig_proto_goTypes, DependencyIndexes: file_datadog_remoteconfig_remoteconfig_proto_depIdxs, + EnumInfos: file_datadog_remoteconfig_remoteconfig_proto_enumTypes, MessageInfos: file_datadog_remoteconfig_remoteconfig_proto_msgTypes, }.Build() File_datadog_remoteconfig_remoteconfig_proto = out.File diff --git a/pkg/proto/pbgo/core/remoteconfig_gen.go b/pkg/proto/pbgo/core/remoteconfig_gen.go index 825ce3ebffad4..ac9385737a4b7 100644 --- a/pkg/proto/pbgo/core/remoteconfig_gen.go +++ b/pkg/proto/pbgo/core/remoteconfig_gen.go @@ -9,9 +9,9 @@ import ( // MarshalMsg implements msgp.Marshaler func (z *Client) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 9 + // map header, size 11 // string "State" - o = append(o, 0x89, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x65) + o = append(o, 0x8b, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x65) if z.State == nil { o = msgp.AppendNil(o) } else { @@ -64,6 +64,20 @@ func (z *Client) MarshalMsg(b []byte) (o []byte, err error) { // string "Capabilities" o = append(o, 0xac, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73) o = msgp.AppendBytes(o, z.Capabilities) + // string "IsUpdater" + o = append(o, 0xa9, 0x49, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72) + o = msgp.AppendBool(o, z.IsUpdater) + // string "ClientUpdater" + o = append(o, 0xad, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x72) + if z.ClientUpdater == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.ClientUpdater.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ClientUpdater") + return + } + } return } @@ -185,6 +199,29 @@ func (z *Client) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Capabilities") return } + case "IsUpdater": + z.IsUpdater, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsUpdater") + return + } + case "ClientUpdater": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.ClientUpdater = nil + } else { + if z.ClientUpdater == nil { + z.ClientUpdater = new(ClientUpdater) + } + bts, err = z.ClientUpdater.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ClientUpdater") + return + } + } default: bts, err = msgp.Skip(bts) if err != nil { @@ -221,7 +258,12 @@ func (z *Client) Msgsize() (s int) { } else { s += z.ClientAgent.Msgsize() } - s += 9 + msgp.Uint64Size + 13 + msgp.BytesPrefixSize + len(z.Capabilities) + s += 9 + msgp.Uint64Size + 13 + msgp.BytesPrefixSize + len(z.Capabilities) + 10 + msgp.BoolSize + 14 + if z.ClientUpdater == nil { + s += msgp.NilSize + } else { + s += z.ClientUpdater.Msgsize() + } return } @@ -946,6 +988,129 @@ func (z *ClientTracer) Msgsize() (s int) { return } +// MarshalMsg implements msgp.Marshaler +func (z *ClientUpdater) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Tags" + o = append(o, 0x82, 0xa4, 0x54, 0x61, 0x67, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Tags))) + for za0001 := range z.Tags { + o = msgp.AppendString(o, z.Tags[za0001]) + } + // string "Packages" + o = append(o, 0xa8, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Packages))) + for za0002 := range z.Packages { + if z.Packages[za0002] == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Packages[za0002].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Packages", za0002) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ClientUpdater) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Tags": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] + } else { + z.Tags = make([]string, zb0002) + } + for za0001 := range z.Tags { + z.Tags[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + case "Packages": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Packages") + return + } + if cap(z.Packages) >= int(zb0003) { + z.Packages = (z.Packages)[:zb0003] + } else { + z.Packages = make([]*PackageState, zb0003) + } + for za0002 := range z.Packages { + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Packages[za0002] = nil + } else { + if z.Packages[za0002] == nil { + z.Packages[za0002] = new(PackageState) + } + bts, err = z.Packages[za0002].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Packages", za0002) + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ClientUpdater) Msgsize() (s int) { + s = 1 + 5 + msgp.ArrayHeaderSize + for za0001 := range z.Tags { + s += msgp.StringPrefixSize + len(z.Tags[za0001]) + } + s += 9 + msgp.ArrayHeaderSize + for za0002 := range z.Packages { + if z.Packages[za0002] == nil { + s += msgp.NilSize + } else { + s += z.Packages[za0002].Msgsize() + } + } + return +} + // MarshalMsg implements msgp.Marshaler func (z *ConfigMetas) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) @@ -1836,6 +2001,34 @@ func (z *DirectorMetas) Msgsize() (s int) { return } +// MarshalMsg implements msgp.Marshaler +func (z ExperimentState) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendInt32(o, int32(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ExperimentState) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 int32 + zb0001, bts, err = msgp.ReadInt32Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ExperimentState(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ExperimentState) Msgsize() (s int) { + s = msgp.Int32Size + return +} + // MarshalMsg implements msgp.Marshaler func (z *File) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) @@ -2871,6 +3064,107 @@ func (z OrgStatusResponse) Msgsize() (s int) { return } +// MarshalMsg implements msgp.Marshaler +func (z *PackageState) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "Package" + o = append(o, 0x86, 0xa7, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65) + o = msgp.AppendString(o, z.Package) + // string "StableVersion" + o = append(o, 0xad, 0x53, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = msgp.AppendString(o, z.StableVersion) + // string "ExperimentVersion" + o = append(o, 0xb1, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = msgp.AppendString(o, z.ExperimentVersion) + // string "ExperimentState" + o = append(o, 0xaf, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65) + o = msgp.AppendInt32(o, int32(z.ExperimentState)) + // string "ExperimentErrorCode" + o = append(o, 0xb3, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65) + o = msgp.AppendUint64(o, z.ExperimentErrorCode) + // string "ExperimentErrorMessage" + o = append(o, 0xb6, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65) + o = msgp.AppendString(o, z.ExperimentErrorMessage) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *PackageState) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Package": + z.Package, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Package") + return + } + case "StableVersion": + z.StableVersion, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StableVersion") + return + } + case "ExperimentVersion": + z.ExperimentVersion, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ExperimentVersion") + return + } + case "ExperimentState": + { + var zb0002 int32 + zb0002, bts, err = msgp.ReadInt32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ExperimentState") + return + } + z.ExperimentState = ExperimentState(zb0002) + } + case "ExperimentErrorCode": + z.ExperimentErrorCode, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ExperimentErrorCode") + return + } + case "ExperimentErrorMessage": + z.ExperimentErrorMessage, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ExperimentErrorMessage") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *PackageState) Msgsize() (s int) { + s = 1 + 8 + msgp.StringPrefixSize + len(z.Package) + 14 + msgp.StringPrefixSize + len(z.StableVersion) + 18 + msgp.StringPrefixSize + len(z.ExperimentVersion) + 16 + msgp.Int32Size + 20 + msgp.Uint64Size + 23 + msgp.StringPrefixSize + len(z.ExperimentErrorMessage) + return +} + // MarshalMsg implements msgp.Marshaler func (z TargetFileHash) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) diff --git a/pkg/proto/pbgo/core/remoteconfig_gen_test.go b/pkg/proto/pbgo/core/remoteconfig_gen_test.go index a1d65b7b98885..2a0a2414aeec4 100644 --- a/pkg/proto/pbgo/core/remoteconfig_gen_test.go +++ b/pkg/proto/pbgo/core/remoteconfig_gen_test.go @@ -356,6 +356,64 @@ func BenchmarkUnmarshalClientTracer(b *testing.B) { } } +func TestMarshalUnmarshalClientUpdater(t *testing.T) { + v := ClientUpdater{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgClientUpdater(b *testing.B) { + v := ClientUpdater{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgClientUpdater(b *testing.B) { + v := ClientUpdater{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalClientUpdater(b *testing.B) { + v := ClientUpdater{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshalConfigMetas(t *testing.T) { v := ConfigMetas{} bts, err := v.MarshalMsg(nil) @@ -994,6 +1052,64 @@ func BenchmarkUnmarshalOrgStatusResponse(b *testing.B) { } } +func TestMarshalUnmarshalPackageState(t *testing.T) { + v := PackageState{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgPackageState(b *testing.B) { + v := PackageState{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgPackageState(b *testing.B) { + v := PackageState{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalPackageState(b *testing.B) { + v := PackageState{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshalTargetFileHash(t *testing.T) { v := TargetFileHash{} bts, err := v.MarshalMsg(nil) diff --git a/pkg/remoteconfig/state/products.go b/pkg/remoteconfig/state/products.go index f9eb786f95323..98fceb6d66ba8 100644 --- a/pkg/remoteconfig/state/products.go +++ b/pkg/remoteconfig/state/products.go @@ -6,6 +6,9 @@ package state var validProducts = map[string]struct{}{ + ProductUpdaterCatalogDD: {}, + ProductUpdaterAgent: {}, + ProductUpdaterTask: {}, ProductAgentConfig: {}, ProductAgentTask: {}, ProductAgentIntegrations: {}, @@ -24,6 +27,12 @@ var validProducts = map[string]struct{}{ } const ( + // ProductUpdaterCatalogDD is the product used to receive the package catalog from datadog + ProductUpdaterCatalogDD = "UPDATER_CATALOG_DD" + // ProductUpdaterAgent is the product used to receive defaults versions to install + ProductUpdaterAgent = "UPDATER_AGENT" + // ProductUpdaterTask is the product used to receive tasks to execute + ProductUpdaterTask = "UPDATER_TASK" // ProductAgentConfig is to receive agent configurations, like the log level ProductAgentConfig = "AGENT_CONFIG" // ProductAgentIntegrations is to receive integrations to schedule diff --git a/pkg/security/agent/agent.go b/pkg/security/agent/agent.go index c1335b0afbf71..d65a1581894a7 100644 --- a/pkg/security/agent/agent.go +++ b/pkg/security/agent/agent.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "io" + "runtime" "sync" "time" @@ -62,8 +63,11 @@ func (rsa *RuntimeSecurityAgent) Start(reporter common.RawReporter, endpoints *c rsa.running.Store(true) // Start the system-probe events listener go rsa.StartEventListener() - // Start activity dumps listener - go rsa.StartActivityDumpListener() + + if runtime.GOOS == "linux" { + // Start activity dumps listener + go rsa.StartActivityDumpListener() + } if rsa.telemetry != nil { // Send Runtime Security Agent telemetry diff --git a/pkg/security/ebpf/c/include/helpers/approvers.h b/pkg/security/ebpf/c/include/helpers/approvers.h index cd9e0fbf0cb30..f709232c18904 100644 --- a/pkg/security/ebpf/c/include/helpers/approvers.h +++ b/pkg/security/ebpf/c/include/helpers/approvers.h @@ -216,4 +216,19 @@ int __attribute__((always_inline)) utime_approvers(struct syscall_cache_t *sysca return basename_approver(syscall, syscall->setattr.dentry, EVENT_UTIME); } +int __attribute__((always_inline)) bpf_approvers(struct syscall_cache_t *syscall) { + int pass_to_userspace = 0; + + if ((syscall->policy.flags & FLAGS) > 0) { + u32 key = 0; + u64 *cmd_bitmask = bpf_map_lookup_elem(&bpf_cmd_approvers, &key); + if (cmd_bitmask != NULL && ((1 << syscall->bpf.cmd) & *cmd_bitmask) > 0) { + monitor_event_approved(syscall->type, FLAG_APPROVER_TYPE); + pass_to_userspace = 1; + } + } + + return pass_to_userspace; +} + #endif diff --git a/pkg/security/ebpf/c/include/hooks/bpf.h b/pkg/security/ebpf/c/include/hooks/bpf.h index 40a344679bc4a..88ce743061ec6 100644 --- a/pkg/security/ebpf/c/include/hooks/bpf.h +++ b/pkg/security/ebpf/c/include/hooks/bpf.h @@ -8,6 +8,7 @@ #include "helpers/discarders.h" #include "helpers/process.h" #include "helpers/syscalls.h" +#include "helpers/approvers.h" __attribute__((always_inline)) void send_bpf_event(void *ctx, struct syscall_cache_t *syscall) { struct bpf_event_t event = { @@ -55,6 +56,7 @@ HOOK_SYSCALL_ENTRY3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, } struct syscall_cache_t syscall = { + .policy = policy, .type = EVENT_BPF, .bpf = { .cmd = cmd, @@ -73,6 +75,10 @@ __attribute__((always_inline)) int sys_bpf_ret(void *ctx, int retval) { return 0; } + if (filter_syscall(syscall, bpf_approvers)) { + return mark_as_discarded(syscall); + } + syscall->bpf.retval = retval; // save file descriptor <-> map_id mapping if applicable diff --git a/pkg/security/ebpf/c/include/maps.h b/pkg/security/ebpf/c/include/maps.h index 13e317d775806..f075cd5c2f061 100644 --- a/pkg/security/ebpf/c/include/maps.h +++ b/pkg/security/ebpf/c/include/maps.h @@ -22,6 +22,7 @@ BPF_ARRAY_MAP(open_flags_approvers, u32, 1) BPF_ARRAY_MAP(selinux_enforce_status, u16, 2) BPF_ARRAY_MAP(splice_entry_flags_approvers, u32, 1) BPF_ARRAY_MAP(splice_exit_flags_approvers, u32, 1) +BPF_ARRAY_MAP(bpf_cmd_approvers, u64, 1) BPF_ARRAY_MAP(syscalls_stats_enabled, u32, 1) BPF_HASH_MAP(activity_dumps_config, u64, struct activity_dump_config, 1) // max entries will be overridden at runtime diff --git a/pkg/security/metrics/metrics.go b/pkg/security/metrics/metrics.go index 123f15ff4f3a1..2f7d1042ab719 100644 --- a/pkg/security/metrics/metrics.go +++ b/pkg/security/metrics/metrics.go @@ -121,6 +121,13 @@ var ( // Tags: - MetricPerfBufferSortingAvgOp = newRuntimeMetric(".perf_buffer.sorting_avg_op") + // MetricPerfBufferInvalidEventsCount is the name of the metric used to count the number of invalid events retrieved from the event stream + // Tags: map, cause + MetricPerfBufferInvalidEventsCount = newRuntimeMetric(".perf_buffer.invalid_events.count") + // MetricPerfBufferInvalidEventsBytes is the name of the metric used to count the number of bytes of invalid events retrieved from the event stream + // Tags: map, cause + MetricPerfBufferInvalidEventsBytes = newRuntimeMetric(".perf_buffer.invalid_events.bytes") + // Process Resolver metrics // MetricProcessResolverCacheSize is the name of the metric used to report the size of the user space diff --git a/pkg/security/probe/doc_generator/backend_doc_gen.go b/pkg/security/probe/doc_generator/backend_doc_gen.go index d058262fa5100..df94e312a9633 100644 --- a/pkg/security/probe/doc_generator/backend_doc_gen.go +++ b/pkg/security/probe/doc_generator/backend_doc_gen.go @@ -36,6 +36,7 @@ func generateBackendJSON(output string) error { reflector.CommentMap = cleanupEasyjson(reflector.CommentMap) schema := reflector.Reflect(&serializers.EventSerializer{}) + schema.ID = "https://github.com/DataDog/datadog-agent/tree/main/pkg/security/serializers" schemaJSON, err := json.MarshalIndent(schema, "", " ") if err != nil { diff --git a/pkg/security/probe/eventstream/monitor.go b/pkg/security/probe/eventstream/monitor.go index dc80b1e4c8691..a9cb7939be5d2 100644 --- a/pkg/security/probe/eventstream/monitor.go +++ b/pkg/security/probe/eventstream/monitor.go @@ -38,6 +38,27 @@ type MapStats struct { Lost *atomic.Uint64 } +type invalidEventStats struct { + bytes *atomic.Uint64 + count *atomic.Uint64 +} + +// InvalidEventCause is an enum that represents the cause of an invalid event +type InvalidEventCause int + +const ( + // InvalidType indicates that the type of an event is invalid + InvalidType InvalidEventCause = iota + maxInvalidEventCause +) + +func (cause InvalidEventCause) String() string { + if cause < 0 || cause >= maxInvalidEventCause { + return "unknown" + } + return [...]string{"invalid_type"}[cause] +} + // NewEventStreamMapStats returns a new MapStats correctly initialized func NewEventStreamMapStats() MapStats { return MapStats{ @@ -86,6 +107,8 @@ type Monitor struct { readLostEvents map[string][]*atomic.Uint64 // sortingErrorStats holds the count of events that indicate that at least 1 event is miss ordered sortingErrorStats map[string][model.MaxKernelEventType]*atomic.Int64 + // badEventsStats tracks statistics for invalid events retrieved from the eventstream + invalidEventStats map[string][maxInvalidEventCause]*invalidEventStats // lastTimestamp is used to track the timestamp of the last event retrieved from the perf map lastTimestamp uint64 @@ -120,6 +143,7 @@ func NewEventStreamMonitor(config *config.Config, eRPC *erpc.ERPC, manager *mana kernelStats: make(map[string][][model.MaxKernelEventType]MapStats), readLostEvents: make(map[string][]*atomic.Uint64), sortingErrorStats: make(map[string][model.MaxKernelEventType]*atomic.Int64), + invalidEventStats: make(map[string][maxInvalidEventCause]*invalidEventStats), onEventLost: onEventLost, } @@ -176,6 +200,7 @@ func NewEventStreamMonitor(config *config.Config, eRPC *erpc.ERPC, manager *mana var stats, kernelStats [][model.MaxKernelEventType]MapStats var usrLostEvents []*atomic.Uint64 var sortingErrorStats [model.MaxKernelEventType]*atomic.Int64 + var invalidEventStats [maxInvalidEventCause]*invalidEventStats for i := 0; i < pbm.numCPU; i++ { stats = append(stats, initEventStreamMapStatsArray()) @@ -187,15 +212,31 @@ func NewEventStreamMonitor(config *config.Config, eRPC *erpc.ERPC, manager *mana sortingErrorStats[i] = atomic.NewInt64(0) } + for i := 0; i < int(maxInvalidEventCause); i++ { + invalidEventStats[i] = newInvalidEventStats() + } + pbm.stats[mapName] = stats pbm.kernelStats[mapName] = kernelStats pbm.readLostEvents[mapName] = usrLostEvents pbm.sortingErrorStats[mapName] = sortingErrorStats + pbm.invalidEventStats[mapName] = invalidEventStats } log.Debugf("monitoring perf ring buffer on %d CPU, %d events", pbm.numCPU, model.MaxKernelEventType) return &pbm, nil } +func newInvalidEventStats() *invalidEventStats { + return &invalidEventStats{ + bytes: atomic.NewUint64(0), + count: atomic.NewUint64(0), + } +} + +func (s *invalidEventStats) getAndReset() (uint64, uint64) { + return s.count.Swap(0), s.bytes.Swap(0) +} + func initEventStreamMapStatsArray() [model.MaxKernelEventType]MapStats { var arr [model.MaxKernelEventType]MapStats for i := 0; i < len(arr); i++ { @@ -403,6 +444,16 @@ func (pbm *Monitor) CountEvent(eventType model.EventType, timestamp uint64, coun pbm.stats[mapName][cpu][eventType].Bytes.Add(size) } +// CountInvalidEvent counts the size of one invalid event of the specified cause +func (pbm *Monitor) CountInvalidEvent(mapName string, cause InvalidEventCause, size uint64) { + // sanity check + if len(pbm.invalidEventStats[mapName]) <= int(cause) { + return + } + pbm.invalidEventStats[mapName][cause].count.Add(1) + pbm.invalidEventStats[mapName][cause].bytes.Add(size) +} + func (pbm *Monitor) sendEventsAndBytesReadStats(client statsd.ClientInterface) error { var count int64 var err error @@ -435,6 +486,24 @@ func (pbm *Monitor) sendEventsAndBytesReadStats(client statsd.ClientInterface) e } } } + + for mapName, causes := range pbm.invalidEventStats { + for cause, stats := range causes { + count, bytes := stats.getAndReset() + tags := []string{fmt.Sprintf("map:%s", mapName), fmt.Sprintf("cause:%s", InvalidEventCause(cause).String())} + if count > 0 { + if err := client.Count(metrics.MetricPerfBufferInvalidEventsCount, int64(count), tags, 1.0); err != nil { + return err + } + } + if bytes > 0 { + if err := client.Count(metrics.MetricPerfBufferInvalidEventsBytes, int64(bytes), tags, 1.0); err != nil { + return err + } + } + } + } + return nil } diff --git a/pkg/security/probe/kfilters/approvers.go b/pkg/security/probe/kfilters/approvers.go index 4c79ca8df4384..3b7637caf0fec 100644 --- a/pkg/security/probe/kfilters/approvers.go +++ b/pkg/security/probe/kfilters/approvers.go @@ -49,11 +49,19 @@ func approveBasenames(tableName string, eventType model.EventType, basenames ... return approvers, nil } -func setFlagsFilter(tableName string, flags ...int) (activeApprover, error) { - var flagsItem ebpf.Uint32MapItem +func intValues[I int32 | int64](fvs rules.FilterValues) []I { + var values []I + for _, v := range fvs { + values = append(values, I(v.Value.(int))) + } + return values +} + +func setFlagsFilter[I int32 | int64](tableName string, flags ...I) (activeApprover, error) { + var flagsItem I for _, flag := range flags { - flagsItem |= ebpf.Uint32MapItem(flag) + flagsItem |= flag } if flagsItem != 0 { @@ -61,14 +69,22 @@ func setFlagsFilter(tableName string, flags ...int) (activeApprover, error) { tableName: tableName, index: uint32(0), value: flagsItem, - zeroValue: ebpf.ZeroUint32MapItem, + zeroValue: I(0), }, nil } return nil, nil } -func approveFlags(tableName string, flags ...int) (activeApprover, error) { +func approveFlags(tableName string, flags ...int32) (activeApprover, error) { + return setFlagsFilter(tableName, flags...) +} + +func approveEnums(tableName string, enums ...int64) (activeApprover, error) { + var flags []int64 + for _, enum := range enums { + flags = append(flags, 1< model.MaxKernelEventType { + p.monitors.eventStreamMonitor.CountInvalidEvent(eventstream.EventStreamMap, eventstream.InvalidType, dataLen) seclog.Errorf("unsupported event type %d", eventType) return } diff --git a/pkg/security/rconfig/policies.go b/pkg/security/rconfig/policies.go index c9c00e224bc8f..5f118a5a56757 100644 --- a/pkg/security/rconfig/policies.go +++ b/pkg/security/rconfig/policies.go @@ -19,7 +19,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/api/security" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/config/remote/client" - "github.com/DataDog/datadog-agent/pkg/config/remote/data" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/security/secl/rules" "github.com/DataDog/datadog-agent/pkg/security/utils" @@ -58,7 +57,7 @@ func NewRCPolicyProvider() (*RCPolicyProvider, error) { c, err := client.NewGRPCClient(ipcAddress, config.GetIPCPort(), func() (string, error) { return security.FetchAuthToken(config.Datadog) }, client.WithAgent(agentName, agentVersion.String()), - client.WithProducts([]data.Product{data.ProductCWSDD, data.ProductCWSCustom}), + client.WithProducts(state.ProductCWSDD, state.ProductCWSCustom), client.WithPollInterval(securityAgentRCPollInterval), client.WithDirectorRootOverride(config.Datadog.GetString("remote_configuration.director_root")), ) diff --git a/pkg/security/rconfig/profiles.go b/pkg/security/rconfig/profiles.go index 001814db15921..5d5ec68bde67e 100644 --- a/pkg/security/rconfig/profiles.go +++ b/pkg/security/rconfig/profiles.go @@ -20,7 +20,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/api/security" "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/config/remote/client" - "github.com/DataDog/datadog-agent/pkg/config/remote/data" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" cgroupModel "github.com/DataDog/datadog-agent/pkg/security/resolvers/cgroup/model" "github.com/DataDog/datadog-agent/pkg/security/utils" @@ -151,7 +150,7 @@ func NewRCProfileProvider() (*RCProfileProvider, error) { c, err := client.NewGRPCClient(ipcAddress, config.GetIPCPort(), func() (string, error) { return security.FetchAuthToken(config.Datadog) }, client.WithAgent(agentName, agentVersion.String()), - client.WithProducts([]data.Product{data.ProductCWSProfile}), + client.WithProducts(state.ProductCWSProfiles), client.WithPollInterval(securityAgentRCPollInterval)) if err != nil { return nil, err diff --git a/pkg/security/serializers/serializers_linux_easyjson.go b/pkg/security/serializers/serializers_linux_easyjson.go index 2a52cb33ef236..42672ced87161 100644 --- a/pkg/security/serializers/serializers_linux_easyjson.go +++ b/pkg/security/serializers/serializers_linux_easyjson.go @@ -1066,31 +1066,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers10( case "source": out.Source = string(in.String()) case "variables": - if in.IsNull() { - in.Skip() - } else { - in.Delim('{') - if !in.IsDelim('}') { - out.Variables = make(Variables) - } else { - out.Variables = nil - } - for !in.IsDelim('}') { - key := string(in.String()) - in.WantColon() - var v11 interface{} - if m, ok := v11.(easyjson.Unmarshaler); ok { - m.UnmarshalEasyJSON(in) - } else if m, ok := v11.(json.Unmarshaler); ok { - _ = m.UnmarshalJSON(in.Raw()) - } else { - v11 = in.Interface() - } - (out.Variables)[key] = v11 - in.WantComma() - } - in.Delim('}') - } + (out.Variables).UnmarshalEasyJSON(in) default: in.SkipRecursive() } @@ -1221,11 +1197,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers10( out.RawString(prefix) { out.RawByte('[') - for v12, v13 := range in.Args { - if v12 > 0 { + for v11, v12 := range in.Args { + if v11 > 0 { out.RawByte(',') } - out.String(string(v13)) + out.String(string(v12)) } out.RawByte(']') } @@ -1240,11 +1216,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers10( out.RawString(prefix) { out.RawByte('[') - for v14, v15 := range in.Envs { - if v14 > 0 { + for v13, v14 := range in.Envs { + if v13 > 0 { out.RawByte(',') } - out.String(string(v15)) + out.String(string(v14)) } out.RawByte(']') } @@ -1277,27 +1253,7 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers10( if len(in.Variables) != 0 { const prefix string = ",\"variables\":" out.RawString(prefix) - { - out.RawByte('{') - v16First := true - for v16Name, v16Value := range in.Variables { - if v16First { - v16First = false - } else { - out.RawByte(',') - } - out.String(string(v16Name)) - out.RawByte(':') - if m, ok := v16Value.(easyjson.Marshaler); ok { - m.MarshalEasyJSON(out) - } else if m, ok := v16Value.(json.Marshaler); ok { - out.Raw(m.MarshalJSON()) - } else { - out.Raw(json.Marshal(v16Value)) - } - } - out.RawByte('}') - } + (in.Variables).MarshalEasyJSON(out) } out.RawByte('}') } @@ -1379,9 +1335,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers11( out.CapEffective = (out.CapEffective)[:0] } for !in.IsDelim(']') { - var v17 string - v17 = string(in.String()) - out.CapEffective = append(out.CapEffective, v17) + var v15 string + v15 = string(in.String()) + out.CapEffective = append(out.CapEffective, v15) in.WantComma() } in.Delim(']') @@ -1402,9 +1358,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers11( out.CapPermitted = (out.CapPermitted)[:0] } for !in.IsDelim(']') { - var v18 string - v18 = string(in.String()) - out.CapPermitted = append(out.CapPermitted, v18) + var v16 string + v16 = string(in.String()) + out.CapPermitted = append(out.CapPermitted, v16) in.WantComma() } in.Delim(']') @@ -1507,11 +1463,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers11( out.RawString("null") } else { out.RawByte('[') - for v19, v20 := range in.CapEffective { - if v19 > 0 { + for v17, v18 := range in.CapEffective { + if v17 > 0 { out.RawByte(',') } - out.String(string(v20)) + out.String(string(v18)) } out.RawByte(']') } @@ -1523,11 +1479,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers11( out.RawString("null") } else { out.RawByte('[') - for v21, v22 := range in.CapPermitted { - if v21 > 0 { + for v19, v20 := range in.CapPermitted { + if v19 > 0 { out.RawByte(',') } - out.String(string(v22)) + out.String(string(v20)) } out.RawByte(']') } @@ -1880,9 +1836,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15( out.Argv = (out.Argv)[:0] } for !in.IsDelim(']') { - var v23 string - v23 = string(in.String()) - out.Argv = append(out.Argv, v23) + var v21 string + v21 = string(in.String()) + out.Argv = append(out.Argv, v21) in.WantComma() } in.Delim(']') @@ -1926,11 +1882,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15( out.RawString(prefix) { out.RawByte('[') - for v24, v25 := range in.Argv { - if v24 > 0 { + for v22, v23 := range in.Argv { + if v22 > 0 { out.RawByte(',') } - out.String(string(v25)) + out.String(string(v23)) } out.RawByte(']') } @@ -2200,9 +2156,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( out.Flags = (out.Flags)[:0] } for !in.IsDelim(']') { - var v26 string - v26 = string(in.String()) - out.Flags = append(out.Flags, v26) + var v24 string + v24 = string(in.String()) + out.Flags = append(out.Flags, v24) in.WantComma() } in.Delim(']') @@ -2263,9 +2219,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( out.Hashes = (out.Hashes)[:0] } for !in.IsDelim(']') { - var v27 string - v27 = string(in.String()) - out.Hashes = append(out.Hashes, v27) + var v25 string + v25 = string(in.String()) + out.Hashes = append(out.Hashes, v25) in.WantComma() } in.Delim(']') @@ -2402,11 +2358,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( out.RawString(prefix) { out.RawByte('[') - for v28, v29 := range in.Flags { - if v28 > 0 { + for v26, v27 := range in.Flags { + if v26 > 0 { out.RawByte(',') } - out.String(string(v29)) + out.String(string(v27)) } out.RawByte(']') } @@ -2441,11 +2397,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( out.RawString(prefix) { out.RawByte('[') - for v30, v31 := range in.Hashes { - if v30 > 0 { + for v28, v29 := range in.Hashes { + if v28 > 0 { out.RawByte(',') } - out.String(string(v31)) + out.String(string(v29)) } out.RawByte(']') } @@ -2578,9 +2534,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers19( out.Flags = (out.Flags)[:0] } for !in.IsDelim(']') { - var v32 string - v32 = string(in.String()) - out.Flags = append(out.Flags, v32) + var v30 string + v30 = string(in.String()) + out.Flags = append(out.Flags, v30) in.WantComma() } in.Delim(']') @@ -2641,9 +2597,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers19( out.Hashes = (out.Hashes)[:0] } for !in.IsDelim(']') { - var v33 string - v33 = string(in.String()) - out.Hashes = append(out.Hashes, v33) + var v31 string + v31 = string(in.String()) + out.Hashes = append(out.Hashes, v31) in.WantComma() } in.Delim(']') @@ -2820,11 +2776,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers19( out.RawString(prefix) { out.RawByte('[') - for v34, v35 := range in.Flags { - if v34 > 0 { + for v32, v33 := range in.Flags { + if v32 > 0 { out.RawByte(',') } - out.String(string(v35)) + out.String(string(v33)) } out.RawByte(']') } @@ -2859,11 +2815,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers19( out.RawString(prefix) { out.RawByte('[') - for v36, v37 := range in.Hashes { - if v36 > 0 { + for v34, v35 := range in.Hashes { + if v34 > 0 { out.RawByte(',') } - out.String(string(v37)) + out.String(string(v35)) } out.RawByte(']') } @@ -3450,9 +3406,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers21( out.Tags = (out.Tags)[:0] } for !in.IsDelim(']') { - var v38 string - v38 = string(in.String()) - out.Tags = append(out.Tags, v38) + var v36 string + v36 = string(in.String()) + out.Tags = append(out.Tags, v36) in.WantComma() } in.Delim(']') @@ -3490,11 +3446,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers21( out.RawString("null") } else { out.RawByte('[') - for v39, v40 := range in.Tags { - if v39 > 0 { + for v37, v38 := range in.Tags { + if v37 > 0 { out.RawByte(',') } - out.String(string(v40)) + out.String(string(v38)) } out.RawByte(']') } @@ -3565,9 +3521,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers23( out.CapEffective = (out.CapEffective)[:0] } for !in.IsDelim(']') { - var v41 string - v41 = string(in.String()) - out.CapEffective = append(out.CapEffective, v41) + var v39 string + v39 = string(in.String()) + out.CapEffective = append(out.CapEffective, v39) in.WantComma() } in.Delim(']') @@ -3588,9 +3544,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers23( out.CapPermitted = (out.CapPermitted)[:0] } for !in.IsDelim(']') { - var v42 string - v42 = string(in.String()) - out.CapPermitted = append(out.CapPermitted, v42) + var v40 string + v40 = string(in.String()) + out.CapPermitted = append(out.CapPermitted, v40) in.WantComma() } in.Delim(']') @@ -3676,11 +3632,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers23( out.RawString("null") } else { out.RawByte('[') - for v43, v44 := range in.CapEffective { - if v43 > 0 { + for v41, v42 := range in.CapEffective { + if v41 > 0 { out.RawByte(',') } - out.String(string(v44)) + out.String(string(v42)) } out.RawByte(']') } @@ -3692,11 +3648,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers23( out.RawString("null") } else { out.RawByte('[') - for v45, v46 := range in.CapPermitted { - if v45 > 0 { + for v43, v44 := range in.CapPermitted { + if v43 > 0 { out.RawByte(',') } - out.String(string(v46)) + out.String(string(v44)) } out.RawByte(']') } @@ -3748,9 +3704,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( out.CapEffective = (out.CapEffective)[:0] } for !in.IsDelim(']') { - var v47 string - v47 = string(in.String()) - out.CapEffective = append(out.CapEffective, v47) + var v45 string + v45 = string(in.String()) + out.CapEffective = append(out.CapEffective, v45) in.WantComma() } in.Delim(']') @@ -3771,9 +3727,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( out.CapPermitted = (out.CapPermitted)[:0] } for !in.IsDelim(']') { - var v48 string - v48 = string(in.String()) - out.CapPermitted = append(out.CapPermitted, v48) + var v46 string + v46 = string(in.String()) + out.CapPermitted = append(out.CapPermitted, v46) in.WantComma() } in.Delim(']') @@ -3799,11 +3755,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( out.RawString("null") } else { out.RawByte('[') - for v49, v50 := range in.CapEffective { - if v49 > 0 { + for v47, v48 := range in.CapEffective { + if v47 > 0 { out.RawByte(',') } - out.String(string(v50)) + out.String(string(v48)) } out.RawByte(']') } @@ -3815,11 +3771,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( out.RawString("null") } else { out.RawByte('[') - for v51, v52 := range in.CapPermitted { - if v51 > 0 { + for v49, v50 := range in.CapPermitted { + if v49 > 0 { out.RawByte(',') } - out.String(string(v52)) + out.String(string(v50)) } out.RawByte(']') } @@ -3931,9 +3887,9 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers26( out.Helpers = (out.Helpers)[:0] } for !in.IsDelim(']') { - var v53 string - v53 = string(in.String()) - out.Helpers = append(out.Helpers, v53) + var v51 string + v51 = string(in.String()) + out.Helpers = append(out.Helpers, v51) in.WantComma() } in.Delim(']') @@ -3998,11 +3954,11 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers26( } { out.RawByte('[') - for v54, v55 := range in.Helpers { - if v54 > 0 { + for v52, v53 := range in.Helpers { + if v52 > 0 { out.RawByte(',') } - out.String(string(v55)) + out.String(string(v53)) } out.RawByte(']') } diff --git a/pkg/security/tests/filters_test.go b/pkg/security/tests/filters_test.go index 7d34f3474f503..7c52f6392a54b 100644 --- a/pkg/security/tests/filters_test.go +++ b/pkg/security/tests/filters_test.go @@ -18,6 +18,9 @@ import ( "time" "unsafe" + "github.com/cilium/ebpf" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/pkg/security/probe" "github.com/DataDog/datadog-agent/pkg/security/secl/compiler/eval" "github.com/DataDog/datadog-agent/pkg/security/secl/model" @@ -718,3 +721,66 @@ func TestFilterDiscarderRetention(t *testing.T) { t.Fatalf("discarder retention (%s) not reached: %s", time.Duration(uint64(probe.DiscardRetention)-uint64(time.Second)), diff) } } + +func TestFilterBpfCmd(t *testing.T) { + SkipIfNotAvailable(t) + + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + + rule := &rules.RuleDefinition{ + ID: "test_bpf_map_create", + Expression: fmt.Sprintf(`bpf.cmd == BPF_MAP_CREATE && process.file.name == "%s"`, path.Base(executable)), + } + + test, err := newTestModule(t, nil, []*rules.RuleDefinition{rule}) + if err != nil { + t.Fatal(err) + } + defer test.Close() + + var m *ebpf.Map + defer func() { + if m != nil { + m.Close() + } + }() + + test.WaitSignal(t, func() error { + m, err = ebpf.NewMap(&ebpf.MapSpec{Name: "test_bpf_map", Type: ebpf.Array, KeySize: 4, ValueSize: 4, MaxEntries: 1}) + if err != nil { + return err + } + return nil + }, func(event *model.Event, rule *rules.Rule) { + assertTriggeredRule(t, rule, "test_bpf_map_create") + }) + + err = test.GetProbeEvent(func() error { + if m.Update(uint32(0), uint32(1), ebpf.UpdateAny) != nil { + return err + } + return nil + }, func(event *model.Event) bool { + cmdIntf, err := event.GetFieldValue("bpf.cmd") + if !assert.NoError(t, err) { + return false + } + cmdInt, ok := cmdIntf.(int) + if !assert.True(t, ok) { + return false + } + cmd := model.BPFCmd(uint64(cmdInt)) + if assert.Equal(t, model.BpfMapCreateCmd, cmd, "should not get a bpf event with cmd other than BPF_MAP_CREATE") { + return false + } + return true + }, 1*time.Second, model.BPFEventType) + if err != nil { + if otherErr, ok := err.(ErrTimeout); !ok { + t.Fatal(otherErr) + } + } +} diff --git a/pkg/serializer/serializer.go b/pkg/serializer/serializer.go index 387a4f09dff53..ed9397e366d25 100644 --- a/pkg/serializer/serializer.go +++ b/pkg/serializer/serializer.go @@ -292,7 +292,7 @@ func (s *Serializer) SendEvents(events event.Events) error { return fmt.Errorf("dropping event payload: %s", err) } - return s.Forwarder.SubmitV1Intake(eventPayloads, extraHeaders) + return s.Forwarder.SubmitV1Intake(eventPayloads, transaction.Events, extraHeaders) } // SendServiceChecks serializes a list of serviceChecks and sends the payload to the forwarder @@ -439,7 +439,8 @@ func (s *Serializer) SendProcessesMetadata(data interface{}) error { if err != nil { return fmt.Errorf("could not compress processes metadata payload: %s", err) } - if err := s.Forwarder.SubmitV1Intake(transaction.NewBytesPayloadsWithoutMetaData([]*[]byte{&compressedPayload}), jsonExtraHeadersWithCompression); err != nil { + if err := s.Forwarder.SubmitV1Intake(transaction.NewBytesPayloadsWithoutMetaData([]*[]byte{&compressedPayload}), + transaction.Events, jsonExtraHeadersWithCompression); err != nil { return err } diff --git a/releasenotes-dca/notes/add_agent_sidecar_injection_webhook-cdea16f8cfd23fdd.yaml b/releasenotes-dca/notes/add_agent_sidecar_injection_webhook-cdea16f8cfd23fdd.yaml new file mode 100644 index 0000000000000..9fae2084618f6 --- /dev/null +++ b/releasenotes-dca/notes/add_agent_sidecar_injection_webhook-cdea16f8cfd23fdd.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG-DCA.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Add agent sidecar injection webhook in `cluster-agent` admission controller. diff --git a/releasenotes-dca/notes/missing-kube_api_version-tag-092f68f355285868.yaml b/releasenotes-dca/notes/missing-kube_api_version-tag-092f68f355285868.yaml new file mode 100644 index 0000000000000..b42b0d15715be --- /dev/null +++ b/releasenotes-dca/notes/missing-kube_api_version-tag-092f68f355285868.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fix missing `kube_api_version` tag on HPA and VPA resources. diff --git a/releasenotes/notes/ProcessPodCheckCleanup-9a1eabfb5c211290.yaml b/releasenotes/notes/ProcessPodCheckCleanup-9a1eabfb5c211290.yaml new file mode 100644 index 0000000000000..43d8d9649e472 --- /dev/null +++ b/releasenotes/notes/ProcessPodCheckCleanup-9a1eabfb5c211290.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +deprecations: + - | + Removal of the pod check from the process agent. The current check will run from the core agent. diff --git a/releasenotes/notes/single-step-config-9b815c41070a10f2.yaml b/releasenotes/notes/single-step-config-9b815c41070a10f2.yaml new file mode 100644 index 0000000000000..840edf0d7dfb9 --- /dev/null +++ b/releasenotes/notes/single-step-config-9b815c41070a10f2.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + APM: Add support for Single Step Instrumentation remote configuration diff --git a/repository.datadog.yml b/repository.datadog.yml index 5ea0d3f7aaa8e..dc0ac21bb6a31 100644 --- a/repository.datadog.yml +++ b/repository.datadog.yml @@ -7,6 +7,7 @@ workflow_type: speculative speculative_max_depth: 3 wait_for_check_timeout_in_minutes: 240 gitlab_jobs_retry_enable: true +skip_labels: true --- schema-version: v1 kind: buildimpactanalysis diff --git a/tasks/build_tags.py b/tasks/build_tags.py index 539b339c6489d..b762361a14a56 100644 --- a/tasks/build_tags.py +++ b/tasks/build_tags.py @@ -106,7 +106,7 @@ IOT_AGENT_TAGS = {"jetson", "otlp", "systemd", "zlib"} # PROCESS_AGENT_TAGS lists the tags necessary to build the process-agent -PROCESS_AGENT_TAGS = AGENT_TAGS.union({"fargateprocess", "orchestrator"}).difference({"otlp", "python", "trivy"}) +PROCESS_AGENT_TAGS = AGENT_TAGS.union({"fargateprocess"}).difference({"otlp", "python", "trivy"}) # PROCESS_AGENT_HEROKU_TAGS lists the tags necessary to build the process-agent for Heroku PROCESS_AGENT_HEROKU_TAGS = PROCESS_AGENT_TAGS.difference( diff --git a/tasks/go_test.py b/tasks/go_test.py index e182da3765128..626e198954f04 100644 --- a/tasks/go_test.py +++ b/tasks/go_test.py @@ -751,6 +751,7 @@ def parse_test_log(log_file): def get_impacted_packages(ctx, build_tags=None): dependencies = create_dependencies(ctx, build_tags) files = get_modified_files(ctx) + modified_packages = { f"github.com/DataDog/datadog-agent/{os.path.dirname(file)}" for file in files @@ -813,7 +814,6 @@ def format_packages(ctx, impacted_packages): for package in packages: match_precision = 0 best_module_path = None - # Since several modules can match the path we take only the most precise one for module_path in DEFAULT_MODULES: package_path = Path(package) @@ -825,7 +825,7 @@ def format_packages(ctx, impacted_packages): # Check if the package is in the target list of the module we want to test targeted = False for target in DEFAULT_MODULES[best_module_path].targets: - if os.path.normpath(os.path.join(best_module_path, target)) in package: + if normpath(os.path.join(best_module_path, target)) in package: targeted = True break if not targeted: @@ -839,7 +839,7 @@ def format_packages(ctx, impacted_packages): if not os.path.exists(package): continue - relative_target = "./" + os.path.relpath(package, best_module_path) + relative_target = "./" + os.path.relpath(package, best_module_path).replace("\\", "/") if best_module_path in modules_to_test: if ( @@ -858,10 +858,11 @@ def format_packages(ctx, impacted_packages): ): # With more packages we can reach the limit of the command line length on Windows modules_to_test[module].targets = DEFAULT_MODULES[module].targets + module_to_remove = [] # Clean up to avoid running tests on package with no Go files matching build tags for module in modules_to_test: res = ctx.run( - f"go list -tags '{' '.join(build_tags)}' {' '.join([os.path.normpath(os.path.join('github.com/DataDog/datadog-agent', module, target)) for target in modules_to_test[module].targets])}", + f"go list -tags '{' '.join(build_tags)}' {' '.join([normpath(os.path.join('github.com/DataDog/datadog-agent', module, target)) for target in modules_to_test[module].targets])}", hide=True, warn=True, ) @@ -869,14 +870,22 @@ def format_packages(ctx, impacted_packages): for package in res.stderr.splitlines(): package_to_remove = os.path.relpath( package.split(" ")[1].strip(":").replace("github.com/DataDog/datadog-agent/", ""), module - ) + ).replace("\\", "/") try: modules_to_test[module].targets.remove(f"./{package_to_remove}") + if len(modules_to_test[module].targets) == 0: + module_to_remove.append(module) except Exception: print("Could not remove ", package_to_remove, ", ignoring...") + for module in module_to_remove: + del modules_to_test[module] print("Running tests for the following modules:") for module in modules_to_test: print(f"- {module}: {modules_to_test[module].targets}") return modules_to_test.values() + + +def normpath(path): # Normpath with forward slashes to avoid issues on Windows + return os.path.normpath(path).replace("\\", "/") diff --git a/test/new-e2e/tests/containers/k8s_test.go b/test/new-e2e/tests/containers/k8s_test.go index c6f573a4a1755..f7d5379d5e34a 100644 --- a/test/new-e2e/tests/containers/k8s_test.go +++ b/test/new-e2e/tests/containers/k8s_test.go @@ -12,9 +12,12 @@ import ( "strings" "time" + "github.com/DataDog/agent-payload/v5/cyclonedx_v1_4" + "github.com/DataDog/agent-payload/v5/sbom" "github.com/DataDog/datadog-agent/pkg/util/pointer" "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" fakeintake "github.com/DataDog/datadog-agent/test/fakeintake/client" + "github.com/samber/lo" "gopkg.in/zorkian/go-datadog-api.v2" "github.com/fatih/color" @@ -802,40 +805,77 @@ func (suite *k8sSuite) TestSBOM() { return } - var sbomID string - for _, id := range sbomIDs { - if strings.HasPrefix(id, "ghcr.io/datadog/apps-nginx-server") { - sbomID = id - break - } + sbomIDs = lo.Filter(sbomIDs, func(id string, _ int) bool { + return strings.HasPrefix(id, "ghcr.io/datadog/apps-nginx-server") + }) + + // Can be replaced by require.NoEmptyf(…) once https://github.com/stretchr/testify/pull/1481 is merged + if !assert.NotEmptyf(c, sbomIDs, "No SBOM for ghcr.io/datadog/apps-nginx-server yet") { + return } + + images := lo.FlatMap(sbomIDs, func(id string, _ int) []*aggregator.SBOMPayload { + images, err := suite.Fakeintake.FilterSBOMs(id) + assert.NoErrorf(c, err, "Failed to query fake intake") + return images + }) + // Can be replaced by require.NoEmptyf(…) once https://github.com/stretchr/testify/pull/1481 is merged - if !assert.NotEmptyf(c, sbomID, "No SBOM for ghcr.io/datadog/apps-nginx-server yet") { + if !assert.NotEmptyf(c, images, "No SBOM payload yet") { return } - images, err := suite.Fakeintake.FilterSBOMs(sbomID) - // Can be replaced by require.NoErrorf(…) once https://github.com/stretchr/testify/pull/1481 is merged - if !assert.NoErrorf(c, err, "Failed to query fake intake") { + images = lo.Filter(images, func(image *aggregator.SBOMPayload, _ int) bool { + return image.Status == sbom.SBOMStatus_SUCCESS + }) + + // Can be replaced by require.NoEmptyf(…) once https://github.com/stretchr/testify/pull/1481 is merged + if !assert.NotEmptyf(c, images, "No successful SBOM yet") { return } + + images = lo.Filter(images, func(image *aggregator.SBOMPayload, _ int) bool { + cyclonedx := image.GetCyclonedx() + return cyclonedx != nil && + cyclonedx.Metadata != nil && + cyclonedx.Metadata.Component != nil + }) + // Can be replaced by require.NoEmptyf(…) once https://github.com/stretchr/testify/pull/1481 is merged - if !assert.NotEmptyf(c, images, "No SBOM yet") { + if !assert.NotEmptyf(c, images, "No SBOM with complete CycloneDX") { return } - expectedTags := []*regexp.Regexp{ - regexp.MustCompile(`^architecture:(amd|arm)64$`), - regexp.MustCompile(`^git\.commit\.sha:`), - regexp.MustCompile(`^git\.repository_url:https://github\.com/DataDog/test-infra-definitions$`), - regexp.MustCompile(`^image_id:ghcr\.io/datadog/apps-nginx-server@sha256:`), - regexp.MustCompile(`^image_name:ghcr\.io/datadog/apps-nginx-server$`), - regexp.MustCompile(`^image_tag:main$`), - regexp.MustCompile(`^os_name:linux$`), - regexp.MustCompile(`^short_image:apps-nginx-server$`), + for _, image := range images { + if !assert.NotNil(c, image.GetCyclonedx().Metadata.Component.Properties) { + continue + } + + expectedTags := []*regexp.Regexp{ + regexp.MustCompile(`^architecture:(amd|arm)64$`), + regexp.MustCompile(`^git\.commit\.sha:`), + regexp.MustCompile(`^git\.repository_url:https://github\.com/DataDog/test-infra-definitions$`), + regexp.MustCompile(`^image_id:ghcr\.io/datadog/apps-nginx-server@sha256:`), + regexp.MustCompile(`^image_name:ghcr\.io/datadog/apps-nginx-server$`), + regexp.MustCompile(`^image_tag:main$`), + regexp.MustCompile(`^os_name:linux$`), + regexp.MustCompile(`^short_image:apps-nginx-server$`), + } + err = assertTags(image.GetTags(), expectedTags) + assert.NoErrorf(c, err, "Tags mismatch") + + properties := lo.Associate(image.GetCyclonedx().Metadata.Component.Properties, func(property *cyclonedx_v1_4.Property) (string, string) { + return property.Name, *property.Value + }) + + if assert.Contains(c, properties, "aquasecurity:trivy:RepoTag") { + assert.Equal(c, "ghcr.io/datadog/apps-nginx-server:main", properties["aquasecurity:trivy:RepoTag"]) + } + + if assert.Contains(c, properties, "aquasecurity:trivy:RepoDigest") { + assert.Contains(c, properties["aquasecurity:trivy:RepoDigest"], "ghcr.io/datadog/apps-nginx-server@sha256:") + } } - err = assertTags(images[len(images)-1].GetTags(), expectedTags) - assert.NoErrorf(c, err, "Tags mismatch") }, 2*time.Minute, 10*time.Second, "Failed finding the container image payload") } diff --git a/test/new-e2e/tests/process/windows_test.go b/test/new-e2e/tests/process/windows_test.go index 00e04b3aba1b4..a06efd7eb7a60 100644 --- a/test/new-e2e/tests/process/windows_test.go +++ b/test/new-e2e/tests/process/windows_test.go @@ -132,6 +132,10 @@ func (s *windowsTestSuite) TestManualProcessDiscoveryCheck() { } func (s *windowsTestSuite) TestManualProcessCheckWithIO() { + s.T().Skip("skipping due to flakiness") + // MsMpEng.exe process missing IO stats, agent process does not always have CPU stats populated as it is restarted multiple times during the test suite run + // Investigation & fix tracked in https://datadoghq.atlassian.net/browse/PROCS-3757 + s.UpdateEnv(awshost.Provisioner( awshost.WithEC2InstanceOptions(ec2.WithOS(os.WindowsDefault)), awshost.WithAgentOptions(agentparams.WithAgentConfig(processCheckConfigStr), agentparams.WithSystemProbeConfig(systemProbeConfigStr)),