From c6f55c72dbf9f2ab6999e04d6ae0cc250d47d4fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Fri, 28 Apr 2023 17:31:59 +0200 Subject: [PATCH 1/5] Refactor SBOM configuration parameters --- .../container_image.d/conf.yaml.default | 5 + .../container_lifecycle.d/conf.yaml.default | 5 + .../dist/conf.d/sbom.d/conf.yaml.default | 5 + pkg/autodiscovery/listeners/staticconfig.go | 124 ++++++++++++++++++ pkg/cli/subcommands/check/command.go | 4 +- .../corechecks/containerimage/check.go | 6 + .../corechecks/containerlifecycle/check.go | 6 + pkg/collector/corechecks/sbom/check.go | 6 + .../corechecks/sbom/processor_test.go | 2 +- pkg/config/config.go | 45 +++---- pkg/config/system_probe.go | 11 +- pkg/sbom/sbom.go | 12 +- pkg/sbom/scanner/scanner.go | 4 +- pkg/util/docker/event_stream.go | 2 +- pkg/util/trivy/trivy.go | 6 +- .../internal/containerd/containerd.go | 2 +- .../internal/containerd/image_sbom_trivy.go | 4 +- .../internal/docker/image_sbom_trivy.go | 4 +- ...ctor-sbom-parameters-4af241e77fc00f83.yaml | 116 ++++++++++++++++ 19 files changed, 326 insertions(+), 43 deletions(-) create mode 100644 cmd/agent/dist/conf.d/container_image.d/conf.yaml.default create mode 100644 cmd/agent/dist/conf.d/container_lifecycle.d/conf.yaml.default create mode 100644 cmd/agent/dist/conf.d/sbom.d/conf.yaml.default create mode 100644 pkg/autodiscovery/listeners/staticconfig.go create mode 100644 releasenotes/notes/refactor-sbom-parameters-4af241e77fc00f83.yaml diff --git a/cmd/agent/dist/conf.d/container_image.d/conf.yaml.default b/cmd/agent/dist/conf.d/container_image.d/conf.yaml.default new file mode 100644 index 0000000000000..1e58dbbaafbbd --- /dev/null +++ b/cmd/agent/dist/conf.d/container_image.d/conf.yaml.default @@ -0,0 +1,5 @@ +ad_identifiers: + - _container_image +init_config: +instances: + - diff --git a/cmd/agent/dist/conf.d/container_lifecycle.d/conf.yaml.default b/cmd/agent/dist/conf.d/container_lifecycle.d/conf.yaml.default new file mode 100644 index 0000000000000..4e60c87b2e456 --- /dev/null +++ b/cmd/agent/dist/conf.d/container_lifecycle.d/conf.yaml.default @@ -0,0 +1,5 @@ +ad_identifiers: + - _container_lifecycle +init_config: +instances: + - diff --git a/cmd/agent/dist/conf.d/sbom.d/conf.yaml.default b/cmd/agent/dist/conf.d/sbom.d/conf.yaml.default new file mode 100644 index 0000000000000..faba9dbb04b91 --- /dev/null +++ b/cmd/agent/dist/conf.d/sbom.d/conf.yaml.default @@ -0,0 +1,5 @@ +ad_identifiers: + - _sbom +init_config: +instances: + - diff --git a/pkg/autodiscovery/listeners/staticconfig.go b/pkg/autodiscovery/listeners/staticconfig.go new file mode 100644 index 0000000000000..81e83473e11a8 --- /dev/null +++ b/pkg/autodiscovery/listeners/staticconfig.go @@ -0,0 +1,124 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +package listeners + +import ( + "context" + + "github.com/DataDog/datadog-agent/pkg/autodiscovery/integration" + "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/containers" +) + +// StaticConfigListener implements a ServiceListener based on static configuration parameters +type StaticConfigListener struct { + newService chan<- Service +} + +// StaticConfigService represents services generated from StaticConfigListener +type StaticConfigService struct { + adIdentifier string +} + +// Make sure StaticConfigService implements the Service interface +var _ Service = &StaticConfigService{} + +func init() { + Register("static config", NewStaticConfigListener) +} + +// NewStaticConfigListener creates a StaticConfigListener +func NewStaticConfigListener(Config) (ServiceListener, error) { + return &StaticConfigListener{}, nil +} + +// Listen starts the goroutine to detect checks based on the config +func (l *StaticConfigListener) Listen(newSvc chan<- Service, delSvc chan<- Service) { + l.newService = newSvc + + go l.createServices() +} + +// Stop has nothing to do in this case +func (l *StaticConfigListener) Stop() { +} + +func (l *StaticConfigListener) createServices() { + for _, staticCheck := range []string{ + "container_image", + "container_lifecycle", + "sbom", + } { + if enabled := config.Datadog.GetBool(staticCheck + ".enabled"); enabled { + l.newService <- &StaticConfigService{adIdentifier: "_" + staticCheck} + } + } +} + +// GetServiceID returns the unique entity name linked to that service +func (s *StaticConfigService) GetServiceID() string { + return s.adIdentifier +} + +// GetTaggerEntity returns the tagger entity +func (s *StaticConfigService) GetTaggerEntity() string { + return "" +} + +// GetADIdentifiers return the single AD identifier for a static config service +func (s *StaticConfigService) GetADIdentifiers(context.Context) ([]string, error) { + return []string{s.adIdentifier}, nil +} + +// GetHosts is not supported +func (s *StaticConfigService) GetHosts(context.Context) (map[string]string, error) { + return nil, ErrNotSupported +} + +// GetPorts returns nil and an error because port is not supported in this listener +func (s *StaticConfigService) GetPorts(context.Context) ([]ContainerPort, error) { + return nil, ErrNotSupported +} + +// GetTags retrieves a container's tags +func (s *StaticConfigService) GetTags() ([]string, error) { + return nil, nil +} + +// GetPid inspect the container and return its pid +// Not relevant in this listener +func (s *StaticConfigService) GetPid(context.Context) (int, error) { + return -1, ErrNotSupported +} + +// GetHostname returns nil and an error because port is not supported in this listener +func (s *StaticConfigService) GetHostname(context.Context) (string, error) { + return "", ErrNotSupported +} + +// IsReady is always true +func (s *StaticConfigService) IsReady(context.Context) bool { + return true +} + +// GetCheckNames is not supported +func (s *StaticConfigService) GetCheckNames(context.Context) []string { + return nil +} + +// HasFilter is not supported +func (s *StaticConfigService) HasFilter(filter containers.FilterType) bool { + return false +} + +// GetExtraConfig is not supported +func (s *StaticConfigService) GetExtraConfig(key string) (string, error) { + return "", ErrNotSupported +} + +// FilterTemplates does nothing. +func (s *StaticConfigService) FilterTemplates(configs map[string]integration.Config) { +} diff --git a/pkg/cli/subcommands/check/command.go b/pkg/cli/subcommands/check/command.go index e08def7d0cc50..95803fc7dc63c 100644 --- a/pkg/cli/subcommands/check/command.go +++ b/pkg/cli/subcommands/check/command.go @@ -182,8 +182,8 @@ func run(log log.Component, config config.Component, sysprobeconfig sysprobeconf // Always disable SBOM collection in `check` command to avoid BoltDB flock issue // and consuming CPU & Memory for asynchronous scans that would not be shown in `agent check` output. - pkgconfig.Datadog.Set("sbom.enabled", "false") - pkgconfig.Datadog.Set("container_image_collection.sbom.enabled", "false") + pkgconfig.Datadog.Set("sbom.host.enabled", "false") + pkgconfig.Datadog.Set("sbom.container_image.enabled", "false") pkgconfig.Datadog.Set("runtime_security_config.sbom.enabled", "false") hostnameDetected, err := hostname.Get(context.TODO()) diff --git a/pkg/collector/corechecks/containerimage/check.go b/pkg/collector/corechecks/containerimage/check.go index fe181a0a41c45..71b762f67345a 100644 --- a/pkg/collector/corechecks/containerimage/check.go +++ b/pkg/collector/corechecks/containerimage/check.go @@ -6,6 +6,7 @@ package containerimage import ( + "errors" "time" yaml "gopkg.in/yaml.v2" @@ -13,6 +14,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/collector/check" core "github.com/DataDog/datadog-agent/pkg/collector/corechecks" + ddConfig "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/workloadmeta" ) @@ -101,6 +103,10 @@ func CheckFactory() check.Check { // Configure parses the check configuration and initializes the container_image check func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error { + if !ddConfig.Datadog.GetBool("container_image.enabled") { + return errors.New("collection of container images is disabled") + } + if err := c.CommonConfigure(integrationConfigDigest, initConfig, config, source); err != nil { return err } diff --git a/pkg/collector/corechecks/containerlifecycle/check.go b/pkg/collector/corechecks/containerlifecycle/check.go index 5c95aec0c4ed6..ca4bfb21cafd9 100644 --- a/pkg/collector/corechecks/containerlifecycle/check.go +++ b/pkg/collector/corechecks/containerlifecycle/check.go @@ -7,6 +7,7 @@ package containerlifecycle import ( "context" + "errors" "time" yaml "gopkg.in/yaml.v2" @@ -14,6 +15,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/collector/check" core "github.com/DataDog/datadog-agent/pkg/collector/corechecks" + ddConfig "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/workloadmeta" ) @@ -50,6 +52,10 @@ type Check struct { // Configure parses the check configuration and initializes the container_lifecycle check func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error { + if !ddConfig.Datadog.GetBool("container_lifecycle.enabled") { + return errors.New("collection of container lifecycle events is disabled") + } + var err error err = c.CommonConfigure(integrationConfigDigest, initConfig, config, source) diff --git a/pkg/collector/corechecks/sbom/check.go b/pkg/collector/corechecks/sbom/check.go index 1b3b640e72902..76108f906cf02 100644 --- a/pkg/collector/corechecks/sbom/check.go +++ b/pkg/collector/corechecks/sbom/check.go @@ -9,6 +9,7 @@ package sbom import ( + "errors" "time" yaml "gopkg.in/yaml.v2" @@ -16,6 +17,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/collector/check" core "github.com/DataDog/datadog-agent/pkg/collector/corechecks" + ddConfig "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/workloadmeta" ) @@ -113,6 +115,10 @@ func CheckFactory() check.Check { // Configure parses the check configuration and initializes the sbom check func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error { + if !ddConfig.Datadog.GetBool("sbom.enabled") { + return errors.New("collection of SBOM is disabled") + } + if err := c.CommonConfigure(integrationConfigDigest, initConfig, config, source); err != nil { return err } diff --git a/pkg/collector/corechecks/sbom/processor_test.go b/pkg/collector/corechecks/sbom/processor_test.go index 87284fbdd2c8e..12d0231abc720 100644 --- a/pkg/collector/corechecks/sbom/processor_test.go +++ b/pkg/collector/corechecks/sbom/processor_test.go @@ -400,7 +400,7 @@ func TestProcessEvents(t *testing.T) { assert.Nil(t, err) defer os.RemoveAll(cacheDir) cfg.Set("sbom.cache_directory", cacheDir) - cfg.Set("container_image_collection.sbom.enabled", true) + cfg.Set("sbom.container_image.enabled", true) _, err = sbomscanner.CreateGlobalScanner(cfg) assert.Nil(t, err) diff --git a/pkg/config/config.go b/pkg/config/config.go index e22ba958d91c5..81013365c3bea 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1082,14 +1082,36 @@ func InitConfig(config Config) { config.BindEnvAndSetDefault("orchestrator_explorer.manifest_collection.buffer_flush_interval", 20*time.Second) // Container lifecycle configuration + config.BindEnvAndSetDefault("container_lifecycle.enabled", false) bindEnvAndSetLogsConfigKeys(config, "container_lifecycle.") // Container image configuration + config.BindEnvAndSetDefault("container_image.enabled", false) bindEnvAndSetLogsConfigKeys(config, "container_image.") // SBOM configuration + config.BindEnvAndSetDefault("sbom.enabled", false) bindEnvAndSetLogsConfigKeys(config, "sbom.") - setupSBOMConfig(config, "sbom-agent") + + config.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, "sbom-agent")) + config.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false) + config.BindEnvAndSetDefault("sbom.cache.enabled", false) + config.BindEnvAndSetDefault("sbom.cache.max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage + config.BindEnvAndSetDefault("sbom.cache.max_cache_entries", 10000) // used by custom cache keys stored in memory + config.BindEnvAndSetDefault("sbom.cache.clean_interval", "30m") // used by custom cache. + + // Container SBOM configuration + config.BindEnvAndSetDefault("sbom.container_image.enabled", false) + config.BindEnvAndSetDefault("sbom.container_image.use_mount", false) + config.BindEnvAndSetDefault("sbom.container_image.scan_interval", 0) // Integer seconds + config.BindEnvAndSetDefault("sbom.container_image.scan_timeout", 10*60) // Integer seconds + config.BindEnvAndSetDefault("sbom.container_image.analyzers", []string{"os"}) + config.BindEnvAndSetDefault("sbom.container_image.check_disk_usage", true) + config.BindEnvAndSetDefault("sbom.container_image.min_available_disk", "1Gb") + + // Host SBOM configuration + config.BindEnvAndSetDefault("sbom.host.enabled", false) + config.BindEnvAndSetDefault("sbom.host.analyzers", []string{"os"}) // Orchestrator Explorer - process agent // DEPRECATED in favor of `orchestrator_explorer.orchestrator_dd_url` setting. If both are set `orchestrator_explorer.orchestrator_dd_url` will take precedence. @@ -1110,16 +1132,6 @@ func InitConfig(config Config) { config.BindEnvAndSetDefault("inventories_max_interval", DefaultInventoriesMaxInterval) // integer seconds config.BindEnvAndSetDefault("inventories_min_interval", DefaultInventoriesMinInterval) // integer seconds - // container_image_collection - config.BindEnvAndSetDefault("container_image_collection.metadata.enabled", false) - config.BindEnvAndSetDefault("container_image_collection.sbom.enabled", false) - config.BindEnvAndSetDefault("container_image_collection.sbom.use_mount", false) - config.BindEnvAndSetDefault("container_image_collection.sbom.scan_interval", 0) // Integer seconds - config.BindEnvAndSetDefault("container_image_collection.sbom.scan_timeout", 10*60) // Integer seconds - config.BindEnvAndSetDefault("container_image_collection.sbom.analyzers", []string{"os"}) - config.BindEnvAndSetDefault("container_image_collection.sbom.check_disk_usage", true) - config.BindEnvAndSetDefault("container_image_collection.sbom.min_available_disk", "1Gb") - // Datadog security agent (common) config.BindEnvAndSetDefault("security_agent.cmd_port", 5010) config.BindEnvAndSetDefault("security_agent.expvar_port", 5011) @@ -1658,17 +1670,6 @@ func setupFipsLogsConfig(config Config, configPrefix string, url string) { config.Set(configPrefix+"logs_dd_url", url) } -func setupSBOMConfig(config Config, cacheDir string) { - config.BindEnvAndSetDefault("sbom.enabled", false) - config.BindEnvAndSetDefault("sbom.analyzers", []string{"os"}) - config.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, cacheDir)) - config.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false) - config.BindEnvAndSetDefault("sbom.use_custom_cache", false) - config.BindEnvAndSetDefault("sbom.custom_cache_max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage - config.BindEnvAndSetDefault("sbom.custom_cache_max_cache_entries", 10000) // used by custom cache keys stored in memory - config.BindEnvAndSetDefault("sbom.cache_clean_interval", "30m") // used by custom cache. -} - // ResolveSecrets merges all the secret values from origin into config. Secret values // are identified by a value of the form "ENC[key]" where key is the secret key. // See: https://github.com/DataDog/datadog-agent/blob/main/docs/agent/secrets.md diff --git a/pkg/config/system_probe.go b/pkg/config/system_probe.go index 8cb3ee0eaed91..cf6e30cb37ba5 100644 --- a/pkg/config/system_probe.go +++ b/pkg/config/system_probe.go @@ -9,6 +9,7 @@ import ( "encoding/json" "os" "path" + "path/filepath" "strings" "time" @@ -59,7 +60,15 @@ func InitSystemProbeConfig(cfg Config) { cfg.BindEnvAndSetDefault("ignore_host_etc", false) cfg.BindEnvAndSetDefault("go_core_dump", false) - setupSBOMConfig(cfg, "sbom-sysprobe") + // SBOM configuration + cfg.BindEnvAndSetDefault("sbom.host.enabled", false) + cfg.BindEnvAndSetDefault("sbom.host.analyzers", []string{"os"}) + cfg.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, "sbom-sysprobe")) + cfg.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false) + cfg.BindEnvAndSetDefault("sbom.cache.enabled", false) + cfg.BindEnvAndSetDefault("sbom.cache.max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage + cfg.BindEnvAndSetDefault("sbom.cache.max_cache_entries", 10000) // used by custom cache keys stored in memory + cfg.BindEnvAndSetDefault("sbom.cache.clean_interval", "30m") // used by custom cache. // Auto exit configuration cfg.BindEnvAndSetDefault("auto_exit.validation_period", 60) diff --git a/pkg/sbom/sbom.go b/pkg/sbom/sbom.go index 9f9fc484e3cde..8eddb87e9c554 100644 --- a/pkg/sbom/sbom.go +++ b/pkg/sbom/sbom.go @@ -33,15 +33,15 @@ type ScanOptions struct { // ScanOptionsFromConfig loads the scanning options from the configuration func ScanOptionsFromConfig(cfg config.Config, containers bool) (scanOpts ScanOptions) { if containers { - scanOpts.CheckDiskUsage = config.Datadog.GetBool("container_image_collection.sbom.check_disk_usage") - scanOpts.MinAvailableDisk = uint64(config.Datadog.GetSizeInBytes("container_image_collection.sbom.min_available_disk")) - scanOpts.Timeout = time.Duration(config.Datadog.GetInt("container_image_collection.sbom.scan_timeout")) * time.Second - scanOpts.WaitAfter = time.Duration(config.Datadog.GetInt("container_image_collection.sbom.scan_interval")) * time.Second - scanOpts.Analyzers = config.Datadog.GetStringSlice("container_image_collection.sbom.analyzers") + scanOpts.CheckDiskUsage = config.Datadog.GetBool("sbom.container_image.check_disk_usage") + scanOpts.MinAvailableDisk = uint64(config.Datadog.GetSizeInBytes("sbom.container_image.min_available_disk")) + scanOpts.Timeout = time.Duration(config.Datadog.GetInt("sbom.container_image.scan_timeout")) * time.Second + scanOpts.WaitAfter = time.Duration(config.Datadog.GetInt("sbom.container_image.scan_interval")) * time.Second + scanOpts.Analyzers = config.Datadog.GetStringSlice("sbom.container_image.analyzers") } if len(scanOpts.Analyzers) == 0 { - scanOpts.Analyzers = config.Datadog.GetStringSlice("sbom.analyzers") + scanOpts.Analyzers = config.Datadog.GetStringSlice("sbom.host.analyzers") } return diff --git a/pkg/sbom/scanner/scanner.go b/pkg/sbom/scanner/scanner.go index 42a43f56e0349..8b84d3eb5bf9b 100644 --- a/pkg/sbom/scanner/scanner.go +++ b/pkg/sbom/scanner/scanner.go @@ -79,7 +79,7 @@ func (s *Scanner) start(ctx context.Context) { return } go func() { - cleanTicker := time.NewTicker(config.Datadog.GetDuration("sbom.cache_clean_interval")) + cleanTicker := time.NewTicker(config.Datadog.GetDuration("sbom.cache.clean_interval")) defer cleanTicker.Stop() s.running = true defer func() { s.running = false }() @@ -174,7 +174,7 @@ func NewScanner(cfg config.Config) *Scanner { // global one, and returns it. Start() needs to be called before any data // collection happens. func CreateGlobalScanner(cfg config.Config) (*Scanner, error) { - if !cfg.GetBool("sbom.enabled") && !cfg.GetBool("container_image_collection.sbom.enabled") && !cfg.GetBool("runtime_security_config.sbom.enabled") { + if !cfg.GetBool("sbom.host.enabled") && !cfg.GetBool("sbom.container_image.enabled") && !cfg.GetBool("runtime_security_config.sbom.enabled") { return nil, nil } diff --git a/pkg/util/docker/event_stream.go b/pkg/util/docker/event_stream.go index e80d947627412..cc36d639b8298 100644 --- a/pkg/util/docker/event_stream.go +++ b/pkg/util/docker/event_stream.go @@ -158,7 +158,7 @@ func eventFilters() filters.Args { res.Add("event", containerEventAction) } - if config.Datadog.GetBool("container_image_collection.metadata.enabled") { + if config.Datadog.GetBool("container_image.enabled") { res.Add("type", events.ImageEventType) for _, imageEventAction := range imageEventActions { res.Add("event", imageEventAction) diff --git a/pkg/util/trivy/trivy.go b/pkg/util/trivy/trivy.go index 0cb142ba21906..67927e296399a 100644 --- a/pkg/util/trivy/trivy.go +++ b/pkg/util/trivy/trivy.go @@ -104,7 +104,7 @@ func DefaultCollectorConfig(cacheLocation string) CollectorConfig { ClearCacheOnClose: true, } - collectorConfig.CacheProvider = cacheProvider(cacheLocation, config.Datadog.GetBool("sbom.use_custom_cache")) + collectorConfig.CacheProvider = cacheProvider(cacheLocation, config.Datadog.GetBool("sbom.cache.enabled")) return collectorConfig } @@ -114,8 +114,8 @@ func cacheProvider(cacheLocation string, useCustomCache bool) func() (cache.Cach return func() (cache.Cache, CacheCleaner, error) { return NewCustomBoltCache( cacheLocation, - config.Datadog.GetInt("sbom.custom_cache_max_cache_entries"), - config.Datadog.GetInt("sbom.custom_cache_max_disk_size"), + config.Datadog.GetInt("sbom.cache.max_cache_entries"), + config.Datadog.GetInt("sbom.cache.max_disk_size"), ) } } diff --git a/pkg/workloadmeta/collectors/internal/containerd/containerd.go b/pkg/workloadmeta/collectors/internal/containerd/containerd.go index 8733b32052ac7..98e2036a0c64f 100644 --- a/pkg/workloadmeta/collectors/internal/containerd/containerd.go +++ b/pkg/workloadmeta/collectors/internal/containerd/containerd.go @@ -396,5 +396,5 @@ func (c *collector) cacheExitInfo(id string, exitCode *uint32, exitTS time.Time) } func imageMetadataCollectionIsEnabled() bool { - return config.Datadog.GetBool("container_image_collection.metadata.enabled") + return config.Datadog.GetBool("container_image.enabled") } diff --git a/pkg/workloadmeta/collectors/internal/containerd/image_sbom_trivy.go b/pkg/workloadmeta/collectors/internal/containerd/image_sbom_trivy.go index 5cf4d2185f12a..1fa7680217b23 100644 --- a/pkg/workloadmeta/collectors/internal/containerd/image_sbom_trivy.go +++ b/pkg/workloadmeta/collectors/internal/containerd/image_sbom_trivy.go @@ -21,7 +21,7 @@ import ( ) func sbomCollectionIsEnabled() bool { - return imageMetadataCollectionIsEnabled() && config.Datadog.GetBool("container_image_collection.sbom.enabled") + return imageMetadataCollectionIsEnabled() && config.Datadog.GetBool("sbom.container_image.enabled") } func (c *collector) startSBOMCollection(ctx context.Context) error { @@ -86,7 +86,7 @@ func (c *collector) extractBOMWithTrivy(ctx context.Context, storedImage *worklo Image: containerdImage, ImageMeta: storedImage, ContainerdClient: c.containerdClient, - FromFilesystem: config.Datadog.GetBool("container_image_collection.sbom.use_mount"), + FromFilesystem: config.Datadog.GetBool("sbom.container_image.use_mount"), } ch := make(chan sbom.ScanResult, 1) diff --git a/pkg/workloadmeta/collectors/internal/docker/image_sbom_trivy.go b/pkg/workloadmeta/collectors/internal/docker/image_sbom_trivy.go index fb4828b723b4b..13577c96ceae0 100644 --- a/pkg/workloadmeta/collectors/internal/docker/image_sbom_trivy.go +++ b/pkg/workloadmeta/collectors/internal/docker/image_sbom_trivy.go @@ -23,11 +23,11 @@ import ( ) func imageMetadataCollectionIsEnabled() bool { - return config.Datadog.GetBool("container_image_collection.metadata.enabled") + return config.Datadog.GetBool("container_image.enabled") } func sbomCollectionIsEnabled() bool { - return imageMetadataCollectionIsEnabled() && config.Datadog.GetBool("container_image_collection.sbom.enabled") + return imageMetadataCollectionIsEnabled() && config.Datadog.GetBool("sbom.container_image.enabled") } func (c *collector) startSBOMCollection(ctx context.Context) error { diff --git a/releasenotes/notes/refactor-sbom-parameters-4af241e77fc00f83.yaml b/releasenotes/notes/refactor-sbom-parameters-4af241e77fc00f83.yaml new file mode 100644 index 0000000000000..dcc9afc1df1e7 --- /dev/null +++ b/releasenotes/notes/refactor-sbom-parameters-4af241e77fc00f83.yaml @@ -0,0 +1,116 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +upgrade: + - | + Refactor the SBOM collection parameters from:: + + conf.d/container_lifecycle.d/conf.yaml existence (A) # to schedule the container lifecycle long running check + conf.d/container_image.d/conf.yaml existence (B) # to schedule the container image metadata long running check + conf.d/sbom.d/conf.yaml existence (C) # to schedule the sbom long running check + + Inside datadog.yaml: + + container_lifecycle: + enabled: (D) # Used to control the start of the container_lifecycle forwarder but has been decommissioned by #16084 (7.45.0-rc) + dd_url: # \ + additional_endpoints: # | + use_compression: # | + compression_level: # > generic parameters for the generic EVP pipeline + … # | + use_v2_api: # / + + container_image: + enabled: (E) # Used to control the start of the container_image forwarder but has been decommissioned by #16084 (7.45.0-rc) + dd_url: # \ + additional_endpoints: # | + use_compression: # | + compression_level: # > generic parameters for the generic EVP pipeline + … # | + use_v2_api: # / + + sbom: + enabled: (F) # control host SBOM collection and do *not* control container related sbom since #16084 (7.45.0-rc) + dd_url: # \ + additional_endpoints: # | + use_compression: # | + compression_level: # > generic parameters for the generic EVP pipeline + … # | + use_v2_api: # / + analyzers: (G) # trivy analyzers user for host SBOM collection + cache_directory: (H) + clear_cache_on_exit: (I) + use_custom_cache: (J) + custom_cache_max_disk_size: (K) + custom_cache_max_cache_entries: (L) + cache_clean_interval: (M) + + container_image_collection: + metadata: + enabled: (N) # Controls the collection of the container image metadata in workload meta + sbom: + enabled: (O) + use_mount: (P) + scan_interval: (Q) + scan_timeout: (R) + analyzers: (S) # trivy analyzers user for containers SBOM collection + check_disk_usage: (T) + min_available_disk: (U) + + to:: + + conf.d/{container_lifecycle,container_image,sbom}.d/conf.yaml don’t need to be created by the users anymore. A default version is always shipped with the agent docker image with an underscore-prefixed ad_identifier that will be synthetised by the agent at runtime based on config {container_lifecycle,container_image,sbom}.enabled parameters. + + Inside datadog.yaml: + + container_lifecycle: + enabled: (A) # Replaces the need of creating a conf.d/container_lifecycle.d/conf.yaml file + dd_url: # \ + additional_endpoints: # | + use_compression: # | + compression_level: # > unchanged generic parameters for the generic EVP pipeline + … # | + use_v2_api: # / + + container_image: + enabled: (B) # Replaces the need of creating a conf.d/container_image.d/conf.yaml file + dd_url: # \ + additional_endpoints: # | + use_compression: # | + compression_level: # > unchanged generic parameters for the generic EVP pipeline + … # | + use_v2_api: # / + + sbom: + enabled: (C) # Replaces the need of creating a conf.d/sbom.d/conf.yaml file + dd_url: # \ + additional_endpoints: # | + use_compression: # | + compression_level: # > unchanged generic parameters for the generic EVP pipeline + … # | + use_v2_api: # / + cache_directory: (H) + clear_cache_on_exit: (I) + cache: # Factorize all settings related to the custom cache + enabled: (J) + max_disk_size: (K) + max_cache_entries: (L) + clean_interval: (M) + + host: # for host SBOM parameters that were directly below `sbom` before. + enabled: (F) # sbom.host.enabled replaces sbom.enabled + analyzers: (G) # sbom.host.analyzers replaces sbom.analyzers + + container_image: # sbom.container_image replaces container_image_collection.sbom + enabled: (O) + use_mount: (P) + scan_interval: (Q) + scan_timeout: (R) + analyzers: (S) # trivy analyzers user for containers SBOM collection + check_disk_usage: (T) + min_available_disk: (U) From d247c244b17118f08c8719b057ed95a4833e4b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Fri, 26 May 2023 09:23:10 +0200 Subject: [PATCH 2/5] Fix --- .github/CODEOWNERS | 11 ++++++++--- pkg/config/autodiscovery/autodiscovery.go | 3 ++- tasks/agent.py | 3 +++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a85ed7f158e54..352e9db1e98a5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -116,10 +116,15 @@ /cmd/agent/subcommands/workloadlist @DataDog/container-integrations /cmd/agent/subcommands/run/internal/clcrunnerapi/ @DataDog/container-integrations @DataDog/agent-shared-components /cmd/agent/windows @DataDog/windows-agent -/cmd/agent/dist/conf.d/jetson.d @DataDog/agent-platform -/cmd/agent/dist/conf.d/oracle-dbm.d @DataDog/database-monitoring -/cmd/agent/dist/conf.d/snmp.d/ @DataDog/network-device-monitoring +/cmp/agent/dist/conf.d/container.d/ @DataDog/container-integrations +/cmp/agent/dist/conf.d/containerd.d/ @DataDog/container-integrations +/cmp/agent/dist/conf.d/container_image.d/ @DataDog/container-integrations +/cmp/agent/dist/conf.d/container_lifecycle.d/ @DataDog/container-integrations +/cmd/agent/dist/conf.d/jetson.d/ @DataDog/agent-platform +/cmd/agent/dist/conf.d/oracle-dbm.d/ @DataDog/database-monitoring /cmd/agent/dist/conf.d/oracle-dbm.d/conf.yaml.default @DataDog/database-monitoring +/cmp/agent/dist/conf.d/sbom.d/ @DataDog/container-integrations +/cmd/agent/dist/conf.d/snmp.d/ @DataDog/network-device-monitoring /cmd/agent/*.manifest @DataDog/agent-platform /cmd/agent/*.mc @DataDog/agent-platform /cmd/agent/*.rc @DataDog/agent-platform diff --git a/pkg/config/autodiscovery/autodiscovery.go b/pkg/config/autodiscovery/autodiscovery.go index 73eba8449bfeb..d3f714f3bd8a1 100644 --- a/pkg/config/autodiscovery/autodiscovery.go +++ b/pkg/config/autodiscovery/autodiscovery.go @@ -73,8 +73,9 @@ func DiscoverComponentsFromEnv() ([]config.ConfigurationProviders, []config.List detectedListeners := []config.Listeners{} // When using automatic discovery of providers/listeners - // We automatically activate the environment listener + // We automatically activate the environment and static config listener detectedListeners = append(detectedListeners, config.Listeners{Name: "environment"}) + detectedListeners = append(detectedListeners, config.Listeners{Name: "static config"}) // Automatic handling of AD providers/listeners should only run in Core agent. if flavor.GetFlavor() != flavor.DefaultAgent { diff --git a/tasks/agent.py b/tasks/agent.py index 018e1d86666bd..7847f6f982d4e 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -43,6 +43,8 @@ AGENT_CORECHECKS = [ "container", "containerd", + "container_image", + "container_lifecycle", "cpu", "cri", "snmp", @@ -57,6 +59,7 @@ "ntp", "oom_kill", "oracle-dbm", + "sbom", "systemd", "tcp_queue_length", "uptime", From abb733da86c44b907853192c255fa06688c40d9a Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Fri, 26 May 2023 16:09:12 +0200 Subject: [PATCH 3/5] Fix host SBOM config --- pkg/collector/corechecks/sbom/check.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/collector/corechecks/sbom/check.go b/pkg/collector/corechecks/sbom/check.go index 76108f906cf02..588797277fe4e 100644 --- a/pkg/collector/corechecks/sbom/check.go +++ b/pkg/collector/corechecks/sbom/check.go @@ -132,7 +132,7 @@ func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig int return err } - c.processor, err = newProcessor(c.workloadmetaStore, sender, c.instance.ChunkSize, time.Duration(c.instance.NewSBOMMaxLatencySeconds)*time.Second, c.instance.HostSBOM) + c.processor, err = newProcessor(c.workloadmetaStore, sender, c.instance.ChunkSize, time.Duration(c.instance.NewSBOMMaxLatencySeconds)*time.Second, ddConfig.Datadog.GetBool("sbom.host.enabled")) if err != nil { return err } From 81d5f7e55cde1d078dedd863f6f3fb34c7cc1422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Fri, 26 May 2023 17:28:22 +0200 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: DeForest Richards <56796055+drichards-87@users.noreply.github.com> --- .../refactor-sbom-parameters-4af241e77fc00f83.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/releasenotes/notes/refactor-sbom-parameters-4af241e77fc00f83.yaml b/releasenotes/notes/refactor-sbom-parameters-4af241e77fc00f83.yaml index dcc9afc1df1e7..5c8aa6355b472 100644 --- a/releasenotes/notes/refactor-sbom-parameters-4af241e77fc00f83.yaml +++ b/releasenotes/notes/refactor-sbom-parameters-4af241e77fc00f83.yaml @@ -12,7 +12,7 @@ upgrade: conf.d/container_lifecycle.d/conf.yaml existence (A) # to schedule the container lifecycle long running check conf.d/container_image.d/conf.yaml existence (B) # to schedule the container image metadata long running check - conf.d/sbom.d/conf.yaml existence (C) # to schedule the sbom long running check + conf.d/sbom.d/conf.yaml existence (C) # to schedule the SBOM long running check Inside datadog.yaml: @@ -35,7 +35,7 @@ upgrade: use_v2_api: # / sbom: - enabled: (F) # control host SBOM collection and do *not* control container related sbom since #16084 (7.45.0-rc) + enabled: (F) # control host SBOM collection and do **not** control container-related SBOM since #16084 (7.45.0-rc) dd_url: # \ additional_endpoints: # | use_compression: # | @@ -64,12 +64,12 @@ upgrade: to:: - conf.d/{container_lifecycle,container_image,sbom}.d/conf.yaml don’t need to be created by the users anymore. A default version is always shipped with the agent docker image with an underscore-prefixed ad_identifier that will be synthetised by the agent at runtime based on config {container_lifecycle,container_image,sbom}.enabled parameters. + conf.d/{container_lifecycle,container_image,sbom}.d/conf.yaml no longer needs to be created. A default version is always shipped with the Agent Docker image with an underscore-prefixed ad_identifier that will be synthesized by the agent at runtime based on config {container_lifecycle,container_image,sbom}.enabled parameters. Inside datadog.yaml: container_lifecycle: - enabled: (A) # Replaces the need of creating a conf.d/container_lifecycle.d/conf.yaml file + enabled: (A) # Replaces the need for creating a conf.d/container_lifecycle.d/conf.yaml file dd_url: # \ additional_endpoints: # | use_compression: # | @@ -78,7 +78,7 @@ upgrade: use_v2_api: # / container_image: - enabled: (B) # Replaces the need of creating a conf.d/container_image.d/conf.yaml file + enabled: (B) # Replaces the need for creating a conf.d/container_image.d/conf.yaml file dd_url: # \ additional_endpoints: # | use_compression: # | @@ -87,7 +87,7 @@ upgrade: use_v2_api: # / sbom: - enabled: (C) # Replaces the need of creating a conf.d/sbom.d/conf.yaml file + enabled: (C) # Replaces the need for creating a conf.d/sbom.d/conf.yaml file dd_url: # \ additional_endpoints: # | use_compression: # | From 29f41637280362b9aa797093edbee576492dbda3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Fri, 26 May 2023 18:51:42 +0200 Subject: [PATCH 5/5] Apply suggestions from code review --- .github/CODEOWNERS | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 352e9db1e98a5..9b711cdae800e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -116,14 +116,14 @@ /cmd/agent/subcommands/workloadlist @DataDog/container-integrations /cmd/agent/subcommands/run/internal/clcrunnerapi/ @DataDog/container-integrations @DataDog/agent-shared-components /cmd/agent/windows @DataDog/windows-agent -/cmp/agent/dist/conf.d/container.d/ @DataDog/container-integrations -/cmp/agent/dist/conf.d/containerd.d/ @DataDog/container-integrations -/cmp/agent/dist/conf.d/container_image.d/ @DataDog/container-integrations -/cmp/agent/dist/conf.d/container_lifecycle.d/ @DataDog/container-integrations +/cmd/agent/dist/conf.d/container.d/ @DataDog/container-integrations +/cmd/agent/dist/conf.d/containerd.d/ @DataDog/container-integrations +/cmd/agent/dist/conf.d/container_image.d/ @DataDog/container-integrations +/cmd/agent/dist/conf.d/container_lifecycle.d/ @DataDog/container-integrations /cmd/agent/dist/conf.d/jetson.d/ @DataDog/agent-platform /cmd/agent/dist/conf.d/oracle-dbm.d/ @DataDog/database-monitoring /cmd/agent/dist/conf.d/oracle-dbm.d/conf.yaml.default @DataDog/database-monitoring -/cmp/agent/dist/conf.d/sbom.d/ @DataDog/container-integrations +/cmd/agent/dist/conf.d/sbom.d/ @DataDog/container-integrations /cmd/agent/dist/conf.d/snmp.d/ @DataDog/network-device-monitoring /cmd/agent/*.manifest @DataDog/agent-platform /cmd/agent/*.mc @DataDog/agent-platform