Refactor SBOM configuration parameters

.github/CODEOWNERS

@@ -116,10 +116,15 @@
 /cmd/agent/subcommands/workloadlist     @DataDog/container-integrations
 /cmd/agent/subcommands/run/internal/clcrunnerapi/       @DataDog/container-integrations @DataDog/agent-shared-components
 /cmd/agent/windows                      @DataDog/windows-agent
-/cmd/agent/dist/conf.d/jetson.d         @DataDog/agent-platform
-/cmd/agent/dist/conf.d/oracle-dbm.d     @DataDog/database-monitoring
-/cmd/agent/dist/conf.d/snmp.d/          @DataDog/network-device-monitoring
+/cmp/agent/dist/conf.d/container.d/     @DataDog/container-integrations
+/cmp/agent/dist/conf.d/containerd.d/    @DataDog/container-integrations
+/cmp/agent/dist/conf.d/container_image.d/      @DataDog/container-integrations
+/cmp/agent/dist/conf.d/container_lifecycle.d/  @DataDog/container-integrations
+/cmd/agent/dist/conf.d/jetson.d/        @DataDog/agent-platform
+/cmd/agent/dist/conf.d/oracle-dbm.d/    @DataDog/database-monitoring
 /cmd/agent/dist/conf.d/oracle-dbm.d/conf.yaml.default @DataDog/database-monitoring
+/cmp/agent/dist/conf.d/sbom.d/          @DataDog/container-integrations
+/cmd/agent/dist/conf.d/snmp.d/          @DataDog/network-device-monitoring
 /cmd/agent/*.manifest                   @DataDog/agent-platform
 /cmd/agent/*.mc                         @DataDog/agent-platform
 /cmd/agent/*.rc                         @DataDog/agent-platform

cmd/agent/dist/conf.d/container_image.d/conf.yaml.default

@@ -0,0 +1,5 @@
+ad_identifiers:
+  - _container_image
+init_config:
+instances:
+  -

cmd/agent/dist/conf.d/container_lifecycle.d/conf.yaml.default

@@ -0,0 +1,5 @@
+ad_identifiers:
+  - _container_lifecycle
+init_config:
+instances:
+  -

cmd/agent/dist/conf.d/sbom.d/conf.yaml.default

@@ -0,0 +1,5 @@
+ad_identifiers:
+  - _sbom
+init_config:
+instances:
+  -

pkg/autodiscovery/listeners/staticconfig.go

@@ -0,0 +1,124 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2023-present Datadog, Inc.
+
+package listeners
+
+import (
+	"context"
+
+	"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
+	"github.com/DataDog/datadog-agent/pkg/config"
+	"github.com/DataDog/datadog-agent/pkg/util/containers"
+)
+
+// StaticConfigListener implements a ServiceListener based on static configuration parameters
+type StaticConfigListener struct {
+	newService chan<- Service
+}
+
+// StaticConfigService represents services generated from StaticConfigListener
+type StaticConfigService struct {
+	adIdentifier string
+}
+
+// Make sure StaticConfigService implements the Service interface
+var _ Service = &StaticConfigService{}
+
+func init() {
+	Register("static config", NewStaticConfigListener)
+}
+
+// NewStaticConfigListener creates a StaticConfigListener
+func NewStaticConfigListener(Config) (ServiceListener, error) {
+	return &StaticConfigListener{}, nil
+}
+
+// Listen starts the goroutine to detect checks based on the config
+func (l *StaticConfigListener) Listen(newSvc chan<- Service, delSvc chan<- Service) {
+	l.newService = newSvc
+
+	go l.createServices()
+}
+
+// Stop has nothing to do in this case
+func (l *StaticConfigListener) Stop() {
+}
+
+func (l *StaticConfigListener) createServices() {
+	for _, staticCheck := range []string{
+		"container_image",
+		"container_lifecycle",
+		"sbom",
+	} {
+		if enabled := config.Datadog.GetBool(staticCheck + ".enabled"); enabled {
+			l.newService <- &StaticConfigService{adIdentifier: "_" + staticCheck}
+		}
+	}
+}
+
+// GetServiceID returns the unique entity name linked to that service
+func (s *StaticConfigService) GetServiceID() string {
+	return s.adIdentifier
+}
+
+// GetTaggerEntity returns the tagger entity
+func (s *StaticConfigService) GetTaggerEntity() string {
+	return ""
+}
+
+// GetADIdentifiers return the single AD identifier for a static config service
+func (s *StaticConfigService) GetADIdentifiers(context.Context) ([]string, error) {
+	return []string{s.adIdentifier}, nil
+}
+
+// GetHosts is not supported
+func (s *StaticConfigService) GetHosts(context.Context) (map[string]string, error) {
+	return nil, ErrNotSupported
+}
+
+// GetPorts returns nil and an error because port is not supported in this listener
+func (s *StaticConfigService) GetPorts(context.Context) ([]ContainerPort, error) {
+	return nil, ErrNotSupported
+}
+
+// GetTags retrieves a container's tags
+func (s *StaticConfigService) GetTags() ([]string, error) {
+	return nil, nil
+}
+
+// GetPid inspect the container and return its pid
+// Not relevant in this listener
+func (s *StaticConfigService) GetPid(context.Context) (int, error) {
+	return -1, ErrNotSupported
+}
+
+// GetHostname returns nil and an error because port is not supported in this listener
+func (s *StaticConfigService) GetHostname(context.Context) (string, error) {
+	return "", ErrNotSupported
+}
+
+// IsReady is always true
+func (s *StaticConfigService) IsReady(context.Context) bool {
+	return true
+}
+
+// GetCheckNames is not supported
+func (s *StaticConfigService) GetCheckNames(context.Context) []string {
+	return nil
+}
+
+// HasFilter is not supported
+func (s *StaticConfigService) HasFilter(filter containers.FilterType) bool {
+	return false
+}
+
+// GetExtraConfig is not supported
+func (s *StaticConfigService) GetExtraConfig(key string) (string, error) {
+	return "", ErrNotSupported
+}
+
+// FilterTemplates does nothing.
+func (s *StaticConfigService) FilterTemplates(configs map[string]integration.Config) {
+}

pkg/cli/subcommands/check/command.go

@@ -182,8 +182,8 @@ func run(log log.Component, config config.Component, sysprobeconfig sysprobeconf
 
 	// Always disable SBOM collection in `check` command to avoid BoltDB flock issue
 	// and consuming CPU & Memory for asynchronous scans that would not be shown in `agent check` output.
-	pkgconfig.Datadog.Set("sbom.enabled", "false")
-	pkgconfig.Datadog.Set("container_image_collection.sbom.enabled", "false")
+	pkgconfig.Datadog.Set("sbom.host.enabled", "false")
+	pkgconfig.Datadog.Set("sbom.container_image.enabled", "false")
 	pkgconfig.Datadog.Set("runtime_security_config.sbom.enabled", "false")
 
 	hostnameDetected, err := hostname.Get(context.TODO())

pkg/collector/corechecks/containerimage/check.go

@@ -6,13 +6,15 @@
 package containerimage
 
 import (
+	"errors"
 	"time"
 
 	yaml "gopkg.in/yaml.v2"
 
 	"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
 	"github.com/DataDog/datadog-agent/pkg/collector/check"
 	core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
+	ddConfig "github.com/DataDog/datadog-agent/pkg/config"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 	"github.com/DataDog/datadog-agent/pkg/workloadmeta"
 )
@@ -101,6 +103,10 @@ func CheckFactory() check.Check {
 
 // Configure parses the check configuration and initializes the container_image check
 func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error {
+	if !ddConfig.Datadog.GetBool("container_image.enabled") {
+		return errors.New("collection of container images is disabled")
+	}
+
 	if err := c.CommonConfigure(integrationConfigDigest, initConfig, config, source); err != nil {
 		return err
 	}

pkg/collector/corechecks/containerlifecycle/check.go

@@ -7,13 +7,15 @@ package containerlifecycle
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	yaml "gopkg.in/yaml.v2"
 
 	"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
 	"github.com/DataDog/datadog-agent/pkg/collector/check"
 	core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
+	ddConfig "github.com/DataDog/datadog-agent/pkg/config"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 	"github.com/DataDog/datadog-agent/pkg/workloadmeta"
 )
@@ -50,6 +52,10 @@ type Check struct {
 
 // Configure parses the check configuration and initializes the container_lifecycle check
 func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error {
+	if !ddConfig.Datadog.GetBool("container_lifecycle.enabled") {
+		return errors.New("collection of container lifecycle events is disabled")
+	}
+
 	var err error
 
 	err = c.CommonConfigure(integrationConfigDigest, initConfig, config, source)

pkg/collector/corechecks/sbom/check.go

@@ -9,13 +9,15 @@
 package sbom
 
 import (
+	"errors"
 	"time"
 
 	yaml "gopkg.in/yaml.v2"
 
 	"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
 	"github.com/DataDog/datadog-agent/pkg/collector/check"
 	core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
+	ddConfig "github.com/DataDog/datadog-agent/pkg/config"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
 	"github.com/DataDog/datadog-agent/pkg/workloadmeta"
 )
@@ -113,6 +115,10 @@ func CheckFactory() check.Check {
 
 // Configure parses the check configuration and initializes the sbom check
 func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error {
+	if !ddConfig.Datadog.GetBool("sbom.enabled") {
+		return errors.New("collection of SBOM is disabled")
+	}
+
 	if err := c.CommonConfigure(integrationConfigDigest, initConfig, config, source); err != nil {
 		return err
 	}

pkg/collector/corechecks/sbom/processor_test.go

@@ -400,7 +400,7 @@ func TestProcessEvents(t *testing.T) {
 	assert.Nil(t, err)
 	defer os.RemoveAll(cacheDir)
 	cfg.Set("sbom.cache_directory", cacheDir)
-	cfg.Set("container_image_collection.sbom.enabled", true)
+	cfg.Set("sbom.container_image.enabled", true)
 	_, err = sbomscanner.CreateGlobalScanner(cfg)
 	assert.Nil(t, err)
 

pkg/config/autodiscovery/autodiscovery.go

@@ -73,8 +73,9 @@ func DiscoverComponentsFromEnv() ([]config.ConfigurationProviders, []config.List
 	detectedListeners := []config.Listeners{}
 
 	// When using automatic discovery of providers/listeners
-	// We automatically activate the environment listener
+	// We automatically activate the environment and static config listener
 	detectedListeners = append(detectedListeners, config.Listeners{Name: "environment"})
+	detectedListeners = append(detectedListeners, config.Listeners{Name: "static config"})
 
 	// Automatic handling of AD providers/listeners should only run in Core agent.
 	if flavor.GetFlavor() != flavor.DefaultAgent {

pkg/config/config.go

@@ -1082,14 +1082,36 @@ func InitConfig(config Config) {
 	config.BindEnvAndSetDefault("orchestrator_explorer.manifest_collection.buffer_flush_interval", 20*time.Second)
 
 	// Container lifecycle configuration
+	config.BindEnvAndSetDefault("container_lifecycle.enabled", false)
 	bindEnvAndSetLogsConfigKeys(config, "container_lifecycle.")
 
 	// Container image configuration
+	config.BindEnvAndSetDefault("container_image.enabled", false)
 	bindEnvAndSetLogsConfigKeys(config, "container_image.")
 
 	// SBOM configuration
+	config.BindEnvAndSetDefault("sbom.enabled", false)
 	bindEnvAndSetLogsConfigKeys(config, "sbom.")
-	setupSBOMConfig(config, "sbom-agent")
+
+	config.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, "sbom-agent"))
+	config.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false)
+	config.BindEnvAndSetDefault("sbom.cache.enabled", false)
+	config.BindEnvAndSetDefault("sbom.cache.max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage
+	config.BindEnvAndSetDefault("sbom.cache.max_cache_entries", 10000)     // used by custom cache keys stored in memory
+	config.BindEnvAndSetDefault("sbom.cache.clean_interval", "30m")        // used by custom cache.
+
+	// Container SBOM configuration
+	config.BindEnvAndSetDefault("sbom.container_image.enabled", false)
+	config.BindEnvAndSetDefault("sbom.container_image.use_mount", false)
+	config.BindEnvAndSetDefault("sbom.container_image.scan_interval", 0)    // Integer seconds
+	config.BindEnvAndSetDefault("sbom.container_image.scan_timeout", 10*60) // Integer seconds
+	config.BindEnvAndSetDefault("sbom.container_image.analyzers", []string{"os"})
+	config.BindEnvAndSetDefault("sbom.container_image.check_disk_usage", true)
+	config.BindEnvAndSetDefault("sbom.container_image.min_available_disk", "1Gb")
+
+	// Host SBOM configuration
+	config.BindEnvAndSetDefault("sbom.host.enabled", false)
+	config.BindEnvAndSetDefault("sbom.host.analyzers", []string{"os"})
 
 	// Orchestrator Explorer - process agent
 	// DEPRECATED in favor of `orchestrator_explorer.orchestrator_dd_url` setting. If both are set `orchestrator_explorer.orchestrator_dd_url` will take precedence.
@@ -1110,16 +1132,6 @@ func InitConfig(config Config) {
 	config.BindEnvAndSetDefault("inventories_max_interval", DefaultInventoriesMaxInterval) // integer seconds
 	config.BindEnvAndSetDefault("inventories_min_interval", DefaultInventoriesMinInterval) // integer seconds
 
-	// container_image_collection
-	config.BindEnvAndSetDefault("container_image_collection.metadata.enabled", false)
-	config.BindEnvAndSetDefault("container_image_collection.sbom.enabled", false)
-	config.BindEnvAndSetDefault("container_image_collection.sbom.use_mount", false)
-	config.BindEnvAndSetDefault("container_image_collection.sbom.scan_interval", 0)    // Integer seconds
-	config.BindEnvAndSetDefault("container_image_collection.sbom.scan_timeout", 10*60) // Integer seconds
-	config.BindEnvAndSetDefault("container_image_collection.sbom.analyzers", []string{"os"})
-	config.BindEnvAndSetDefault("container_image_collection.sbom.check_disk_usage", true)
-	config.BindEnvAndSetDefault("container_image_collection.sbom.min_available_disk", "1Gb")
-
 	// Datadog security agent (common)
 	config.BindEnvAndSetDefault("security_agent.cmd_port", 5010)
 	config.BindEnvAndSetDefault("security_agent.expvar_port", 5011)
@@ -1658,17 +1670,6 @@ func setupFipsLogsConfig(config Config, configPrefix string, url string) {
 	config.Set(configPrefix+"logs_dd_url", url)
 }
 
-func setupSBOMConfig(config Config, cacheDir string) {
-	config.BindEnvAndSetDefault("sbom.enabled", false)
-	config.BindEnvAndSetDefault("sbom.analyzers", []string{"os"})
-	config.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, cacheDir))
-	config.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false)
-	config.BindEnvAndSetDefault("sbom.use_custom_cache", false)
-	config.BindEnvAndSetDefault("sbom.custom_cache_max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage
-	config.BindEnvAndSetDefault("sbom.custom_cache_max_cache_entries", 10000)     // used by custom cache keys stored in memory
-	config.BindEnvAndSetDefault("sbom.cache_clean_interval", "30m")               // used by custom cache.
-}
-
 // ResolveSecrets merges all the secret values from origin into config. Secret values
 // are identified by a value of the form "ENC[key]" where key is the secret key.
 // See: https://github.com/DataDog/datadog-agent/blob/main/docs/agent/secrets.md

pkg/config/system_probe.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -59,7 +60,15 @@ func InitSystemProbeConfig(cfg Config) {
 	cfg.BindEnvAndSetDefault("ignore_host_etc", false)
 	cfg.BindEnvAndSetDefault("go_core_dump", false)
 
-	setupSBOMConfig(cfg, "sbom-sysprobe")
+	// SBOM configuration
+	cfg.BindEnvAndSetDefault("sbom.host.enabled", false)
+	cfg.BindEnvAndSetDefault("sbom.host.analyzers", []string{"os"})
+	cfg.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, "sbom-sysprobe"))
+	cfg.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false)
+	cfg.BindEnvAndSetDefault("sbom.cache.enabled", false)
+	cfg.BindEnvAndSetDefault("sbom.cache.max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage
+	cfg.BindEnvAndSetDefault("sbom.cache.max_cache_entries", 10000)     // used by custom cache keys stored in memory
+	cfg.BindEnvAndSetDefault("sbom.cache.clean_interval", "30m")        // used by custom cache.
 
 	// Auto exit configuration
 	cfg.BindEnvAndSetDefault("auto_exit.validation_period", 60)

pkg/sbom/sbom.go

@@ -33,15 +33,15 @@ type ScanOptions struct {
 // ScanOptionsFromConfig loads the scanning options from the configuration
 func ScanOptionsFromConfig(cfg config.Config, containers bool) (scanOpts ScanOptions) {
 	if containers {
-		scanOpts.CheckDiskUsage = config.Datadog.GetBool("container_image_collection.sbom.check_disk_usage")
-		scanOpts.MinAvailableDisk = uint64(config.Datadog.GetSizeInBytes("container_image_collection.sbom.min_available_disk"))
-		scanOpts.Timeout = time.Duration(config.Datadog.GetInt("container_image_collection.sbom.scan_timeout")) * time.Second
-		scanOpts.WaitAfter = time.Duration(config.Datadog.GetInt("container_image_collection.sbom.scan_interval")) * time.Second
-		scanOpts.Analyzers = config.Datadog.GetStringSlice("container_image_collection.sbom.analyzers")
+		scanOpts.CheckDiskUsage = config.Datadog.GetBool("sbom.container_image.check_disk_usage")
+		scanOpts.MinAvailableDisk = uint64(config.Datadog.GetSizeInBytes("sbom.container_image.min_available_disk"))
+		scanOpts.Timeout = time.Duration(config.Datadog.GetInt("sbom.container_image.scan_timeout")) * time.Second
+		scanOpts.WaitAfter = time.Duration(config.Datadog.GetInt("sbom.container_image.scan_interval")) * time.Second
+		scanOpts.Analyzers = config.Datadog.GetStringSlice("sbom.container_image.analyzers")
 	}
 
 	if len(scanOpts.Analyzers) == 0 {
-		scanOpts.Analyzers = config.Datadog.GetStringSlice("sbom.analyzers")
+		scanOpts.Analyzers = config.Datadog.GetStringSlice("sbom.host.analyzers")
 	}
 
 	return