Skip to content

Commit

Permalink
Refactor SBOM configuration parameters (#16847)
Browse files Browse the repository at this point in the history
  • Loading branch information
L3n41c authored May 26, 2023
1 parent a2fea06 commit 161899d
Show file tree
Hide file tree
Showing 22 changed files with 340 additions and 48 deletions.
11 changes: 8 additions & 3 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -116,10 +116,15 @@
/cmd/agent/subcommands/workloadlist @DataDog/container-integrations
/cmd/agent/subcommands/run/internal/clcrunnerapi/ @DataDog/container-integrations @DataDog/agent-shared-components
/cmd/agent/windows @DataDog/windows-agent
/cmd/agent/dist/conf.d/jetson.d @DataDog/agent-platform
/cmd/agent/dist/conf.d/oracle-dbm.d @DataDog/database-monitoring
/cmd/agent/dist/conf.d/snmp.d/ @DataDog/network-device-monitoring
/cmd/agent/dist/conf.d/container.d/ @DataDog/container-integrations
/cmd/agent/dist/conf.d/containerd.d/ @DataDog/container-integrations
/cmd/agent/dist/conf.d/container_image.d/ @DataDog/container-integrations
/cmd/agent/dist/conf.d/container_lifecycle.d/ @DataDog/container-integrations
/cmd/agent/dist/conf.d/jetson.d/ @DataDog/agent-platform
/cmd/agent/dist/conf.d/oracle-dbm.d/ @DataDog/database-monitoring
/cmd/agent/dist/conf.d/oracle-dbm.d/conf.yaml.default @DataDog/database-monitoring
/cmd/agent/dist/conf.d/sbom.d/ @DataDog/container-integrations
/cmd/agent/dist/conf.d/snmp.d/ @DataDog/network-device-monitoring
/cmd/agent/*.manifest @DataDog/agent-platform
/cmd/agent/*.mc @DataDog/agent-platform
/cmd/agent/*.rc @DataDog/agent-platform
Expand Down
5 changes: 5 additions & 0 deletions cmd/agent/dist/conf.d/container_image.d/conf.yaml.default
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
ad_identifiers:
- _container_image
init_config:
instances:
-
5 changes: 5 additions & 0 deletions cmd/agent/dist/conf.d/container_lifecycle.d/conf.yaml.default
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
ad_identifiers:
- _container_lifecycle
init_config:
instances:
-
5 changes: 5 additions & 0 deletions cmd/agent/dist/conf.d/sbom.d/conf.yaml.default
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
ad_identifiers:
- _sbom
init_config:
instances:
-
124 changes: 124 additions & 0 deletions pkg/autodiscovery/listeners/staticconfig.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2023-present Datadog, Inc.

package listeners

import (
"context"

"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
"github.com/DataDog/datadog-agent/pkg/config"
"github.com/DataDog/datadog-agent/pkg/util/containers"
)

// StaticConfigListener implements a ServiceListener based on static configuration parameters
type StaticConfigListener struct {
newService chan<- Service
}

// StaticConfigService represents services generated from StaticConfigListener
type StaticConfigService struct {
adIdentifier string
}

// Make sure StaticConfigService implements the Service interface
var _ Service = &StaticConfigService{}

func init() {
Register("static config", NewStaticConfigListener)
}

// NewStaticConfigListener creates a StaticConfigListener
func NewStaticConfigListener(Config) (ServiceListener, error) {
return &StaticConfigListener{}, nil
}

// Listen starts the goroutine to detect checks based on the config
func (l *StaticConfigListener) Listen(newSvc chan<- Service, delSvc chan<- Service) {
l.newService = newSvc

go l.createServices()
}

// Stop has nothing to do in this case
func (l *StaticConfigListener) Stop() {
}

func (l *StaticConfigListener) createServices() {
for _, staticCheck := range []string{
"container_image",
"container_lifecycle",
"sbom",
} {
if enabled := config.Datadog.GetBool(staticCheck + ".enabled"); enabled {
l.newService <- &StaticConfigService{adIdentifier: "_" + staticCheck}
}
}
}

// GetServiceID returns the unique entity name linked to that service
func (s *StaticConfigService) GetServiceID() string {
return s.adIdentifier
}

// GetTaggerEntity returns the tagger entity
func (s *StaticConfigService) GetTaggerEntity() string {
return ""
}

// GetADIdentifiers return the single AD identifier for a static config service
func (s *StaticConfigService) GetADIdentifiers(context.Context) ([]string, error) {
return []string{s.adIdentifier}, nil
}

// GetHosts is not supported
func (s *StaticConfigService) GetHosts(context.Context) (map[string]string, error) {
return nil, ErrNotSupported
}

// GetPorts returns nil and an error because port is not supported in this listener
func (s *StaticConfigService) GetPorts(context.Context) ([]ContainerPort, error) {
return nil, ErrNotSupported
}

// GetTags retrieves a container's tags
func (s *StaticConfigService) GetTags() ([]string, error) {
return nil, nil
}

// GetPid inspect the container and return its pid
// Not relevant in this listener
func (s *StaticConfigService) GetPid(context.Context) (int, error) {
return -1, ErrNotSupported
}

// GetHostname returns nil and an error because port is not supported in this listener
func (s *StaticConfigService) GetHostname(context.Context) (string, error) {
return "", ErrNotSupported
}

// IsReady is always true
func (s *StaticConfigService) IsReady(context.Context) bool {
return true
}

// GetCheckNames is not supported
func (s *StaticConfigService) GetCheckNames(context.Context) []string {
return nil
}

// HasFilter is not supported
func (s *StaticConfigService) HasFilter(filter containers.FilterType) bool {
return false
}

// GetExtraConfig is not supported
func (s *StaticConfigService) GetExtraConfig(key string) (string, error) {
return "", ErrNotSupported
}

// FilterTemplates does nothing.
func (s *StaticConfigService) FilterTemplates(configs map[string]integration.Config) {
}
4 changes: 2 additions & 2 deletions pkg/cli/subcommands/check/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,8 @@ func run(log log.Component, config config.Component, sysprobeconfig sysprobeconf

// Always disable SBOM collection in `check` command to avoid BoltDB flock issue
// and consuming CPU & Memory for asynchronous scans that would not be shown in `agent check` output.
pkgconfig.Datadog.Set("sbom.enabled", "false")
pkgconfig.Datadog.Set("container_image_collection.sbom.enabled", "false")
pkgconfig.Datadog.Set("sbom.host.enabled", "false")
pkgconfig.Datadog.Set("sbom.container_image.enabled", "false")
pkgconfig.Datadog.Set("runtime_security_config.sbom.enabled", "false")

hostnameDetected, err := hostname.Get(context.TODO())
Expand Down
6 changes: 6 additions & 0 deletions pkg/collector/corechecks/containerimage/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
package containerimage

import (
"errors"
"time"

yaml "gopkg.in/yaml.v2"

"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
"github.com/DataDog/datadog-agent/pkg/collector/check"
core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
ddConfig "github.com/DataDog/datadog-agent/pkg/config"
"github.com/DataDog/datadog-agent/pkg/util/log"
"github.com/DataDog/datadog-agent/pkg/workloadmeta"
)
Expand Down Expand Up @@ -101,6 +103,10 @@ func CheckFactory() check.Check {

// Configure parses the check configuration and initializes the container_image check
func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error {
if !ddConfig.Datadog.GetBool("container_image.enabled") {
return errors.New("collection of container images is disabled")
}

if err := c.CommonConfigure(integrationConfigDigest, initConfig, config, source); err != nil {
return err
}
Expand Down
6 changes: 6 additions & 0 deletions pkg/collector/corechecks/containerlifecycle/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ package containerlifecycle

import (
"context"
"errors"
"time"

yaml "gopkg.in/yaml.v2"

"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
"github.com/DataDog/datadog-agent/pkg/collector/check"
core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
ddConfig "github.com/DataDog/datadog-agent/pkg/config"
"github.com/DataDog/datadog-agent/pkg/util/log"
"github.com/DataDog/datadog-agent/pkg/workloadmeta"
)
Expand Down Expand Up @@ -50,6 +52,10 @@ type Check struct {

// Configure parses the check configuration and initializes the container_lifecycle check
func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error {
if !ddConfig.Datadog.GetBool("container_lifecycle.enabled") {
return errors.New("collection of container lifecycle events is disabled")
}

var err error

err = c.CommonConfigure(integrationConfigDigest, initConfig, config, source)
Expand Down
8 changes: 7 additions & 1 deletion pkg/collector/corechecks/sbom/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@
package sbom

import (
"errors"
"time"

yaml "gopkg.in/yaml.v2"

"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
"github.com/DataDog/datadog-agent/pkg/collector/check"
core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
ddConfig "github.com/DataDog/datadog-agent/pkg/config"
"github.com/DataDog/datadog-agent/pkg/util/log"
"github.com/DataDog/datadog-agent/pkg/workloadmeta"
)
Expand Down Expand Up @@ -112,6 +114,10 @@ func CheckFactory() check.Check {

// Configure parses the check configuration and initializes the sbom check
func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error {
if !ddConfig.Datadog.GetBool("sbom.enabled") {
return errors.New("collection of SBOM is disabled")
}

if err := c.CommonConfigure(integrationConfigDigest, initConfig, config, source); err != nil {
return err
}
Expand All @@ -125,7 +131,7 @@ func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig int
return err
}

c.processor, err = newProcessor(c.workloadmetaStore, sender, c.instance.ChunkSize, time.Duration(c.instance.NewSBOMMaxLatencySeconds)*time.Second, c.instance.HostSBOM)
c.processor, err = newProcessor(c.workloadmetaStore, sender, c.instance.ChunkSize, time.Duration(c.instance.NewSBOMMaxLatencySeconds)*time.Second, ddConfig.Datadog.GetBool("sbom.host.enabled"))
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/collector/corechecks/sbom/processor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,7 @@ func TestProcessEvents(t *testing.T) {
assert.Nil(t, err)
defer os.RemoveAll(cacheDir)
cfg.Set("sbom.cache_directory", cacheDir)
cfg.Set("container_image_collection.sbom.enabled", true)
cfg.Set("sbom.container_image.enabled", true)
_, err = sbomscanner.CreateGlobalScanner(cfg)
assert.Nil(t, err)

Expand Down
3 changes: 2 additions & 1 deletion pkg/config/autodiscovery/autodiscovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,9 @@ func DiscoverComponentsFromEnv() ([]config.ConfigurationProviders, []config.List
detectedListeners := []config.Listeners{}

// When using automatic discovery of providers/listeners
// We automatically activate the environment listener
// We automatically activate the environment and static config listener
detectedListeners = append(detectedListeners, config.Listeners{Name: "environment"})
detectedListeners = append(detectedListeners, config.Listeners{Name: "static config"})

// Automatic handling of AD providers/listeners should only run in Core agent.
if flavor.GetFlavor() != flavor.DefaultAgent {
Expand Down
45 changes: 23 additions & 22 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1098,14 +1098,36 @@ func InitConfig(config Config) {
config.BindEnvAndSetDefault("orchestrator_explorer.manifest_collection.buffer_flush_interval", 20*time.Second)

// Container lifecycle configuration
config.BindEnvAndSetDefault("container_lifecycle.enabled", false)
bindEnvAndSetLogsConfigKeys(config, "container_lifecycle.")

// Container image configuration
config.BindEnvAndSetDefault("container_image.enabled", false)
bindEnvAndSetLogsConfigKeys(config, "container_image.")

// SBOM configuration
config.BindEnvAndSetDefault("sbom.enabled", false)
bindEnvAndSetLogsConfigKeys(config, "sbom.")
setupSBOMConfig(config, "sbom-agent")

config.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, "sbom-agent"))
config.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false)
config.BindEnvAndSetDefault("sbom.cache.enabled", false)
config.BindEnvAndSetDefault("sbom.cache.max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage
config.BindEnvAndSetDefault("sbom.cache.max_cache_entries", 10000) // used by custom cache keys stored in memory
config.BindEnvAndSetDefault("sbom.cache.clean_interval", "30m") // used by custom cache.

// Container SBOM configuration
config.BindEnvAndSetDefault("sbom.container_image.enabled", false)
config.BindEnvAndSetDefault("sbom.container_image.use_mount", false)
config.BindEnvAndSetDefault("sbom.container_image.scan_interval", 0) // Integer seconds
config.BindEnvAndSetDefault("sbom.container_image.scan_timeout", 10*60) // Integer seconds
config.BindEnvAndSetDefault("sbom.container_image.analyzers", []string{"os"})
config.BindEnvAndSetDefault("sbom.container_image.check_disk_usage", true)
config.BindEnvAndSetDefault("sbom.container_image.min_available_disk", "1Gb")

// Host SBOM configuration
config.BindEnvAndSetDefault("sbom.host.enabled", false)
config.BindEnvAndSetDefault("sbom.host.analyzers", []string{"os"})

// Orchestrator Explorer - process agent
// DEPRECATED in favor of `orchestrator_explorer.orchestrator_dd_url` setting. If both are set `orchestrator_explorer.orchestrator_dd_url` will take precedence.
Expand All @@ -1127,16 +1149,6 @@ func InitConfig(config Config) {
config.BindEnvAndSetDefault("inventories_max_interval", DefaultInventoriesMaxInterval) // integer seconds
config.BindEnvAndSetDefault("inventories_min_interval", DefaultInventoriesMinInterval) // integer seconds

// container_image_collection
config.BindEnvAndSetDefault("container_image_collection.metadata.enabled", false)
config.BindEnvAndSetDefault("container_image_collection.sbom.enabled", false)
config.BindEnvAndSetDefault("container_image_collection.sbom.use_mount", false)
config.BindEnvAndSetDefault("container_image_collection.sbom.scan_interval", 0) // Integer seconds
config.BindEnvAndSetDefault("container_image_collection.sbom.scan_timeout", 10*60) // Integer seconds
config.BindEnvAndSetDefault("container_image_collection.sbom.analyzers", []string{"os"})
config.BindEnvAndSetDefault("container_image_collection.sbom.check_disk_usage", true)
config.BindEnvAndSetDefault("container_image_collection.sbom.min_available_disk", "1Gb")

// Datadog security agent (common)
config.BindEnvAndSetDefault("security_agent.cmd_port", 5010)
config.BindEnvAndSetDefault("security_agent.expvar_port", 5011)
Expand Down Expand Up @@ -1677,17 +1689,6 @@ func setupFipsLogsConfig(config Config, configPrefix string, url string) {
config.Set(configPrefix+"logs_dd_url", url)
}

func setupSBOMConfig(config Config, cacheDir string) {
config.BindEnvAndSetDefault("sbom.enabled", false)
config.BindEnvAndSetDefault("sbom.analyzers", []string{"os"})
config.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, cacheDir))
config.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false)
config.BindEnvAndSetDefault("sbom.use_custom_cache", false)
config.BindEnvAndSetDefault("sbom.custom_cache_max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage
config.BindEnvAndSetDefault("sbom.custom_cache_max_cache_entries", 10000) // used by custom cache keys stored in memory
config.BindEnvAndSetDefault("sbom.cache_clean_interval", "30m") // used by custom cache.
}

// ResolveSecrets merges all the secret values from origin into config. Secret values
// are identified by a value of the form "ENC[key]" where key is the secret key.
// See: https://github.com/DataDog/datadog-agent/blob/main/docs/agent/secrets.md
Expand Down
11 changes: 10 additions & 1 deletion pkg/config/system_probe.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"encoding/json"
"os"
"path"
"path/filepath"
"strings"
"time"

Expand Down Expand Up @@ -59,7 +60,15 @@ func InitSystemProbeConfig(cfg Config) {
cfg.BindEnvAndSetDefault("ignore_host_etc", false)
cfg.BindEnvAndSetDefault("go_core_dump", false)

setupSBOMConfig(cfg, "sbom-sysprobe")
// SBOM configuration
cfg.BindEnvAndSetDefault("sbom.host.enabled", false)
cfg.BindEnvAndSetDefault("sbom.host.analyzers", []string{"os"})
cfg.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, "sbom-sysprobe"))
cfg.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false)
cfg.BindEnvAndSetDefault("sbom.cache.enabled", false)
cfg.BindEnvAndSetDefault("sbom.cache.max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage
cfg.BindEnvAndSetDefault("sbom.cache.max_cache_entries", 10000) // used by custom cache keys stored in memory
cfg.BindEnvAndSetDefault("sbom.cache.clean_interval", "30m") // used by custom cache.

// Auto exit configuration
cfg.BindEnvAndSetDefault("auto_exit.validation_period", 60)
Expand Down
Loading

0 comments on commit 161899d

Please sign in to comment.