Skip to content

Commit

Permalink
Refactor SBOM configuration parameters
Browse files Browse the repository at this point in the history
  • Loading branch information
L3n41c committed Apr 29, 2023
1 parent b100d09 commit c6f55c7
Show file tree
Hide file tree
Showing 19 changed files with 326 additions and 43 deletions.
5 changes: 5 additions & 0 deletions cmd/agent/dist/conf.d/container_image.d/conf.yaml.default
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
ad_identifiers:
- _container_image
init_config:
instances:
-
5 changes: 5 additions & 0 deletions cmd/agent/dist/conf.d/container_lifecycle.d/conf.yaml.default
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
ad_identifiers:
- _container_lifecycle
init_config:
instances:
-
5 changes: 5 additions & 0 deletions cmd/agent/dist/conf.d/sbom.d/conf.yaml.default
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
ad_identifiers:
- _sbom
init_config:
instances:
-
124 changes: 124 additions & 0 deletions pkg/autodiscovery/listeners/staticconfig.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2023-present Datadog, Inc.

package listeners

import (
"context"

"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
"github.com/DataDog/datadog-agent/pkg/config"
"github.com/DataDog/datadog-agent/pkg/util/containers"
)

// StaticConfigListener implements a ServiceListener based on static configuration parameters
type StaticConfigListener struct {
newService chan<- Service
}

// StaticConfigService represents services generated from StaticConfigListener
type StaticConfigService struct {
adIdentifier string
}

// Make sure StaticConfigService implements the Service interface
var _ Service = &StaticConfigService{}

func init() {
Register("static config", NewStaticConfigListener)
}

// NewStaticConfigListener creates a StaticConfigListener
func NewStaticConfigListener(Config) (ServiceListener, error) {
return &StaticConfigListener{}, nil
}

// Listen starts the goroutine to detect checks based on the config
func (l *StaticConfigListener) Listen(newSvc chan<- Service, delSvc chan<- Service) {
l.newService = newSvc

go l.createServices()
}

// Stop has nothing to do in this case
func (l *StaticConfigListener) Stop() {
}

func (l *StaticConfigListener) createServices() {
for _, staticCheck := range []string{
"container_image",
"container_lifecycle",
"sbom",
} {
if enabled := config.Datadog.GetBool(staticCheck + ".enabled"); enabled {
l.newService <- &StaticConfigService{adIdentifier: "_" + staticCheck}
}
}
}

// GetServiceID returns the unique entity name linked to that service
func (s *StaticConfigService) GetServiceID() string {
return s.adIdentifier
}

// GetTaggerEntity returns the tagger entity
func (s *StaticConfigService) GetTaggerEntity() string {
return ""
}

// GetADIdentifiers return the single AD identifier for a static config service
func (s *StaticConfigService) GetADIdentifiers(context.Context) ([]string, error) {
return []string{s.adIdentifier}, nil
}

// GetHosts is not supported
func (s *StaticConfigService) GetHosts(context.Context) (map[string]string, error) {
return nil, ErrNotSupported
}

// GetPorts returns nil and an error because port is not supported in this listener
func (s *StaticConfigService) GetPorts(context.Context) ([]ContainerPort, error) {
return nil, ErrNotSupported
}

// GetTags retrieves a container's tags
func (s *StaticConfigService) GetTags() ([]string, error) {
return nil, nil
}

// GetPid inspect the container and return its pid
// Not relevant in this listener
func (s *StaticConfigService) GetPid(context.Context) (int, error) {
return -1, ErrNotSupported
}

// GetHostname returns nil and an error because port is not supported in this listener
func (s *StaticConfigService) GetHostname(context.Context) (string, error) {
return "", ErrNotSupported
}

// IsReady is always true
func (s *StaticConfigService) IsReady(context.Context) bool {
return true
}

// GetCheckNames is not supported
func (s *StaticConfigService) GetCheckNames(context.Context) []string {
return nil
}

// HasFilter is not supported
func (s *StaticConfigService) HasFilter(filter containers.FilterType) bool {
return false
}

// GetExtraConfig is not supported
func (s *StaticConfigService) GetExtraConfig(key string) (string, error) {
return "", ErrNotSupported
}

// FilterTemplates does nothing.
func (s *StaticConfigService) FilterTemplates(configs map[string]integration.Config) {
}
4 changes: 2 additions & 2 deletions pkg/cli/subcommands/check/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,8 @@ func run(log log.Component, config config.Component, sysprobeconfig sysprobeconf

// Always disable SBOM collection in `check` command to avoid BoltDB flock issue
// and consuming CPU & Memory for asynchronous scans that would not be shown in `agent check` output.
pkgconfig.Datadog.Set("sbom.enabled", "false")
pkgconfig.Datadog.Set("container_image_collection.sbom.enabled", "false")
pkgconfig.Datadog.Set("sbom.host.enabled", "false")
pkgconfig.Datadog.Set("sbom.container_image.enabled", "false")
pkgconfig.Datadog.Set("runtime_security_config.sbom.enabled", "false")

hostnameDetected, err := hostname.Get(context.TODO())
Expand Down
6 changes: 6 additions & 0 deletions pkg/collector/corechecks/containerimage/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
package containerimage

import (
"errors"
"time"

yaml "gopkg.in/yaml.v2"

"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
"github.com/DataDog/datadog-agent/pkg/collector/check"
core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
ddConfig "github.com/DataDog/datadog-agent/pkg/config"
"github.com/DataDog/datadog-agent/pkg/util/log"
"github.com/DataDog/datadog-agent/pkg/workloadmeta"
)
Expand Down Expand Up @@ -101,6 +103,10 @@ func CheckFactory() check.Check {

// Configure parses the check configuration and initializes the container_image check
func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error {
if !ddConfig.Datadog.GetBool("container_image.enabled") {
return errors.New("collection of container images is disabled")
}

if err := c.CommonConfigure(integrationConfigDigest, initConfig, config, source); err != nil {
return err
}
Expand Down
6 changes: 6 additions & 0 deletions pkg/collector/corechecks/containerlifecycle/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ package containerlifecycle

import (
"context"
"errors"
"time"

yaml "gopkg.in/yaml.v2"

"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
"github.com/DataDog/datadog-agent/pkg/collector/check"
core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
ddConfig "github.com/DataDog/datadog-agent/pkg/config"
"github.com/DataDog/datadog-agent/pkg/util/log"
"github.com/DataDog/datadog-agent/pkg/workloadmeta"
)
Expand Down Expand Up @@ -50,6 +52,10 @@ type Check struct {

// Configure parses the check configuration and initializes the container_lifecycle check
func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error {
if !ddConfig.Datadog.GetBool("container_lifecycle.enabled") {
return errors.New("collection of container lifecycle events is disabled")
}

var err error

err = c.CommonConfigure(integrationConfigDigest, initConfig, config, source)
Expand Down
6 changes: 6 additions & 0 deletions pkg/collector/corechecks/sbom/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@
package sbom

import (
"errors"
"time"

yaml "gopkg.in/yaml.v2"

"github.com/DataDog/datadog-agent/pkg/autodiscovery/integration"
"github.com/DataDog/datadog-agent/pkg/collector/check"
core "github.com/DataDog/datadog-agent/pkg/collector/corechecks"
ddConfig "github.com/DataDog/datadog-agent/pkg/config"
"github.com/DataDog/datadog-agent/pkg/util/log"
"github.com/DataDog/datadog-agent/pkg/workloadmeta"
)
Expand Down Expand Up @@ -113,6 +115,10 @@ func CheckFactory() check.Check {

// Configure parses the check configuration and initializes the sbom check
func (c *Check) Configure(integrationConfigDigest uint64, config, initConfig integration.Data, source string) error {
if !ddConfig.Datadog.GetBool("sbom.enabled") {
return errors.New("collection of SBOM is disabled")
}

if err := c.CommonConfigure(integrationConfigDigest, initConfig, config, source); err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/collector/corechecks/sbom/processor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ func TestProcessEvents(t *testing.T) {
assert.Nil(t, err)
defer os.RemoveAll(cacheDir)
cfg.Set("sbom.cache_directory", cacheDir)
cfg.Set("container_image_collection.sbom.enabled", true)
cfg.Set("sbom.container_image.enabled", true)
_, err = sbomscanner.CreateGlobalScanner(cfg)
assert.Nil(t, err)

Expand Down
45 changes: 23 additions & 22 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1082,14 +1082,36 @@ func InitConfig(config Config) {
config.BindEnvAndSetDefault("orchestrator_explorer.manifest_collection.buffer_flush_interval", 20*time.Second)

// Container lifecycle configuration
config.BindEnvAndSetDefault("container_lifecycle.enabled", false)
bindEnvAndSetLogsConfigKeys(config, "container_lifecycle.")

// Container image configuration
config.BindEnvAndSetDefault("container_image.enabled", false)
bindEnvAndSetLogsConfigKeys(config, "container_image.")

// SBOM configuration
config.BindEnvAndSetDefault("sbom.enabled", false)
bindEnvAndSetLogsConfigKeys(config, "sbom.")
setupSBOMConfig(config, "sbom-agent")

config.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, "sbom-agent"))
config.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false)
config.BindEnvAndSetDefault("sbom.cache.enabled", false)
config.BindEnvAndSetDefault("sbom.cache.max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage
config.BindEnvAndSetDefault("sbom.cache.max_cache_entries", 10000) // used by custom cache keys stored in memory
config.BindEnvAndSetDefault("sbom.cache.clean_interval", "30m") // used by custom cache.

// Container SBOM configuration
config.BindEnvAndSetDefault("sbom.container_image.enabled", false)
config.BindEnvAndSetDefault("sbom.container_image.use_mount", false)
config.BindEnvAndSetDefault("sbom.container_image.scan_interval", 0) // Integer seconds
config.BindEnvAndSetDefault("sbom.container_image.scan_timeout", 10*60) // Integer seconds
config.BindEnvAndSetDefault("sbom.container_image.analyzers", []string{"os"})
config.BindEnvAndSetDefault("sbom.container_image.check_disk_usage", true)
config.BindEnvAndSetDefault("sbom.container_image.min_available_disk", "1Gb")

// Host SBOM configuration
config.BindEnvAndSetDefault("sbom.host.enabled", false)
config.BindEnvAndSetDefault("sbom.host.analyzers", []string{"os"})

// Orchestrator Explorer - process agent
// DEPRECATED in favor of `orchestrator_explorer.orchestrator_dd_url` setting. If both are set `orchestrator_explorer.orchestrator_dd_url` will take precedence.
Expand All @@ -1110,16 +1132,6 @@ func InitConfig(config Config) {
config.BindEnvAndSetDefault("inventories_max_interval", DefaultInventoriesMaxInterval) // integer seconds
config.BindEnvAndSetDefault("inventories_min_interval", DefaultInventoriesMinInterval) // integer seconds

// container_image_collection
config.BindEnvAndSetDefault("container_image_collection.metadata.enabled", false)
config.BindEnvAndSetDefault("container_image_collection.sbom.enabled", false)
config.BindEnvAndSetDefault("container_image_collection.sbom.use_mount", false)
config.BindEnvAndSetDefault("container_image_collection.sbom.scan_interval", 0) // Integer seconds
config.BindEnvAndSetDefault("container_image_collection.sbom.scan_timeout", 10*60) // Integer seconds
config.BindEnvAndSetDefault("container_image_collection.sbom.analyzers", []string{"os"})
config.BindEnvAndSetDefault("container_image_collection.sbom.check_disk_usage", true)
config.BindEnvAndSetDefault("container_image_collection.sbom.min_available_disk", "1Gb")

// Datadog security agent (common)
config.BindEnvAndSetDefault("security_agent.cmd_port", 5010)
config.BindEnvAndSetDefault("security_agent.expvar_port", 5011)
Expand Down Expand Up @@ -1658,17 +1670,6 @@ func setupFipsLogsConfig(config Config, configPrefix string, url string) {
config.Set(configPrefix+"logs_dd_url", url)
}

func setupSBOMConfig(config Config, cacheDir string) {
config.BindEnvAndSetDefault("sbom.enabled", false)
config.BindEnvAndSetDefault("sbom.analyzers", []string{"os"})
config.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, cacheDir))
config.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false)
config.BindEnvAndSetDefault("sbom.use_custom_cache", false)
config.BindEnvAndSetDefault("sbom.custom_cache_max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage
config.BindEnvAndSetDefault("sbom.custom_cache_max_cache_entries", 10000) // used by custom cache keys stored in memory
config.BindEnvAndSetDefault("sbom.cache_clean_interval", "30m") // used by custom cache.
}

// ResolveSecrets merges all the secret values from origin into config. Secret values
// are identified by a value of the form "ENC[key]" where key is the secret key.
// See: https://github.com/DataDog/datadog-agent/blob/main/docs/agent/secrets.md
Expand Down
11 changes: 10 additions & 1 deletion pkg/config/system_probe.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"encoding/json"
"os"
"path"
"path/filepath"
"strings"
"time"

Expand Down Expand Up @@ -59,7 +60,15 @@ func InitSystemProbeConfig(cfg Config) {
cfg.BindEnvAndSetDefault("ignore_host_etc", false)
cfg.BindEnvAndSetDefault("go_core_dump", false)

setupSBOMConfig(cfg, "sbom-sysprobe")
// SBOM configuration
cfg.BindEnvAndSetDefault("sbom.host.enabled", false)
cfg.BindEnvAndSetDefault("sbom.host.analyzers", []string{"os"})
cfg.BindEnvAndSetDefault("sbom.cache_directory", filepath.Join(defaultRunPath, "sbom-sysprobe"))
cfg.BindEnvAndSetDefault("sbom.clear_cache_on_exit", false)
cfg.BindEnvAndSetDefault("sbom.cache.enabled", false)
cfg.BindEnvAndSetDefault("sbom.cache.max_disk_size", 1000*1000*100) // used by custom cache: max disk space used by cached objects. Not equal to max disk usage
cfg.BindEnvAndSetDefault("sbom.cache.max_cache_entries", 10000) // used by custom cache keys stored in memory
cfg.BindEnvAndSetDefault("sbom.cache.clean_interval", "30m") // used by custom cache.

// Auto exit configuration
cfg.BindEnvAndSetDefault("auto_exit.validation_period", 60)
Expand Down
12 changes: 6 additions & 6 deletions pkg/sbom/sbom.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,15 @@ type ScanOptions struct {
// ScanOptionsFromConfig loads the scanning options from the configuration
func ScanOptionsFromConfig(cfg config.Config, containers bool) (scanOpts ScanOptions) {
if containers {
scanOpts.CheckDiskUsage = config.Datadog.GetBool("container_image_collection.sbom.check_disk_usage")
scanOpts.MinAvailableDisk = uint64(config.Datadog.GetSizeInBytes("container_image_collection.sbom.min_available_disk"))
scanOpts.Timeout = time.Duration(config.Datadog.GetInt("container_image_collection.sbom.scan_timeout")) * time.Second
scanOpts.WaitAfter = time.Duration(config.Datadog.GetInt("container_image_collection.sbom.scan_interval")) * time.Second
scanOpts.Analyzers = config.Datadog.GetStringSlice("container_image_collection.sbom.analyzers")
scanOpts.CheckDiskUsage = config.Datadog.GetBool("sbom.container_image.check_disk_usage")
scanOpts.MinAvailableDisk = uint64(config.Datadog.GetSizeInBytes("sbom.container_image.min_available_disk"))
scanOpts.Timeout = time.Duration(config.Datadog.GetInt("sbom.container_image.scan_timeout")) * time.Second
scanOpts.WaitAfter = time.Duration(config.Datadog.GetInt("sbom.container_image.scan_interval")) * time.Second
scanOpts.Analyzers = config.Datadog.GetStringSlice("sbom.container_image.analyzers")
}

if len(scanOpts.Analyzers) == 0 {
scanOpts.Analyzers = config.Datadog.GetStringSlice("sbom.analyzers")
scanOpts.Analyzers = config.Datadog.GetStringSlice("sbom.host.analyzers")
}

return
Expand Down
4 changes: 2 additions & 2 deletions pkg/sbom/scanner/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func (s *Scanner) start(ctx context.Context) {
return
}
go func() {
cleanTicker := time.NewTicker(config.Datadog.GetDuration("sbom.cache_clean_interval"))
cleanTicker := time.NewTicker(config.Datadog.GetDuration("sbom.cache.clean_interval"))
defer cleanTicker.Stop()
s.running = true
defer func() { s.running = false }()
Expand Down Expand Up @@ -174,7 +174,7 @@ func NewScanner(cfg config.Config) *Scanner {
// global one, and returns it. Start() needs to be called before any data
// collection happens.
func CreateGlobalScanner(cfg config.Config) (*Scanner, error) {
if !cfg.GetBool("sbom.enabled") && !cfg.GetBool("container_image_collection.sbom.enabled") && !cfg.GetBool("runtime_security_config.sbom.enabled") {
if !cfg.GetBool("sbom.host.enabled") && !cfg.GetBool("sbom.container_image.enabled") && !cfg.GetBool("runtime_security_config.sbom.enabled") {
return nil, nil
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/util/docker/event_stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ func eventFilters() filters.Args {
res.Add("event", containerEventAction)
}

if config.Datadog.GetBool("container_image_collection.metadata.enabled") {
if config.Datadog.GetBool("container_image.enabled") {
res.Add("type", events.ImageEventType)
for _, imageEventAction := range imageEventActions {
res.Add("event", imageEventAction)
Expand Down
Loading

0 comments on commit c6f55c7

Please sign in to comment.