Skip to content

Commit

Permalink
Fix the issue that we cannot modify apm_config.obfuscation.* throug…
Browse files Browse the repository at this point in the history 
…h env vars (#32318)
  • Loading branch information
@keisku @louis-cqrl
keisku authored and louis-cqrl committed Dec 25, 2024
1 parent 9a88be4 commit 3b83d08
Show file tree
Hide file tree
Showing 8 changed files with 168 additions and 154 deletions.
131 changes: 29 additions & 102 deletions comp/trace/config/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -405,110 +405,37 @@ func applyDatadogConfig(c *config.AgentConfig, core corecompcfg.Component) error
c.TelemetryConfig.Endpoints = appendEndpoints(c.TelemetryConfig.Endpoints, "apm_config.telemetry.additional_endpoints")
}
c.Obfuscation = new(config.ObfuscationConfig)
if core.IsSet("apm_config.obfuscation") {
cfg := pkgconfigsetup.Datadog()
var o config.ObfuscationConfig
err := structure.UnmarshalKey(cfg, "apm_config.obfuscation", &o)
if err == nil {
c.Obfuscation = &o
if o.RemoveStackTraces {
if err = addReplaceRule(c, "error.stack", `(?s).*`, "?"); err != nil {
return err
}
}
}
}
{
// Obfuscation of database statements will be ON by default. Any new obfuscators should likely be
// enabled by default as well. This can be explicitly disabled with the agent config. Any changes
// to obfuscation options or defaults must be reflected in the public docs.
c.Obfuscation.ES.Enabled = true
c.Obfuscation.OpenSearch.Enabled = true
c.Obfuscation.Mongo.Enabled = true
c.Obfuscation.Memcached.Enabled = true
c.Obfuscation.Redis.Enabled = true
c.Obfuscation.CreditCards.Enabled = true
c.Obfuscation.Cache.Enabled = true

// TODO(x): There is an issue with pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation"), probably coming from Viper,
// where it returns false even is "apm_config.obfuscation.credit_cards.enabled" is set via an environment
// variable, so we need a temporary workaround by specifically setting env. var. accessible fields.
if core.IsSet("apm_config.obfuscation.credit_cards.enabled") {
c.Obfuscation.CreditCards.Enabled = core.GetBool("apm_config.obfuscation.credit_cards.enabled")
}
if core.IsSet("apm_config.obfuscation.credit_cards.luhn") {
c.Obfuscation.CreditCards.Luhn = core.GetBool("apm_config.obfuscation.credit_cards.luhn")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.elasticsearch.enabled") {
c.Obfuscation.ES.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.elasticsearch.enabled")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.elasticsearch.keep_values") {
c.Obfuscation.ES.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.elasticsearch.keep_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.elasticsearch.obfuscate_sql_values") {
c.Obfuscation.ES.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.elasticsearch.obfuscate_sql_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.opensearch.enabled") {
c.Obfuscation.OpenSearch.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.opensearch.enabled")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.opensearch.keep_values") {
c.Obfuscation.OpenSearch.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.opensearch.keep_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.opensearch.obfuscate_sql_values") {
c.Obfuscation.OpenSearch.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.opensearch.obfuscate_sql_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.http.remove_query_string") {
c.Obfuscation.HTTP.RemoveQueryString = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.http.remove_query_string")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.http.remove_paths_with_digits") {
c.Obfuscation.HTTP.RemovePathDigits = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.http.remove_paths_with_digits")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.memcached.enabled") {
c.Obfuscation.Memcached.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.memcached.enabled")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.memcached.keep_command") {
c.Obfuscation.Memcached.KeepCommand = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.memcached.keep_command")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.mongodb.enabled") {
c.Obfuscation.Mongo.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.mongodb.enabled")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.mongodb.keep_values") {
c.Obfuscation.Mongo.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.mongodb.keep_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.mongodb.obfuscate_sql_values") {
c.Obfuscation.Mongo.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.mongodb.obfuscate_sql_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.redis.enabled") {
c.Obfuscation.Redis.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.redis.enabled")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.redis.remove_all_args") {
c.Obfuscation.Redis.RemoveAllArgs = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.redis.remove_all_args")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.remove_stack_traces") {
c.Obfuscation.RemoveStackTraces = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.remove_stack_traces")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.sql_exec_plan.enabled") {
c.Obfuscation.SQLExecPlan.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.sql_exec_plan.enabled")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.sql_exec_plan.keep_values") {
c.Obfuscation.SQLExecPlan.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.sql_exec_plan.keep_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.sql_exec_plan.obfuscate_sql_values") {
c.Obfuscation.SQLExecPlan.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.sql_exec_plan.obfuscate_sql_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.sql_exec_plan_normalize.enabled") {
c.Obfuscation.SQLExecPlanNormalize.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.sql_exec_plan_normalize.enabled")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.sql_exec_plan_normalize.keep_values") {
c.Obfuscation.SQLExecPlanNormalize.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.sql_exec_plan_normalize.keep_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.sql_exec_plan_normalize.obfuscate_sql_values") {
c.Obfuscation.SQLExecPlanNormalize.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.sql_exec_plan_normalize.obfuscate_sql_values")
}
if pkgconfigsetup.Datadog().IsSet("apm_config.obfuscation.cache.enabled") {
c.Obfuscation.Cache.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.cache.enabled")
c.Obfuscation.ES.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.elasticsearch.enabled")
c.Obfuscation.ES.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.elasticsearch.keep_values")
c.Obfuscation.ES.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.elasticsearch.obfuscate_sql_values")
c.Obfuscation.OpenSearch.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.opensearch.enabled")
c.Obfuscation.OpenSearch.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.opensearch.keep_values")
c.Obfuscation.OpenSearch.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.opensearch.obfuscate_sql_values")
c.Obfuscation.Mongo.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.mongodb.enabled")
c.Obfuscation.Mongo.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.mongodb.keep_values")
c.Obfuscation.Mongo.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.mongodb.obfuscate_sql_values")
c.Obfuscation.SQLExecPlan.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.sql_exec_plan.enabled")
c.Obfuscation.SQLExecPlan.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.sql_exec_plan.keep_values")
c.Obfuscation.SQLExecPlan.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.sql_exec_plan.obfuscate_sql_values")
c.Obfuscation.SQLExecPlanNormalize.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.sql_exec_plan_normalize.enabled")
c.Obfuscation.SQLExecPlanNormalize.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.sql_exec_plan_normalize.keep_values")
c.Obfuscation.SQLExecPlanNormalize.ObfuscateSQLValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.sql_exec_plan_normalize.obfuscate_sql_values")
c.Obfuscation.HTTP.RemoveQueryString = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.http.remove_query_string")
c.Obfuscation.HTTP.RemovePathDigits = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.http.remove_paths_with_digits")
c.Obfuscation.RemoveStackTraces = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.remove_stack_traces")
if c.Obfuscation.RemoveStackTraces {
if err = addReplaceRule(c, "error.stack", `(?s).*`, "?"); err != nil {
return err
}
}
c.Obfuscation.Memcached.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.memcached.enabled")
c.Obfuscation.Memcached.KeepCommand = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.memcached.keep_command")
c.Obfuscation.Redis.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.redis.enabled")
c.Obfuscation.Redis.RemoveAllArgs = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.redis.remove_all_args")
c.Obfuscation.CreditCards.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.credit_cards.enabled")
c.Obfuscation.CreditCards.Luhn = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.credit_cards.luhn")
c.Obfuscation.CreditCards.KeepValues = pkgconfigsetup.Datadog().GetStringSlice("apm_config.obfuscation.credit_cards.keep_values")
c.Obfuscation.Cache.Enabled = pkgconfigsetup.Datadog().GetBool("apm_config.obfuscation.cache.enabled")

if core.IsSet("apm_config.filter_tags.require") {
tags := core.GetStringSlice("apm_config.filter_tags.require")
Expand Down
23 changes: 13 additions & 10 deletions pkg/collector/python/datadog_agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ var (
// the GIL is always locked when calling c code from python which means that the exported functions in this file
// will only ever be called by one goroutine at a time
obfuscator *obfuscate.Obfuscator
obfuscaterConfig obfuscate.Config // For testing purposes
obfuscatorLoader sync.Once
)

Expand All @@ -253,21 +254,23 @@ var (
// will definitely be initialized by the time one of the python checks runs
func lazyInitObfuscator() *obfuscate.Obfuscator {
obfuscatorLoader.Do(func() {
var cfg obfuscate.Config
if err := structure.UnmarshalKey(pkgconfigsetup.Datadog(), "apm_config.obfuscation", &cfg); err != nil {
if err := structure.UnmarshalKey(pkgconfigsetup.Datadog(), "apm_config.obfuscation", &obfuscaterConfig); err != nil {
log.Errorf("Failed to unmarshal apm_config.obfuscation: %s", err.Error())
cfg = obfuscate.Config{}
obfuscaterConfig = obfuscate.Config{}
}
if !cfg.SQLExecPlan.Enabled {
cfg.SQLExecPlan = defaultSQLPlanObfuscateSettings
if !obfuscaterConfig.SQLExecPlan.Enabled {
obfuscaterConfig.SQLExecPlan = defaultSQLPlanObfuscateSettings
}
if !cfg.SQLExecPlanNormalize.Enabled {
cfg.SQLExecPlanNormalize = defaultSQLPlanNormalizeSettings
if !obfuscaterConfig.SQLExecPlanNormalize.Enabled {
obfuscaterConfig.SQLExecPlanNormalize = defaultSQLPlanNormalizeSettings
}
if !cfg.Mongo.Enabled {
cfg.Mongo = defaultMongoObfuscateSettings
if len(obfuscaterConfig.Mongo.KeepValues) == 0 {
obfuscaterConfig.Mongo.KeepValues = defaultMongoObfuscateSettings.KeepValues
}
obfuscator = obfuscate.NewObfuscator(cfg)
if len(obfuscaterConfig.Mongo.ObfuscateSQLValues) == 0 {
obfuscaterConfig.Mongo.ObfuscateSQLValues = defaultMongoObfuscateSettings.ObfuscateSQLValues
}
obfuscator = obfuscate.NewObfuscator(obfuscaterConfig)
})
return obfuscator
}
Expand Down
4 changes: 4 additions & 0 deletions pkg/collector/python/datadog_agent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,7 @@ func TestSetExternalTags(t *testing.T) {
func TestEmitAgentTelemetry(t *testing.T) {
testEmitAgentTelemetry(t)
}

func TestObfuscaterConfig(t *testing.T) {
testObfuscaterConfig(t)
}
48 changes: 48 additions & 0 deletions pkg/collector/python/test_datadog_agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ package python
import (
"context"
"math/rand/v2"
"strings"
"sync"
"testing"
"time"
Expand All @@ -19,6 +20,9 @@ import (
yaml "gopkg.in/yaml.v2"

"github.com/DataDog/datadog-agent/pkg/collector/externalhost"
pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model"
pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup"
"github.com/DataDog/datadog-agent/pkg/obfuscate"
"github.com/DataDog/datadog-agent/pkg/util"
"github.com/DataDog/datadog-agent/pkg/util/hostname"
"github.com/DataDog/datadog-agent/pkg/util/kubernetes/clustername"
Expand Down Expand Up @@ -122,3 +126,47 @@ func testEmitAgentTelemetry(t *testing.T) {

assert.True(t, true)
}

func testObfuscaterConfig(t *testing.T) {
pkgconfigmodel.CleanOverride(t)
conf := pkgconfigmodel.NewConfig("datadog", "DD", strings.NewReplacer(".", "_")) // nolint: forbidigo // legit use case
pkgconfigsetup.InitConfig(conf)
o := lazyInitObfuscator()
o.Stop()
expected := obfuscate.Config{
ES: obfuscate.JSONConfig{
Enabled: true,
KeepValues: []string{},
ObfuscateSQLValues: []string{},
},
OpenSearch: obfuscate.JSONConfig{
Enabled: true,
KeepValues: []string{},
ObfuscateSQLValues: []string{},
},
Mongo: defaultMongoObfuscateSettings,
SQLExecPlan: defaultSQLPlanObfuscateSettings,
SQLExecPlanNormalize: defaultSQLPlanNormalizeSettings,
HTTP: obfuscate.HTTPConfig{
RemoveQueryString: false,
RemovePathDigits: false,
},
Redis: obfuscate.RedisConfig{
Enabled: true,
RemoveAllArgs: false,
},
Memcached: obfuscate.MemcachedConfig{
Enabled: true,
KeepCommand: false,
},
CreditCard: obfuscate.CreditCardsConfig{
Enabled: true,
Luhn: false,
KeepValues: []string{},
},
Cache: obfuscate.CacheConfig{
Enabled: true,
},
}
assert.Equal(t, expected, obfuscaterConfig)
}
12 changes: 6 additions & 6 deletions pkg/config/config_template.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1235,28 +1235,28 @@ api_key:
# remove_stack_traces: false
#
# sql_exec_plan:
## @param DD_APM_SQL_EXEC_PLAN_ENABLED - boolean - optional
## @param DD_APM_OBFUSCATION_SQL_EXEC_PLAN_ENABLED - boolean - optional
## Enables obfuscation rules for JSON query execution plans. Disabled by default.
# enabled: false
## @param DD_APM_SQL_EXEC_PLAN_KEEP_VALUES - object - optional
## @param DD_APM_OBFUSCATION_SQL_EXEC_PLAN_KEEP_VALUES - object - optional
## List of keys that should not be obfuscated.
# keep_values:
# - id1
## @param DD_APM_SQL_EXEC_PLAN_OBFUSCATE_SQL_VALUES - boolean - optional
## @param DD_APM_OBFUSCATION_SQL_EXEC_PLAN_OBFUSCATE_SQL_VALUES - boolean - optional
## The set of keys for which their values will be passed through SQL obfuscation
# obfuscate_sql_values:
# - val1
#
# sql_exec_plan_normalize:
## @param DD_APM_SQL_EXEC_PLAN_NORMALIZE_ENABLED - boolean - optional
## @param DD_APM_OBFUSCATION_SQL_EXEC_PLAN_NORMALIZE_ENABLED - boolean - optional
## Enables obfuscation rules for JSON query execution plans, including cost and row estimates.
## Produces a normalized execution plan. Disabled by default.
# enabled: false
## @param DD_APM_SQL_EXEC_PLAN_NORMALIZE_KEEP_VALUES - object - optional
## @param DD_APM_OBFUSCATION_SQL_EXEC_PLAN_NORMALIZE_KEEP_VALUES - object - optional
## List of keys that should not be obfuscated.
# keep_values:
# - id1
## @param DD_APM_SQL_EXEC_PLAN_NORMALIZE_OBFUSCATE_SQL_VALUES - boolean - optional
## @param DD_APM_OBFUSCATION_SQL_EXEC_PLAN_NORMALIZE_OBFUSCATE_SQL_VALUES - boolean - optional
## The set of keys for which their values will be passed through SQL obfuscation
# obfuscate_sql_values:
# - val1
Expand Down
Loading

0 comments on commit 3b83d08

Please sign in to comment.