Skip to content
This repository has been archived by the owner on Jul 27, 2023. It is now read-only.

Move INI config parsing to common agent config util #264

Closed
wants to merge 4 commits into from
Closed

Move INI config parsing to common agent config util #264

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d1c4f62
INI configuration parsing is now done using common util
sunhay Mar 7, 2019
e988395
Handle agent 5 proxies and update related tests
sunhay Mar 7, 2019
bae750c
Set statsd host if configured
sunhay Mar 7, 2019
b79ff8e
Add log message if we detect both config types
sunhay Mar 7, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
841 changes: 681 additions & 160 deletions Gopkg.lock
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions Gopkg.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
[[constraint]]
name = "github.com/DataDog/datadog-agent"
revision = "4aa90bd394b62b0d3e3f15703c6e459c1c477fbc"
revision = "87fcbdb53630c978b8561d2c2272b285123c3d42"

[[constraint]]
name = "github.com/DataDog/gopsutil"
revision = "c8f74f1344dd41bc49ec4d3c2377c526aaedfd20"

[[constraint]]
name = "github.com/gogo/protobuf"
revision = "d76fbc1373015ced59b43ac267f28d546b955683"
version = "~1.0.0"
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is required due to a change in datadog-agent. We'll have to test this well.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Any way to pin this to a hard version?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like to keep it in-sync with datadog-agent:
https://github.com/DataDog/datadog-agent/blob/master/Gopkg.toml#L41


[[constraint]]
name = "github.com/iovisor/gobpf"
Expand Down
8 changes: 1 addition & 7 deletions cmd/agent/main_common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,7 @@ func runAgent(exit chan bool) {
}()
}

agentConf, err := config.NewIfExists(opts.ddConfigPath)
if err != nil {
log.Criticalf("Error reading dd-agent config: %s", err)
os.Exit(1)
}

cfg, err := config.NewAgentConfig(agentConf, opts.configPath, opts.netConfigPath)
cfg, err := config.NewAgentConfig(opts.ddConfigPath, opts.configPath, opts.netConfigPath)
if err != nil {
log.Criticalf("Error parsing config: %s", err)
os.Exit(1)
Expand Down
153 changes: 7 additions & 146 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
ecsutil "github.com/DataDog/datadog-agent/pkg/util/ecs"

log "github.com/cihub/seelog"
"github.com/go-ini/ini"
)

var (
Expand Down Expand Up @@ -216,123 +215,20 @@ func NewDefaultAgentConfig() *AgentConfig {

// NewAgentConfig returns an AgentConfig using a configuration file. It can be nil
// if there is no file available. In this case we'll configure only via environment.
func NewAgentConfig(agentIni *File, yamlPath, netYamlPath string) (*AgentConfig, error) {
func NewAgentConfig(iniPath, yamlPath, netYamlPath string) (*AgentConfig, error) {
var err error
cfg := NewDefaultAgentConfig()

var ns string
var section *ini.Section
if agentIni != nil {
section, _ = agentIni.GetSection("Main")
}

// Pull from the ini Agent config by default.
if section != nil {
a, err := agentIni.Get("Main", "api_key")
if err != nil {
if util.PathExists(yamlPath) { // For Agents >= 6 we will have a YAML config file to use.
sunhay marked this conversation as resolved.
Show resolved Hide resolved
if err := cfg.loadProcessConfig("", yamlPath); err != nil {
return nil, err
}
ak := strings.Split(a, ",")
cfg.APIEndpoints[0].APIKey = ak[0]
if len(ak) > 1 {
for i := 1; i < len(ak); i++ {
cfg.APIEndpoints = append(cfg.APIEndpoints, APIEndpoint{APIKey: ak[i]})
}
}

cfg.LogLevel = strings.ToLower(agentIni.GetDefault("Main", "log_level", "INFO"))
cfg.proxy, err = getProxySettings(section)
if err != nil {
log.Errorf("error parsing proxy settings, not using a proxy: %s", err)
}

v, _ := agentIni.Get("Main", "process_agent_enabled")
if enabled, err := isAffirmative(v); enabled {
cfg.Enabled = true
cfg.EnabledChecks = processChecks
} else if !enabled && err == nil { // Only want to disable the process agent if it's explicitly disabled
cfg.Enabled = false
}

cfg.StatsdHost = agentIni.GetDefault("Main", "bind_host", cfg.StatsdHost)
// non_local_traffic is a shorthand in dd-agent configuration that is
// equivalent to setting `bind_host: 0.0.0.0`. Respect this flag
// since it defaults to true in Docker and saves us a command-line param
v, _ = agentIni.Get("Main", "non_local_traffic")
if enabled, _ := isAffirmative(v); enabled {
cfg.StatsdHost = "0.0.0.0"
}
cfg.StatsdPort = agentIni.GetIntDefault("Main", "dogstatsd_port", cfg.StatsdPort)

// All process-agent specific config lives under [process.config] section.
// NOTE: we truncate either endpoints or APIEndpoints if the lengths don't match
ns = "process.config"
endpoints := agentIni.GetStrArrayDefault(ns, "endpoint", ",", []string{defaultEndpoint})
if len(endpoints) < len(cfg.APIEndpoints) {
log.Warnf("found %d api keys and %d endpoints", len(cfg.APIEndpoints), len(endpoints))
cfg.APIEndpoints = cfg.APIEndpoints[:len(endpoints)]
} else if len(endpoints) > len(cfg.APIEndpoints) {
log.Warnf("found %d api keys and %d endpoints", len(cfg.APIEndpoints), len(endpoints))
endpoints = endpoints[:len(cfg.APIEndpoints)]
}
for i, e := range endpoints {
u, err := url.Parse(e)
if err != nil {
return nil, fmt.Errorf("invalid endpoint URL: %s", err)
}
cfg.APIEndpoints[i].Endpoint = u
}

cfg.QueueSize = agentIni.GetIntDefault(ns, "queue_size", cfg.QueueSize)
cfg.AllowRealTime = agentIni.GetBool(ns, "allow_real_time", cfg.AllowRealTime)
cfg.LogFile = agentIni.GetDefault(ns, "log_file", cfg.LogFile)
cfg.DDAgentPy = agentIni.GetDefault(ns, "dd_agent_py", cfg.DDAgentPy)
cfg.DDAgentPyEnv = agentIni.GetStrArrayDefault(ns, "dd_agent_py_env", ",", cfg.DDAgentPyEnv)

blacklistPats := agentIni.GetStrArrayDefault(ns, "blacklist", ",", []string{})
blacklist := make([]*regexp.Regexp, 0, len(blacklistPats))
for _, b := range blacklistPats {
r, err := regexp.Compile(b)
if err == nil {
blacklist = append(blacklist, r)
}
}
cfg.Blacklist = blacklist

// DataScrubber
cfg.Scrubber.Enabled = agentIni.GetBool(ns, "scrub_args", true)
customSensitiveWords := agentIni.GetStrArrayDefault(ns, "custom_sensitive_words", ",", []string{})
cfg.Scrubber.AddCustomSensitiveWords(customSensitiveWords)
cfg.Scrubber.StripAllArguments = agentIni.GetBool(ns, "strip_proc_arguments", false)

batchSize := agentIni.GetIntDefault(ns, "proc_limit", cfg.MaxPerMessage)
if batchSize <= maxMessageBatch {
cfg.MaxPerMessage = batchSize
} else {
log.Warn("Overriding the configured item count per message limit because it exceeds maximum")
cfg.MaxPerMessage = maxMessageBatch
}

// Checks intervals can be overridden by configuration.
for checkName, defaultInterval := range cfg.CheckIntervals {
key := fmt.Sprintf("%s_interval", checkName)
interval := agentIni.GetDurationDefault(ns, key, time.Second, defaultInterval)
if interval != defaultInterval {
log.Infof("Overriding check interval for %s to %s", checkName, interval)
cfg.CheckIntervals[checkName] = interval
}
}

// windows args config
cfg.Windows.ArgsRefreshInterval = agentIni.GetIntDefault(ns, "windows_args_refresh_interval", cfg.Windows.ArgsRefreshInterval)
cfg.Windows.AddNewArgs = agentIni.GetBool(ns, "windows_add_new_args", true)
}

// For Agents >= 6 we will have a YAML config file to use.
if util.PathExists(yamlPath) {
if err := cfg.loadProcessYamlConfig(yamlPath); err != nil {
} else if util.PathExists(iniPath) { // For Agent 5, we will have an INI config file to use
if err := cfg.loadProcessConfig(iniPath, ""); err != nil {
return nil, err
}
} else {
log.Infof("no process config file detected, continuing with defaults + environment variable overrides")
}

// For network tracing, there is an additional config file that is shared with the network-tracer
Expand Down Expand Up @@ -603,41 +499,6 @@ func getHostname(ddAgentPy, ddAgentBin string, ddAgentEnv []string) (string, err
return hostname, err
}

// getProxySettings returns a url.Url for the proxy configuration from datadog.conf, if available.
// In the case of invalid settings an error is logged and nil is returned. If settings are missing,
// meaning we don't want a proxy, then nil is returned with no error.
func getProxySettings(m *ini.Section) (proxyFunc, error) {
var host string
scheme := "http"
if v := m.Key("proxy_host").MustString(""); v != "" {
// accept either http://myproxy.com or myproxy.com
if i := strings.Index(v, "://"); i != -1 {
// when available, parse the scheme from the url
scheme = v[0:i]
host = v[i+3:]
} else {
host = v
}
}

if host == "" {
return nil, nil
}

port := defaultProxyPort
if v := m.Key("proxy_port").MustInt(-1); v != -1 {
port = v
}
var user, password string
if v := m.Key("proxy_user").MustString(""); v != "" {
user = v
}
if v := m.Key("proxy_password").MustString(""); v != "" {
password = v
}
return constructProxy(host, scheme, port, user, password)
}

// proxyFromEnv parses out the proxy configuration from the ENV variables in a
// similar way to getProxySettings and, if enough values are available, returns
// a new proxy URL value. If the environment is not set for this then the
Expand Down
Loading