[beta] Add remote config (#1215)

* [add-remote-config] initial commit (#1126)

* Draft of service activation through RC

* various refactoring and updates

* use CWS_DD product, update logger, add statecallbacks

---------

Co-authored-by: Sylvain Baubeau <[email protected]>

* Use agent config for service activation (#1134)

* [remote-config] set up test config

* change product to AgentConfig

* small edits

* Refactor and bump controller-tools version

* Test with mocked Config

* Test e2e

* Merge configs

* Merge configs, send Acknowledges

* Add service initialisation in reconcilier

* Fix tags initialisation

* Fix validation tests

* Fix build test error

---------

Co-authored-by: Celene <[email protected]>

* Add usm service activation (#1141)

* Add usm service activation

* Read CWS config from system-probe and CSPM from security-agent

* Read CWS config from system-probe and CSPM from security-agent

* Add DDSystemProbeServiceMonitoringEnabled to coreAgent container

* move initialization to be independent of agent reconciler (#1142)

* cleanup unneeded changes (#1149)

* [remote-config] add envvars missing from core agent for system probe features (#1170)

* [remote-config] sysprobe socket fixes

* add sysprobe envvars to core container

* add sys probe enabled envvar to where missing

* Fix angeConfig struct (#1187)

* [remote-config] store RC config in dda status (#1196)

* [remote-config] store RC config in dda status

* add layer to config stored in status, apply updates in reconcile feature code

* fix feature config references

* small comment update

* a little cleanup

* update modules

* fix licenses

* fix tests

* updated generated files

* rm two licenses

* small fixes

* properly merge remoteconfig config

* simplify

* update comment

---------

Co-authored-by: Sylvain Baubeau <[email protected]>
Co-authored-by: Momar TOURÉ <[email protected]>

.golangci.toml

@@ -103,6 +103,7 @@
     "perfsprint",
     "inamedparam",
     "nakedret",
+    "gomoddirectives",
   ]
 
 [issues]

LICENSE-3rdparty.csv

@@ -1,19 +1,33 @@
 Component,Origin,License
 core,cloud.google.com/go/compute/metadata,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/config/model,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/config/remote,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/proto,Apache-2.0
 core,github.com/DataDog/datadog-agent/pkg/remoteconfig/state,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/util/backoff,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/util/cache,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/util/grpc,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/util/http,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/util/log,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/util/scrubber,Apache-2.0
+core,github.com/DataDog/datadog-agent/pkg/util/uuid,Apache-2.0
 core,github.com/DataDog/datadog-api-client-go/v2,Apache-2.0
 core,github.com/DataDog/datadog-go/v5/statsd,MIT
 core,github.com/DataDog/datadog-operator,Apache-2.0
 core,github.com/DataDog/extendeddaemonset,Apache-2.0
 core,github.com/DataDog/go-tuf,BSD-3-Clause
 core,github.com/DataDog/gostackparse,Apache-2.0
+core,github.com/DataDog/viper,MIT
 core,github.com/DataDog/zstd,BSD-3-Clause
+core,github.com/Masterminds/semver,MIT
 core,github.com/Masterminds/semver/v3,MIT
 core,github.com/PuerkitoBio/purell,BSD-3-Clause
 core,github.com/PuerkitoBio/urlesc,BSD-3-Clause
+core,github.com/benbjohnson/clock,MIT
 core,github.com/beorn7/perks/quantile,MIT
 core,github.com/cenkalti/backoff,MIT
 core,github.com/cespare/xxhash/v2,MIT
+core,github.com/cihub/seelog,BSD-3-Clause
 core,github.com/davecgh/go-spew/spew,ISC
 core,github.com/emicklei/go-restful,MIT
 core,github.com/evanphx/json-patch,BSD-3-Clause
@@ -32,14 +46,22 @@ core,github.com/google/go-cmp/cmp,BSD-3-Clause
 core,github.com/google/gofuzz,Apache-2.0
 core,github.com/google/pprof/profile,Apache-2.0
 core,github.com/google/uuid,BSD-3-Clause
+core,github.com/grpc-ecosystem/go-grpc-middleware,Apache-2.0
+core,github.com/grpc-ecosystem/grpc-gateway,BSD-3-Clause
+core,github.com/hashicorp/hcl,MPL-2.0
 core,github.com/imdario/mergo,BSD-3-Clause
 core,github.com/josharian/intern,MIT
 core,github.com/json-iterator/go,MIT
+core,github.com/magiconair/properties,BSD-2-Clause
 core,github.com/mailru/easyjson,MIT
 core,github.com/matttproud/golang_protobuf_extensions/pbutil,Apache-2.0
+core,github.com/mitchellh/mapstructure,MIT
 core,github.com/modern-go/concurrent,Apache-2.0
 core,github.com/modern-go/reflect2,Apache-2.0
 core,github.com/munnerz/goautoneg,BSD-3-Clause
+core,github.com/patrickmn/go-cache,MIT
+core,github.com/pelletier/go-toml,Apache-2.0
+core,github.com/philhofer/fwd,MIT
 core,github.com/pkg/errors,BSD-2-Clause
 core,github.com/pmezard/go-difflib/difflib,BSD-3-Clause
 core,github.com/prometheus/client_golang/prometheus,Apache-2.0
@@ -51,11 +73,17 @@ core,github.com/richardartoul/molecule,MIT
 core,github.com/richardartoul/molecule/src/codec,Apache-2.0
 core,github.com/richardartoul/molecule/src/protowire,BSD-3-Clause
 core,github.com/secure-systems-lab/go-securesystemslib/cjson,MIT
+core,github.com/shirou/gopsutil/v3,BSD-3-Clause
 core,github.com/spaolacci/murmur3,BSD-3-Clause
+core,github.com/spf13/afero,Apache-2.0
+core,github.com/spf13/cast,MIT
+core,github.com/spf13/jwalterweatherman,MIT
 core,github.com/spf13/pflag,BSD-3-Clause
 core,github.com/stretchr/objx,MIT
 core,github.com/stretchr/testify,MIT
+core,github.com/tinylib/msgp/msgp,MIT
 core,github.com/zorkian/go-datadog-api,BSD-3-Clause
+core,go.etcd.io/bbolt,MIT
 core,go.uber.org/atomic,MIT
 core,go.uber.org/multierr,MIT
 core,go.uber.org/zap,MIT
@@ -66,6 +94,10 @@ core,golang.org/x/term,BSD-3-Clause
 core,golang.org/x/text,BSD-3-Clause
 core,golang.org/x/time/rate,BSD-3-Clause
 core,gomodules.xyz/jsonpatch/v2,Apache-2.0
+core,google.golang.org/genproto/googleapis/api,Apache-2.0
+core,google.golang.org/genproto/googleapis/rpc/status,Apache-2.0
+core,google.golang.org/genproto/protobuf/field_mask,Apache-2.0
+core,google.golang.org/grpc,Apache-2.0
 core,google.golang.org/protobuf,BSD-3-Clause
 core,gopkg.in/DataDog/dd-trace-go.v1,Apache-2.0
 core,gopkg.in/inf.v0,BSD-3-Clause

apis/datadoghq/v2alpha1/datadogagent_types.go

@@ -1268,29 +1268,6 @@ const (
 	SingleContainerStrategy ContainerStrategyType = "single"
 )
 
-// DatadogAgentStatus defines the observed state of DatadogAgent.
-// +k8s:openapi-gen=true
-type DatadogAgentStatus struct {
-	// Conditions Represents the latest available observations of a DatadogAgent's current state.
-	// +optional
-	// +listType=map
-	// +listMapKey=type
-	Conditions []metav1.Condition `json:"conditions"`
-	// The actual state of the Agent as a daemonset or an extended daemonset.
-	// +optional
-	// +listType=atomic
-	AgentList []*commonv1.DaemonSetStatus `json:"agentList,omitempty"`
-	// The combined actual state of all Agents as daemonsets or extended daemonsets.
-	// +optional
-	Agent *commonv1.DaemonSetStatus `json:"agent,omitempty"`
-	// The actual state of the Cluster Agent as a deployment.
-	// +optional
-	ClusterAgent *commonv1.DeploymentStatus `json:"clusterAgent,omitempty"`
-	// The actual state of the Cluster Checks Runner as a deployment.
-	// +optional
-	ClusterChecksRunner *commonv1.DeploymentStatus `json:"clusterChecksRunner,omitempty"`
-}
-
 // FIPSConfig contains the FIPS configuration.
 // +k8s:openapi-gen=true
 type FIPSConfig struct {
@@ -1327,6 +1304,38 @@ type FIPSConfig struct {
 	CustomFIPSConfig *CustomConfig `json:"customFIPSConfig,omitempty"`
 }
 
+// RemoteConfigConfiguration stores the configuration received from RemoteConfig.
+// +k8s:openapi-gen=true
+type RemoteConfigConfiguration struct {
+	Features *DatadogFeatures `json:"features,omitempty"`
+}
+
+// DatadogAgentStatus defines the observed state of DatadogAgent.
+// +k8s:openapi-gen=true
+type DatadogAgentStatus struct {
+	// Conditions Represents the latest available observations of a DatadogAgent's current state.
+	// +optional
+	// +listType=map
+	// +listMapKey=type
+	Conditions []metav1.Condition `json:"conditions"`
+	// The actual state of the Agent as a daemonset or an extended daemonset.
+	// +optional
+	// +listType=atomic
+	AgentList []*commonv1.DaemonSetStatus `json:"agentList,omitempty"`
+	// The combined actual state of all Agents as daemonsets or extended daemonsets.
+	// +optional
+	Agent *commonv1.DaemonSetStatus `json:"agent,omitempty"`
+	// The actual state of the Cluster Agent as a deployment.
+	// +optional
+	ClusterAgent *commonv1.DeploymentStatus `json:"clusterAgent,omitempty"`
+	// The actual state of the Cluster Checks Runner as a deployment.
+	// +optional
+	ClusterChecksRunner *commonv1.DeploymentStatus `json:"clusterChecksRunner,omitempty"`
+	// RemoteConfigConfiguration stores the configuration received from RemoteConfig.
+	// +optional
+	RemoteConfigConfiguration *RemoteConfigConfiguration `json:"remoteConfigConfiguration,omitempty"`
+}
+
 // DatadogAgent Deployment with the Datadog Operator.
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status