First version to support Agent profiles (#976)

* [override/daemonset] Apply only if it's not empty

* [config/rbac/role] Add profiles

* Add basic version of profiles manager

* [controllers/datadogagent] Create DaemonSets according to profiles

* [override/container] Don't override resource if not specified

* [agentprofile] Add support for overrides in all node agent containers

* [agentprofile] Take into account namespace when setting DS name

config/rbac/role.yaml

@@ -696,3 +696,29 @@ rules:
     - get
     - patch
     - update
+- apiGroups:
+    - datadoghq.com
+  resources:
+    - datadogagentprofiles
+  verbs:
+    - create
+    - delete
+    - get
+    - list
+    - patch
+    - update
+    - watch
+- apiGroups:
+    - datadoghq.com
+  resources:
+    - datadogagentprofiles/status
+  verbs:
+    - get
+    - patch
+    - update
+- apiGroups:
+    - datadoghq.com
+  resources:
+    - datadogagentprofiles/finalizers
+  verbs:
+    - update

controllers/datadogagent/controller_reconcile_agent.go

@@ -7,26 +7,32 @@ package datadogagent
 
 import (
 	"context"
+	"fmt"
 	"time"
 
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
 	"github.com/DataDog/datadog-operator/apis/datadoghq/common/v1"
+	"github.com/DataDog/datadog-operator/apis/datadoghq/v1alpha1"
 	datadoghqv2alpha1 "github.com/DataDog/datadog-operator/apis/datadoghq/v2alpha1"
 	apiutils "github.com/DataDog/datadog-operator/apis/utils"
+	ddacomponent "github.com/DataDog/datadog-operator/controllers/datadogagent/component"
 	componentagent "github.com/DataDog/datadog-operator/controllers/datadogagent/component/agent"
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/feature"
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/override"
+	"github.com/DataDog/datadog-operator/pkg/agentprofile"
 	"github.com/DataDog/datadog-operator/pkg/controller/utils/datadog"
-
+	"github.com/DataDog/datadog-operator/pkg/kubernetes"
 	edsv1alpha1 "github.com/DataDog/extendeddaemonset/api/v1alpha1"
-	"github.com/go-logr/logr"
-	appsv1 "k8s.io/api/apps/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/types"
-	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
-func (r *Reconciler) reconcileV2Agent(logger logr.Logger, requiredComponents feature.RequiredComponents, features []feature.Feature, dda *datadoghqv2alpha1.DatadogAgent, resourcesManager feature.ResourceManagers, newStatus *datadoghqv2alpha1.DatadogAgentStatus, requiredContainers []common.AgentContainerName) (reconcile.Result, error) {
+func (r *Reconciler) reconcileV2Agent(logger logr.Logger, requiredComponents feature.RequiredComponents, features []feature.Feature, dda *datadoghqv2alpha1.DatadogAgent, resourcesManager feature.ResourceManagers, newStatus *datadoghqv2alpha1.DatadogAgentStatus, requiredContainers []common.AgentContainerName, profile *v1alpha1.DatadogAgentProfile) (reconcile.Result, error) {
 	var result reconcile.Result
 	var eds *edsv1alpha1.ExtendedDaemonSet
 	var daemonset *appsv1.DaemonSet
@@ -41,6 +47,8 @@ func (r *Reconciler) reconcileV2Agent(logger logr.Logger, requiredComponents fea
 	agentEnabled := requiredComponents.Agent.IsEnabled()
 
 	if r.options.ExtendedDaemonsetOptions.Enabled {
+		// TODO: handle profiles like we do for DaemonSets below
+
 		// Start by creating the Default Agent extendeddaemonset
 		eds = componentagent.NewDefaultAgentExtendedDaemonset(dda, &r.options.ExtendedDaemonsetOptions, requiredContainers)
 		podManagers = feature.NewPodTemplateManagers(&eds.Spec.Template)
@@ -96,13 +104,23 @@ func (r *Reconciler) reconcileV2Agent(logger logr.Logger, requiredComponents fea
 	}
 
 	// If Override is defined for the node agent component, apply the override on the PodTemplateSpec, it will cascade to container.
+	var componentOverrides []*datadoghqv2alpha1.DatadogAgentComponentOverride
 	if componentOverride, ok := dda.Spec.Override[datadoghqv2alpha1.NodeAgentComponentName]; ok {
+		componentOverrides = append(componentOverrides, componentOverride)
+	}
+
+	// Apply overrides from profiles last, so they can override what's defined in the DDA.
+	overrideFromProfile := agentprofile.ComponentOverrideFromProfile(profile)
+	componentOverrides = append(componentOverrides, &overrideFromProfile)
+
+	for _, componentOverride := range componentOverrides {
 		if apiutils.BoolValue(componentOverride.Disabled) {
 			disabledByOverride = true
 		}
 		override.PodTemplateSpec(logger, podManagers, componentOverride, datadoghqv2alpha1.NodeAgentComponentName, dda.Name)
 		override.DaemonSet(daemonset, componentOverride)
 	}
+
 	if disabledByOverride {
 		if agentEnabled {
 			// The override supersedes what's set in requiredComponents; update status to reflect the conflict
@@ -182,3 +200,49 @@ func (r *Reconciler) cleanupV2ExtendedDaemonSet(logger logr.Logger, dda *datadog
 
 	return reconcile.Result{}, nil
 }
+
+func (r *Reconciler) cleanupDaemonSetsForProfilesThatNoLongerApply(ctx context.Context, dda *datadoghqv2alpha1.DatadogAgent, daemonSetNamesAppliedProfiles map[string]struct{}) error {
+	daemonSets, err := r.agentDaemonSetsCreatedByOperator(ctx)
+	if err != nil {
+		return err
+	}
+
+	defaultDaemonSetName := ddacomponent.GetAgentName(dda) // TODO: take into account name override
+	for _, daemonSet := range daemonSets {
+		_, belongsToActiveProfile := daemonSetNamesAppliedProfiles[daemonSet.Name]
+
+		if belongsToActiveProfile || daemonSet.Name == defaultDaemonSetName {
+			continue
+		}
+
+		if err = r.client.Delete(ctx, &daemonSet); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// TODO: add specific labels to the daemonset created by the operator that belong to a profile so that we can filter more easily
+func (r *Reconciler) agentDaemonSetsCreatedByOperator(ctx context.Context) ([]appsv1.DaemonSet, error) {
+	daemonSetList := appsv1.DaemonSetList{}
+
+	err := r.client.List(
+		ctx,
+		&daemonSetList,
+		client.HasLabels{
+			fmt.Sprintf(
+				"%s=%s,%s=%s",
+				kubernetes.AppKubernetesNameLabelKey,
+				"datadog-agent-deployment",
+				kubernetes.AppKubernetesManageByLabelKey,
+				"datadog-operator",
+			),
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return daemonSetList.Items, nil
+}

controllers/datadogagent/controller_reconcile_v2.go

@@ -15,15 +15,18 @@ import (
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/errors"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	apicommon "github.com/DataDog/datadog-operator/apis/datadoghq/common"
 	commonv1 "github.com/DataDog/datadog-operator/apis/datadoghq/common/v1"
+	datadoghqv1alpha1 "github.com/DataDog/datadog-operator/apis/datadoghq/v1alpha1"
 	datadoghqv2alpha1 "github.com/DataDog/datadog-operator/apis/datadoghq/v2alpha1"
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/dependencies"
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/feature"
 	"github.com/DataDog/datadog-operator/controllers/datadogagent/override"
+	"github.com/DataDog/datadog-operator/pkg/agentprofile"
 	"github.com/DataDog/datadog-operator/pkg/controller/utils"
 	"github.com/DataDog/datadog-operator/pkg/kubernetes"
 )
@@ -141,9 +144,27 @@ func (r *Reconciler) reconcileInstanceV2(ctx context.Context, logger logr.Logger
 	}
 
 	requiredContainers := requiredComponents.Agent.Containers
-	result, err = r.reconcileV2Agent(logger, requiredComponents, features, instance, resourceManagers, newStatus, requiredContainers)
-	if utils.ShouldReturn(result, err) {
-		return r.updateStatusIfNeededV2(logger, instance, newStatus, result, err)
+
+	profiles, err := r.profilesToApply(ctx)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	daemonSetNamesAppliedProfiles := map[string]struct{}{} // TODO: do the same for EDS
+	for _, profile := range profiles {
+		result, err = r.reconcileV2Agent(logger, requiredComponents, features, instance, resourceManagers, newStatus, requiredContainers, &profile)
+		if utils.ShouldReturn(result, err) {
+			return r.updateStatusIfNeededV2(logger, instance, newStatus, result, err)
+		}
+		profileNamespacedName := types.NamespacedName{
+			Namespace: profile.Namespace,
+			Name:      profile.Name,
+		}
+		daemonSetNamesAppliedProfiles[agentprofile.DaemonSetName(profileNamespacedName)] = struct{}{}
+	}
+	// TODO: do the same for EDS. Also make sure that this doesn't delete anything that it isn't supposed to when there's an error in the for above that doesn't return
+	if err = r.cleanupDaemonSetsForProfilesThatNoLongerApply(ctx, instance, daemonSetNamesAppliedProfiles); err != nil {
+		return reconcile.Result{}, err
 	}
 
 	result, err = r.reconcileV2ClusterChecksRunner(logger, requiredComponents, features, instance, resourceManagers, newStatus)
@@ -252,3 +273,12 @@ func (r *Reconciler) updateMetricsForwardersFeatures(dda *datadoghqv2alpha1.Data
 	// 	r.forwarders.SetEnabledFeatures(dda, features)
 	// }
 }
+
+func (r *Reconciler) profilesToApply(ctx context.Context) ([]datadoghqv1alpha1.DatadogAgentProfile, error) {
+	profilesList := datadoghqv1alpha1.DatadogAgentProfileList{}
+	err := r.client.List(ctx, &profilesList)
+	if err != nil {
+		return nil, err
+	}
+	return agentprofile.ProfilesToApply(profilesList.Items), nil
+}

controllers/datadogagent/override/container.go

@@ -109,7 +109,27 @@ func overrideContainer(container *corev1.Container, override *v2alpha1.DatadogAg
 	}
 
 	if override.Resources != nil {
-		container.Resources = *override.Resources
+		for resource, quantity := range override.Resources.Requests {
+			if quantity.IsZero() {
+				continue
+			}
+
+			if container.Resources.Requests == nil {
+				container.Resources.Requests = corev1.ResourceList{}
+			}
+			container.Resources.Requests[resource] = quantity
+		}
+
+		for resource, quantity := range override.Resources.Limits {
+			if quantity.IsZero() {
+				continue
+			}
+
+			if container.Resources.Limits == nil {
+				container.Resources.Limits = corev1.ResourceList{}
+			}
+			container.Resources.Limits[resource] = quantity
+		}
 	}
 
 	if override.Command != nil {

controllers/datadogagent/override/container_test.go

@@ -7,11 +7,12 @@ package override
 
 import (
 	"fmt"
+	"reflect"
+	"testing"
+
 	"github.com/DataDog/datadog-operator/apis/datadoghq/common"
 	apiutils "github.com/DataDog/datadog-operator/apis/utils"
 	"k8s.io/apimachinery/pkg/api/resource"
-	"reflect"
-	"testing"
 
 	commonv1 "github.com/DataDog/datadog-operator/apis/datadoghq/common/v1"
 	"github.com/DataDog/datadog-operator/apis/datadoghq/v2alpha1"
@@ -176,7 +177,7 @@ func TestContainer(t *testing.T) {
 			},
 		},
 		{
-			name:          "override resources",
+			name:          "override resources - when there are none defined",
 			containerName: commonv1.CoreAgentContainerName,
 			existingManager: func() *fake.PodTemplateManagers {
 				return fake.NewPodTemplateManagers(t, corev1.PodTemplateSpec{
@@ -210,6 +211,57 @@ func TestContainer(t *testing.T) {
 				})
 			},
 		},
+		{
+			name:          "override resources - when there are some defined",
+			containerName: commonv1.CoreAgentContainerName,
+			existingManager: func() *fake.PodTemplateManagers {
+				return fake.NewPodTemplateManagers(t, corev1.PodTemplateSpec{
+					Spec: corev1.PodSpec{
+						Containers: []corev1.Container{
+							{
+								Name: string(commonv1.CoreAgentContainerName),
+								Resources: corev1.ResourceRequirements{
+									Limits: map[corev1.ResourceName]resource.Quantity{
+										corev1.ResourceCPU:    *resource.NewQuantity(2, resource.DecimalSI), // Not overridden, should be kept
+										corev1.ResourceMemory: *resource.NewQuantity(2048, resource.DecimalSI),
+									},
+									Requests: map[corev1.ResourceName]resource.Quantity{
+										corev1.ResourceCPU:    *resource.NewQuantity(1, resource.DecimalSI),
+										corev1.ResourceMemory: *resource.NewQuantity(1024, resource.DecimalSI), // Not overridden, should be kept
+									},
+								},
+							},
+						},
+					},
+				})
+			},
+			override: v2alpha1.DatadogAgentGenericContainer{
+				Resources: &corev1.ResourceRequirements{
+					Limits: map[corev1.ResourceName]resource.Quantity{
+						corev1.ResourceMemory: *resource.NewQuantity(4096, resource.DecimalSI),
+					},
+					Requests: map[corev1.ResourceName]resource.Quantity{
+						corev1.ResourceCPU: *resource.NewQuantity(2, resource.DecimalSI),
+					},
+				},
+			},
+			validateManager: func(t *testing.T, manager *fake.PodTemplateManagers, containerName string) {
+				assertContainerMatch(t, manager.PodTemplateSpec().Spec.Containers, containerName, func(container corev1.Container) bool {
+					return reflect.DeepEqual(
+						corev1.ResourceRequirements{
+							Limits: map[corev1.ResourceName]resource.Quantity{
+								corev1.ResourceCPU:    *resource.NewQuantity(2, resource.DecimalSI),    // Not overridden
+								corev1.ResourceMemory: *resource.NewQuantity(4096, resource.DecimalSI), // Overridden
+							},
+							Requests: map[corev1.ResourceName]resource.Quantity{
+								corev1.ResourceCPU:    *resource.NewQuantity(2, resource.DecimalSI),    // Overridden
+								corev1.ResourceMemory: *resource.NewQuantity(1024, resource.DecimalSI), // Not overridden
+							},
+						},
+						container.Resources)
+				})
+			},
+		},
 		{
 			name:          "override command",
 			containerName: commonv1.CoreAgentContainerName,

controllers/datadogagent/override/daemonset.go

@@ -13,14 +13,14 @@ import (
 
 // DaemonSet overrides a DaemonSet according to the given override options
 func DaemonSet(daemonSet *v1.DaemonSet, override *v2alpha1.DatadogAgentComponentOverride) {
-	if override.Name != nil {
+	if override.Name != nil && *override.Name != "" {
 		daemonSet.Name = *override.Name
 	}
 }
 
 // ExtendedDaemonSet overrides an ExtendedDaemonSet according to the given override options
 func ExtendedDaemonSet(eds *edsv1alpha1.ExtendedDaemonSet, override *v2alpha1.DatadogAgentComponentOverride) {
-	if override.Name != nil {
+	if override.Name != nil && *override.Name != "" {
 		eds.Name = *override.Name
 	}
 }

controllers/datadogagent_controller.go

@@ -190,6 +190,8 @@ func (r *DatadogAgentReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Owns(r.PlatformInfo.CreatePDBObject()).
 		Owns(&networkingv1.NetworkPolicy{})
 
+	builder.Watches(&source.Kind{Type: &datadoghqv1alpha1.DatadogAgentProfile{}}, &handler.EnqueueRequestForObject{})
+
 	// DatadogAgent is namespaced whereas ClusterRole and ClusterRoleBinding are
 	// cluster-scoped. That means that DatadogAgent cannot be their owner, and
 	// we cannot use .Owns().