pkg/clusteragent/admission: introduce deployment patcher

cmd/cluster-agent/app/app.go

@@ -37,6 +37,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/clusteragent"
 	admissionpkg "github.com/DataDog/datadog-agent/pkg/clusteragent/admission"
 	"github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate"
+	admissionpatch "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/patch"
 	"github.com/DataDog/datadog-agent/pkg/clusteragent/clusterchecks"
 	"github.com/DataDog/datadog-agent/pkg/collector"
 	"github.com/DataDog/datadog-agent/pkg/config"
@@ -277,6 +278,7 @@ func start(cmd *cobra.Command, args []string) error {
 		}
 	}
 
+	clusterName := clustername.GetClusterName(context.TODO(), hname)
 	if config.Datadog.GetBool("orchestrator_explorer.enabled") {
 		// Generate and persist a cluster ID
 		// this must be a UUID, and ideally be stable for the lifetime of a cluster,
@@ -287,7 +289,6 @@ func start(cmd *cobra.Command, args []string) error {
 			log.Errorf("Failed to generate or retrieve the cluster ID")
 		}
 
-		clusterName := clustername.GetClusterName(context.TODO(), hname)
 		if clusterName == "" {
 			log.Warn("Failed to auto-detect a Kubernetes cluster name. We recommend you set it manually via the cluster_name config option")
 		}
@@ -350,6 +351,17 @@ func start(cmd *cobra.Command, args []string) error {
 	}
 
 	if config.Datadog.GetBool("admission_controller.enabled") {
+		if config.Datadog.GetBool("admission_controller.auto_instrumentation.patcher.enabled") {
+			patchCtx := admissionpatch.ControllerContext{
+				IsLeaderFunc: le.IsLeader,
+				Client:       apiCl.Cl,
+				StopCh:       stopCh,
+			}
+			admissionpatch.StartControllers(patchCtx, clusterName)
+		} else {
+			log.Info("Auto instrumentation patcher is disabled")
+		}
+
 		admissionCtx := admissionpkg.ControllerContext{
 			IsLeaderFunc:        le.IsLeader,
 			LeaderSubscribeFunc: le.Subscribe,

go.mod

@@ -276,7 +276,7 @@ require (
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/emicklei/go-restful v2.16.0+incompatible // indirect
 	github.com/emicklei/go-restful-swagger12 v0.0.0-20201014110547-68ccff494617 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/ghodss/yaml v1.0.0 // indirect

pkg/clusteragent/admission/common/const.go

@@ -14,4 +14,10 @@ const (
 
 	// InjectionModeLabelKey pod label to chose the config injection at the pod level.
 	InjectionModeLabelKey = "admission.datadoghq.com/config.mode"
+
+	// LibVersionAnnotKeyFormat is the format of the library version annotation
+	LibVersionAnnotKeyFormat = "admission.datadoghq.com/%s-lib.version"
+
+	// LibConfigV1AnnotKeyFormat is the format of the library config annotation
+	LibConfigV1AnnotKeyFormat = "admission.datadoghq.com/%s-lib.config.v1"
 )

pkg/clusteragent/admission/common/lib_config.go

@@ -0,0 +1,92 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2016-present Datadog, Inc.
+
+//go:build kubeapiserver
+// +build kubeapiserver
+
+package common
+
+import (
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+// LibConfig holds the APM library configuration
+type LibConfig struct {
+	Version         int    `yaml:"version,omitempty" json:"version,omitempty"` // config schema version, not config version
+	ServiceLanguage string `yaml:"service_language,omitempty" json:"service_language,omitempty"`
+
+	Tracing             *bool    `yaml:"tracing_enabled,omitempty" json:"tracing_enabled,omitempty"`
+	LogInjection        *bool    `yaml:"log_injection_enabled,omitempty" json:"log_injection_enabled,omitempty"`
+	HealthMetrics       *bool    `yaml:"health_metrics_enabled,omitempty" json:"health_metrics_enabled,omitempty"`
+	RuntimeMetrics      *bool    `yaml:"runtime_metrics_enabled,omitempty" json:"runtime_metrics_enabled,omitempty"`
+	TracingSamplingRate *float64 `yaml:"tracing_sampling_rate,omitempty" json:"tracing_sampling_rate,omitempty"`
+	TracingRateLimit    *int     `yaml:"tracing_rate_limit,omitempty" json:"tracing_rate_limit,omitempty"`
+	TracingTags         []string `yaml:"tracing_tags,omitempty" json:"tracing_tags,omitempty"`
+}
+
+// ToEnvs converts the config fields into environment variables
+func (lc LibConfig) ToEnvs() []corev1.EnvVar {
+	var envs []corev1.EnvVar
+	if val, defined := checkFormatVal(lc.Tracing); defined {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "DD_TRACE_ENABLED",
+			Value: val,
+		})
+	}
+	if val, defined := checkFormatVal(lc.LogInjection); defined {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "DD_LOGS_INJECTION",
+			Value: val,
+		})
+	}
+	if val, defined := checkFormatVal(lc.HealthMetrics); defined {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "DD_TRACE_HEALTH_METRICS_ENABLED",
+			Value: val,
+		})
+	}
+	if val, defined := checkFormatVal(lc.RuntimeMetrics); defined {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "DD_RUNTIME_METRICS_ENABLED",
+			Value: val,
+		})
+	}
+	if val, defined := checkFormatFloat(lc.TracingSamplingRate); defined {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "DD_TRACE_SAMPLE_RATE",
+			Value: val,
+		})
+	}
+	if val, defined := checkFormatVal(lc.TracingRateLimit); defined {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "DD_TRACE_RATE_LIMIT",
+			Value: val,
+		})
+	}
+	if lc.TracingTags != nil {
+		envs = append(envs, corev1.EnvVar{
+			Name:  "DD_TAGS",
+			Value: strings.Join(lc.TracingTags, ","),
+		})
+	}
+	return envs
+}
+
+func checkFormatVal[T int | bool](val *T) (string, bool) {
+	if val == nil {
+		return "", false
+	}
+	return fmt.Sprintf("%v", *val), true
+}
+
+func checkFormatFloat(val *float64) (string, bool) {
+	if val == nil {
+		return "", false
+	}
+	return fmt.Sprintf("%.2f", *val), true
+}

pkg/clusteragent/admission/mutate/auto_instrumentation.go

@@ -9,11 +9,13 @@
 package mutate
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strconv"
 	"strings"
 
+	"github.com/DataDog/datadog-agent/pkg/clusteragent/admission/common"
 	"github.com/DataDog/datadog-agent/pkg/clusteragent/admission/metrics"
 	"github.com/DataDog/datadog-agent/pkg/config"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
@@ -50,9 +52,8 @@ const (
 )
 
 var (
-	libVersionAnnotationKeyFormat = "admission.datadoghq.com/%s-lib.version"
-	customLibAnnotationKeyFormat  = "admission.datadoghq.com/%s-lib.custom-image"
-	supportedLanguages            = []language{java, js, python}
+	customLibAnnotationKeyFormat = "admission.datadoghq.com/%s-lib.custom-image"
+	supportedLanguages           = []language{java, js, python}
 )
 
 // InjectAutoInstrumentation injects APM libraries into pods
@@ -90,7 +91,7 @@ func extractLibInfo(pod *corev1.Pod, containerRegistry string) (language, string
 			return lang, image, true
 		}
 
-		libVersionAnnotation := strings.ToLower(fmt.Sprintf(libVersionAnnotationKeyFormat, lang))
+		libVersionAnnotation := strings.ToLower(fmt.Sprintf(common.LibVersionAnnotKeyFormat, lang))
 		if version, found := podAnnotations[libVersionAnnotation]; found {
 			image := fmt.Sprintf("%s/dd-lib-%s-init:%s", containerRegistry, lang, version)
 			return lang, image, true
@@ -107,36 +108,33 @@ func injectAutoInstruConfig(pod *corev1.Pod, lang language, image string) error
 		metrics.LibInjectionAttempts.Inc(langStr, strconv.FormatBool(injected))
 	}()
 
+	var langEnvKey string
+	var langEnvFunc envValFunc
 	switch lang {
 	case java:
-		injectLibInitContainer(pod, image)
-		err := injectLibConfig(pod, javaToolOptionsKey, javaEnvValFunc)
-		if err != nil {
-			metrics.LibInjectionErrors.Inc(langStr)
-			return err
-		}
-
+		langEnvKey = javaToolOptionsKey
+		langEnvFunc = javaEnvValFunc
 	case js:
-		injectLibInitContainer(pod, image)
-		err := injectLibConfig(pod, nodeOptionsKey, jsEnvValFunc)
-		if err != nil {
-			metrics.LibInjectionErrors.Inc(langStr)
-			return err
-		}
-
+		langEnvKey = nodeOptionsKey
+		langEnvFunc = jsEnvValFunc
 	case python:
-		injectLibInitContainer(pod, image)
-		err := injectLibConfig(pod, pythonPathKey, pythonEnvValFunc)
-		if err != nil {
-			metrics.LibInjectionErrors.Inc(langStr)
-			return err
-		}
-
+		langEnvKey = pythonPathKey
+		langEnvFunc = pythonEnvValFunc
 	default:
 		metrics.LibInjectionErrors.Inc(langStr)
 		return fmt.Errorf("language %q is not supported. Supported languages are %v", lang, supportedLanguages)
 	}
-
+	injectLibInitContainer(pod, image)
+	err := injectLibRequirements(pod, langEnvKey, langEnvFunc)
+	if err != nil {
+		metrics.LibInjectionErrors.Inc(langStr)
+		return err
+	}
+	err = injectLibConfig(pod, lang)
+	if err != nil {
+		metrics.LibInjectionErrors.Inc(langStr)
+		return err
+	}
 	injectLibVolume(pod)
 	injected = true
 
@@ -160,7 +158,8 @@ func injectLibInitContainer(pod *corev1.Pod, image string) {
 	}, pod.Spec.InitContainers...)
 }
 
-func injectLibConfig(pod *corev1.Pod, envKey string, envVal envValFunc) error {
+// injectLibRequirements injects the minimal config requirements to enable instrumentation
+func injectLibRequirements(pod *corev1.Pod, envKey string, envVal envValFunc) error {
 	for i, ctr := range pod.Spec.Containers {
 		index := envIndex(ctr.Env, envKey)
 		if index < 0 {
@@ -182,6 +181,26 @@ func injectLibConfig(pod *corev1.Pod, envKey string, envVal envValFunc) error {
 	return nil
 }
 
+// injectLibConfig injects additional library configuration extracted from pod annotations
+func injectLibConfig(pod *corev1.Pod, lang language) error {
+	configAnnotKey := fmt.Sprintf(common.LibConfigV1AnnotKeyFormat, lang)
+	confString, found := pod.GetAnnotations()[configAnnotKey]
+	if !found {
+		log.Debugf("Config annotation key %q not found on pod %s, skipping config injection", configAnnotKey, podString(pod))
+		return nil
+	}
+	log.Infof("Config annotation key %q found on pod %s, config: %q", configAnnotKey, podString(pod), confString)
+	var libConfig common.LibConfig
+	err := json.Unmarshal([]byte(confString), &libConfig)
+	if err != nil {
+		return fmt.Errorf("invalid json config in annotation %s=%s: %w", configAnnotKey, confString, err)
+	}
+	for _, env := range libConfig.ToEnvs() {
+		_ = injectEnv(pod, env)
+	}
+	return nil
+}
+
 func injectLibVolume(pod *corev1.Pod) {
 	pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
 		Name: volumeName,

pkg/clusteragent/admission/mutate/auto_instrumentation_test.go

@@ -115,12 +115,12 @@ func TestInjectAutoInstruConfig(t *testing.T) {
 			if err != nil {
 				return
 			}
-			assertLibConfig(t, tt.pod, tt.image, tt.expectedEnvKey, tt.expectedEnvVal)
+			assertLibReq(t, tt.pod, tt.image, tt.expectedEnvKey, tt.expectedEnvVal)
 		})
 	}
 }
 
-func assertLibConfig(t *testing.T, pod *corev1.Pod, image, envKey, envVal string) {
+func assertLibReq(t *testing.T, pod *corev1.Pod, image, envKey, envVal string) {
 	// Empty dir volume
 	volumeFound := false
 	for _, volume := range pod.Spec.Volumes {
@@ -220,3 +220,58 @@ func TestExtractLibInfo(t *testing.T) {
 		})
 	}
 }
+
+func TestInjectLibConfig(t *testing.T) {
+	tests := []struct {
+		name         string
+		pod          *corev1.Pod
+		lang         language
+		wantErr      bool
+		expectedEnvs []corev1.EnvVar
+	}{
+		{
+			name:    "nominal case",
+			pod:     fakePodWithAnnotation("admission.datadoghq.com/java-lib.config.v1", `{"version":1,"service_language":"java","runtime_metrics_enabled":true,"tracing_rate_limit":50}`),
+			lang:    java,
+			wantErr: false,
+			expectedEnvs: []corev1.EnvVar{
+				{
+					Name:  "DD_RUNTIME_METRICS_ENABLED",
+					Value: "true",
+				},
+				{
+					Name:  "DD_TRACE_RATE_LIMIT",
+					Value: "50",
+				},
+			},
+		},
+		{
+			name:         "invalid json",
+			pod:          fakePodWithAnnotation("admission.datadoghq.com/java-lib.config.v1", "invalid"),
+			lang:         java,
+			wantErr:      true,
+			expectedEnvs: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := injectLibConfig(tt.pod, tt.lang)
+			require.False(t, (err != nil) != tt.wantErr)
+			if err != nil {
+				return
+			}
+			container := tt.pod.Spec.Containers[0]
+			envCount := 0
+			for _, expectEnv := range tt.expectedEnvs {
+				for _, contEnv := range container.Env {
+					if expectEnv.Name == contEnv.Name {
+						require.Equal(t, expectEnv.Value, contEnv.Value)
+						envCount++
+						break
+					}
+				}
+			}
+			require.Equal(t, len(tt.expectedEnvs), envCount)
+		})
+	}
+}

pkg/clusteragent/admission/mutate/test_utils.go

@@ -86,13 +86,15 @@ func fakePodWithLabel(k, v string) *corev1.Pod {
 }
 
 func fakePodWithAnnotation(k, v string) *corev1.Pod {
-	return &corev1.Pod{
+	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
+			Name: "pod",
 			Annotations: map[string]string{
 				k: v,
 			},
 		},
 	}
+	return withContainer(pod, "-container")
 }
 
 func fakePodWithEnv(name, env string) *corev1.Pod {