From 593c137a1db537f340ffb990c7d30386d7bec289 Mon Sep 17 00:00:00 2001 From: "agent-platform-auto-pr[bot]" <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:29:14 +0000 Subject: [PATCH 01/43] [test-infra-definitions][automated] Bump test-infra-definitions to 4b4112f5f64d18c33cc15eb1bc8eccb603b71a93 (#32288) Co-authored-by: L3n41c Co-authored-by: pducolin Co-authored-by: adel121 --- .gitlab/common/test_infra_version.yml | 2 +- go.mod | 20 +++--- go.sum | 40 +++++------ test/new-e2e/go.mod | 44 ++++++------ test/new-e2e/go.sum | 84 +++++++++++------------ test/new-e2e/tests/containers/k8s_test.go | 10 ++- test/new-e2e/tests/orchestrator/apply.go | 7 ++ 7 files changed, 110 insertions(+), 97 deletions(-) diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index 7d435a4bc3a4b..71934be35d15a 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -4,4 +4,4 @@ variables: # and check the job creating the image to make sure you have the right SHA prefix TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 9c7c5005ca28 + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 4b4112f5f64d diff --git a/go.mod b/go.mod index e8175e8cf0efb..30981dbec9ce2 100644 --- a/go.mod +++ b/go.mod @@ -375,19 +375,19 @@ require ( github.com/armon/go-metrics v0.4.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect - github.com/aws/aws-sdk-go-v2 v1.32.5 - github.com/aws/aws-sdk-go-v2/config v1.28.5 - github.com/aws/aws-sdk-go-v2/credentials v1.17.46 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2/config v1.28.6 + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/service/ebs v1.27.0 // indirect github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect github.com/aws/smithy-go v1.22.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect diff --git a/go.sum b/go.sum index 44b70bde70c36..26375333ef0cf 100644 --- a/go.sum +++ b/go.sum @@ -321,24 +321,24 @@ github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= -github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= -github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14/go.mod h1:9NCTOURS8OpxvoAVHq79LK81/zC78hfRWFn+aL0SPcY= github.com/aws/aws-sdk-go-v2/config v1.19.0/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= -github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= -github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= @@ -352,8 +352,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhv github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38/go.mod h1:epIZoRSSbRIwLPJU5F+OldHhwZPBdpDeQkRdCeY3+00= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6/go.mod h1:lnc2taBsR9nTlz9meD+lhFZZ9EWY712QHrRflWpTcOA= github.com/aws/aws-sdk-go-v2/service/kms v1.37.6 h1:CZImQdb1QbU9sGgJ9IswhVkxAcjkkD1eQTMA1KHWk+E= github.com/aws/aws-sdk-go-v2/service/kms v1.37.6/go.mod h1:YJDdlK0zsyxVBxGU48AR/Mi8DMrGdc1E3Yij4fNrONA= @@ -363,14 +363,14 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2/go.mod h1:Zjfqt7KhQK+PO1bbOsFNzK github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.6 h1:1KDMKvOKNrpD667ORbZ/+4OgvUoaok1gg/MLzrHF9fw= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.6/go.mod h1:DmtyfCfONhOyVAJ6ZMTrDSFIeyCBlEO93Qkfhxwbxu0= github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index 008a6b32a4753..2bd2138c3ea69 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -45,7 +45,7 @@ replace ( require ( github.com/DataDog/agent-payload/v5 v5.0.138 github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/def v0.56.2 - github.com/DataDog/datadog-agent/pkg/util/optional v0.59.0 + github.com/DataDog/datadog-agent/pkg/util/optional v0.59.1 github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.0 github.com/DataDog/datadog-agent/pkg/util/testutil v0.56.2 @@ -58,9 +58,9 @@ require ( // `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version // Example: github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB // => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB - github.com/DataDog/test-infra-definitions v0.0.0-20241211124138-9c7c5005ca28 - github.com/aws/aws-sdk-go-v2 v1.32.5 - github.com/aws/aws-sdk-go-v2/config v1.28.5 + github.com/DataDog/test-infra-definitions v0.0.0-20241217110507-4b4112f5f64d + github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 github.com/aws/aws-sdk-go-v2/service/eks v1.51.0 github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 @@ -109,22 +109,22 @@ require ( github.com/alessio/shellescape v1.4.2 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect - github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ecs v1.47.4 + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.36.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ecs v1.52.2 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect github.com/aws/smithy-go v1.22.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect @@ -217,8 +217,8 @@ require ( github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect github.com/pulumi/esc v0.10.0 // indirect github.com/pulumi/pulumi-command/sdk v1.0.1 // indirect - github.com/pulumi/pulumi-docker/sdk/v4 v4.5.5 // indirect - github.com/pulumi/pulumi-libvirt/sdk v0.4.7 // indirect + github.com/pulumi/pulumi-docker/sdk/v4 v4.5.7 // indirect + github.com/pulumi/pulumi-libvirt/sdk v0.5.3 // indirect github.com/pulumi/pulumi-random/sdk/v4 v4.16.7 // indirect github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 // indirect github.com/pulumiverse/pulumi-time/sdk v0.1.0 // indirect @@ -284,7 +284,7 @@ require ( github.com/DataDog/datadog-agent/pkg/trace v0.56.0-rc.3 github.com/DataDog/datadog-go/v5 v5.6.0 github.com/aws/aws-sdk-go v1.55.5 - github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 github.com/aws/session-manager-plugin v0.0.0-20241119210807-82dc72922492 github.com/digitalocean/go-libvirt v0.0.0-20240812180835-9c6c0a310c6c github.com/hairyhenderson/go-codeowners v0.7.0 @@ -309,7 +309,7 @@ require ( github.com/pulumi/pulumi-azure-native-sdk/managedidentity/v2 v2.73.1 // indirect github.com/pulumi/pulumi-azure-native-sdk/network/v2 v2.73.1 // indirect github.com/pulumi/pulumi-azure-native-sdk/v2 v2.73.1 // indirect - github.com/pulumi/pulumi-eks/sdk/v3 v3.3.0 // indirect + github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 // indirect github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 // indirect github.com/twinj/uuid v0.0.0-20151029044442-89173bcdda19 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index cbbd971dd8ac3..b08902fb2e378 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -17,8 +17,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= -github.com/DataDog/test-infra-definitions v0.0.0-20241211124138-9c7c5005ca28 h1:LaZgAke+RN4wBKNl+R10ewdtKe/C2MJCbp9ozXKlLP8= -github.com/DataDog/test-infra-definitions v0.0.0-20241211124138-9c7c5005ca28/go.mod h1:blPG0VXBgk1oXm2+KHMTMyR0sNI2jv51FACAYPNQvNo= +github.com/DataDog/test-infra-definitions v0.0.0-20241217110507-4b4112f5f64d h1:F1yqGiWtXVsHkMiNUhs8bgaoZ1WlV3rYGRZ1CtCxpm8= +github.com/DataDog/test-infra-definitions v0.0.0-20241217110507-4b4112f5f64d/go.mod h1:1PAUwGjC25ACjfft4HrLEmHliuajlvjzcLFWpuqAIyk= github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= @@ -52,50 +52,50 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= -github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= -github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= -github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE= github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 h1:k97fGog9Tl0woxTiSIHN14Qs5ehqK6GXejUwkhJYyL0= github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0/go.mod h1:mzj8EEjIHSN2oZRXiw1Dd+uB4HZTl7hC8nBzX9IZMWw= -github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 h1:VDQaVwGOokbd3VUbHF+wupiffdrbAZPdQnr5XZMJqrs= -github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2/go.mod h1:lvUlMghKYmSxSfv0vU7pdU/8jSY+s0zpG8xXhaGKCw0= -github.com/aws/aws-sdk-go-v2/service/ecs v1.47.4 h1:CTkPGE8fiElvLtYWl/U+Eu5+1fVXiZbJUjyVCRSRgxk= -github.com/aws/aws-sdk-go-v2/service/ecs v1.47.4/go.mod h1:sMFLFhL27cKYa/eQYZp4asvIwHsnJWrAzTUpy9AQdnU= +github.com/aws/aws-sdk-go-v2/service/ecr v1.36.7 h1:R+5XKIJga2K9Dkj0/iQ6fD/MBGo02oxGGFTc512lK/Q= +github.com/aws/aws-sdk-go-v2/service/ecr v1.36.7/go.mod h1:fDPQV/6ONOQOjvtKhtypIy1wcGLcKYtoK/lvZ9fyDGQ= +github.com/aws/aws-sdk-go-v2/service/ecs v1.52.2 h1:LRM6z+wmXqAgCvuH36RR+Wf8SZZhvOVjt6f5r38V2II= +github.com/aws/aws-sdk-go-v2/service/ecs v1.52.2/go.mod h1:Ghi1OWUv4+VMEULWiHsKH2gNA3KAcMoLWsvU0eRXvIA= github.com/aws/aws-sdk-go-v2/service/eks v1.51.0 h1:BYyB+byjQ7oyupe3v+YjTp1yfmfNEwChYA2naCc85xI= github.com/aws/aws-sdk-go-v2/service/eks v1.51.0/go.mod h1:oaPCqTzAe8C5RQZJGRD4RENcV7A4n99uGxbD4rULbNg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= -github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0 h1:xA6XhTF7PE89BCNHJbQi8VvPzcgMtmGC5dr8S8N7lHk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.66.0/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6/go.mod h1:ngUiVRCco++u+soRRVBIvBZxSMMvOVMXA4PJ36JLfSw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2 h1:z6Pq4+jtKlhK4wWJGHRGwMLGjC1HZwAO3KJr/Na0tSU= github.com/aws/aws-sdk-go-v2/service/ssm v1.55.2/go.mod h1:DSmu/VZzpQlAubWBbAvNpt+S4k/XweglJi4XaDGyvQk= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= github.com/aws/session-manager-plugin v0.0.0-20241119210807-82dc72922492 h1:Ihams/fjKo4iWwM313ng2gCJWoetsL7ZQkXhOTmVUq4= github.com/aws/session-manager-plugin v0.0.0-20241119210807-82dc72922492/go.mod h1:7n17tunRPUsniNBu5Ja9C7WwJWTdOzaLqr/H0Ns3uuI= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= @@ -421,16 +421,16 @@ github.com/pulumi/pulumi-azure-native-sdk/v2 v2.73.1 h1:yzXxwwq3tHdtSOi5vjKmKXq7 github.com/pulumi/pulumi-azure-native-sdk/v2 v2.73.1/go.mod h1:ChjIUNDNeN6jI33ZOivHUFqM6purDiLP01mghMGe1Fs= github.com/pulumi/pulumi-command/sdk v1.0.1 h1:ZuBSFT57nxg/fs8yBymUhKLkjJ6qmyN3gNvlY/idiN0= github.com/pulumi/pulumi-command/sdk v1.0.1/go.mod h1:C7sfdFbUIoXKoIASfXUbP/U9xnwPfxvz8dBpFodohlA= -github.com/pulumi/pulumi-docker/sdk/v4 v4.5.5 h1:7OjAfgLz5PAy95ynbgPAlWls5WBe4I/QW/61TdPWRlQ= -github.com/pulumi/pulumi-docker/sdk/v4 v4.5.5/go.mod h1:XZKLFXbw13olxuztlWnmVUPYZp2a+BqzqhuMl0j/Ow8= -github.com/pulumi/pulumi-eks/sdk/v3 v3.3.0 h1:F3xAOBZ/In4PqydTsKeg3tou/c5FZ+JTp5dQO0oMjqE= -github.com/pulumi/pulumi-eks/sdk/v3 v3.3.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= +github.com/pulumi/pulumi-docker/sdk/v4 v4.5.7 h1:cuIl5YyIghqtnFMGsdtPOeaNSix5S2CrqO0/UZ1Yjsc= +github.com/pulumi/pulumi-docker/sdk/v4 v4.5.7/go.mod h1:f2ek887nKRSwNtqTqCFENJSOH0PXm1b3FhzSXYL0IyM= +github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 h1:s2Cpu6E2lmADNUbutbJGm6O+O9j0mBLlrhQmc40ukt0= +github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 h1:21oSj+TKlKTzQcxN9Hik7iSNNHPUQXN4s3itOnahy/w= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0/go.mod h1:YaEZms1NgXFqGhObKVofcAeWXu2V+3t/BAXdHQZq7fU= github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 h1:quqoGsLbF7lpGpGU4mi5WfVLIAo4gfvoQeYYmemx1Dg= github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3/go.mod h1:9dBA6+rtpKmyZB3k1XryUOHDOuNdoTODFKEEZZCtrz8= -github.com/pulumi/pulumi-libvirt/sdk v0.4.7 h1:/BBnqqx/Gbg2vINvJxXIVb58THXzw2lSqFqxlRSXH9M= -github.com/pulumi/pulumi-libvirt/sdk v0.4.7/go.mod h1:VKvjhAm1sGtzKZruYwIhgascabEx7+oVVRCoxp/cPi4= +github.com/pulumi/pulumi-libvirt/sdk v0.5.3 h1:CiUGTweLLIxbAbADxxnwPv4BK8pxXfU8urokJvK1ihM= +github.com/pulumi/pulumi-libvirt/sdk v0.5.3/go.mod h1:gAhyIZKtzs4rknrl8fu8BQnyqijAmViFbaUkyuHt4xY= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7 h1:39rhOe/PTUGMYia8pR5T2wbxxMt2pwrlonf0ncYKSzE= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7/go.mod h1:cxxDhJzUPt/YElfvlWa15Q4NGF6XXS8kUs4OQsCxSBk= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 h1:tXemWrzeVTqG8zq6hBdv1TdPFXjgZ+dob63a/6GlF1o= diff --git a/test/new-e2e/tests/containers/k8s_test.go b/test/new-e2e/tests/containers/k8s_test.go index 872714c799525..84a65c936561e 100644 --- a/test/new-e2e/tests/containers/k8s_test.go +++ b/test/new-e2e/tests/containers/k8s_test.go @@ -469,7 +469,10 @@ func (suite *k8sSuite) TestNginx() { `^pod_name:nginx-[[:alnum:]]+-[[:alnum:]]+$`, `^pod_phase:running$`, `^short_image:apps-nginx-server$`, - `^email:team-container-platform@datadoghq.com$`, + `^domain:deployment$`, + `^mail:team-container-platform@datadoghq.com$`, + `^org:agent-org$`, + `^parent-name:nginx$`, `^team:contp$`, }, AcceptUnexpectedTags: true, @@ -544,7 +547,10 @@ func (suite *k8sSuite) TestNginx() { `^pod_name:nginx-[[:alnum:]]+-[[:alnum:]]+$`, `^pod_phase:running$`, `^short_image:apps-nginx-server$`, - `^email:team-container-platform@datadoghq.com$`, + `^domain:deployment$`, + `^mail:team-container-platform@datadoghq.com$`, + `^org:agent-org$`, + `^parent-name:nginx$`, `^team:contp$`, }, Message: `GET / HTTP/1\.1`, diff --git a/test/new-e2e/tests/orchestrator/apply.go b/test/new-e2e/tests/orchestrator/apply.go index ee4e8668ab40c..45b3d792102a7 100644 --- a/test/new-e2e/tests/orchestrator/apply.go +++ b/test/new-e2e/tests/orchestrator/apply.go @@ -18,6 +18,7 @@ import ( dogstatsdstandalone "github.com/DataDog/test-infra-definitions/components/datadog/dogstatsd-standalone" fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" localKubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" + "github.com/DataDog/test-infra-definitions/components/kubernetes/vpa" resAws "github.com/DataDog/test-infra-definitions/resources/aws" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" @@ -86,6 +87,12 @@ func createCluster(ctx *pulumi.Context) (*resAws.Environment, *localKubernetes.C if err != nil { return nil, nil, nil, err } + + // Deploy VPA CRD + if _, err := vpa.DeployCRD(&awsEnv, kindKubeProvider); err != nil { + return nil, nil, nil, err + } + return &awsEnv, kindCluster, kindKubeProvider, nil } From 2b44f9730152dbff11d2c2499775fd6701360f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 17 Dec 2024 17:00:46 +0100 Subject: [PATCH 02/43] omnibus: vcredist: move the actual file copy to the recipe (#32299) --- omnibus/config/software/python3.rb | 1 - omnibus/config/software/vc_redist_14.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 6e5636064bad2..2fe37a032fc24 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -81,7 +81,6 @@ license "Python-2.0" command "XCOPY /YEHIR *.* \"#{windows_safe_path(python_3_embedded)}\"" - command "copy /y \"#{windows_safe_path(vcrt140_root)}\\*.dll\" \"#{windows_safe_path(python_3_embedded)}\"" # Install pip python = "#{windows_safe_path(python_3_embedded)}\\python.exe" diff --git a/omnibus/config/software/vc_redist_14.rb b/omnibus/config/software/vc_redist_14.rb index ecc8067943f96..226a1145bb993 100644 --- a/omnibus/config/software/vc_redist_14.rb +++ b/omnibus/config/software/vc_redist_14.rb @@ -23,5 +23,5 @@ else source_msm = "Microsoft_VC141_CRT_x64.msm" end - command "powershell -C \"#{windows_safe_path(script_root)} -file #{source_msm} -targetDir .\\expanded\"" + command "powershell -C \"#{windows_safe_path(script_root)} -file #{source_msm} -targetDir \"#{windows_safe_path(python_3_embedded)}\"\"" end From f5794905d1fc472991da3edecf4d0957461df92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lian=20Raimbault?= <161456554+CelianR@users.noreply.github.com> Date: Tue, 17 Dec 2024 11:07:13 -0500 Subject: [PATCH 03/43] Don't run notify gitlab ci changes on merge queue (#32293) --- .gitlab/notify/notify.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab/notify/notify.yml b/.gitlab/notify/notify.yml index f40187465a50f..13131a7fecb4a 100644 --- a/.gitlab/notify/notify.yml +++ b/.gitlab/notify/notify.yml @@ -80,8 +80,8 @@ notify_github: script: - !reference [.setup_agent_github_app] # Python 3.12 changes default behavior how packages are installed. - # In particular, --break-system-packages command line option is - # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 + # In particular, --break-system-packages command line option is + # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages - messagefile="$(mktemp)" - echo "Use this command from [test-infra-definitions](https://github.com/DataDog/test-infra-definitions) to manually test this PR changes on a VM:" >> "$messagefile" @@ -98,6 +98,7 @@ notify_gitlab_ci_changes: needs: [compute_gitlab_ci_config] tags: ["arch:amd64"] rules: + - !reference [.except_mergequeue] - changes: paths: - .gitlab-ci.yml @@ -105,8 +106,8 @@ notify_gitlab_ci_changes: compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 script: # Python 3.12 changes default behavior how packages are installed. - # In particular, --break-system-packages command line option is - # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 + # In particular, --break-system-packages command line option is + # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages - !reference [.setup_agent_github_app] - inv -e notify.gitlab-ci-diff --from-diff artifacts/diff.gitlab-ci.yml --pr-comment From 5cde5c4d2dc3c143ff6efc6ef70a73fc6e248e9e Mon Sep 17 00:00:00 2001 From: Alexandre Menasria <47357713+amenasria@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:24:10 +0100 Subject: [PATCH 04/43] [CI] Refactor the Agent integration tests (#31801) --- tasks/__init__.py | 2 +- tasks/agent.py | 88 ++++---------------- tasks/cluster_agent.py | 39 +++------ tasks/dogstatsd.py | 34 +++----- tasks/gointegrationtest.py | 161 +++++++++++++++++++++++++++++++++++++ tasks/gotest.py | 36 --------- tasks/trace_agent.py | 20 ++--- 7 files changed, 208 insertions(+), 172 deletions(-) create mode 100644 tasks/gointegrationtest.py diff --git a/tasks/__init__.py b/tasks/__init__.py index d59a06de7463b..0dffed23dacaf 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -87,13 +87,13 @@ tidy, tidy_all, ) +from tasks.gointegrationtest import integration_tests from tasks.gotest import ( check_otel_build, check_otel_module_versions, e2e_tests, get_impacted_packages, get_modified_packages, - integration_tests, lint_go, send_unit_tests_stats, test, diff --git a/tasks/agent.py b/tasks/agent.py index d4e5d13230885..19cd6170b96c6 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -17,6 +17,11 @@ from tasks.build_tags import add_fips_tags, filter_incompatible_tags, get_build_tags, get_default_build_tags from tasks.devcontainer import run_on_devcontainer from tasks.flavor import AgentFlavor +from tasks.gointegrationtest import ( + CORE_AGENT_LINUX_IT_CONF, + CORE_AGENT_WINDOWS_IT_CONF, + containerized_integration_tests, +) from tasks.libs.common.utils import ( REPO_PATH, bin_name, @@ -578,79 +583,18 @@ def integration_tests(ctx, race=False, remote_docker=False, go_mod="readonly", t """ Run integration tests for the Agent """ - if sys.platform == 'win32': - return _windows_integration_tests(ctx, race=race, go_mod=go_mod, timeout=timeout) - else: - # TODO: See if these will function on Windows - return _linux_integration_tests(ctx, race=race, remote_docker=remote_docker, go_mod=go_mod, timeout=timeout) - - -def _windows_integration_tests(ctx, race=False, go_mod="readonly", timeout=""): - test_args = { - "go_mod": go_mod, - "go_build_tags": " ".join(get_default_build_tags(build="test")), - "race_opt": "-race" if race else "", - "exec_opts": "", - "timeout_opt": f"-timeout {timeout}" if timeout else "", - } - - go_cmd = 'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args) # noqa: FS002 - - tests = [ - { - # Run eventlog tests with the Windows API, which depend on the EventLog service - "dir": "./pkg/util/winutil/", - 'prefix': './eventlog/...', - 'extra_args': '-evtapi Windows', - }, - { - # Run eventlog tailer tests with the Windows API, which depend on the EventLog service - "dir": ".", - 'prefix': './pkg/logs/tailers/windowsevent/...', - 'extra_args': '-evtapi Windows', - }, - { - # Run eventlog check tests with the Windows API, which depend on the EventLog service - "dir": ".", - # Don't include submodules, since the `-evtapi` flag is not defined in them - 'prefix': './comp/checks/windowseventlog/windowseventlogimpl/check', - 'extra_args': '-evtapi Windows', - }, - ] - - for test in tests: - with ctx.cd(f"{test['dir']}"): - ctx.run(f"{go_cmd} {test['prefix']} {test['extra_args']}") - - -def _linux_integration_tests(ctx, race=False, remote_docker=False, go_mod="readonly", timeout=""): - test_args = { - "go_mod": go_mod, - "go_build_tags": " ".join(get_default_build_tags(build="test")), - "race_opt": "-race" if race else "", - "exec_opts": "", - "timeout_opt": f"-timeout {timeout}" if timeout else "", - } - - # since Go 1.13, the -exec flag of go test could add some parameters such as -test.timeout - # to the call, we don't want them because while calling invoke below, invoke - # thinks that the parameters are for it to interpret. - # we're calling an intermediate script which only pass the binary name to the invoke task. - if remote_docker: - test_args["exec_opts"] = f"-exec \"{os.getcwd()}/test/integration/dockerize_tests.sh\"" - - go_cmd = 'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args) # noqa: FS002 - - prefixes = [ - "./test/integration/config_providers/...", - "./test/integration/corechecks/...", - "./test/integration/listeners/...", - "./test/integration/util/kubelet/...", - ] - - for prefix in prefixes: - ctx.run(f"{go_cmd} {prefix}") + return containerized_integration_tests( + ctx, CORE_AGENT_WINDOWS_IT_CONF, race=race, go_mod=go_mod, timeout=timeout + ) + return containerized_integration_tests( + ctx, + CORE_AGENT_LINUX_IT_CONF, + race=race, + remote_docker=remote_docker, + go_mod=go_mod, + timeout=timeout, + ) def check_supports_python_version(check_dir, python): diff --git a/tasks/cluster_agent.py b/tasks/cluster_agent.py index 8bbde5be51ece..b929270831a58 100644 --- a/tasks/cluster_agent.py +++ b/tasks/cluster_agent.py @@ -12,10 +12,10 @@ from invoke import task from invoke.exceptions import Exit -from tasks.build_tags import get_build_tags, get_default_build_tags +from tasks.build_tags import get_default_build_tags from tasks.cluster_agent_helpers import build_common, clean_common, refresh_assets_common, version_common from tasks.cws_instrumentation import BIN_PATH as CWS_INSTRUMENTATION_BIN_PATH -from tasks.libs.common.utils import TestsNotSupportedError +from tasks.gointegrationtest import CLUSTER_AGENT_IT_CONF, containerized_integration_tests from tasks.libs.releasing.version import load_release_versions # constants @@ -92,33 +92,14 @@ def integration_tests(ctx, race=False, remote_docker=False, go_mod="readonly", t """ Run integration tests for cluster-agent """ - if sys.platform == 'win32': - raise TestsNotSupportedError('Cluster Agent integration tests are not supported on Windows') - - # We need docker for the kubeapiserver integration tests - tags = get_default_build_tags(build="cluster-agent") + ["docker", "test"] - - go_build_tags = " ".join(get_build_tags(tags, [])) - race_opt = "-race" if race else "" - exec_opts = "" - timeout_opt = f"-timeout {timeout}" if timeout else "" - - # since Go 1.13, the -exec flag of go test could add some parameters such as -test.timeout - # to the call, we don't want them because while calling invoke below, invoke - # thinks that the parameters are for it to interpret. - # we're calling an intermediate script which only pass the binary name to the invoke task. - if remote_docker: - exec_opts = f"-exec \"{os.getcwd()}/test/integration/dockerize_tests.sh\"" - - go_cmd = f'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}' - - prefixes = [ - "./test/integration/util/kube_apiserver", - "./test/integration/util/leaderelection", - ] - - for prefix in prefixes: - ctx.run(f"{go_cmd} {prefix}") + containerized_integration_tests( + ctx, + CLUSTER_AGENT_IT_CONF, + race=race, + remote_docker=remote_docker, + go_mod=go_mod, + timeout=timeout, + ) @task diff --git a/tasks/dogstatsd.py b/tasks/dogstatsd.py index 08dbe80a85993..311560a3d118e 100644 --- a/tasks/dogstatsd.py +++ b/tasks/dogstatsd.py @@ -11,7 +11,8 @@ from tasks.build_tags import filter_incompatible_tags, get_build_tags, get_default_build_tags from tasks.flavor import AgentFlavor -from tasks.libs.common.utils import REPO_PATH, TestsNotSupportedError, bin_name, get_build_flags, get_root +from tasks.gointegrationtest import DOGSTATSD_IT_CONF, containerized_integration_tests +from tasks.libs.common.utils import REPO_PATH, bin_name, get_build_flags, get_root from tasks.windows_resources import build_messagetable, build_rc, versioninfo_vars # constants @@ -174,29 +175,14 @@ def integration_tests(ctx, race=False, remote_docker=False, go_mod="readonly", t """ Run integration tests for dogstatsd """ - if sys.platform == 'win32': - raise TestsNotSupportedError('DogStatsD integration tests are not supported on Windows') - - go_build_tags = " ".join(get_default_build_tags(build="test")) - race_opt = "-race" if race else "" - exec_opts = "" - timeout_opt = f"-timeout {timeout}" if timeout else "" - - # since Go 1.13, the -exec flag of go test could add some parameters such as -test.timeout - # to the call, we don't want them because while calling invoke below, invoke - # thinks that the parameters are for it to interpret. - # we're calling an intermediate script which only pass the binary name to the invoke task. - if remote_docker: - exec_opts = f"-exec \"{os.getcwd()}/test/integration/dockerize_tests.sh\"" - - go_cmd = f'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}' - - prefixes = [ - "./test/integration/dogstatsd/...", - ] - - for prefix in prefixes: - ctx.run(f"{go_cmd} {prefix}") + containerized_integration_tests( + ctx, + DOGSTATSD_IT_CONF, + race=race, + remote_docker=remote_docker, + go_mod=go_mod, + timeout=timeout, + ) @task diff --git a/tasks/gointegrationtest.py b/tasks/gointegrationtest.py new file mode 100644 index 0000000000000..1d5cc56c8b9cf --- /dev/null +++ b/tasks/gointegrationtest.py @@ -0,0 +1,161 @@ +import os +import sys +import traceback +from dataclasses import dataclass + +from invoke import Context, task +from invoke.exceptions import Exit + +from tasks.build_tags import get_default_build_tags +from tasks.libs.common.utils import TestsNotSupportedError, gitlab_section + + +@dataclass +class IntegrationTest: + """ + Integration tests + """ + + prefix: str + dir: str = None + extra_args: str = None + + +@dataclass +class IntegrationTestsConfig: + """ + Integration tests configuration + """ + + name: str + go_build_tags: list[str] + tests: list[IntegrationTest] + env: dict[str, str] = None + is_windows_supported: bool = True + + +CORE_AGENT_LINUX_IT_CONF = IntegrationTestsConfig( + name="Core Agent Linux", + go_build_tags=get_default_build_tags(build="test"), + tests=[ + IntegrationTest(prefix="./test/integration/config_providers/..."), + IntegrationTest(prefix="./test/integration/corechecks/..."), + IntegrationTest(prefix="./test/integration/listeners/..."), + IntegrationTest(prefix="./test/integration/util/kubelet/..."), + ], + is_windows_supported=False, +) + +CORE_AGENT_WINDOWS_IT_CONF = IntegrationTestsConfig( + name="Core Agent Windows", + go_build_tags=get_default_build_tags(build="test"), + tests=[ + # Run eventlog tests with the Windows API, which depend on the EventLog service + IntegrationTest(dir="./pkg/util/winutil/", prefix="./eventlog/...", extra_args="-evtapi Windows"), + # Run eventlog tailer tests with the Windows API, which depend on the EventLog service + IntegrationTest(dir=".", prefix="./pkg/logs/tailers/windowsevent/...", extra_args="-evtapi Windows"), + # Run eventlog check tests with the Windows API, which depend on the EventLog service + # Don't include submodules, since the `-evtapi` flag is not defined in them + IntegrationTest( + dir=".", prefix="./comp/checks/windowseventlog/windowseventlogimpl/check", extra_args="-evtapi Windows" + ), + ], +) + +DOGSTATSD_IT_CONF = IntegrationTestsConfig( + name="DogStatsD", + go_build_tags=get_default_build_tags(build="test"), + tests=[IntegrationTest(prefix="./test/integration/dogstatsd/...")], + is_windows_supported=False, +) + +CLUSTER_AGENT_IT_CONF = IntegrationTestsConfig( + name="Cluster Agent", + go_build_tags=get_default_build_tags(build="cluster-agent") + ["docker", "test"], + tests=[ + IntegrationTest(prefix="./test/integration/util/kube_apiserver"), + IntegrationTest(prefix="./test/integration/util/leaderelection"), + ], + is_windows_supported=False, +) + +TRACE_AGENT_IT_CONF = IntegrationTestsConfig( + name="Trace Agent", + go_build_tags=get_default_build_tags(build="test"), + tests=[IntegrationTest(prefix="./cmd/trace-agent/test/testsuite/...")], + env={"INTEGRATION": "yes"}, + is_windows_supported=False, +) + + +def containerized_integration_tests( + ctx: Context, + integration_tests_config: IntegrationTestsConfig, + race=False, + remote_docker=False, + go_mod="readonly", + timeout="", +): + if sys.platform == 'win32' and not integration_tests_config.is_windows_supported: + raise TestsNotSupportedError(f'{integration_tests_config.name} integration tests are not supported on Windows') + test_args = { + "go_mod": go_mod, + "go_build_tags": " ".join(integration_tests_config.go_build_tags), + "race_opt": "-race" if race else "", + "exec_opts": "", + "timeout_opt": f"-timeout {timeout}" if timeout else "", + } + + # since Go 1.13, the -exec flag of go test could add some parameters such as -test.timeout + # to the call, we don't want them because while calling invoke below, invoke + # thinks that the parameters are for it to interpret. + # we're calling an intermediate script which only pass the binary name to the invoke task. + if remote_docker: + test_args["exec_opts"] = f"-exec \"{os.getcwd()}/test/integration/dockerize_tests.sh\"" + + go_cmd = 'go test {timeout_opt} -mod={go_mod} {race_opt} -tags "{go_build_tags}" {exec_opts}'.format(**test_args) # noqa: FS002 + + for it in integration_tests_config.tests: + if it.dir: + with ctx.cd(f"{it.dir}"): + ctx.run(f"{go_cmd} {it.prefix}", env=integration_tests_config.env) + else: + ctx.run(f"{go_cmd} {it.prefix}", env=integration_tests_config.env) + + +@task +def integration_tests(ctx, race=False, remote_docker=False, timeout=""): + """ + Run all the available integration tests + """ + core_agent_conf = CORE_AGENT_WINDOWS_IT_CONF if sys.platform == 'win32' else CORE_AGENT_LINUX_IT_CONF + tests = { + "Agent Core": lambda: containerized_integration_tests( + ctx, core_agent_conf, race=race, remote_docker=remote_docker, timeout=timeout + ), + "DogStatsD": lambda: containerized_integration_tests( + ctx, DOGSTATSD_IT_CONF, race=race, remote_docker=remote_docker, timeout=timeout + ), + "Cluster Agent": lambda: containerized_integration_tests( + ctx, CLUSTER_AGENT_IT_CONF, race=race, remote_docker=remote_docker, timeout=timeout + ), + "Trace Agent": lambda: containerized_integration_tests(ctx, TRACE_AGENT_IT_CONF, race=race, timeout=timeout), + } + tests_failures = {} + for t_name, t in tests.items(): + with gitlab_section(f"Running the {t_name} integration tests", collapsed=True, echo=True): + try: + t() + except TestsNotSupportedError as e: + print(f"Skipping {t_name}: {e}") + except Exception: + # Keep printing the traceback not to have to wait until all tests are done to see what failed + traceback.print_exc() + # Storing the traceback to print it at the end without directly raising the exception + tests_failures[t_name] = traceback.format_exc() + if tests_failures: + print("The following integration tests failed:") + for t_name in tests_failures: + print(f"- {t_name}") + print("See the above logs to get the full traceback.") + raise Exit(code=1) diff --git a/tasks/gotest.py b/tasks/gotest.py index 7ca411e7521ea..4aa9a4b3e151f 100644 --- a/tasks/gotest.py +++ b/tasks/gotest.py @@ -11,7 +11,6 @@ import os import re import sys -import traceback from collections import defaultdict from collections.abc import Iterable from datetime import datetime @@ -22,13 +21,10 @@ from invoke.context import Context from invoke.exceptions import Exit -from tasks.agent import integration_tests as agent_integration_tests from tasks.build_tags import compute_build_tags_for_flavor -from tasks.cluster_agent import integration_tests as dca_integration_tests from tasks.collector import OCB_VERSION from tasks.coverage import PROFILE_COV, CodecovWorkaround from tasks.devcontainer import run_on_devcontainer -from tasks.dogstatsd import integration_tests as dsd_integration_tests from tasks.flavor import AgentFlavor from tasks.libs.common.color import color_message from tasks.libs.common.datadog_api import create_count, send_metrics @@ -36,7 +32,6 @@ from tasks.libs.common.gomodules import get_default_modules from tasks.libs.common.junit_upload_core import enrich_junitxml, produce_junit_tar from tasks.libs.common.utils import ( - TestsNotSupportedError, clean_nested_paths, get_build_flags, gitlab_section, @@ -46,7 +41,6 @@ from tasks.modules import GoModule, get_module_by_path from tasks.test_core import ModuleTestResult, process_input_args, process_module_results, test_core from tasks.testwasher import TestWasher -from tasks.trace_agent import integration_tests as trace_integration_tests from tasks.update_go import PATTERN_MAJOR_MINOR_BUGFIX GO_TEST_RESULT_TMP_JSON = 'module_test_output.json' @@ -405,36 +399,6 @@ def test( print(f"Tests final status (including re-runs): {color_message('ALL TESTS PASSED', 'green')}") -@task -def integration_tests(ctx, race=False, remote_docker=False, timeout=""): - """ - Run all the available integration tests - """ - tests = { - "Agent": lambda: agent_integration_tests(ctx, race=race, remote_docker=remote_docker, timeout=timeout), - "DogStatsD": lambda: dsd_integration_tests(ctx, race=race, remote_docker=remote_docker, timeout=timeout), - "Cluster Agent": lambda: dca_integration_tests(ctx, race=race, remote_docker=remote_docker, timeout=timeout), - "Trace Agent": lambda: trace_integration_tests(ctx, race=race, timeout=timeout), - } - tests_failures = {} - for t_name, t in tests.items(): - with gitlab_section(f"Running the {t_name} integration tests", collapsed=True, echo=True): - try: - t() - except TestsNotSupportedError as e: - print(f"Skipping {t_name}: {e}") - except Exception: - # Keep printing the traceback not to have to wait until all tests are done to see what failed - traceback.print_exc() - # Storing the traceback to print it at the end without directly raising the exception - tests_failures[t_name] = traceback.format_exc() - if tests_failures: - print("Integration tests failed:") - for t_name, t_failure in tests_failures.items(): - print(f"{t_name}:\n{t_failure}") - raise Exit(code=1) - - @task def e2e_tests(ctx, target="gitlab", agent_image="", dca_image="", argo_workflow="default"): """ diff --git a/tasks/trace_agent.py b/tasks/trace_agent.py index d65ae6861fcd8..0ec1dd1956aa8 100644 --- a/tasks/trace_agent.py +++ b/tasks/trace_agent.py @@ -5,7 +5,8 @@ from tasks.build_tags import add_fips_tags, filter_incompatible_tags, get_build_tags, get_default_build_tags from tasks.flavor import AgentFlavor -from tasks.libs.common.utils import REPO_PATH, TestsNotSupportedError, bin_name, get_build_flags +from tasks.gointegrationtest import TRACE_AGENT_IT_CONF, containerized_integration_tests +from tasks.libs.common.utils import REPO_PATH, bin_name, get_build_flags from tasks.windows_resources import build_messagetable, build_rc, versioninfo_vars BIN_PATH = os.path.join(".", "bin", "trace-agent") @@ -81,15 +82,14 @@ def integration_tests(ctx, race=False, go_mod="readonly", timeout="10m"): """ Run integration tests for trace agent """ - if sys.platform == 'win32': - raise TestsNotSupportedError('Trace Agent integration tests are not supported on Windows') - - go_build_tags = " ".join(get_default_build_tags(build="test")) - race_opt = "-race" if race else "" - timeout_opt = f"-timeout {timeout}" if timeout else "" - - go_cmd = f'go test {timeout_opt} -mod={go_mod} {race_opt} -v -tags "{go_build_tags}"' - ctx.run(f"{go_cmd} ./cmd/trace-agent/test/testsuite/...", env={"INTEGRATION": "yes"}) + containerized_integration_tests( + ctx, + TRACE_AGENT_IT_CONF, + race=race, + remote_docker=False, + go_mod=go_mod, + timeout=timeout, + ) @task From 381a3cb19124c62dccb064ea35c326b26bc3077f Mon Sep 17 00:00:00 2001 From: Kevin Gosse Date: Tue, 17 Dec 2024 17:37:35 +0100 Subject: [PATCH 05/43] Capture additional entries from the event log (#31559) Co-authored-by: Ursula Chen <58821586+urseberry@users.noreply.github.com> --- pkg/flare/archive_win.go | 2 +- .../notes/add-events-to-flare-b10cb7bf1aaf4d50.yaml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/add-events-to-flare-b10cb7bf1aaf4d50.yaml diff --git a/pkg/flare/archive_win.go b/pkg/flare/archive_win.go index e62f23d58a8dc..96afe5ac17a94 100644 --- a/pkg/flare/archive_win.go +++ b/pkg/flare/archive_win.go @@ -31,7 +31,7 @@ var ( eventLogChannelsToExport = map[string]string{ "System": "Event/System/Provider[@Name=\"Service Control Manager\"]", - "Application": "Event/System/Provider[@Name=\"datadog-trace-agent\" or @Name=\"DatadogAgent\"]", + "Application": "Event/System/Provider[@Name=\"datadog-trace-agent\" or @Name=\"DatadogAgent\" or @Name=\".NET Runtime\" or @Name=\"Application Error\"]", "Microsoft-Windows-WMI-Activity/Operational": "*", } execTimeout = 30 * time.Second diff --git a/releasenotes/notes/add-events-to-flare-b10cb7bf1aaf4d50.yaml b/releasenotes/notes/add-events-to-flare-b10cb7bf1aaf4d50.yaml new file mode 100644 index 0000000000000..9bff569f5f774 --- /dev/null +++ b/releasenotes/notes/add-events-to-flare-b10cb7bf1aaf4d50.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + On Windows, Agent flares now include event logs for .NET applications. From 2ad528f546105c0b0b24271a5bec8d4db2cd31ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Tue, 17 Dec 2024 18:32:53 +0100 Subject: [PATCH 06/43] Add e2e test for VPA metrics from KSM check (#31872) --- test/new-e2e/tests/containers/k8s_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/new-e2e/tests/containers/k8s_test.go b/test/new-e2e/tests/containers/k8s_test.go index 84a65c936561e..ea2e14c8d3742 100644 --- a/test/new-e2e/tests/containers/k8s_test.go +++ b/test/new-e2e/tests/containers/k8s_test.go @@ -801,6 +801,24 @@ func (suite *k8sSuite) TestCPU() { }) } +func (suite *k8sSuite) TestKSM() { + suite.testMetric(&testMetricArgs{ + Filter: testMetricFilterArgs{ + Name: "kubernetes_state.vpa.count", + }, + Expect: testMetricExpectArgs{ + Tags: &[]string{ + `^kube_cluster_name:` + regexp.QuoteMeta(suite.clusterName) + `$`, + `^kube_namespace:workload-(?:nginx|redis)$`, + }, + Value: &testMetricExpectValueArgs{ + Max: 1, + Min: 1, + }, + }, + }) +} + func (suite *k8sSuite) TestDogstatsdInAgent() { // Test with UDS suite.testDogstatsd(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDS) From c2f284215d18bcf9fc155ec362459a5f30af0773 Mon Sep 17 00:00:00 2001 From: Gabriel Dos Santos <91925154+gabedos@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:53:06 -0500 Subject: [PATCH 07/43] chore(dca): Config component migration in webhook and config (#32247) --- .../admission/controllers/webhook/config.go | 20 +++--- .../controllers/webhook/controller_base.go | 5 +- .../webhook/controller_base_test.go | 4 +- .../controllers/webhook/controller_v1_test.go | 69 +++++++++--------- .../webhook/controller_v1beta1_test.go | 70 ++++++++++--------- .../mutate/autoscaling/autoscaling.go | 6 +- pkg/clusteragent/admission/start.go | 2 +- 7 files changed, 91 insertions(+), 85 deletions(-) diff --git a/pkg/clusteragent/admission/controllers/webhook/config.go b/pkg/clusteragent/admission/controllers/webhook/config.go index 90cec93a8d8db..4f5e549434557 100644 --- a/pkg/clusteragent/admission/controllers/webhook/config.go +++ b/pkg/clusteragent/admission/controllers/webhook/config.go @@ -13,7 +13,7 @@ import ( "fmt" "strings" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver/common" ) @@ -36,21 +36,21 @@ type Config struct { } // NewConfig creates a webhook controller configuration -func NewConfig(admissionV1Enabled, namespaceSelectorEnabled, matchConditionsSupported bool) Config { +func NewConfig(admissionV1Enabled, namespaceSelectorEnabled, matchConditionsSupported bool, datadogConfig config.Component) Config { return Config{ - webhookName: pkgconfigsetup.Datadog().GetString("admission_controller.webhook_name"), - secretName: pkgconfigsetup.Datadog().GetString("admission_controller.certificate.secret_name"), - validationEnabled: pkgconfigsetup.Datadog().GetBool("admission_controller.validation.enabled"), - mutationEnabled: pkgconfigsetup.Datadog().GetBool("admission_controller.mutation.enabled"), + webhookName: datadogConfig.GetString("admission_controller.webhook_name"), + secretName: datadogConfig.GetString("admission_controller.certificate.secret_name"), + validationEnabled: datadogConfig.GetBool("admission_controller.validation.enabled"), + mutationEnabled: datadogConfig.GetBool("admission_controller.mutation.enabled"), namespace: common.GetResourcesNamespace(), admissionV1Enabled: admissionV1Enabled, namespaceSelectorEnabled: namespaceSelectorEnabled, matchConditionsSupported: matchConditionsSupported, - svcName: pkgconfigsetup.Datadog().GetString("admission_controller.service_name"), + svcName: datadogConfig.GetString("admission_controller.service_name"), svcPort: int32(443), - timeout: pkgconfigsetup.Datadog().GetInt32("admission_controller.timeout_seconds"), - failurePolicy: pkgconfigsetup.Datadog().GetString("admission_controller.failure_policy"), - reinvocationPolicy: pkgconfigsetup.Datadog().GetString("admission_controller.reinvocation_policy"), + timeout: datadogConfig.GetInt32("admission_controller.timeout_seconds"), + failurePolicy: datadogConfig.GetString("admission_controller.failure_policy"), + reinvocationPolicy: datadogConfig.GetString("admission_controller.reinvocation_policy"), } } diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_base.go b/pkg/clusteragent/admission/controllers/webhook/controller_base.go index 79114dac7397f..5505337ba5838 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_base.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_base.go @@ -35,7 +35,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate/tagsfromlabels" "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/validate/kubernetesadmissionevents" "github.com/DataDog/datadog-agent/pkg/clusteragent/autoscaling/workload" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -125,7 +124,7 @@ func (c *controllerBase) generateWebhooks(wmeta workloadmeta.Component, pa workl configWebhook.NewWebhook(wmeta, injectionFilter, datadogConfig), tagsfromlabels.NewWebhook(wmeta, datadogConfig, injectionFilter), agentsidecar.NewWebhook(datadogConfig), - autoscaling.NewWebhook(pa), + autoscaling.NewWebhook(pa, datadogConfig), } webhooks = append(webhooks, mutatingWebhooks...) @@ -137,7 +136,7 @@ func (c *controllerBase) generateWebhooks(wmeta workloadmeta.Component, pa workl log.Errorf("failed to register APM Instrumentation webhook: %v", err) } - isCWSInstrumentationEnabled := pkgconfigsetup.Datadog().GetBool("admission_controller.cws_instrumentation.enabled") + isCWSInstrumentationEnabled := datadogConfig.GetBool("admission_controller.cws_instrumentation.enabled") if isCWSInstrumentationEnabled { cws, err := cwsinstrumentation.NewCWSInstrumentation(wmeta, datadogConfig) if err == nil { diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_base_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_base_test.go index c202f1ab38f25..d5612b96c74e0 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_base_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_base_test.go @@ -36,7 +36,7 @@ func TestNewController(t *testing.T) { factory.Admissionregistration(), func() bool { return true }, make(chan struct{}), - v1Cfg, + getV1Cfg(t), wmeta, nil, datadogConfig, @@ -53,7 +53,7 @@ func TestNewController(t *testing.T) { factory.Admissionregistration(), func() bool { return true }, make(chan struct{}), - v1beta1Cfg, + getV1beta1Cfg(t), wmeta, nil, datadogConfig, diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go index aa2a3efc4e68e..6a52255c42e29 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1_test.go @@ -41,10 +41,11 @@ const ( tick = 50 * time.Millisecond ) -var v1Cfg = NewConfig(true, false, false) +func getV1Cfg(t *testing.T) Config { return NewConfig(true, false, false, configComp.NewMock(t)) } func TestSecretNotFoundV1(t *testing.T) { f := newFixtureV1(t) + v1Cfg := getV1Cfg(t) stopCh := make(chan struct{}) defer close(stopCh) @@ -63,6 +64,7 @@ func TestSecretNotFoundV1(t *testing.T) { func TestCreateWebhookV1(t *testing.T) { f := newFixtureV1(t) + v1Cfg := getV1Cfg(t) data, err := certificate.GenerateSecretData(time.Now(), time.Now().Add(365*24*time.Hour), []string{"my.svc.dns"}) if err != nil { @@ -97,6 +99,7 @@ func TestCreateWebhookV1(t *testing.T) { func TestUpdateOutdatedWebhookV1(t *testing.T) { f := newFixtureV1(t) + v1Cfg := getV1Cfg(t) data, err := certificate.GenerateSecretData(time.Now(), time.Now().Add(365*24*time.Hour), []string{"my.svc.dns"}) if err != nil { @@ -166,7 +169,7 @@ func TestAdmissionControllerFailureModeV1(t *testing.T) { for _, value := range []string{"Ignore", "ignore", "BadVal", ""} { mockConfig.SetWithoutSource("admission_controller.failure_policy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) validatingWebhookSkeleton := c.getValidatingWebhookSkeleton("foo", "/bar", []admiv1.OperationType{admiv1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1.Ignore, *validatingWebhookSkeleton.FailurePolicy) @@ -176,7 +179,7 @@ func TestAdmissionControllerFailureModeV1(t *testing.T) { for _, value := range []string{"Fail", "fail"} { mockConfig.SetWithoutSource("admission_controller.failure_policy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) validatingWebhookSkeleton := c.getValidatingWebhookSkeleton("foo", "/bar", []admiv1.OperationType{admiv1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1.Fail, *validatingWebhookSkeleton.FailurePolicy) @@ -192,7 +195,7 @@ func TestAdmissionControllerReinvocationPolicyV1(t *testing.T) { for _, value := range []string{"IfNeeded", "ifneeded", "Never", "never", "wrong", ""} { mockConfig.SetWithoutSource("admission_controller.reinvocationpolicy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) mutatingWebhookSkeleton := c.getMutatingWebhookSkeleton("foo", "/bar", []admiv1.OperationType{admiv1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1.IfNeededReinvocationPolicy, *mutatingWebhookSkeleton.ReinvocationPolicy) @@ -241,7 +244,7 @@ func TestGenerateTemplatesV1(t *testing.T) { tests := []struct { name string setupConfig func(model.Config) - configFunc func() Config + configFunc func(model.Config) Config want func() []admiv1.MutatingWebhook }{ { @@ -253,7 +256,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -276,7 +279,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -295,7 +298,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.standard.tags", "/injecttags", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -318,7 +321,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.standard.tags", "/injecttags", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -337,7 +340,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.lib.injection", "/injectlib", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -360,7 +363,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook("datadog.webhook.lib.injection", "/injectlib", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -378,7 +381,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -402,7 +405,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -435,7 +438,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.namespace_selector_fallback", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", nil, &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -461,7 +464,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.inject_tags.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook( "datadog.webhook.agent.config", @@ -511,7 +514,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { webhook := webhook( "datadog.webhook.agent.config", @@ -559,7 +562,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", true) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(true, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -602,7 +605,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(true, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, false, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -641,7 +644,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", true) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -684,7 +687,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -726,7 +729,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "misconfigured") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -746,7 +749,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -779,7 +782,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -799,7 +802,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -819,7 +822,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -850,7 +853,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -870,7 +873,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -899,7 +902,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey1\": \"labelVal1\"}}, \"NamespaceSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -928,7 +931,7 @@ func TestGenerateTemplatesV1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\":{\"labelKey1\": \"labelVal1\"}}} , {\"ObjectSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1.MutatingWebhook { return []admiv1.MutatingWebhook{} }, @@ -947,7 +950,7 @@ func TestGenerateTemplatesV1(t *testing.T) { tt.setupConfig(mockConfig) c := &ControllerV1{} - c.config = tt.configFunc() + c.config = tt.configFunc(mockConfig) c.webhooks = c.generateWebhooks(wmeta, nil, mockConfig, nil) c.generateTemplates() @@ -1051,7 +1054,7 @@ func TestGetValidatingWebhookSkeletonV1(t *testing.T) { } c := &ControllerV1{} - c.config = NewConfig(false, tt.namespaceSelector, false) + c.config = NewConfig(false, tt.namespaceSelector, false, mockConfig) nsSelector, objSelector := common.DefaultLabelSelectors(tt.namespaceSelector) @@ -1157,7 +1160,7 @@ func TestGetMutatingWebhookSkeletonV1(t *testing.T) { } c := &ControllerV1{} - c.config = NewConfig(false, tt.namespaceSelector, false) + c.config = NewConfig(false, tt.namespaceSelector, false, mockConfig) nsSelector, objSelector := common.DefaultLabelSelectors(tt.namespaceSelector) @@ -1188,7 +1191,7 @@ func (f *fixtureV1) createController() (*ControllerV1, informers.SharedInformerF factory.Admissionregistration().V1().MutatingWebhookConfigurations(), func() bool { return true }, make(chan struct{}), - v1Cfg, + getV1Cfg(f.t), wmeta, nil, datadogConfig, diff --git a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go index bd5daf906a2fb..62fa7713cd809 100644 --- a/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go +++ b/pkg/clusteragent/admission/controllers/webhook/controller_v1beta1_test.go @@ -36,10 +36,11 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/kubernetes/certificate" ) -var v1beta1Cfg = NewConfig(false, false, false) +func getV1beta1Cfg(t *testing.T) Config { return NewConfig(false, false, false, configComp.NewMock(t)) } func TestSecretNotFoundV1beta1(t *testing.T) { f := newFixtureV1beta1(t) + v1beta1Cfg := getV1beta1Cfg(t) stopCh := make(chan struct{}) defer close(stopCh) @@ -58,6 +59,7 @@ func TestSecretNotFoundV1beta1(t *testing.T) { func TestCreateWebhookV1beta1(t *testing.T) { f := newFixtureV1beta1(t) + v1beta1Cfg := getV1beta1Cfg(t) data, err := certificate.GenerateSecretData(time.Now(), time.Now().Add(365*24*time.Hour), []string{"my.svc.dns"}) if err != nil { @@ -92,6 +94,8 @@ func TestCreateWebhookV1beta1(t *testing.T) { func TestUpdateOutdatedWebhookV1beta1(t *testing.T) { f := newFixtureV1beta1(t) + v1Cfg := getV1Cfg(t) + v1beta1Cfg := getV1beta1Cfg(t) data, err := certificate.GenerateSecretData(time.Now(), time.Now().Add(365*24*time.Hour), []string{"my.svc.dns"}) if err != nil { @@ -161,7 +165,7 @@ func TestAdmissionControllerFailureModeV1beta1(t *testing.T) { for _, value := range []string{"Ignore", "ignore", "BadVal", ""} { mockConfig.SetWithoutSource("admission_controller.failure_policy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) validatingWebhookSkeleton := c.getValidatingWebhookSkeleton("foo", "/bar", []admiv1beta1.OperationType{admiv1beta1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1beta1.Ignore, *validatingWebhookSkeleton.FailurePolicy) @@ -171,7 +175,7 @@ func TestAdmissionControllerFailureModeV1beta1(t *testing.T) { for _, value := range []string{"Fail", "fail"} { mockConfig.SetWithoutSource("admission_controller.failure_policy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) validatingWebhookSkeleton := c.getValidatingWebhookSkeleton("foo", "/bar", []admiv1beta1.OperationType{admiv1beta1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1beta1.Fail, *validatingWebhookSkeleton.FailurePolicy) @@ -187,7 +191,7 @@ func TestAdmissionControllerReinvocationPolicyV1beta1(t *testing.T) { for _, value := range []string{"IfNeeded", "ifneeded", "Never", "never", "wrong", ""} { mockConfig.SetWithoutSource("admission_controller.reinvocationpolicy", value) - c.config = NewConfig(true, false, false) + c.config = NewConfig(true, false, false, mockConfig) mutatingWebhookSkeleton := c.getMutatingWebhookSkeleton("foo", "/bar", []admiv1beta1.OperationType{admiv1beta1.Create}, map[string][]string{"": {"pods"}}, nil, nil, nil) assert.Equal(t, admiv1beta1.IfNeededReinvocationPolicy, *mutatingWebhookSkeleton.ReinvocationPolicy) @@ -236,7 +240,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { tests := []struct { name string setupConfig func(model.Config) - configFunc func() Config + configFunc func(model.Config) Config want func() []admiv1beta1.MutatingWebhook }{ { @@ -248,7 +252,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -271,7 +275,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -290,7 +294,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.standard.tags", "/injecttags", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -313,7 +317,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.standard.tags", "/injecttags", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -332,7 +336,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.lib.injection", "/injectlib", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -355,7 +359,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", true) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook("datadog.webhook.lib.injection", "/injectlib", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -373,7 +377,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -397,7 +401,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", &metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ @@ -430,7 +434,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhookConfig := webhook("datadog.webhook.agent.config", "/injectconfig", nil, &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -456,7 +460,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook( "datadog.webhook.agent.config", @@ -505,7 +509,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.auto_instrumentation.enabled", false) mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.enabled", false) }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { webhook := webhook( "datadog.webhook.agent.config", @@ -552,7 +556,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", true) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -595,7 +599,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(false, false, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, false, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -634,7 +638,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", true) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -677,7 +681,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.cws_instrumentation.mutate_unlabelled", false) mockConfig.SetWithoutSource("cluster_agent.service_account_name", "datadog-cluster-agent") }, - configFunc: func() Config { return NewConfig(false, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(false, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.cws.pod.instrumentation", @@ -719,7 +723,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "misconfigured") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -739,7 +743,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -772,7 +776,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -792,7 +796,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -812,7 +816,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -843,7 +847,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -863,7 +867,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey\": \"labelVal\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -892,7 +896,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"ObjectSelector\": {\"MatchLabels\": {\"labelKey1\": \"labelVal1\"}}, \"NamespaceSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { podWebhook := webhook( "datadog.webhook.agent.sidecar", @@ -921,7 +925,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { mockConfig.SetWithoutSource("admission_controller.agent_sidecar.selectors", "[{\"NamespaceSelector\": {\"MatchLabels\":{\"labelKey1\": \"labelVal1\"}}} , {\"ObjectSelector\": {\"MatchLabels\": {\"labelKey2\": \"labelVal2\"}}}]") mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", "[]") }, - configFunc: func() Config { return NewConfig(true, true, false) }, + configFunc: func(mockConfig model.Config) Config { return NewConfig(true, true, false, mockConfig) }, want: func() []admiv1beta1.MutatingWebhook { return []admiv1beta1.MutatingWebhook{} }, @@ -940,7 +944,7 @@ func TestGenerateTemplatesV1beta1(t *testing.T) { tt.setupConfig(mockConfig) c := &ControllerV1beta1{} - c.config = tt.configFunc() + c.config = tt.configFunc(mockConfig) c.webhooks = c.generateWebhooks(wmeta, nil, mockConfig, nil) c.generateTemplates() @@ -1044,7 +1048,7 @@ func TestGetValidatingWebhookSkeletonV1beta1(t *testing.T) { } c := &ControllerV1beta1{} - c.config = NewConfig(false, tt.namespaceSelector, false) + c.config = NewConfig(false, tt.namespaceSelector, false, mockConfig) nsSelector, objSelector := common.DefaultLabelSelectors(tt.namespaceSelector) @@ -1150,7 +1154,7 @@ func TestGetMutatingWebhookSkeletonV1beta1(t *testing.T) { } c := &ControllerV1beta1{} - c.config = NewConfig(false, tt.namespaceSelector, false) + c.config = NewConfig(false, tt.namespaceSelector, false, mockConfig) nsSelector, objSelector := common.DefaultLabelSelectors(tt.namespaceSelector) @@ -1181,7 +1185,7 @@ func (f *fixtureV1beta1) createController() (*ControllerV1beta1, informers.Share factory.Admissionregistration().V1beta1().MutatingWebhookConfigurations(), func() bool { return true }, make(chan struct{}), - v1beta1Cfg, + getV1beta1Cfg(f.t), wmeta, nil, datadogConfig, diff --git a/pkg/clusteragent/admission/mutate/autoscaling/autoscaling.go b/pkg/clusteragent/admission/mutate/autoscaling/autoscaling.go index b4e7225ef710e..4ac5396506357 100644 --- a/pkg/clusteragent/admission/mutate/autoscaling/autoscaling.go +++ b/pkg/clusteragent/admission/mutate/autoscaling/autoscaling.go @@ -16,10 +16,10 @@ import ( "k8s.io/client-go/dynamic" "github.com/DataDog/datadog-agent/cmd/cluster-agent/admission" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/common" mutatecommon "github.com/DataDog/datadog-agent/pkg/clusteragent/admission/mutate/common" "github.com/DataDog/datadog-agent/pkg/clusteragent/autoscaling/workload" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" ) const ( @@ -39,10 +39,10 @@ type Webhook struct { } // NewWebhook returns a new Webhook -func NewWebhook(patcher workload.PodPatcher) *Webhook { +func NewWebhook(patcher workload.PodPatcher, datadogConfig config.Component) *Webhook { return &Webhook{ name: webhookName, - isEnabled: pkgconfigsetup.Datadog().GetBool("autoscaling.workload.enabled"), + isEnabled: datadogConfig.GetBool("autoscaling.workload.enabled"), endpoint: webhookEndpoint, resources: map[string][]string{"": {"pods"}}, operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, diff --git a/pkg/clusteragent/admission/start.go b/pkg/clusteragent/admission/start.go index aee1183a3133d..ef69edfd4a203 100644 --- a/pkg/clusteragent/admission/start.go +++ b/pkg/clusteragent/admission/start.go @@ -79,7 +79,7 @@ func StartControllers(ctx ControllerContext, wmeta workloadmeta.Component, pa wo return webhooks, err } - webhookConfig := webhook.NewConfig(v1Enabled, nsSelectorEnabled, matchConditionsSupported) + webhookConfig := webhook.NewConfig(v1Enabled, nsSelectorEnabled, matchConditionsSupported, datadogConfig) webhookController := webhook.NewController( ctx.Client, ctx.SecretInformers.Core().V1().Secrets(), From a582565b880ad6d75587090d9417ab6150fd7cc5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:31:24 +0000 Subject: [PATCH 08/43] Bump actions/upload-artifact from 4.4.0 to 4.4.3 (#32041) Co-authored-by: chouetz --- .github/workflows/cws-btfhub-sync.yml | 2 +- .github/workflows/docs-dev.yml | 2 +- .github/workflows/serverless-benchmarks.yml | 4 ++-- .github/workflows/serverless-binary-size.yml | 2 +- .github/workflows/serverless-integration.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml index b5f1ba4df7b0c..795953b22a354 100644 --- a/.github/workflows/cws-btfhub-sync.yml +++ b/.github/workflows/cws-btfhub-sync.yml @@ -97,7 +97,7 @@ jobs: inv -e security-agent.generate-btfhub-constants --archive-path=./dev/dist/archive --output-path=./"$ARTIFACT_NAME".json --force-refresh - name: Upload artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: ${{ steps.artifact-name.outputs.ARTIFACT_NAME }} path: ./${{ steps.artifact-name.outputs.ARTIFACT_NAME }}.json diff --git a/.github/workflows/docs-dev.yml b/.github/workflows/docs-dev.yml index 8602adc372c1f..1eab6460db4b6 100644 --- a/.github/workflows/docs-dev.yml +++ b/.github/workflows/docs-dev.yml @@ -50,7 +50,7 @@ jobs: - name: Build documentation run: invoke docs.build - - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: documentation path: site diff --git a/.github/workflows/serverless-benchmarks.yml b/.github/workflows/serverless-benchmarks.yml index 195f3441c4591..8ad83b34d36d1 100644 --- a/.github/workflows/serverless-benchmarks.yml +++ b/.github/workflows/serverless-benchmarks.yml @@ -48,7 +48,7 @@ jobs: ./pkg/serverless/... | tee "$TEMP_RUNNER"/benchmark.log - name: Upload result artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: baseline.log path: ${{runner.temp}}/benchmark.log @@ -87,7 +87,7 @@ jobs: ./pkg/serverless/... | tee "$TEMP_RUNNER"/benchmark.log - name: Upload result artifact - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: current.log path: ${{runner.temp}}/benchmark.log diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml index 5bf7a69b36177..7be692d81d51a 100644 --- a/.github/workflows/serverless-binary-size.yml +++ b/.github/workflows/serverless-binary-size.yml @@ -152,7 +152,7 @@ jobs: done - name: Archive dependency graphs - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 if: steps.should.outputs.should_run == 'true' with: name: dependency-graphs diff --git a/.github/workflows/serverless-integration.yml b/.github/workflows/serverless-integration.yml index f94ed5ca56b52..ace5e88fbda98 100644 --- a/.github/workflows/serverless-integration.yml +++ b/.github/workflows/serverless-integration.yml @@ -80,7 +80,7 @@ jobs: - name: Archive raw logs if: always() - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: rawlogs-${{ matrix.suite }}-${{ matrix.architecture }} path: ${{ steps.rawlogs.outputs.dir }} From 6fb76d59dcac4a66e7ed9bb68845dfe1ed653a7d Mon Sep 17 00:00:00 2001 From: Scott Opell Date: Tue, 17 Dec 2024 14:22:39 -0500 Subject: [PATCH 09/43] Sets new lower limits based on latest SMP nightly build (#32302) --- test/regression/cases/quality_gate_idle/experiment.yaml | 2 +- .../cases/quality_gate_idle_all_features/experiment.yaml | 2 +- test/regression/cases/quality_gate_logs/experiment.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/regression/cases/quality_gate_idle/experiment.yaml b/test/regression/cases/quality_gate_idle/experiment.yaml index 6555258f34601..1c120a0d23187 100644 --- a/test/regression/cases/quality_gate_idle/experiment.yaml +++ b/test/regression/cases/quality_gate_idle/experiment.yaml @@ -36,7 +36,7 @@ checks: description: "Memory usage quality gate. This puts a bound on the total agent memory usage." bounds: series: total_rss_bytes - upper_bound: "390.0 MiB" + upper_bound: "365.0 MiB" report_links: - text: "bounds checks dashboard" diff --git a/test/regression/cases/quality_gate_idle_all_features/experiment.yaml b/test/regression/cases/quality_gate_idle_all_features/experiment.yaml index 624b68caeab20..2b16f4988c0b0 100644 --- a/test/regression/cases/quality_gate_idle_all_features/experiment.yaml +++ b/test/regression/cases/quality_gate_idle_all_features/experiment.yaml @@ -47,7 +47,7 @@ checks: description: "Memory usage quality gate. This puts a bound on the total agent memory usage." bounds: series: total_rss_bytes - upper_bound: "795.0 MiB" + upper_bound: "755.0 MiB" report_links: - text: "bounds checks dashboard" diff --git a/test/regression/cases/quality_gate_logs/experiment.yaml b/test/regression/cases/quality_gate_logs/experiment.yaml index b1b2d9ee9c02b..761cae4678a8e 100644 --- a/test/regression/cases/quality_gate_logs/experiment.yaml +++ b/test/regression/cases/quality_gate_logs/experiment.yaml @@ -29,7 +29,7 @@ checks: description: "Memory usage" bounds: series: total_rss_bytes - upper_bound: 440MiB + upper_bound: 420MiB - name: lost_bytes description: "Allowable bytes not polled by log Agent" From 007c8aa999a3b6aa436edf18c9b4c32ea546afb9 Mon Sep 17 00:00:00 2001 From: Maxime Riaud <65339037+misteriaud@users.noreply.github.com> Date: Tue, 17 Dec 2024 15:53:53 -0600 Subject: [PATCH 10/43] Reapply "[ASCII-2586] Migrating SecurityAgent to use IPC cert" (#32313) --- cmd/security-agent/api/server.go | 48 ++++--------------- cmd/security-agent/main_windows.go | 6 ++- .../subcommands/start/command.go | 9 ++-- 3 files changed, 19 insertions(+), 44 deletions(-) diff --git a/cmd/security-agent/api/server.go b/cmd/security-agent/api/server.go index e776628e5ccd4..307a4e4c6cd43 100644 --- a/cmd/security-agent/api/server.go +++ b/cmd/security-agent/api/server.go @@ -12,9 +12,6 @@ package api import ( "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" stdLog "log" "net" "net/http" @@ -23,10 +20,10 @@ import ( "github.com/gorilla/mux" "github.com/DataDog/datadog-agent/cmd/security-agent/api/agent" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/core/settings" "github.com/DataDog/datadog-agent/comp/core/status" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" - "github.com/DataDog/datadog-agent/pkg/api/security" "github.com/DataDog/datadog-agent/pkg/api/util" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -35,19 +32,21 @@ import ( // Server implements security agent API server type Server struct { - listener net.Listener - agent *agent.Agent + listener net.Listener + agent *agent.Agent + tlsConfig *tls.Config } // NewServer creates a new Server instance -func NewServer(statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component) (*Server, error) { +func NewServer(statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) (*Server, error) { listener, err := newListener() if err != nil { return nil, err } return &Server{ - listener: listener, - agent: agent.NewAgent(statusComponent, settings, wmeta), + listener: listener, + agent: agent.NewAgent(statusComponent, settings, wmeta), + tlsConfig: at.GetTLSServerConfig(), }, nil } @@ -62,43 +61,16 @@ func (s *Server) Start() error { // Validate token for every request r.Use(validateToken) - err := util.CreateAndSetAuthToken(pkgconfigsetup.Datadog()) - if err != nil { - return err - } - - hosts := []string{"127.0.0.1", "localhost"} - _, rootCertPEM, rootKey, err := security.GenerateRootCert(hosts, 2048) - if err != nil { - return fmt.Errorf("unable to start TLS server") - } - - // PEM encode the private key - rootKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey), - }) - - // Create a TLS cert using the private key and certificate - rootTLSCert, err := tls.X509KeyPair(rootCertPEM, rootKeyPEM) - if err != nil { - return fmt.Errorf("invalid key pair: %v", err) - } - - tlsConfig := tls.Config{ - Certificates: []tls.Certificate{rootTLSCert}, - MinVersion: tls.VersionTLS13, - } - // Use a stack depth of 4 on top of the default one to get a relevant filename in the stdlib logWriter, _ := pkglogsetup.NewLogWriter(4, log.ErrorLvl) srv := &http.Server{ Handler: r, ErrorLog: stdLog.New(logWriter, "Error from the agent http API server: ", 0), // log errors to seelog, - TLSConfig: &tlsConfig, + TLSConfig: s.tlsConfig, WriteTimeout: pkgconfigsetup.Datadog().GetDuration("server_timeout") * time.Second, } - tlsListener := tls.NewListener(s.listener, &tlsConfig) + tlsListener := tls.NewListener(s.listener, s.tlsConfig) go srv.Serve(tlsListener) //nolint:errcheck return nil diff --git a/cmd/security-agent/main_windows.go b/cmd/security-agent/main_windows.go index b81223e6aebc1..a729e7a2003f0 100644 --- a/cmd/security-agent/main_windows.go +++ b/cmd/security-agent/main_windows.go @@ -25,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands/start" "github.com/DataDog/datadog-agent/comp/agent/autoexit" "github.com/DataDog/datadog-agent/comp/agent/autoexit/autoexitimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" @@ -91,10 +92,11 @@ func (s *service) Run(svcctx context.Context) error { params := &cliParams{} err := fxutil.OneShot( func(log log.Component, config config.Component, _ secrets.Component, _ statsd.Component, _ sysprobeconfig.Component, - telemetry telemetry.Component, _ workloadmeta.Component, _ *cliParams, statusComponent status.Component, _ autoexit.Component, settings settings.Component, wmeta workloadmeta.Component) error { + telemetry telemetry.Component, _ workloadmeta.Component, _ *cliParams, statusComponent status.Component, _ autoexit.Component, + settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) error { defer start.StopAgent(log) - err := start.RunAgent(log, config, telemetry, statusComponent, settings, wmeta) + err := start.RunAgent(log, config, telemetry, statusComponent, settings, wmeta, at) if err != nil { if errors.Is(err, start.ErrAllComponentsDisabled) { // If all components are disabled, we should exit cleanly diff --git a/cmd/security-agent/subcommands/start/command.go b/cmd/security-agent/subcommands/start/command.go index 12a93fc4560ff..2a3dd883dcd84 100644 --- a/cmd/security-agent/subcommands/start/command.go +++ b/cmd/security-agent/subcommands/start/command.go @@ -29,6 +29,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands/runtime" "github.com/DataDog/datadog-agent/comp/agent/autoexit" "github.com/DataDog/datadog-agent/comp/agent/autoexit/autoexitimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" @@ -201,10 +202,10 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { // TODO(components): note how workloadmeta is passed anonymously, it is still required as it is used // as a global. This should eventually be fixed and all workloadmeta interactions should be via the // injected instance. -func start(log log.Component, config config.Component, _ secrets.Component, _ statsd.Component, _ sysprobeconfig.Component, telemetry telemetry.Component, statusComponent status.Component, _ pid.Component, _ autoexit.Component, settings settings.Component, wmeta workloadmeta.Component) error { +func start(log log.Component, config config.Component, _ secrets.Component, _ statsd.Component, _ sysprobeconfig.Component, telemetry telemetry.Component, statusComponent status.Component, _ pid.Component, _ autoexit.Component, settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) error { defer StopAgent(log) - err := RunAgent(log, config, telemetry, statusComponent, settings, wmeta) + err := RunAgent(log, config, telemetry, statusComponent, settings, wmeta, at) if errors.Is(err, ErrAllComponentsDisabled) || errors.Is(err, errNoAPIKeyConfigured) { return nil } @@ -256,7 +257,7 @@ var ErrAllComponentsDisabled = errors.New("all security-agent component are disa var errNoAPIKeyConfigured = errors.New("no API key configured") // RunAgent initialized resources and starts API server -func RunAgent(log log.Component, config config.Component, telemetry telemetry.Component, statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component) (err error) { +func RunAgent(log log.Component, config config.Component, telemetry telemetry.Component, statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) (err error) { if err := coredump.Setup(config); err != nil { log.Warnf("Can't setup core dumps: %v, core dumps might not be available after a crash", err) } @@ -299,7 +300,7 @@ func RunAgent(log log.Component, config config.Component, telemetry telemetry.Co } }() - srv, err = api.NewServer(statusComponent, settings, wmeta) + srv, err = api.NewServer(statusComponent, settings, wmeta, at) if err != nil { return log.Errorf("Error while creating api server, exiting: %v", err) } From 04763f785605e993831458a37faaccc8d950cdf8 Mon Sep 17 00:00:00 2001 From: Kyle Ames Date: Tue, 17 Dec 2024 17:14:13 -0500 Subject: [PATCH 11/43] [RC] Optimize retrieval of raw target files in ClientGetConfigs (#31987) --- pkg/config/remote/service/service.go | 151 +++++++++++----------- pkg/config/remote/service/service_test.go | 42 ++++-- pkg/config/remote/uptane/client.go | 31 +++++ 3 files changed, 140 insertions(+), 84 deletions(-) diff --git a/pkg/config/remote/service/service.go b/pkg/config/remote/service/service.go index a379ac5b387ec..84377477da44c 100644 --- a/pkg/config/remote/service/service.go +++ b/pkg/config/remote/service/service.go @@ -111,50 +111,21 @@ func (s *Service) getNewDirectorRoots(uptane uptaneClient, currentVersion uint64 return roots, nil } -func (s *Service) getTargetFiles(uptane uptaneClient, products []rdata.Product, cachedTargetFiles []*pbgo.TargetFileMeta) ([]*pbgo.File, error) { - productSet := make(map[rdata.Product]struct{}) - for _, product := range products { - productSet[product] = struct{}{} - } - targets, err := uptane.Targets() +func (s *Service) getTargetFiles(uptaneClient uptaneClient, targetFilePaths []string) ([]*pbgo.File, error) { + files, err := uptaneClient.TargetFiles(targetFilePaths) if err != nil { return nil, err } - cachedTargets := make(map[string]data.FileMeta) - for _, cachedTarget := range cachedTargetFiles { - hashes := make(data.Hashes) - for _, hash := range cachedTarget.Hashes { - h, err := hex.DecodeString(hash.Hash) - if err != nil { - return nil, err - } - hashes[hash.Algorithm] = h - } - cachedTargets[cachedTarget.Path] = data.FileMeta{ - Hashes: hashes, - Length: cachedTarget.Length, - } - } + var configFiles []*pbgo.File - for targetPath, targetMeta := range targets { - configPathMeta, err := rdata.ParseConfigPath(targetPath) - if err != nil { - return nil, err - } - if _, inClientProducts := productSet[rdata.Product(configPathMeta.Product)]; inClientProducts { - if notEqualErr := tufutil.FileMetaEqual(cachedTargets[targetPath], targetMeta.FileMeta); notEqualErr == nil { - continue - } - fileContents, err := uptane.TargetFile(targetPath) - if err != nil { - return nil, err - } - configFiles = append(configFiles, &pbgo.File{ - Path: targetPath, - Raw: fileContents, - }) - } + for path, contents := range files { + // Note: This unconditionally succeeds as long as we don't change bufferDestination earlier + configFiles = append(configFiles, &pbgo.File{ + Path: path, + Raw: contents, + }) } + return configFiles, nil } @@ -211,6 +182,7 @@ type uptaneClient interface { StoredOrgUUID() (string, error) Targets() (data.TargetFiles, error) TargetFile(path string) ([]byte, error) + TargetFiles(files []string) (map[string][]byte, error) TargetsMeta() ([]byte, error) TargetsCustom() ([]byte, error) TUFVersionState() (uptane.TUFVersions, error) @@ -828,36 +800,30 @@ func (s *CoreAgentService) ClientGetConfigs(_ context.Context, request *pbgo.Cli if err != nil { return nil, err } - targetsRaw, err := s.uptane.TargetsMeta() + + directorTargets, err := s.uptane.Targets() if err != nil { return nil, err } - targetFiles, err := s.getTargetFiles(s.uptane, rdata.StringListToProduct(request.Client.Products), request.CachedTargetFiles) + matchedClientConfigs, err := executeTracerPredicates(request.Client, directorTargets) if err != nil { return nil, err } - directorTargets, err := s.uptane.Targets() + neededFiles, err := filterNeededTargetFiles(matchedClientConfigs, request.CachedTargetFiles, directorTargets) if err != nil { return nil, err } - matchedClientConfigs, err := executeTracerPredicates(request.Client, directorTargets) + + targetFiles, err := s.getTargetFiles(s.uptane, neededFiles) if err != nil { return nil, err } - // filter files to only return the ones that predicates marked for this client - matchedConfigsMap := make(map[string]interface{}) - for _, configPointer := range matchedClientConfigs { - matchedConfigsMap[configPointer] = struct{}{} - } - filteredFiles := make([]*pbgo.File, 0, len(matchedClientConfigs)) - for _, targetFile := range targetFiles { - if _, ok := matchedConfigsMap[targetFile.Path]; ok { - filteredFiles = append(filteredFiles, targetFile) - } + targetsRaw, err := s.uptane.TargetsMeta() + if err != nil { + return nil, err } - canonicalTargets, err := enforceCanonicalJSON(targetsRaw) if err != nil { return nil, err @@ -866,11 +832,42 @@ func (s *CoreAgentService) ClientGetConfigs(_ context.Context, request *pbgo.Cli return &pbgo.ClientGetConfigsResponse{ Roots: roots, Targets: canonicalTargets, - TargetFiles: filteredFiles, + TargetFiles: targetFiles, ClientConfigs: matchedClientConfigs, }, nil } +func filterNeededTargetFiles(neededConfigs []string, cachedTargetFiles []*pbgo.TargetFileMeta, tufTargets data.TargetFiles) ([]string, error) { + // Build an O(1) lookup of cached target files + cachedTargetsMap := make(map[string]data.FileMeta) + for _, cachedTarget := range cachedTargetFiles { + hashes := make(data.Hashes) + for _, hash := range cachedTarget.Hashes { + h, err := hex.DecodeString(hash.Hash) + if err != nil { + return nil, err + } + hashes[hash.Algorithm] = h + } + cachedTargetsMap[cachedTarget.Path] = data.FileMeta{ + Hashes: hashes, + Length: cachedTarget.Length, + } + } + + // We don't need to pull the raw contents if the client already has the exact version of the file cached + filteredList := make([]string, 0, len(neededConfigs)) + for _, path := range neededConfigs { + if notEqualErr := tufutil.FileMetaEqual(cachedTargetsMap[path], tufTargets[path].FileMeta); notEqualErr == nil { + continue + } + + filteredList = append(filteredList, path) + } + + return filteredList, nil +} + // ConfigGetState returns the state of the configuration and the director repos in the local store func (s *CoreAgentService) ConfigGetState() (*pbgo.GetStateConfigResponse, error) { state, err := s.uptane.State() @@ -1117,29 +1114,14 @@ func (c *HTTPClient) getUpdate( if tufVersions.DirectorTargets == currentTargetsVersion { return nil, nil } - roots, err := c.getNewDirectorRoots(c.uptane, currentRootVersion, tufVersions.DirectorRoot) - if err != nil { - return nil, err - } - targetsRaw, err := c.uptane.TargetsMeta() - if err != nil { - return nil, err - } - targetFiles, err := c.getTargetFiles(c.uptane, rdata.StringListToProduct(products), cachedTargetFiles) - if err != nil { - return nil, err - } - - canonicalTargets, err := enforceCanonicalJSON(targetsRaw) - if err != nil { - return nil, err - } + // Filter out files that either: + // - don't correspond to the product list the client is requesting + // - have expired directorTargets, err := c.uptane.Targets() if err != nil { return nil, err } - productsMap := make(map[string]struct{}) for _, product := range products { productsMap[product] = struct{}{} @@ -1165,14 +1147,33 @@ func (c *HTTPClient) getUpdate( configs = append(configs, path) } + span.SetTag("configs.returned", configs) + span.SetTag("configs.expired", expiredConfigs) + // Gather the files and map-ify them for the state data structure + targetFiles, err := c.getTargetFiles(c.uptane, configs) + if err != nil { + return nil, err + } fileMap := make(map[string][]byte, len(targetFiles)) for _, f := range targetFiles { fileMap[f.Path] = f.Raw } - span.SetTag("configs.returned", configs) - span.SetTag("configs.expired", expiredConfigs) + // Gather some TUF metadata files we need to send down + roots, err := c.getNewDirectorRoots(c.uptane, currentRootVersion, tufVersions.DirectorRoot) + if err != nil { + return nil, err + } + targetsRaw, err := c.uptane.TargetsMeta() + if err != nil { + return nil, err + } + canonicalTargets, err := enforceCanonicalJSON(targetsRaw) + if err != nil { + return nil, err + } + return &state.Update{ TUFRoots: roots, TUFTargets: canonicalTargets, diff --git a/pkg/config/remote/service/service_test.go b/pkg/config/remote/service/service_test.go index 40d8302db3192..34175eea2feba 100644 --- a/pkg/config/remote/service/service_test.go +++ b/pkg/config/remote/service/service_test.go @@ -116,6 +116,11 @@ func (m *mockUptane) TargetFile(path string) ([]byte, error) { return args.Get(0).([]byte), args.Error(1) } +func (m *mockUptane) TargetFiles(files []string) (map[string][]byte, error) { + args := m.Called(files) + return args.Get(0).(map[string][]byte), args.Error(1) +} + func (m *mockUptane) TargetsMeta() ([]byte, error) { args := m.Called() return args.Get(0).([]byte), args.Error(1) @@ -486,8 +491,8 @@ func TestService(t *testing.T) { }, nil) uptaneClient.On("DirectorRoot", uint64(3)).Return(root3, nil) uptaneClient.On("DirectorRoot", uint64(4)).Return(root4, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/1").Return(fileAPM1, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/2").Return(fileAPM2, nil) + + uptaneClient.On("TargetFiles", mock.MatchedBy(listsEqual([]string{"datadog/2/APM_SAMPLING/id/1", "datadog/2/APM_SAMPLING/id/2"}))).Return(map[string][]byte{"datadog/2/APM_SAMPLING/id/1": fileAPM1, "datadog/2/APM_SAMPLING/id/2": fileAPM2}, nil) uptaneClient.On("Update", lastConfigResponse).Return(nil) api.On("Fetch", mock.Anything, &pbgo.LatestConfigsRequest{ Hostname: service.hostname, @@ -513,7 +518,7 @@ func TestService(t *testing.T) { configResponse, err := service.ClientGetConfigs(context.Background(), &pbgo.ClientGetConfigsRequest{Client: client}) assert.NoError(t, err) assert.ElementsMatch(t, [][]byte{canonicalRoot3, canonicalRoot4}, configResponse.Roots) - assert.ElementsMatch(t, []*pbgo.File{{Path: "datadog/2/APM_SAMPLING/id/1", Raw: fileAPM1}, {Path: "datadog/2/APM_SAMPLING/id/2", Raw: fileAPM2}}, configResponse.TargetFiles) + assert.ElementsMatch(t, []*pbgo.File{{Path: "datadog/2/APM_SAMPLING/id/1", Raw: fileAPM1}, {Path: "datadog/2/APM_SAMPLING/id/2", Raw: fileAPM2}}, configResponse.TargetFiles, nil) assert.Equal(t, canonicalTargets, configResponse.Targets) assert.ElementsMatch(t, configResponse.ClientConfigs, @@ -597,8 +602,7 @@ func TestServiceClientPredicates(t *testing.T) { DirectorRoot: 1, DirectorTargets: 5, }, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/1").Return([]byte(``), nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/2").Return([]byte(``), nil) + uptaneClient.On("TargetFiles", mock.MatchedBy(listsEqual([]string{"datadog/2/APM_SAMPLING/id/1", "datadog/2/APM_SAMPLING/id/2"}))).Return(map[string][]byte{"datadog/2/APM_SAMPLING/id/1": []byte(``), "datadog/2/APM_SAMPLING/id/2": []byte(``)}, nil) uptaneClient.On("Update", lastConfigResponse).Return(nil) api.On("Fetch", mock.Anything, &pbgo.LatestConfigsRequest{ Hostname: service.hostname, @@ -887,8 +891,7 @@ func TestConfigExpiration(t *testing.T) { DirectorRoot: 1, DirectorTargets: 5, }, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/1").Return([]byte(``), nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/2").Return([]byte(``), nil) + uptaneClient.On("TargetFiles", []string{"datadog/2/APM_SAMPLING/id/1"}).Return(map[string][]byte{"datadog/2/APM_SAMPLING/id/1": []byte(``), "datadog/2/APM_SAMPLING/id/2": []byte(``)}, nil) uptaneClient.On("Update", lastConfigResponse).Return(nil) api.On("Fetch", mock.Anything, &pbgo.LatestConfigsRequest{ Hostname: service.hostname, @@ -1190,7 +1193,7 @@ func TestHTTPClientRecentUpdate(t *testing.T) { }, nil, ) - uptaneClient.On("TargetFile", "datadog/2/TESTING1/id/1").Return([]byte(`testing_1`), nil) + uptaneClient.On("TargetFiles", []string{"datadog/2/TESTING1/id/1"}).Return(map[string][]byte{"datadog/2/TESTING1/id/1": []byte(`testing_1`)}, nil) client := setupCDNClient(t, uptaneClient) defer client.Close() @@ -1236,7 +1239,7 @@ func TestHTTPClientUpdateSuccess(t *testing.T) { }, nil, ) - uptaneClient.On("TargetFile", "datadog/2/TESTING1/id/1").Return([]byte(`testing_1`), nil) + uptaneClient.On("TargetFiles", []string{"datadog/2/TESTING1/id/1"}).Return(map[string][]byte{"datadog/2/TESTING1/id/1": []byte(`testing_1`)}, nil) updateErr := fmt.Errorf("uh oh") if tt.updateSucceeds { @@ -1261,3 +1264,24 @@ func TestHTTPClientUpdateSuccess(t *testing.T) { }) } } + +func listsEqual(mustMatch []string) func(candidate []string) bool { + return func(candidate []string) bool { + if len(candidate) != len(mustMatch) { + return false + } + + candidateSet := make(map[string]struct{}) + for _, item := range candidate { + candidateSet[item] = struct{}{} + } + + for _, item := range mustMatch { + if _, ok := candidateSet[item]; !ok { + return false + } + } + + return true + } +} diff --git a/pkg/config/remote/uptane/client.go b/pkg/config/remote/uptane/client.go index b4da1c675b296..fcd1d3834b478 100644 --- a/pkg/config/remote/uptane/client.go +++ b/pkg/config/remote/uptane/client.go @@ -342,6 +342,37 @@ func (c *Client) TargetFile(path string) ([]byte, error) { return c.unsafeTargetFile(path) } +// TargetFiles returns the content of various multiple target files if the repository is in a +// verified state. +func (c *Client) TargetFiles(targetFiles []string) (map[string][]byte, error) { + c.Lock() + defer c.Unlock() + + err := c.verify() + if err != nil { + return nil, err + } + + // Build the storage space + destinations := make(map[string]client.Destination) + for _, path := range targetFiles { + destinations[path] = &bufferDestination{} + } + + err = c.directorTUFClient.DownloadBatch(destinations) + if err != nil { + return nil, err + } + + // Build the return type + files := make(map[string][]byte) + for path, contents := range destinations { + files[path] = contents.(*bufferDestination).Bytes() + } + + return files, nil +} + // TargetsMeta returns the current raw targets.json meta of this uptane client func (c *Client) TargetsMeta() ([]byte, error) { c.Lock() From 7992c127770f6b3c17efc6d6a50ce9aa9d1a7732 Mon Sep 17 00:00:00 2001 From: Maxime Riaud <65339037+misteriaud@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:02:24 -0600 Subject: [PATCH 12/43] [ASCII-2584] Migrating CoreAgent to use IPC cert (#31843) --- comp/api/api/apiimpl/api_test.go | 5 ++-- comp/api/api/apiimpl/security.go | 48 ------------------------------ comp/api/api/apiimpl/server.go | 31 ++++--------------- comp/api/api/apiimpl/server_cmd.go | 14 ++------- comp/api/api/apiimpl/server_ipc.go | 5 ++-- 5 files changed, 13 insertions(+), 90 deletions(-) diff --git a/comp/api/api/apiimpl/api_test.go b/comp/api/api/apiimpl/api_test.go index d92e8f1e4fa48..b248dc9531330 100644 --- a/comp/api/api/apiimpl/api_test.go +++ b/comp/api/api/apiimpl/api_test.go @@ -20,7 +20,7 @@ import ( "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/api/api/apiimpl/observability" api "github.com/DataDog/datadog-agent/comp/api/api/def" - "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl" "github.com/DataDog/datadog-agent/comp/collector/collector" "github.com/DataDog/datadog-agent/comp/core/autodiscovery" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/autodiscoveryimpl" @@ -76,7 +76,7 @@ func getTestAPIServer(t *testing.T, params config.MockParams) testdeps { demultiplexerimpl.MockModule(), fx.Supply(optional.NewNoneOption[rcservice.Component]()), fx.Supply(optional.NewNoneOption[rcservicemrf.Component]()), - fetchonlyimpl.MockModule(), + createandfetchimpl.Module(), fx.Supply(context.Background()), taggermock.Module(), fx.Provide(func(mock taggermock.Mock) tagger.Component { @@ -166,7 +166,6 @@ func TestStartBothServersWithObservability(t *testing.T) { req, err := http.NewRequest(http.MethodGet, url, nil) require.NoError(t, err) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authTokenValue)) resp, err := util.GetClient(false).Do(req) require.NoError(t, err) defer resp.Body.Close() diff --git a/comp/api/api/apiimpl/security.go b/comp/api/api/apiimpl/security.go index f7176bbfbf99c..4b8b17d2ba33d 100644 --- a/comp/api/api/apiimpl/security.go +++ b/comp/api/api/apiimpl/security.go @@ -7,16 +7,9 @@ package apiimpl import ( "crypto/subtle" - "crypto/tls" - "crypto/x509" - "encoding/pem" "errors" - "fmt" "net/http" - "runtime" - "strings" - "github.com/DataDog/datadog-agent/pkg/api/security" "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -44,44 +37,3 @@ func parseToken(token string) (interface{}, error) { // type. return struct{}{}, nil } - -func buildSelfSignedKeyPair(additionalHostIdentities ...string) ([]byte, []byte) { - hosts := []string{"127.0.0.1", "localhost", "::1"} - hosts = append(hosts, additionalHostIdentities...) - _, rootCertPEM, rootKey, err := security.GenerateRootCert(hosts, 2048) - if err != nil { - return nil, nil - } - - // PEM encode the private key - rootKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey), - }) - - // Create and return TLS private cert and key - return rootCertPEM, rootKeyPEM -} - -func initializeTLS(additionalHostIdentities ...string) (*tls.Certificate, *x509.CertPool, error) { - // print the caller to identify what is calling this function - if _, file, line, ok := runtime.Caller(1); ok { - log.Infof("[%s:%d] Initializing TLS certificates for hosts %v", file, line, strings.Join(additionalHostIdentities, ", ")) - } - - cert, key := buildSelfSignedKeyPair(additionalHostIdentities...) - if cert == nil { - return nil, nil, errors.New("unable to generate certificate") - } - pair, err := tls.X509KeyPair(cert, key) - if err != nil { - return nil, nil, fmt.Errorf("unable to generate TLS key pair: %v", err) - } - - tlsCertPool := x509.NewCertPool() - ok := tlsCertPool.AppendCertsFromPEM(cert) - if !ok { - return nil, nil, fmt.Errorf("unable to add new certificate to pool") - } - - return &pair, tlsCertPool, nil -} diff --git a/comp/api/api/apiimpl/server.go b/comp/api/api/apiimpl/server.go index 762a67b8a4a63..6def35781eb1a 100644 --- a/comp/api/api/apiimpl/server.go +++ b/comp/api/api/apiimpl/server.go @@ -11,6 +11,7 @@ import ( stdLog "log" "net" "net/http" + "strconv" "github.com/DataDog/datadog-agent/comp/api/api/apiimpl/observability" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -47,34 +48,11 @@ func (server *apiServer) startServers() error { return fmt.Errorf("unable to get IPC address and port: %v", err) } - additionalHostIdentities := []string{apiAddr} - - ipcServerHost, ipcServerHostPort, ipcServerEnabled := getIPCServerAddressPort() - if ipcServerEnabled { - additionalHostIdentities = append(additionalHostIdentities, ipcServerHost) - } - - tlsKeyPair, tlsCertPool, err := initializeTLS(additionalHostIdentities...) - if err != nil { - return fmt.Errorf("unable to initialize TLS: %v", err) - } - - // tls.Config is written to when serving, so it has to be cloned for each server - tlsConfig := func() *tls.Config { - return &tls.Config{ - Certificates: []tls.Certificate{*tlsKeyPair}, - NextProtos: []string{"h2"}, - MinVersion: tls.VersionTLS12, - } - } - tmf := observability.NewTelemetryMiddlewareFactory(server.telemetry) // start the CMD server if err := server.startCMDServer( apiAddr, - tlsConfig(), - tlsCertPool, tmf, server.cfg, ); err != nil { @@ -82,8 +60,11 @@ func (server *apiServer) startServers() error { } // start the IPC server - if ipcServerEnabled { - if err := server.startIPCServer(ipcServerHostPort, tlsConfig(), tmf); err != nil { + if ipcServerPort := server.cfg.GetInt("agent_ipc.port"); ipcServerPort > 0 { + ipcServerHost := server.cfg.GetString("agent_ipc.host") + ipcServerHostPort := net.JoinHostPort(ipcServerHost, strconv.Itoa(ipcServerPort)) + + if err := server.startIPCServer(ipcServerHostPort, tmf); err != nil { // if we fail to start the IPC server, we should stop the CMD server server.stopServers() return fmt.Errorf("unable to start IPC API server: %v", err) diff --git a/comp/api/api/apiimpl/server_cmd.go b/comp/api/api/apiimpl/server_cmd.go index 2b806aa8726e3..2eca0fe7abd09 100644 --- a/comp/api/api/apiimpl/server_cmd.go +++ b/comp/api/api/apiimpl/server_cmd.go @@ -7,8 +7,6 @@ package apiimpl import ( "context" - "crypto/tls" - "crypto/x509" "fmt" "net/http" "time" @@ -35,8 +33,6 @@ const cmdServerShortName string = "CMD" func (server *apiServer) startCMDServer( cmdAddr string, - tlsConfig *tls.Config, - tlsCertPool *x509.CertPool, tmf observability.TelemetryMiddlewareFactory, cfg config.Component, ) (err error) { @@ -54,7 +50,7 @@ func (server *apiServer) startCMDServer( maxMessageSize := cfg.GetInt("cluster_agent.cluster_tagger.grpc_max_message_size") opts := []grpc.ServerOption{ - grpc.Creds(credentials.NewClientTLSFromCert(tlsCertPool, cmdAddr)), + grpc.Creds(credentials.NewTLS(server.authToken.GetTLSServerConfig())), grpc.StreamInterceptor(grpc_auth.StreamServerInterceptor(authInterceptor)), grpc.UnaryInterceptor(grpc_auth.UnaryServerInterceptor(authInterceptor)), grpc.MaxRecvMsgSize(maxMessageSize), @@ -79,11 +75,7 @@ func (server *apiServer) startCMDServer( autodiscovery: server.autoConfig, }) - dcreds := credentials.NewTLS(&tls.Config{ - ServerName: cmdAddr, - RootCAs: tlsCertPool, - }) - dopts := []grpc.DialOption{grpc.WithTransportCredentials(dcreds)} + dopts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(server.authToken.GetTLSClientConfig()))} // starting grpc gateway ctx := context.Background() @@ -133,7 +125,7 @@ func (server *apiServer) startCMDServer( srv := grpcutil.NewMuxedGRPCServer( cmdAddr, - tlsConfig, + server.authToken.GetTLSServerConfig(), s, grpcutil.TimeoutHandlerFunc(cmdMuxHandler, time.Duration(pkgconfigsetup.Datadog().GetInt64("server_timeout"))*time.Second), ) diff --git a/comp/api/api/apiimpl/server_ipc.go b/comp/api/api/apiimpl/server_ipc.go index 10e17509f75e3..634ec1810f35a 100644 --- a/comp/api/api/apiimpl/server_ipc.go +++ b/comp/api/api/apiimpl/server_ipc.go @@ -6,7 +6,6 @@ package apiimpl import ( - "crypto/tls" "net/http" "time" @@ -18,7 +17,7 @@ import ( const ipcServerName string = "IPC API Server" const ipcServerShortName string = "IPC" -func (server *apiServer) startIPCServer(ipcServerAddr string, tlsConfig *tls.Config, tmf observability.TelemetryMiddlewareFactory) (err error) { +func (server *apiServer) startIPCServer(ipcServerAddr string, tmf observability.TelemetryMiddlewareFactory) (err error) { server.ipcListener, err = getListener(ipcServerAddr) if err != nil { return err @@ -39,7 +38,7 @@ func (server *apiServer) startIPCServer(ipcServerAddr string, tlsConfig *tls.Con ipcServer := &http.Server{ Addr: ipcServerAddr, Handler: http.TimeoutHandler(ipcMuxHandler, time.Duration(pkgconfigsetup.Datadog().GetInt64("server_timeout"))*time.Second, "timeout"), - TLSConfig: tlsConfig, + TLSConfig: server.authToken.GetTLSServerConfig(), } startServer(server.ipcListener, ipcServer, ipcServerName) From 1c2598522c37ad6b0dc338331c8b02d3e48c1cf1 Mon Sep 17 00:00:00 2001 From: Maxime Riaud <65339037+misteriaud@users.noreply.github.com> Date: Tue, 17 Dec 2024 17:02:29 -0600 Subject: [PATCH 13/43] [ASCII-2587] Migrating TraceAgent to use IPC cert (#31847) --- cmd/agent/subcommands/flare/command.go | 22 ++++++++++++++------- cmd/agent/subcommands/flare/command_test.go | 18 +++++++++++++---- cmd/agent/subcommands/secret/command.go | 2 +- comp/trace/agent/impl/agent.go | 4 ++++ comp/trace/agent/impl/run.go | 3 +++ comp/trace/bundle_test.go | 4 ++++ comp/trace/status/statusimpl/status.go | 2 +- pkg/config/fetcher/from_processes.go | 2 +- pkg/flare/archive.go | 2 +- pkg/trace/api/api_test.go | 18 ++++++++++++++--- pkg/trace/api/debug_server.go | 18 ++++++++++++----- pkg/trace/info/info.go | 6 ++++-- pkg/trace/info/info_test.go | 8 ++++---- 13 files changed, 80 insertions(+), 29 deletions(-) diff --git a/cmd/agent/subcommands/flare/command.go b/cmd/agent/subcommands/flare/command.go index 1b4fb2d3bc1e3..a6da5391ae5d5 100644 --- a/cmd/agent/subcommands/flare/command.go +++ b/cmd/agent/subcommands/flare/command.go @@ -175,10 +175,18 @@ func readProfileData(seconds int) (flare.ProfileData, error) { type pprofGetter func(path string) ([]byte, error) - tcpGet := func(portConfig string) pprofGetter { - pprofURL := fmt.Sprintf("http://127.0.0.1:%d/debug/pprof", pkgconfigsetup.Datadog().GetInt(portConfig)) + tcpGet := func(portConfig string, onHTTPS bool) pprofGetter { + endpoint := url.URL{ + Scheme: "http", + Host: net.JoinHostPort("127.0.0.1", strconv.Itoa(pkgconfigsetup.Datadog().GetInt(portConfig))), + Path: "/debug/pprof", + } + if onHTTPS { + endpoint.Scheme = "https" + } + return func(path string) ([]byte, error) { - return util.DoGet(c, pprofURL+path, util.LeaveConnectionOpen) + return util.DoGet(c, endpoint.String()+path, util.LeaveConnectionOpen) } } @@ -228,15 +236,15 @@ func readProfileData(seconds int) (flare.ProfileData, error) { } agentCollectors := map[string]agentProfileCollector{ - "core": serviceProfileCollector(tcpGet("expvar_port"), seconds), - "security-agent": serviceProfileCollector(tcpGet("security_agent.expvar_port"), seconds), + "core": serviceProfileCollector(tcpGet("expvar_port", false), seconds), + "security-agent": serviceProfileCollector(tcpGet("security_agent.expvar_port", false), seconds), } if pkgconfigsetup.Datadog().GetBool("process_config.enabled") || pkgconfigsetup.Datadog().GetBool("process_config.container_collection.enabled") || pkgconfigsetup.Datadog().GetBool("process_config.process_collection.enabled") { - agentCollectors["process"] = serviceProfileCollector(tcpGet("process_config.expvar_port"), seconds) + agentCollectors["process"] = serviceProfileCollector(tcpGet("process_config.expvar_port", false), seconds) } if pkgconfigsetup.Datadog().GetBool("apm_config.enabled") { @@ -249,7 +257,7 @@ func readProfileData(seconds int) (flare.ProfileData, error) { traceCpusec = 4 } - agentCollectors["trace"] = serviceProfileCollector(tcpGet("apm_config.debug.port"), traceCpusec) + agentCollectors["trace"] = serviceProfileCollector(tcpGet("apm_config.debug.port", true), traceCpusec) } if pkgconfigsetup.SystemProbe().GetBool("system_probe_config.enabled") { diff --git a/cmd/agent/subcommands/flare/command_test.go b/cmd/agent/subcommands/flare/command_test.go index a27304e95b133..4c96cae079aa9 100644 --- a/cmd/agent/subcommands/flare/command_test.go +++ b/cmd/agent/subcommands/flare/command_test.go @@ -29,6 +29,7 @@ type commandTestSuite struct { suite.Suite sysprobeSocketPath string tcpServer *httptest.Server + tcpTLSServer *httptest.Server unixServer *httptest.Server systemProbeServer *httptest.Server } @@ -42,13 +43,17 @@ func (c *commandTestSuite) SetupSuite() { // This should be called by each test that requires them. func (c *commandTestSuite) startTestServers() { t := c.T() - c.tcpServer, c.unixServer, c.systemProbeServer = c.getPprofTestServer() + c.tcpServer, c.tcpTLSServer, c.unixServer, c.systemProbeServer = c.getPprofTestServer() t.Cleanup(func() { if c.tcpServer != nil { c.tcpServer.Close() c.tcpServer = nil } + if c.tcpTLSServer != nil { + c.tcpTLSServer.Close() + c.tcpTLSServer = nil + } if c.unixServer != nil { c.unixServer.Close() c.unixServer = nil @@ -82,12 +87,13 @@ func newMockHandler() http.HandlerFunc { }) } -func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, unixServer *httptest.Server, sysProbeServer *httptest.Server) { +func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, tcpTLSServer *httptest.Server, unixServer *httptest.Server, sysProbeServer *httptest.Server) { var err error t := c.T() handler := newMockHandler() tcpServer = httptest.NewServer(handler) + tcpTLSServer = httptest.NewTLSServer(handler) if runtime.GOOS == "linux" { unixServer = httptest.NewUnstartedServer(handler) unixServer.Listener, err = net.Listen("unix", c.sysprobeSocketPath) @@ -101,7 +107,7 @@ func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, uni sysProbeServer.Start() } - return tcpServer, unixServer, sysProbeServer + return tcpServer, tcpTLSServer, unixServer, sysProbeServer } func TestCommandTestSuite(t *testing.T) { @@ -116,10 +122,14 @@ func (c *commandTestSuite) TestReadProfileData() { require.NoError(t, err) port := u.Port() + u, err = url.Parse(c.tcpTLSServer.URL) + require.NoError(t, err) + httpsPort := u.Port() + mockConfig := configmock.New(t) mockConfig.SetWithoutSource("expvar_port", port) mockConfig.SetWithoutSource("apm_config.enabled", true) - mockConfig.SetWithoutSource("apm_config.debug.port", port) + mockConfig.SetWithoutSource("apm_config.debug.port", httpsPort) mockConfig.SetWithoutSource("apm_config.receiver_timeout", "10") mockConfig.SetWithoutSource("process_config.expvar_port", port) mockConfig.SetWithoutSource("security_agent.expvar_port", port) diff --git a/cmd/agent/subcommands/secret/command.go b/cmd/agent/subcommands/secret/command.go index e24f95ca6d693..d8b2a7048114f 100644 --- a/cmd/agent/subcommands/secret/command.go +++ b/cmd/agent/subcommands/secret/command.go @@ -101,7 +101,7 @@ func traceAgentSecretRefresh(conf config.Component) ([]byte, error) { c := apiutil.GetClient(false) c.Timeout = conf.GetDuration("server_timeout") * time.Second - url := fmt.Sprintf("http://127.0.0.1:%d/secret/refresh", port) + url := fmt.Sprintf("https://127.0.0.1:%d/secret/refresh", port) res, err := apiutil.DoGet(c, url, apiutil.CloseConnection) if err != nil { return nil, fmt.Errorf("could not contact trace-agent: %s", err) diff --git a/comp/trace/agent/impl/agent.go b/comp/trace/agent/impl/agent.go index b3d783288ceb6..de9237dedeea5 100644 --- a/comp/trace/agent/impl/agent.go +++ b/comp/trace/agent/impl/agent.go @@ -24,6 +24,7 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/core/secrets" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" @@ -68,6 +69,7 @@ type dependencies struct { Statsd statsd.Component Tagger tagger.Component Compressor compression.Component + At authtoken.Component } var _ traceagent.Component = (*component)(nil) @@ -93,6 +95,7 @@ type component struct { params *Params tagger tagger.Component telemetryCollector telemetry.TelemetryCollector + at authtoken.Component wg *sync.WaitGroup } @@ -115,6 +118,7 @@ func NewAgent(deps dependencies) (traceagent.Component, error) { params: deps.Params, telemetryCollector: deps.TelemetryCollector, tagger: deps.Tagger, + at: deps.At, wg: &sync.WaitGroup{}, } statsdCl, err := setupMetrics(deps.Statsd, c.config, c.telemetryCollector) diff --git a/comp/trace/agent/impl/run.go b/comp/trace/agent/impl/run.go index 20ecbc6496a1b..f40b36e29ae7f 100644 --- a/comp/trace/agent/impl/run.go +++ b/comp/trace/agent/impl/run.go @@ -98,6 +98,9 @@ func runAgentSidekicks(ag component) error { })) } + // Configure the Trace Agent Debug server to use the IPC certificate + ag.Agent.DebugServer.SetTLSConfig(ag.at.GetTLSServerConfig()) + log.Infof("Trace agent running on host %s", tracecfg.Hostname) if pcfg := profilingConfig(tracecfg); pcfg != nil { if err := profiling.Start(*pcfg); err != nil { diff --git a/comp/trace/bundle_test.go b/comp/trace/bundle_test.go index e9874a4b40077..692e7c255f473 100644 --- a/comp/trace/bundle_test.go +++ b/comp/trace/bundle_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" coreconfig "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" @@ -45,6 +47,7 @@ func TestBundleDependencies(t *testing.T) { zstdfx.Module(), taggerfx.Module(tagger.Params{}), fx.Supply(&traceagentimpl.Params{}), + createandfetchimpl.Module(), ) } @@ -75,6 +78,7 @@ func TestMockBundleDependencies(t *testing.T) { fx.Invoke(func(_ traceagent.Component) {}), MockBundle(), taggerfx.Module(tagger.Params{}), + fetchonlyimpl.MockModule(), )) require.NotNil(t, cfg.Object()) diff --git a/comp/trace/status/statusimpl/status.go b/comp/trace/status/statusimpl/status.go index e476ee0281d7a..00a8730b87da8 100644 --- a/comp/trace/status/statusimpl/status.go +++ b/comp/trace/status/statusimpl/status.go @@ -95,7 +95,7 @@ func (s statusProvider) populateStatus() map[string]interface{} { port := s.Config.GetInt("apm_config.debug.port") c := client() - url := fmt.Sprintf("http://localhost:%d/debug/vars", port) + url := fmt.Sprintf("https://localhost:%d/debug/vars", port) resp, err := apiutil.DoGet(c, url, apiutil.CloseConnection) if err != nil { return map[string]interface{}{ diff --git a/pkg/config/fetcher/from_processes.go b/pkg/config/fetcher/from_processes.go index 5b8088f41d970..95c24e283fb46 100644 --- a/pkg/config/fetcher/from_processes.go +++ b/pkg/config/fetcher/from_processes.go @@ -71,7 +71,7 @@ func TraceAgentConfig(config config.Reader) (string, error) { c := util.GetClient(false) c.Timeout = config.GetDuration("server_timeout") * time.Second - ipcAddressWithPort := fmt.Sprintf("http://127.0.0.1:%d/config", port) + ipcAddressWithPort := fmt.Sprintf("https://127.0.0.1:%d/config", port) client := settingshttp.NewClient(c, ipcAddressWithPort, "trace-agent", settingshttp.NewHTTPClientOptions(util.CloseConnection)) return client.FullConfig() diff --git a/pkg/flare/archive.go b/pkg/flare/archive.go index c3e5dffe00ca8..b61c6d9dd6ed2 100644 --- a/pkg/flare/archive.go +++ b/pkg/flare/archive.go @@ -214,7 +214,7 @@ func getExpVar(fb flaretypes.FlareBuilder) error { apmDebugPort := pkgconfigsetup.Datadog().GetInt("apm_config.debug.port") f := filepath.Join("expvar", "trace-agent") - resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/debug/vars", apmDebugPort)) + resp, err := http.Get(fmt.Sprintf("https://127.0.0.1:%d/debug/vars", apmDebugPort)) if err != nil { return fb.AddFile(f, []byte(fmt.Sprintf("Error retrieving vars: %v", err))) } diff --git a/pkg/trace/api/api_test.go b/pkg/trace/api/api_test.go index 9a468dc164487..3f3b2c5b3a20b 100644 --- a/pkg/trace/api/api_test.go +++ b/pkg/trace/api/api_test.go @@ -1039,14 +1039,26 @@ func TestExpvar(t *testing.T) { } c := newTestReceiverConfig() - c.DebugServerPort = 5012 + c.DebugServerPort = 6789 info.InitInfo(c) + + // Starting a TLS httptest server to retrieve tlsCert + ts := httptest.NewTLSServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + tlsConfig := ts.TLS.Clone() + // Setting a client with the proper TLS configuration + client := ts.Client() + ts.Close() + + // Starting Debug Server s := NewDebugServer(c) + s.SetTLSConfig(tlsConfig) + + // Starting the Debug server s.Start() defer s.Stop() - resp, err := http.Get("http://127.0.0.1:5012/debug/vars") - assert.NoError(t, err) + resp, err := client.Get(fmt.Sprintf("https://127.0.0.1:%d/debug/vars", c.DebugServerPort)) + require.NoError(t, err) defer resp.Body.Close() t.Run("read-expvars", func(t *testing.T) { diff --git a/pkg/trace/api/debug_server.go b/pkg/trace/api/debug_server.go index 828d5357330eb..6fd2f39cc011a 100644 --- a/pkg/trace/api/debug_server.go +++ b/pkg/trace/api/debug_server.go @@ -9,6 +9,7 @@ package api import ( "context" + "crypto/tls" "expvar" "fmt" "net" @@ -29,9 +30,10 @@ const ( // DebugServer serves /debug/* endpoints type DebugServer struct { - conf *config.AgentConfig - server *http.Server - mux *http.ServeMux + conf *config.AgentConfig + server *http.Server + mux *http.ServeMux + tlsConfig *tls.Config } // NewDebugServer returns a debug server @@ -53,13 +55,14 @@ func (ds *DebugServer) Start() { WriteTimeout: defaultTimeout, Handler: ds.setupMux(), } - listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", ds.conf.DebugServerPort)) + listener, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(ds.conf.DebugServerPort))) if err != nil { log.Errorf("Error creating debug server listener: %s", err) return } + tlsListener := tls.NewListener(listener, ds.tlsConfig) go func() { - if err := ds.server.Serve(listener); err != nil && err != http.ErrServerClosed { + if err := ds.server.Serve(tlsListener); err != nil && err != http.ErrServerClosed { log.Errorf("Could not start debug server: %s. Debug server disabled.", err) } }() @@ -82,6 +85,11 @@ func (ds *DebugServer) AddRoute(route string, handler http.Handler) { ds.mux.Handle(route, handler) } +// SetTLSConfig adds the provided tls.Config to the internal http.Server +func (ds *DebugServer) SetTLSConfig(config *tls.Config) { + ds.tlsConfig = config +} + func (ds *DebugServer) setupMux() *http.ServeMux { ds.mux.HandleFunc("/debug/pprof/", pprof.Index) ds.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) diff --git a/pkg/trace/info/info.go b/pkg/trace/info/info.go index df86c40587fdc..4198100ee6a5b 100644 --- a/pkg/trace/info/info.go +++ b/pkg/trace/info/info.go @@ -8,6 +8,7 @@ package info import ( "bytes" + "crypto/tls" "encoding/json" "expvar" // automatically publish `/debug/vars` on HTTP port "fmt" @@ -236,8 +237,9 @@ func getProgramBanner(version string) (string, string) { // If error is nil, means the program is running. // If not, it displays a pretty-printed message anyway (for support) func Info(w io.Writer, conf *config.AgentConfig) error { - url := fmt.Sprintf("http://127.0.0.1:%d/debug/vars", conf.DebugServerPort) - client := http.Client{Timeout: 3 * time.Second} + url := fmt.Sprintf("https://127.0.0.1:%d/debug/vars", conf.DebugServerPort) + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := http.Client{Timeout: 3 * time.Second, Transport: tr} resp, err := client.Get(url) if err != nil { // OK, here, we can't even make an http call on the agent port, diff --git a/pkg/trace/info/info_test.go b/pkg/trace/info/info_test.go index 01c369cac403e..25b7cb42b13ff 100644 --- a/pkg/trace/info/info_test.go +++ b/pkg/trace/info/info_test.go @@ -63,7 +63,7 @@ func (h *testServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func testServer(t *testing.T, testFile string) *httptest.Server { t.Helper() - server := httptest.NewServer(&testServerHandler{t: t, testFile: testFile}) + server := httptest.NewTLSServer(&testServerHandler{t: t, testFile: testFile}) t.Logf("test server (serving fake yet valid data) listening on %s", server.URL) return server } @@ -94,7 +94,7 @@ func (h *testServerWarningHandler) ServeHTTP(w http.ResponseWriter, r *http.Requ } func testServerWarning(t *testing.T) *httptest.Server { - server := httptest.NewServer(&testServerWarningHandler{t: t}) + server := httptest.NewTLSServer(&testServerWarningHandler{t: t}) t.Logf("test server (serving data containing worrying values) listening on %s", server.URL) return server } @@ -119,7 +119,7 @@ func (h *testServerErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques } func testServerError(t *testing.T) *httptest.Server { - server := httptest.NewServer(&testServerErrorHandler{t: t}) + server := httptest.NewTLSServer(&testServerErrorHandler{t: t}) t.Logf("test server (serving bad data to trigger errors) listening on %s", server.URL) return server } @@ -331,7 +331,7 @@ func TestError(t *testing.T) { assert.Equal(len(lines[1]), len(lines[2])) assert.Equal("", lines[3]) assert.Regexp(regexp.MustCompile(`^ Error: .*$`), lines[4]) - assert.Equal(fmt.Sprintf(" URL: http://127.0.0.1:%d/debug/vars", port), lines[5]) + assert.Equal(fmt.Sprintf(" URL: https://127.0.0.1:%d/debug/vars", port), lines[5]) assert.Equal("", lines[6]) assert.Equal("", lines[7]) } From 26052f9b0fdb75103002f7090cab64a7c9a12aa2 Mon Sep 17 00:00:00 2001 From: Keisuke Umegaki <41987730+keisku@users.noreply.github.com> Date: Wed, 18 Dec 2024 09:53:55 +0900 Subject: [PATCH 14/43] typo: `DD_APM_OBFUSCATION_CACHE_ENABLED` in Agent Main Configuration File (#32291) --- pkg/config/config_template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/config_template.yaml b/pkg/config/config_template.yaml index a858e50ce8b7f..5b4403be1a654 100644 --- a/pkg/config/config_template.yaml +++ b/pkg/config/config_template.yaml @@ -1261,7 +1261,7 @@ api_key: # obfuscate_sql_values: # - val1 # cache: - ## @param DD_APM_CACHE_ENABLED - boolean - optional + ## @param DD_APM_OBFUSCATION_CACHE_ENABLED - boolean - optional ## Enables caching obfuscated statements. Currently supported for SQL and MongoDB queries. ## Enabled by default. # enabled: true From 9f73ec4f6622e92c48f9008b7797ef5aeb9dc28f Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Wed, 18 Dec 2024 08:59:55 +0200 Subject: [PATCH 15/43] npm: tests: Fixed testing gotchas (#32190) --- pkg/network/tracer/tracer_linux_test.go | 57 +++++++++++++------------ 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/pkg/network/tracer/tracer_linux_test.go b/pkg/network/tracer/tracer_linux_test.go index 29a8b6e34965a..46f33ac7fa405 100644 --- a/pkg/network/tracer/tracer_linux_test.go +++ b/pkg/network/tracer/tracer_linux_test.go @@ -109,7 +109,7 @@ func (s *TracerSuite) TestTCPRemoveEntries() { defer c2.Close() assert.EventuallyWithT(t, func(ct *assert.CollectT) { - conn, ok := findConnection(c2.LocalAddr(), c2.RemoteAddr(), getConnections(t, tr)) + conn, ok := findConnection(c2.LocalAddr(), c2.RemoteAddr(), getConnections(ct, tr)) if !assert.True(ct, ok) { return } @@ -123,9 +123,9 @@ func (s *TracerSuite) TestTCPRemoveEntries() { }, 3*time.Second, 100*time.Millisecond) // Make sure the first connection got cleaned up - assert.Eventually(t, func() bool { - _, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), getConnections(t, tr)) - return !ok + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + _, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), getConnections(ct, tr)) + require.False(ct, ok) }, 5*time.Second, 100*time.Millisecond) } @@ -735,10 +735,10 @@ func (s *TracerSuite) TestGatewayLookupEnabled() { dnsServerAddr := &net.UDPAddr{IP: dnsAddr, Port: 53} var conn *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(ct *assert.CollectT) { var ok bool - conn, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok + conn, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(ct, tr)) + require.True(ct, ok, "connection not found") }, 3*time.Second, 100*time.Millisecond) require.NotNil(t, conn.Via, "connection is missing via: %s", conn) @@ -791,10 +791,10 @@ func (s *TracerSuite) TestGatewayLookupSubnetLookupError() { dnsClientAddr := &net.UDPAddr{IP: net.ParseIP(clientIP), Port: clientPort} dnsServerAddr := &net.UDPAddr{IP: destAddr, Port: 53} var c *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(ct *assert.CollectT) { var ok bool - c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok + c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(ct, tr)) + require.True(ct, ok, "connection not found") }, 3*time.Second, 100*time.Millisecond, "connection not found") require.Nil(t, c.Via) @@ -804,10 +804,10 @@ func (s *TracerSuite) TestGatewayLookupSubnetLookupError() { }, 6*time.Second, 100*time.Millisecond, "failed to send dns query") dnsClientAddr = &net.UDPAddr{IP: net.ParseIP(clientIP), Port: clientPort} - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(ct *assert.CollectT) { var ok bool - c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok + c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(ct, tr)) + require.True(ct, ok, "connection not found") }, 3*time.Second, 100*time.Millisecond, "connection not found") require.Nil(t, c.Via) @@ -1529,25 +1529,25 @@ func testUDPReusePort(t *testing.T, udpnet string, ip string) { assert.EventuallyWithT(t, func(ct *assert.CollectT) { //nolint:revive // TODO // use t instead of ct because getConnections uses require (not assert), and we get a better error message that way - connections := getConnections(t, tr) + connections := getConnections(ct, tr) incoming, ok := findConnection(c.RemoteAddr(), c.LocalAddr(), connections) - if assert.True(t, ok, "unable to find incoming connection") { - assert.Equal(t, network.INCOMING, incoming.Direction) + if assert.True(ct, ok, "unable to find incoming connection") { + assert.Equal(ct, network.INCOMING, incoming.Direction) // make sure the inverse values are seen for the other message - assert.Equal(t, serverMessageSize, int(incoming.Monotonic.SentBytes), "incoming sent") - assert.Equal(t, clientMessageSize, int(incoming.Monotonic.RecvBytes), "incoming recv") - assert.True(t, incoming.IntraHost, "incoming intrahost") + assert.Equal(ct, serverMessageSize, int(incoming.Monotonic.SentBytes), "incoming sent") + assert.Equal(ct, clientMessageSize, int(incoming.Monotonic.RecvBytes), "incoming recv") + assert.True(ct, incoming.IntraHost, "incoming intrahost") } outgoing, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), connections) - if assert.True(t, ok, "unable to find outgoing connection") { - assert.Equal(t, network.OUTGOING, outgoing.Direction) + if assert.True(ct, ok, "unable to find outgoing connection") { + assert.Equal(ct, network.OUTGOING, outgoing.Direction) - assert.Equal(t, clientMessageSize, int(outgoing.Monotonic.SentBytes), "outgoing sent") - assert.Equal(t, serverMessageSize, int(outgoing.Monotonic.RecvBytes), "outgoing recv") - assert.True(t, outgoing.IntraHost, "outgoing intrahost") + assert.Equal(ct, clientMessageSize, int(outgoing.Monotonic.SentBytes), "outgoing sent") + assert.Equal(ct, serverMessageSize, int(outgoing.Monotonic.RecvBytes), "outgoing recv") + assert.True(ct, outgoing.IntraHost, "outgoing intrahost") } }, 3*time.Second, 100*time.Millisecond) @@ -1660,8 +1660,8 @@ func (s *TracerSuite) TestSendfileRegression() { t.Logf("looking for connections %+v <-> %+v", c.LocalAddr(), c.RemoteAddr()) var outConn, inConn *network.ConnectionStats - assert.Eventually(t, func() bool { - conns := getConnections(t, tr) + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + conns := getConnections(ct, tr) t.Log(conns) if outConn == nil { outConn = network.FirstConnection(conns, network.ByType(connType), network.ByFamily(family), network.ByTuple(c.LocalAddr(), c.RemoteAddr())) @@ -1669,7 +1669,8 @@ func (s *TracerSuite) TestSendfileRegression() { if inConn == nil { inConn = network.FirstConnection(conns, network.ByType(connType), network.ByFamily(family), network.ByTuple(c.RemoteAddr(), c.LocalAddr())) } - return outConn != nil && inConn != nil + require.NotNil(ct, outConn) + require.NotNil(ct, inConn) }, 3*time.Second, 100*time.Millisecond, "couldn't find connections used by sendfile(2)") if assert.NotNil(t, outConn, "couldn't find outgoing connection used by sendfile(2)") { @@ -2433,7 +2434,7 @@ LOOP: require.NoError(t, c.Close(), "error closing client connection") require.EventuallyWithT(t, func(collect *assert.CollectT) { - conn, found := findConnection(c.LocalAddr(), srv.Addr(), getConnections(t, tr)) + conn, found := findConnection(c.LocalAddr(), srv.Addr(), getConnections(collect, tr)) require.True(collect, found, "could not find connection") require.True(collect, conn.IsClosed, "connection should be closed") // after closing the client connection, the duration should be From 332025d2eca869ea264ad9abde503853757d5f70 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Wed, 18 Dec 2024 09:26:17 +0100 Subject: [PATCH 16/43] fix(release.cleanup): Checkout the default branch (#32234) --- tasks/libs/ciproviders/github_api.py | 4 +++- tasks/release.py | 17 ++++++----------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/tasks/libs/ciproviders/github_api.py b/tasks/libs/ciproviders/github_api.py index 8a4d5e69027d1..54a3ba49a43f1 100644 --- a/tasks/libs/ciproviders/github_api.py +++ b/tasks/libs/ciproviders/github_api.py @@ -332,7 +332,9 @@ def workflow_run_for_ref_after_date(self, workflow_name, ref, oldest_date): return sorted(recent_runs, key=lambda run: run.created_at, reverse=True) - def latest_release(self) -> str: + def latest_release(self, major_version=7) -> str: + if major_version == 6: + return max((r for r in self.get_releases() if r.title.startswith('6.53')), key=lambda r: r.created_at).title release = self._repository.get_latest_release() return release.title diff --git a/tasks/release.py b/tasks/release.py index 077834b279923..4e5805b0ffb76 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -837,10 +837,6 @@ def _update_last_stable(_, version, major_version: int = 7): return release_json["current_milestone"] -def _get_agent6_latest_release(gh): - return max((r for r in gh.get_releases() if r.title.startswith('6.53')), key=lambda r: r.created_at).title - - @task def cleanup(ctx, release_branch): """Perform the post release cleanup steps @@ -850,13 +846,13 @@ def cleanup(ctx, release_branch): - Updates the release.json last_stable fields """ - with agent_context(ctx, release_branch): + # This task will create a PR to update the last_stable field in release.json + # It must create the PR against the default branch (6 or 7), so setting the context on it + main_branch = get_default_branch() + with agent_context(ctx, main_branch): gh = GithubAPI() major_version = get_version_major(release_branch) - if major_version == 6: - latest_release = _get_agent6_latest_release(gh) - else: - latest_release = gh.latest_release() + latest_release = gh.latest_release(major_version) match = VERSION_RE.search(latest_release) if not match: raise Exit(f'Unexpected version fetched from github {latest_release}', code=1) @@ -865,7 +861,6 @@ def cleanup(ctx, release_branch): current_milestone = _update_last_stable(ctx, version, major_version=major_version) # create pull request to update last stable version - main_branch = get_default_branch() cleanup_branch = f"release/{version}-cleanup" ctx.run(f"git checkout -b {cleanup_branch}") ctx.run("git add release.json") @@ -1034,7 +1029,7 @@ def get_active_release_branch(ctx, release_branch): with agent_context(ctx, branch=release_branch): gh = GithubAPI() - next_version = get_next_version(gh, latest_release=_get_agent6_latest_release(gh) if is_agent6(ctx) else None) + next_version = get_next_version(gh, latest_release=gh.latest_release(6) if is_agent6(ctx) else None) release_branch = gh.get_branch(next_version.branch()) if release_branch: print(f"{release_branch.name}") From 469bd66d6ad9bbb0b11af7b487d6f86c991782b6 Mon Sep 17 00:00:00 2001 From: louis-cqrl <93274433+louis-cqrl@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:02:37 +0100 Subject: [PATCH 17/43] [ASCII-2621] Update Scrubber package (#32311) --- comp/core/flare/helpers/builder_test.go | 8 +- comp/core/log/mock/go.mod | 1 + .../otlp/components/statsprocessor/go.mod | 1 - .../otlp/components/statsprocessor/go.sum | 2 - pkg/config/teeconfig/go.mod | 1 + pkg/gohai/go.mod | 1 - pkg/gohai/go.sum | 2 - pkg/logs/util/testutils/go.mod | 1 + pkg/orchestrator/model/go.mod | 2 +- pkg/orchestrator/model/go.sum | 2 - pkg/trace/go.mod | 1 - pkg/trace/go.sum | 2 - pkg/trace/stats/oteltest/go.mod | 1 - pkg/trace/stats/oteltest/go.sum | 2 - pkg/util/cgroups/go.mod | 1 - pkg/util/cgroups/go.sum | 2 - pkg/util/defaultpaths/go.mod | 2 +- pkg/util/defaultpaths/go.sum | 2 - pkg/util/filesystem/go.mod | 1 - pkg/util/filesystem/go.sum | 2 - pkg/util/hostname/validate/go.mod | 1 - pkg/util/hostname/validate/go.sum | 2 - pkg/util/log/go.mod | 1 - pkg/util/log/go.sum | 2 - pkg/util/scrubber/default_test.go | 180 +++--------------- pkg/util/scrubber/go.mod | 3 +- pkg/util/scrubber/go.sum | 2 - pkg/util/scrubber/json_scrubber_test.go | 27 +++ pkg/util/scrubber/test/conf.yaml | 16 +- pkg/util/scrubber/test/conf_scrubbed.yaml | 16 +- pkg/util/scrubber/yaml_scrubber.go | 16 +- pkg/util/scrubber/yaml_scrubber_test.go | 107 +++++++++++ pkg/util/system/go.mod | 1 - pkg/util/system/go.sum | 2 - pkg/util/uuid/go.mod | 2 +- pkg/util/uuid/go.sum | 2 - pkg/util/winutil/go.mod | 1 - pkg/util/winutil/go.sum | 2 - 38 files changed, 198 insertions(+), 222 deletions(-) diff --git a/comp/core/flare/helpers/builder_test.go b/comp/core/flare/helpers/builder_test.go index 26508575eef92..ccc544eb6ecb9 100644 --- a/comp/core/flare/helpers/builder_test.go +++ b/comp/core/flare/helpers/builder_test.go @@ -213,8 +213,8 @@ func TestAddFileYamlDetection(t *testing.T) { - abcdef - abcdef`) redacted := `instances: -- host: 127.0.0.1 - token: "********"` + - host: 127.0.0.1 + token: "********"` fb.AddFile("test.yaml", clear) assertFileContent(t, fb, redacted, "test.yaml") @@ -297,8 +297,8 @@ func TestCopyFileYamlDetection(t *testing.T) { - abcdef - abcdef`) redacted := `instances: -- host: 127.0.0.1 - token: "********"` + - host: 127.0.0.1 + token: "********"` path1 := filepath.Join(t.TempDir(), "test.yaml") os.WriteFile(path1, []byte(input), os.ModePerm) diff --git a/comp/core/log/mock/go.mod b/comp/core/log/mock/go.mod index 8b744f1e85ca2..e4f0f87cabdcb 100644 --- a/comp/core/log/mock/go.mod +++ b/comp/core/log/mock/go.mod @@ -58,6 +58,7 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/config/structure => ../../../../pkg/config/structure diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index 05eb43fdf4629..4ad268c10dfcb 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -97,7 +97,6 @@ require ( google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index 25a9e6fccac3c..a19fb370fa6c4 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -245,8 +245,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/config/teeconfig/go.mod b/pkg/config/teeconfig/go.mod index 2484fb9cd55e3..daec9f976da68 100644 --- a/pkg/config/teeconfig/go.mod +++ b/pkg/config/teeconfig/go.mod @@ -33,6 +33,7 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/gohai/go.mod b/pkg/gohai/go.mod index 45edc82c08f38..e7dee142ec26f 100644 --- a/pkg/gohai/go.mod +++ b/pkg/gohai/go.mod @@ -26,7 +26,6 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/gohai/go.sum b/pkg/gohai/go.sum index 99c1b368ae068..828633a9495c7 100644 --- a/pkg/gohai/go.sum +++ b/pkg/gohai/go.sum @@ -44,7 +44,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/logs/util/testutils/go.mod b/pkg/logs/util/testutils/go.mod index 46e01043de4bf..1c02f6f2f8775 100644 --- a/pkg/logs/util/testutils/go.mod +++ b/pkg/logs/util/testutils/go.mod @@ -95,4 +95,5 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/orchestrator/model/go.mod b/pkg/orchestrator/model/go.mod index 96f90197c3fe2..f941c7960e5ae 100644 --- a/pkg/orchestrator/model/go.mod +++ b/pkg/orchestrator/model/go.mod @@ -17,7 +17,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/orchestrator/model/go.sum b/pkg/orchestrator/model/go.sum index 3c93bebd8c6dd..01c4e89a188d8 100644 --- a/pkg/orchestrator/model/go.sum +++ b/pkg/orchestrator/model/go.sum @@ -19,7 +19,5 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index fa1ca487a3dc8..51564f328f09e 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -110,7 +110,6 @@ require ( golang.org/x/text v0.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index 4613790f7cc34..e1b7335487809 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -402,8 +402,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 306e0217b22ad..c3759db9cc74b 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -83,7 +83,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/grpc v1.67.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index 25a9e6fccac3c..a19fb370fa6c4 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -245,8 +245,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/cgroups/go.mod b/pkg/util/cgroups/go.mod index 47a3c2c368f5f..0be6406d65503 100644 --- a/pkg/util/cgroups/go.mod +++ b/pkg/util/cgroups/go.mod @@ -31,7 +31,6 @@ require ( go.uber.org/atomic v1.11.0 // indirect golang.org/x/sys v0.28.0 // indirect google.golang.org/protobuf v1.35.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/cgroups/go.sum b/pkg/util/cgroups/go.sum index 69f8c3e58e44b..d2a0373b5e095 100644 --- a/pkg/util/cgroups/go.sum +++ b/pkg/util/cgroups/go.sum @@ -38,7 +38,5 @@ google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/defaultpaths/go.mod b/pkg/util/defaultpaths/go.mod index ad84aa45db6e9..0f0c498c8154e 100644 --- a/pkg/util/defaultpaths/go.mod +++ b/pkg/util/defaultpaths/go.mod @@ -22,7 +22,7 @@ require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/util/defaultpaths/go.sum b/pkg/util/defaultpaths/go.sum index 767c365bf98df..82beecfd023fb 100644 --- a/pkg/util/defaultpaths/go.sum +++ b/pkg/util/defaultpaths/go.sum @@ -21,7 +21,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/filesystem/go.mod b/pkg/util/filesystem/go.mod index 292bb06f164b9..31b6f1fbd4ab0 100644 --- a/pkg/util/filesystem/go.mod +++ b/pkg/util/filesystem/go.mod @@ -29,7 +29,6 @@ require ( github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/filesystem/go.sum b/pkg/util/filesystem/go.sum index 14cc56a3421e8..ebdb96f4beb1d 100644 --- a/pkg/util/filesystem/go.sum +++ b/pkg/util/filesystem/go.sum @@ -36,7 +36,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/hostname/validate/go.mod b/pkg/util/hostname/validate/go.mod index 6ec1ae79a177d..9c1a227476e2c 100644 --- a/pkg/util/hostname/validate/go.mod +++ b/pkg/util/hostname/validate/go.mod @@ -19,7 +19,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/hostname/validate/go.sum b/pkg/util/hostname/validate/go.sum index c57375d7e46df..d45d69118e85e 100644 --- a/pkg/util/hostname/validate/go.sum +++ b/pkg/util/hostname/validate/go.sum @@ -17,7 +17,5 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/log/go.mod b/pkg/util/log/go.mod index 26f759935f92e..a439a62283f76 100644 --- a/pkg/util/log/go.mod +++ b/pkg/util/log/go.mod @@ -17,7 +17,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.uber.org/multierr v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/log/go.sum b/pkg/util/log/go.sum index f46b720dc7374..ffca41d546584 100644 --- a/pkg/util/log/go.sum +++ b/pkg/util/log/go.sum @@ -23,7 +23,5 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/scrubber/default_test.go b/pkg/util/scrubber/default_test.go index 86134fbc90328..7c093d9e40300 100644 --- a/pkg/util/scrubber/default_test.go +++ b/pkg/util/scrubber/default_test.go @@ -6,16 +6,13 @@ package scrubber import ( - "encoding/json" "os" "path/filepath" - "reflect" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" ) func assertClean(t *testing.T, contents, cleanContents string) { @@ -26,96 +23,6 @@ func assertClean(t *testing.T, contents, cleanContents string) { assert.Equal(t, strings.TrimSpace(cleanContents), strings.TrimSpace(cleanedString)) } -func TestConfigScrubbedValidYaml(t *testing.T) { - wd, _ := os.Getwd() - - inputConf := filepath.Join(wd, "test", "conf.yaml") - inputConfData, err := os.ReadFile(inputConf) - require.NoError(t, err) - - outputConf := filepath.Join(wd, "test", "conf_scrubbed.yaml") - outputConfData, err := os.ReadFile(outputConf) - require.NoError(t, err) - - cleaned, err := ScrubBytes([]byte(inputConfData)) - require.NoError(t, err) - - // First test that the a scrubbed yaml is still a valid yaml - var out interface{} - err = yaml.Unmarshal(cleaned, &out) - assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") - - // We replace windows line break by linux so the tests pass on every OS - trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) - trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) - - assert.Equal(t, trimmedOutput, trimmedCleaned) -} - -func TestConfigScrubbedYaml(t *testing.T) { - wd, _ := os.Getwd() - - inputConf := filepath.Join(wd, "test", "conf_multiline.yaml") - inputConfData, err := os.ReadFile(inputConf) - require.NoError(t, err) - - outputConf := filepath.Join(wd, "test", "conf_multiline_scrubbed.yaml") - outputConfData, err := os.ReadFile(outputConf) - require.NoError(t, err) - - cleaned, err := ScrubYaml([]byte(inputConfData)) - require.NoError(t, err) - - // First test that the a scrubbed yaml is still a valid yaml - var out interface{} - err = yaml.Unmarshal(cleaned, &out) - assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") - - // We replace windows line break by linux so the tests pass on every OS - trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) - trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) - - assert.Equal(t, trimmedOutput, trimmedCleaned) -} - -func TestConfigScrubbedJson(t *testing.T) { - wd, _ := os.Getwd() - - inputConf := filepath.Join(wd, "test", "config.json") - inputConfData, err := os.ReadFile(inputConf) - require.NoError(t, err) - cleaned, err := ScrubJSON([]byte(inputConfData)) - require.NoError(t, err) - // First test that the a scrubbed json is still valid - var actualOutJSON map[string]interface{} - err = json.Unmarshal(cleaned, &actualOutJSON) - assert.NoError(t, err, "Could not load JSON configuration after being scrubbed") - - outputConf := filepath.Join(wd, "test", "config_scrubbed.json") - outputConfData, err := os.ReadFile(outputConf) - require.NoError(t, err) - var expectedOutJSON map[string]interface{} - err = json.Unmarshal(outputConfData, &expectedOutJSON) - require.NoError(t, err) - assert.Equal(t, reflect.DeepEqual(expectedOutJSON, actualOutJSON), true) -} - -func TestEmptyYaml(t *testing.T) { - cleaned, err := ScrubYaml(nil) - require.NoError(t, err) - assert.Equal(t, "", string(cleaned)) - - cleaned, err = ScrubYaml([]byte("")) - require.NoError(t, err) - assert.Equal(t, "", string(cleaned)) -} - -func TestEmptyYamlString(t *testing.T) { - cleaned, err := ScrubYamlString("") - require.NoError(t, err) - assert.Equal(t, "", string(cleaned)) -} - func TestConfigStripApiKey(t *testing.T) { assertClean(t, `api_key: aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb`, @@ -569,39 +476,6 @@ func TestAddStrippedKeys(t *testing.T) { assertClean(t, contents, `foobar: "********"`) } -func TestAddStrippedKeysExceptions(t *testing.T) { - t.Run("single key", func(t *testing.T) { - contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'` - - AddStrippedKeys([]string{"api_key"}) - - scrubbed, err := ScrubYamlString(contents) - require.Nil(t, err) - require.YAMLEq(t, `api_key: '***************************aaaaa'`, scrubbed) - }) - - t.Run("multiple keys", func(t *testing.T) { - contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' -some_other_key: 'bbbb' -app_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaacccc' -yet_another_key: 'dddd'` - - keys := []string{"api_key", "some_other_key", "app_key"} - AddStrippedKeys(keys) - - // check that AddStrippedKeys didn't modify the parameter slice - assert.Equal(t, []string{"api_key", "some_other_key", "app_key"}, keys) - - scrubbed, err := ScrubYamlString(contents) - require.Nil(t, err) - expected := `api_key: '***************************aaaaa' -some_other_key: '********' -app_key: '***********************************acccc' -yet_another_key: 'dddd'` - require.YAMLEq(t, expected, scrubbed) - }) -} - func TestAddStrippedKeysNewReplacer(t *testing.T) { contents := `foobar: baz` AddStrippedKeys([]string{"foobar"}) @@ -686,33 +560,6 @@ network_devices: log_level: info`) } -func TestConfigFile(t *testing.T) { - cleanedConfigFile := `dd_url: https://app.datadoghq.com - -api_key: "***************************aaaaa" - -proxy: http://user:********@host:port - - - - - - -dogstatsd_port : 8125 - - -log_level: info -` - - wd, _ := os.Getwd() - filePath := filepath.Join(wd, "test", "datadog.yaml") - cleaned, err := ScrubFile(filePath) - assert.NoError(t, err) - cleanedString := string(cleaned) - - assert.Equal(t, cleanedConfigFile, cleanedString) -} - func TestBearerToken(t *testing.T) { assertClean(t, `Bearer 2fe663014abcd1850076f6d68c0355666db98758262870811cace007cd4a62ba`, @@ -785,3 +632,30 @@ func TestScrubCommandsEnv(t *testing.T) { }) } } + +func TestConfigFile(t *testing.T) { + cleanedConfigFile := `dd_url: https://app.datadoghq.com + +api_key: "***************************aaaaa" + +proxy: http://user:********@host:port + + + + + + +dogstatsd_port : 8125 + + +log_level: info +` + + wd, _ := os.Getwd() + filePath := filepath.Join(wd, "test", "datadog.yaml") + cleaned, err := ScrubFile(filePath) + assert.NoError(t, err) + cleanedString := string(cleaned) + + assert.Equal(t, cleanedConfigFile, cleanedString) +} diff --git a/pkg/util/scrubber/go.mod b/pkg/util/scrubber/go.mod index 6f10e986954d3..ff0eb09afc0ef 100644 --- a/pkg/util/scrubber/go.mod +++ b/pkg/util/scrubber/go.mod @@ -5,14 +5,13 @@ go 1.22.0 require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 github.com/stretchr/testify v1.10.0 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/util/scrubber/go.sum b/pkg/util/scrubber/go.sum index def44d1759f5c..878d5c29e2e5d 100644 --- a/pkg/util/scrubber/go.sum +++ b/pkg/util/scrubber/go.sum @@ -14,7 +14,5 @@ github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/scrubber/json_scrubber_test.go b/pkg/util/scrubber/json_scrubber_test.go index 007da8d3f34cb..b45d0a8d08f78 100644 --- a/pkg/util/scrubber/json_scrubber_test.go +++ b/pkg/util/scrubber/json_scrubber_test.go @@ -6,9 +6,14 @@ package scrubber import ( + "encoding/json" + "os" + "path/filepath" + "reflect" "regexp" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -68,3 +73,25 @@ func TestScrubJSON(t *testing.T) { require.Equal(t, expected, string(actual)) }) } + +func TestConfigScrubbedJson(t *testing.T) { + wd, _ := os.Getwd() + + inputConf := filepath.Join(wd, "test", "config.json") + inputConfData, err := os.ReadFile(inputConf) + require.NoError(t, err) + cleaned, err := ScrubJSON([]byte(inputConfData)) + require.NoError(t, err) + // First test that the a scrubbed json is still valid + var actualOutJSON map[string]interface{} + err = json.Unmarshal(cleaned, &actualOutJSON) + assert.NoError(t, err, "Could not load JSON configuration after being scrubbed") + + outputConf := filepath.Join(wd, "test", "config_scrubbed.json") + outputConfData, err := os.ReadFile(outputConf) + require.NoError(t, err) + var expectedOutJSON map[string]interface{} + err = json.Unmarshal(outputConfData, &expectedOutJSON) + require.NoError(t, err) + assert.Equal(t, reflect.DeepEqual(expectedOutJSON, actualOutJSON), true) +} diff --git a/pkg/util/scrubber/test/conf.yaml b/pkg/util/scrubber/test/conf.yaml index cf7fc3d34d3f5..d2bb11a2a9fe3 100644 --- a/pkg/util/scrubber/test/conf.yaml +++ b/pkg/util/scrubber/test/conf.yaml @@ -14,7 +14,7 @@ list_api_key: [aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, aaaaaaaaaaaaaaaaaaaaaaaaaaaabbb list_app_key: [aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb] dict_api_key: {one: aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, two: aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb} -dict_app_key: {two: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, two: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb} +dict_app_key: {one: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, two: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb} additional_endpoints: "https://app.datadoghq.com": @@ -29,15 +29,15 @@ random_url_key1: http://user:p@ssw0r)@host:port random_url_key2: protocol://user:p@ssw0r)@host:port random_url_key3: "http://user:password@host:port" random_domain_key: 'user:password@host:port' -random_url_key1: | +random_url_key4: | http://user:password@host:port -random_url_key2: > +random_url_key5: > http://user:password@host:port -random_url_key3: 'http://user:password@host:port' -random_url_key4: 'mongodb+s.r-v://user:password@host:port' -random_url_key5: 'mongodb+srv://user:pass-with-hyphen@abc.example.com/database' -random_url_key6: 'http://user-with-hyphen:pass-with-hyphen@abc.example.com/database' -random_url_key7: 'http://user-with-hyphen:pass@abc.example.com/database' +random_url_key6: 'http://user:password@host:port' +random_url_key7: 'mongodb+s.r-v://user:password@host:port' +random_url_key8: 'mongodb+srv://user:pass-with-hyphen@abc.example.com/database' +random_url_key9: 'http://user-with-hyphen:pass-with-hyphen@abc.example.com/database' +random_url_key10: 'http://user-with-hyphen:pass@abc.example.com/database' mysql_password1: password mysql_password3: "password" diff --git a/pkg/util/scrubber/test/conf_scrubbed.yaml b/pkg/util/scrubber/test/conf_scrubbed.yaml index c4c987bba5df4..5bb34c750ac3c 100644 --- a/pkg/util/scrubber/test/conf_scrubbed.yaml +++ b/pkg/util/scrubber/test/conf_scrubbed.yaml @@ -14,7 +14,7 @@ list_api_key: ["***************************abbbb", "***************************a list_app_key: ["***********************************abbbb", "***********************************abbbb"] dict_api_key: {one: "***************************abbbb", two: "***************************abbbb"} -dict_app_key: {two: "***********************************abbbb", two: "***********************************abbbb"} +dict_app_key: {one: "***********************************abbbb", two: "***********************************abbbb"} additional_endpoints: "https://app.datadoghq.com": @@ -29,15 +29,15 @@ random_url_key1: http://user:********@host:port random_url_key2: protocol://user:********@host:port random_url_key3: "http://user:********@host:port" random_domain_key: 'user:********@host:port' -random_url_key1: | +random_url_key4: | http://user:********@host:port -random_url_key2: > +random_url_key5: > http://user:********@host:port -random_url_key3: 'http://user:********@host:port' -random_url_key4: 'mongodb+s.r-v://user:********@host:port' -random_url_key5: 'mongodb+srv://user:********@abc.example.com/database' -random_url_key6: 'http://user-with-hyphen:********@abc.example.com/database' -random_url_key7: 'http://user-with-hyphen:********@abc.example.com/database' +random_url_key6: 'http://user:********@host:port' +random_url_key7: 'mongodb+s.r-v://user:********@host:port' +random_url_key8: 'mongodb+srv://user:********@abc.example.com/database' +random_url_key9: 'http://user-with-hyphen:********@abc.example.com/database' +random_url_key10: 'http://user-with-hyphen:********@abc.example.com/database' mysql_password1: "********" mysql_password3: "********" diff --git a/pkg/util/scrubber/yaml_scrubber.go b/pkg/util/scrubber/yaml_scrubber.go index f01d93e064ec5..90f144c6617eb 100644 --- a/pkg/util/scrubber/yaml_scrubber.go +++ b/pkg/util/scrubber/yaml_scrubber.go @@ -6,10 +6,11 @@ package scrubber import ( + "bytes" "fmt" "os" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type scrubCallback = func(string, interface{}) (bool, interface{}) @@ -110,13 +111,16 @@ func (c *Scrubber) ScrubYaml(input []byte) ([]byte, error) { // if we can't load the yaml run the default scrubber on the input if len(input) != 0 && err == nil { c.ScrubDataObj(data) - newInput, err := yaml.Marshal(data) - if err == nil { - input = newInput - } else { - // Since the scrubber is a dependency of the logger we can use it here. + + var buffer bytes.Buffer + encoder := yaml.NewEncoder(&buffer) + encoder.SetIndent(2) + if err := encoder.Encode(&data); err != nil { fmt.Fprintf(os.Stderr, "error scrubbing YAML, falling back on text scrubber: %s\n", err) + } else { + input = buffer.Bytes() } + encoder.Close() } return c.ScrubBytes(input) } diff --git a/pkg/util/scrubber/yaml_scrubber_test.go b/pkg/util/scrubber/yaml_scrubber_test.go index 68f45dbcb9aaa..d576472186bd2 100644 --- a/pkg/util/scrubber/yaml_scrubber_test.go +++ b/pkg/util/scrubber/yaml_scrubber_test.go @@ -6,9 +6,15 @@ package scrubber import ( + "os" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "gopkg.in/yaml.v3" ) func TestScrubDataObj(t *testing.T) { @@ -62,3 +68,104 @@ func TestScrubDataObj(t *testing.T) { }) } } + +func TestConfigScrubbedValidYaml(t *testing.T) { + wd, _ := os.Getwd() + + inputConf := filepath.Join(wd, "test", "conf.yaml") + inputConfData, err := os.ReadFile(inputConf) + require.NoError(t, err) + + outputConf := filepath.Join(wd, "test", "conf_scrubbed.yaml") + outputConfData, err := os.ReadFile(outputConf) + require.NoError(t, err) + + cleaned, err := ScrubBytes([]byte(inputConfData)) + require.NoError(t, err) + + // First test that the a scrubbed yaml is still a valid yaml + var out interface{} + err = yaml.Unmarshal(cleaned, &out) + assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") + + // We replace windows line break by linux so the tests pass on every OS + trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) + trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) + + assert.Equal(t, trimmedOutput, trimmedCleaned) +} + +func TestConfigScrubbedYaml(t *testing.T) { + wd, _ := os.Getwd() + + inputConf := filepath.Join(wd, "test", "conf_multiline.yaml") + inputConfData, err := os.ReadFile(inputConf) + require.NoError(t, err) + + outputConf := filepath.Join(wd, "test", "conf_multiline_scrubbed.yaml") + outputConfData, err := os.ReadFile(outputConf) + require.NoError(t, err) + + cleaned, err := ScrubYaml([]byte(inputConfData)) + require.NoError(t, err) + + // First test that the a scrubbed yaml is still a valid yaml + var out interface{} + err = yaml.Unmarshal(cleaned, &out) + assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") + + // We replace windows line break by linux so the tests pass on every OS + trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) + trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) + + assert.Equal(t, trimmedOutput, trimmedCleaned) +} + +func TestEmptyYaml(t *testing.T) { + cleaned, err := ScrubYaml(nil) + require.NoError(t, err) + assert.Equal(t, "", string(cleaned)) + + cleaned, err = ScrubYaml([]byte("")) + require.NoError(t, err) + assert.Equal(t, "", string(cleaned)) +} + +func TestEmptyYamlString(t *testing.T) { + cleaned, err := ScrubYamlString("") + require.NoError(t, err) + assert.Equal(t, "", string(cleaned)) +} + +func TestAddStrippedKeysExceptions(t *testing.T) { + t.Run("single key", func(t *testing.T) { + contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'` + + AddStrippedKeys([]string{"api_key"}) + + scrubbed, err := ScrubYamlString(contents) + require.Nil(t, err) + require.YAMLEq(t, `api_key: '***************************aaaaa'`, scrubbed) + }) + + t.Run("multiple keys", func(t *testing.T) { + contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +some_other_key: 'bbbb' +app_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaacccc' +yet_another_key: 'dddd'` + + keys := []string{"api_key", "some_other_key", "app_key"} + AddStrippedKeys(keys) + + // check that AddStrippedKeys didn't modify the parameter slice + assert.Equal(t, []string{"api_key", "some_other_key", "app_key"}, keys) + + scrubbed, err := ScrubYamlString(contents) + require.Nil(t, err) + expected := `api_key: '***************************aaaaa' +some_other_key: '********' +app_key: '***********************************acccc' +yet_another_key: 'dddd'` + require.YAMLEq(t, expected, scrubbed) + }) +} diff --git a/pkg/util/system/go.mod b/pkg/util/system/go.mod index 78e3380d124ed..69183e1155599 100644 --- a/pkg/util/system/go.mod +++ b/pkg/util/system/go.mod @@ -37,7 +37,6 @@ require ( github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/system/go.sum b/pkg/util/system/go.sum index 47e0acb25ee1f..5eac83e6a0bd2 100644 --- a/pkg/util/system/go.sum +++ b/pkg/util/system/go.sum @@ -45,7 +45,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/uuid/go.mod b/pkg/util/uuid/go.mod index ad2e7ef2c7f0a..0f8d47fc3bb8c 100644 --- a/pkg/util/uuid/go.mod +++ b/pkg/util/uuid/go.mod @@ -28,7 +28,7 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/util/uuid/go.sum b/pkg/util/uuid/go.sum index 2e5f8b7759901..7399205747094 100644 --- a/pkg/util/uuid/go.sum +++ b/pkg/util/uuid/go.sum @@ -44,7 +44,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/winutil/go.mod b/pkg/util/winutil/go.mod index 1facedf93843c..172db988efae7 100644 --- a/pkg/util/winutil/go.mod +++ b/pkg/util/winutil/go.mod @@ -21,7 +21,6 @@ require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/winutil/go.sum b/pkg/util/winutil/go.sum index 99e864f633ef6..788135f619bbb 100644 --- a/pkg/util/winutil/go.sum +++ b/pkg/util/winutil/go.sum @@ -21,7 +21,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From a53dc62289591cb1d1abe3cb028699f8bc4f5896 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Wed, 18 Dec 2024 10:10:38 +0100 Subject: [PATCH 18/43] [CWS] Add test on container scope variables (#32319) --- pkg/security/tests/container_test.go | 70 ++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/pkg/security/tests/container_test.go b/pkg/security/tests/container_test.go index 7a659481da74b..15d896f1718ef 100644 --- a/pkg/security/tests/container_test.go +++ b/pkg/security/tests/container_test.go @@ -182,3 +182,73 @@ func TestContainerFlagsPodman(t *testing.T) { }) }) } + +func TestContainerVariables(t *testing.T) { + SkipIfNotAvailable(t) + + ruleDefs := []*rules.RuleDefinition{ + { + ID: "test_container_set_variable", + Expression: `container.id != "" && open.file.path == "{{.Root}}/test-open"`, + Actions: []*rules.ActionDefinition{ + { + Set: &rules.SetDefinition{ + Scope: "container", + Value: 1, + Name: "foo", + }, + }, + }, + }, + { + ID: "test_container_check_variable", + Expression: `container.id != "" && open.file.path == "{{.Root}}/test-open2" && ${container.foo} == 1`, + }, + } + test, err := newTestModule(t, nil, ruleDefs) + if err != nil { + t.Fatal(err) + } + defer test.Close() + + testFile, _, err := test.Path("test-open") + if err != nil { + t.Fatal(err) + } + + testFile2, _, err := test.Path("test-open2") + if err != nil { + t.Fatal(err) + } + + dockerWrapper, err := newDockerCmdWrapper(test.Root(), test.Root(), "ubuntu", "") + if err != nil { + t.Skip("Skipping created time in containers tests: Docker not available") + return + } + defer dockerWrapper.stop() + + dockerWrapper.Run(t, "container-variables", func(t *testing.T, _ wrapperType, cmdFunc func(cmd string, args []string, envs []string) *exec.Cmd) { + test.WaitSignal(t, func() error { + cmd := cmdFunc("touch", []string{testFile}, nil) + return cmd.Run() + }, func(event *model.Event, rule *rules.Rule) { + assertTriggeredRule(t, rule, "test_container_set_variable") + assertFieldEqual(t, event, "open.file.path", testFile) + assertFieldNotEmpty(t, event, "container.id", "container id shouldn't be empty") + + test.validateOpenSchema(t, event) + }) + + test.WaitSignal(t, func() error { + cmd := cmdFunc("touch", []string{testFile2}, nil) + return cmd.Run() + }, func(event *model.Event, rule *rules.Rule) { + assertTriggeredRule(t, rule, "test_container_check_variable") + assertFieldEqual(t, event, "open.file.path", testFile2) + assertFieldNotEmpty(t, event, "container.id", "container id shouldn't be empty") + + test.validateOpenSchema(t, event) + }) + }) +} From 6d0ce2d61353b6fca280b60e157c06642515fc9c Mon Sep 17 00:00:00 2001 From: Alex Lopez Date: Wed, 18 Dec 2024 10:13:12 +0100 Subject: [PATCH 19/43] Revert "[ASCII-2587] Migrating TraceAgent to use IPC cert" (#32320) --- cmd/agent/subcommands/flare/command.go | 22 +++++++-------------- cmd/agent/subcommands/flare/command_test.go | 18 ++++------------- cmd/agent/subcommands/secret/command.go | 2 +- comp/trace/agent/impl/agent.go | 4 ---- comp/trace/agent/impl/run.go | 3 --- comp/trace/bundle_test.go | 4 ---- comp/trace/status/statusimpl/status.go | 2 +- pkg/config/fetcher/from_processes.go | 2 +- pkg/flare/archive.go | 2 +- pkg/trace/api/api_test.go | 18 +++-------------- pkg/trace/api/debug_server.go | 18 +++++------------ pkg/trace/info/info.go | 6 ++---- pkg/trace/info/info_test.go | 8 ++++---- 13 files changed, 29 insertions(+), 80 deletions(-) diff --git a/cmd/agent/subcommands/flare/command.go b/cmd/agent/subcommands/flare/command.go index a6da5391ae5d5..1b4fb2d3bc1e3 100644 --- a/cmd/agent/subcommands/flare/command.go +++ b/cmd/agent/subcommands/flare/command.go @@ -175,18 +175,10 @@ func readProfileData(seconds int) (flare.ProfileData, error) { type pprofGetter func(path string) ([]byte, error) - tcpGet := func(portConfig string, onHTTPS bool) pprofGetter { - endpoint := url.URL{ - Scheme: "http", - Host: net.JoinHostPort("127.0.0.1", strconv.Itoa(pkgconfigsetup.Datadog().GetInt(portConfig))), - Path: "/debug/pprof", - } - if onHTTPS { - endpoint.Scheme = "https" - } - + tcpGet := func(portConfig string) pprofGetter { + pprofURL := fmt.Sprintf("http://127.0.0.1:%d/debug/pprof", pkgconfigsetup.Datadog().GetInt(portConfig)) return func(path string) ([]byte, error) { - return util.DoGet(c, endpoint.String()+path, util.LeaveConnectionOpen) + return util.DoGet(c, pprofURL+path, util.LeaveConnectionOpen) } } @@ -236,15 +228,15 @@ func readProfileData(seconds int) (flare.ProfileData, error) { } agentCollectors := map[string]agentProfileCollector{ - "core": serviceProfileCollector(tcpGet("expvar_port", false), seconds), - "security-agent": serviceProfileCollector(tcpGet("security_agent.expvar_port", false), seconds), + "core": serviceProfileCollector(tcpGet("expvar_port"), seconds), + "security-agent": serviceProfileCollector(tcpGet("security_agent.expvar_port"), seconds), } if pkgconfigsetup.Datadog().GetBool("process_config.enabled") || pkgconfigsetup.Datadog().GetBool("process_config.container_collection.enabled") || pkgconfigsetup.Datadog().GetBool("process_config.process_collection.enabled") { - agentCollectors["process"] = serviceProfileCollector(tcpGet("process_config.expvar_port", false), seconds) + agentCollectors["process"] = serviceProfileCollector(tcpGet("process_config.expvar_port"), seconds) } if pkgconfigsetup.Datadog().GetBool("apm_config.enabled") { @@ -257,7 +249,7 @@ func readProfileData(seconds int) (flare.ProfileData, error) { traceCpusec = 4 } - agentCollectors["trace"] = serviceProfileCollector(tcpGet("apm_config.debug.port", true), traceCpusec) + agentCollectors["trace"] = serviceProfileCollector(tcpGet("apm_config.debug.port"), traceCpusec) } if pkgconfigsetup.SystemProbe().GetBool("system_probe_config.enabled") { diff --git a/cmd/agent/subcommands/flare/command_test.go b/cmd/agent/subcommands/flare/command_test.go index 4c96cae079aa9..a27304e95b133 100644 --- a/cmd/agent/subcommands/flare/command_test.go +++ b/cmd/agent/subcommands/flare/command_test.go @@ -29,7 +29,6 @@ type commandTestSuite struct { suite.Suite sysprobeSocketPath string tcpServer *httptest.Server - tcpTLSServer *httptest.Server unixServer *httptest.Server systemProbeServer *httptest.Server } @@ -43,17 +42,13 @@ func (c *commandTestSuite) SetupSuite() { // This should be called by each test that requires them. func (c *commandTestSuite) startTestServers() { t := c.T() - c.tcpServer, c.tcpTLSServer, c.unixServer, c.systemProbeServer = c.getPprofTestServer() + c.tcpServer, c.unixServer, c.systemProbeServer = c.getPprofTestServer() t.Cleanup(func() { if c.tcpServer != nil { c.tcpServer.Close() c.tcpServer = nil } - if c.tcpTLSServer != nil { - c.tcpTLSServer.Close() - c.tcpTLSServer = nil - } if c.unixServer != nil { c.unixServer.Close() c.unixServer = nil @@ -87,13 +82,12 @@ func newMockHandler() http.HandlerFunc { }) } -func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, tcpTLSServer *httptest.Server, unixServer *httptest.Server, sysProbeServer *httptest.Server) { +func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, unixServer *httptest.Server, sysProbeServer *httptest.Server) { var err error t := c.T() handler := newMockHandler() tcpServer = httptest.NewServer(handler) - tcpTLSServer = httptest.NewTLSServer(handler) if runtime.GOOS == "linux" { unixServer = httptest.NewUnstartedServer(handler) unixServer.Listener, err = net.Listen("unix", c.sysprobeSocketPath) @@ -107,7 +101,7 @@ func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, tcp sysProbeServer.Start() } - return tcpServer, tcpTLSServer, unixServer, sysProbeServer + return tcpServer, unixServer, sysProbeServer } func TestCommandTestSuite(t *testing.T) { @@ -122,14 +116,10 @@ func (c *commandTestSuite) TestReadProfileData() { require.NoError(t, err) port := u.Port() - u, err = url.Parse(c.tcpTLSServer.URL) - require.NoError(t, err) - httpsPort := u.Port() - mockConfig := configmock.New(t) mockConfig.SetWithoutSource("expvar_port", port) mockConfig.SetWithoutSource("apm_config.enabled", true) - mockConfig.SetWithoutSource("apm_config.debug.port", httpsPort) + mockConfig.SetWithoutSource("apm_config.debug.port", port) mockConfig.SetWithoutSource("apm_config.receiver_timeout", "10") mockConfig.SetWithoutSource("process_config.expvar_port", port) mockConfig.SetWithoutSource("security_agent.expvar_port", port) diff --git a/cmd/agent/subcommands/secret/command.go b/cmd/agent/subcommands/secret/command.go index d8b2a7048114f..e24f95ca6d693 100644 --- a/cmd/agent/subcommands/secret/command.go +++ b/cmd/agent/subcommands/secret/command.go @@ -101,7 +101,7 @@ func traceAgentSecretRefresh(conf config.Component) ([]byte, error) { c := apiutil.GetClient(false) c.Timeout = conf.GetDuration("server_timeout") * time.Second - url := fmt.Sprintf("https://127.0.0.1:%d/secret/refresh", port) + url := fmt.Sprintf("http://127.0.0.1:%d/secret/refresh", port) res, err := apiutil.DoGet(c, url, apiutil.CloseConnection) if err != nil { return nil, fmt.Errorf("could not contact trace-agent: %s", err) diff --git a/comp/trace/agent/impl/agent.go b/comp/trace/agent/impl/agent.go index de9237dedeea5..b3d783288ceb6 100644 --- a/comp/trace/agent/impl/agent.go +++ b/comp/trace/agent/impl/agent.go @@ -24,7 +24,6 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/fx" - "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/core/secrets" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" @@ -69,7 +68,6 @@ type dependencies struct { Statsd statsd.Component Tagger tagger.Component Compressor compression.Component - At authtoken.Component } var _ traceagent.Component = (*component)(nil) @@ -95,7 +93,6 @@ type component struct { params *Params tagger tagger.Component telemetryCollector telemetry.TelemetryCollector - at authtoken.Component wg *sync.WaitGroup } @@ -118,7 +115,6 @@ func NewAgent(deps dependencies) (traceagent.Component, error) { params: deps.Params, telemetryCollector: deps.TelemetryCollector, tagger: deps.Tagger, - at: deps.At, wg: &sync.WaitGroup{}, } statsdCl, err := setupMetrics(deps.Statsd, c.config, c.telemetryCollector) diff --git a/comp/trace/agent/impl/run.go b/comp/trace/agent/impl/run.go index f40b36e29ae7f..20ecbc6496a1b 100644 --- a/comp/trace/agent/impl/run.go +++ b/comp/trace/agent/impl/run.go @@ -98,9 +98,6 @@ func runAgentSidekicks(ag component) error { })) } - // Configure the Trace Agent Debug server to use the IPC certificate - ag.Agent.DebugServer.SetTLSConfig(ag.at.GetTLSServerConfig()) - log.Infof("Trace agent running on host %s", tracecfg.Hostname) if pcfg := profilingConfig(tracecfg); pcfg != nil { if err := profiling.Start(*pcfg); err != nil { diff --git a/comp/trace/bundle_test.go b/comp/trace/bundle_test.go index 692e7c255f473..e9874a4b40077 100644 --- a/comp/trace/bundle_test.go +++ b/comp/trace/bundle_test.go @@ -13,8 +13,6 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/fx" - "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl" - "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" coreconfig "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" @@ -47,7 +45,6 @@ func TestBundleDependencies(t *testing.T) { zstdfx.Module(), taggerfx.Module(tagger.Params{}), fx.Supply(&traceagentimpl.Params{}), - createandfetchimpl.Module(), ) } @@ -78,7 +75,6 @@ func TestMockBundleDependencies(t *testing.T) { fx.Invoke(func(_ traceagent.Component) {}), MockBundle(), taggerfx.Module(tagger.Params{}), - fetchonlyimpl.MockModule(), )) require.NotNil(t, cfg.Object()) diff --git a/comp/trace/status/statusimpl/status.go b/comp/trace/status/statusimpl/status.go index 00a8730b87da8..e476ee0281d7a 100644 --- a/comp/trace/status/statusimpl/status.go +++ b/comp/trace/status/statusimpl/status.go @@ -95,7 +95,7 @@ func (s statusProvider) populateStatus() map[string]interface{} { port := s.Config.GetInt("apm_config.debug.port") c := client() - url := fmt.Sprintf("https://localhost:%d/debug/vars", port) + url := fmt.Sprintf("http://localhost:%d/debug/vars", port) resp, err := apiutil.DoGet(c, url, apiutil.CloseConnection) if err != nil { return map[string]interface{}{ diff --git a/pkg/config/fetcher/from_processes.go b/pkg/config/fetcher/from_processes.go index 95c24e283fb46..5b8088f41d970 100644 --- a/pkg/config/fetcher/from_processes.go +++ b/pkg/config/fetcher/from_processes.go @@ -71,7 +71,7 @@ func TraceAgentConfig(config config.Reader) (string, error) { c := util.GetClient(false) c.Timeout = config.GetDuration("server_timeout") * time.Second - ipcAddressWithPort := fmt.Sprintf("https://127.0.0.1:%d/config", port) + ipcAddressWithPort := fmt.Sprintf("http://127.0.0.1:%d/config", port) client := settingshttp.NewClient(c, ipcAddressWithPort, "trace-agent", settingshttp.NewHTTPClientOptions(util.CloseConnection)) return client.FullConfig() diff --git a/pkg/flare/archive.go b/pkg/flare/archive.go index b61c6d9dd6ed2..c3e5dffe00ca8 100644 --- a/pkg/flare/archive.go +++ b/pkg/flare/archive.go @@ -214,7 +214,7 @@ func getExpVar(fb flaretypes.FlareBuilder) error { apmDebugPort := pkgconfigsetup.Datadog().GetInt("apm_config.debug.port") f := filepath.Join("expvar", "trace-agent") - resp, err := http.Get(fmt.Sprintf("https://127.0.0.1:%d/debug/vars", apmDebugPort)) + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/debug/vars", apmDebugPort)) if err != nil { return fb.AddFile(f, []byte(fmt.Sprintf("Error retrieving vars: %v", err))) } diff --git a/pkg/trace/api/api_test.go b/pkg/trace/api/api_test.go index 3f3b2c5b3a20b..9a468dc164487 100644 --- a/pkg/trace/api/api_test.go +++ b/pkg/trace/api/api_test.go @@ -1039,26 +1039,14 @@ func TestExpvar(t *testing.T) { } c := newTestReceiverConfig() - c.DebugServerPort = 6789 + c.DebugServerPort = 5012 info.InitInfo(c) - - // Starting a TLS httptest server to retrieve tlsCert - ts := httptest.NewTLSServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) - tlsConfig := ts.TLS.Clone() - // Setting a client with the proper TLS configuration - client := ts.Client() - ts.Close() - - // Starting Debug Server s := NewDebugServer(c) - s.SetTLSConfig(tlsConfig) - - // Starting the Debug server s.Start() defer s.Stop() - resp, err := client.Get(fmt.Sprintf("https://127.0.0.1:%d/debug/vars", c.DebugServerPort)) - require.NoError(t, err) + resp, err := http.Get("http://127.0.0.1:5012/debug/vars") + assert.NoError(t, err) defer resp.Body.Close() t.Run("read-expvars", func(t *testing.T) { diff --git a/pkg/trace/api/debug_server.go b/pkg/trace/api/debug_server.go index 6fd2f39cc011a..828d5357330eb 100644 --- a/pkg/trace/api/debug_server.go +++ b/pkg/trace/api/debug_server.go @@ -9,7 +9,6 @@ package api import ( "context" - "crypto/tls" "expvar" "fmt" "net" @@ -30,10 +29,9 @@ const ( // DebugServer serves /debug/* endpoints type DebugServer struct { - conf *config.AgentConfig - server *http.Server - mux *http.ServeMux - tlsConfig *tls.Config + conf *config.AgentConfig + server *http.Server + mux *http.ServeMux } // NewDebugServer returns a debug server @@ -55,14 +53,13 @@ func (ds *DebugServer) Start() { WriteTimeout: defaultTimeout, Handler: ds.setupMux(), } - listener, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(ds.conf.DebugServerPort))) + listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", ds.conf.DebugServerPort)) if err != nil { log.Errorf("Error creating debug server listener: %s", err) return } - tlsListener := tls.NewListener(listener, ds.tlsConfig) go func() { - if err := ds.server.Serve(tlsListener); err != nil && err != http.ErrServerClosed { + if err := ds.server.Serve(listener); err != nil && err != http.ErrServerClosed { log.Errorf("Could not start debug server: %s. Debug server disabled.", err) } }() @@ -85,11 +82,6 @@ func (ds *DebugServer) AddRoute(route string, handler http.Handler) { ds.mux.Handle(route, handler) } -// SetTLSConfig adds the provided tls.Config to the internal http.Server -func (ds *DebugServer) SetTLSConfig(config *tls.Config) { - ds.tlsConfig = config -} - func (ds *DebugServer) setupMux() *http.ServeMux { ds.mux.HandleFunc("/debug/pprof/", pprof.Index) ds.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) diff --git a/pkg/trace/info/info.go b/pkg/trace/info/info.go index 4198100ee6a5b..df86c40587fdc 100644 --- a/pkg/trace/info/info.go +++ b/pkg/trace/info/info.go @@ -8,7 +8,6 @@ package info import ( "bytes" - "crypto/tls" "encoding/json" "expvar" // automatically publish `/debug/vars` on HTTP port "fmt" @@ -237,9 +236,8 @@ func getProgramBanner(version string) (string, string) { // If error is nil, means the program is running. // If not, it displays a pretty-printed message anyway (for support) func Info(w io.Writer, conf *config.AgentConfig) error { - url := fmt.Sprintf("https://127.0.0.1:%d/debug/vars", conf.DebugServerPort) - tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} - client := http.Client{Timeout: 3 * time.Second, Transport: tr} + url := fmt.Sprintf("http://127.0.0.1:%d/debug/vars", conf.DebugServerPort) + client := http.Client{Timeout: 3 * time.Second} resp, err := client.Get(url) if err != nil { // OK, here, we can't even make an http call on the agent port, diff --git a/pkg/trace/info/info_test.go b/pkg/trace/info/info_test.go index 25b7cb42b13ff..01c369cac403e 100644 --- a/pkg/trace/info/info_test.go +++ b/pkg/trace/info/info_test.go @@ -63,7 +63,7 @@ func (h *testServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func testServer(t *testing.T, testFile string) *httptest.Server { t.Helper() - server := httptest.NewTLSServer(&testServerHandler{t: t, testFile: testFile}) + server := httptest.NewServer(&testServerHandler{t: t, testFile: testFile}) t.Logf("test server (serving fake yet valid data) listening on %s", server.URL) return server } @@ -94,7 +94,7 @@ func (h *testServerWarningHandler) ServeHTTP(w http.ResponseWriter, r *http.Requ } func testServerWarning(t *testing.T) *httptest.Server { - server := httptest.NewTLSServer(&testServerWarningHandler{t: t}) + server := httptest.NewServer(&testServerWarningHandler{t: t}) t.Logf("test server (serving data containing worrying values) listening on %s", server.URL) return server } @@ -119,7 +119,7 @@ func (h *testServerErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques } func testServerError(t *testing.T) *httptest.Server { - server := httptest.NewTLSServer(&testServerErrorHandler{t: t}) + server := httptest.NewServer(&testServerErrorHandler{t: t}) t.Logf("test server (serving bad data to trigger errors) listening on %s", server.URL) return server } @@ -331,7 +331,7 @@ func TestError(t *testing.T) { assert.Equal(len(lines[1]), len(lines[2])) assert.Equal("", lines[3]) assert.Regexp(regexp.MustCompile(`^ Error: .*$`), lines[4]) - assert.Equal(fmt.Sprintf(" URL: https://127.0.0.1:%d/debug/vars", port), lines[5]) + assert.Equal(fmt.Sprintf(" URL: http://127.0.0.1:%d/debug/vars", port), lines[5]) assert.Equal("", lines[6]) assert.Equal("", lines[7]) } From 6caf7d7f3a2286388a7bfbc08e72f7558932c2a0 Mon Sep 17 00:00:00 2001 From: Yoann Ghigoff Date: Wed, 18 Dec 2024 10:58:15 +0100 Subject: [PATCH 20/43] Commit data when writing `auth_token` file (#32244) --- pkg/api/security/security.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/api/security/security.go b/pkg/api/security/security.go index 40b1310c2f79c..93867572572d0 100644 --- a/pkg/api/security/security.go +++ b/pkg/api/security/security.go @@ -239,7 +239,7 @@ func validateAuthToken(authToken string) error { // writes auth token(s) to a file with the same permissions as datadog.yaml func saveAuthToken(token, tokenPath string) error { log.Infof("Saving a new authentication token in %s", tokenPath) - if err := os.WriteFile(tokenPath, []byte(token), 0o600); err != nil { + if err := writeFileAndSync(tokenPath, []byte(token), 0o600); err != nil { return err } @@ -256,3 +256,21 @@ func saveAuthToken(token, tokenPath string) error { log.Infof("Wrote auth token in %s", tokenPath) return nil } + +// call the file.Sync method to flush the file to disk and catch potential I/O errors +func writeFileAndSync(name string, data []byte, perm os.FileMode) error { + f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + _, err = f.Write(data) + // only sync data if the write was successful + if err == nil { + err = f.Sync() + } + // but always close the file and report the first error that occurred + if err1 := f.Close(); err1 != nil && err == nil { + err = err1 + } + return err +} From 48763aacf6fe5a19d9d4c8c29be6cd9872b1fb65 Mon Sep 17 00:00:00 2001 From: Kevin Fairise <132568982+KevinFairise2@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:04:32 +0100 Subject: [PATCH 21/43] Make go.mod files unowned by default (#32121) Co-authored-by: Pierre Gimalac --- .github/CODEOWNERS | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8c82bee16308e..ec5ac92f6a2e5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -271,6 +271,7 @@ /go.mod # do not notify anyone /go.sum # do not notify anyone + /Makefile.trace @DataDog/agent-apm /omnibus/ @DataDog/agent-delivery @@ -647,3 +648,16 @@ /internal/third_party/client-go @DataDog/container-platform /internal/third_party/kubernetes @DataDog/container-integrations /internal/third_party/golang/ @DataDog/container-integrations + +# With the introduction of go.work, dependencies bump modify go.mod and go.sum in a lot of file. +# Which bring a lot of team in the review each time. To make it smoother we no longer consider go.mod and go.sum owned by teams. +# Each team can individually decide to update CODEOWNERS to be requested for a review on each modification of their go.mod/sum +/**/go.mod # do not notify anyone +/**/go.sum # do not notify anyone + +# Add here modules that need explicit review from the team owning them +/internal/tools/**/go.mod @DataDog/agent-devx-loops +/internal/tools/**/go.sum @DataDog/agent-devx-loops + +/pkg/util/scrubber/go.mod @DataDog/agent-shared-components +/pkg/util/scrubber/go.sum @DataDog/agent-shared-components From 459aa24aad5a8903db6b58266b7497a7ecf33509 Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Wed, 18 Dec 2024 11:33:38 +0100 Subject: [PATCH 22/43] Stop lint and test tasks on keyboard interrupt (#32201) --- tasks/go.py | 7 ++++++- tasks/gotest.py | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tasks/go.py b/tasks/go.py index 13f451cce050f..5bb3fcad7d656 100644 --- a/tasks/go.py +++ b/tasks/go.py @@ -78,11 +78,16 @@ def lint_module(target): concurrency_arg = "" if concurrency is None else f"--concurrency {concurrency}" tags_arg = " ".join(sorted(set(tags))) timeout_arg_value = "25m0s" if not timeout else f"{timeout}m0s" - return ctx.run( + res = ctx.run( f'golangci-lint run {verbosity} --timeout {timeout_arg_value} {concurrency_arg} --build-tags "{tags_arg}" --path-prefix "{module_path}" {golangci_lint_kwargs} {target}/...', env=env, warn=True, ) + # early stop on SIGINT: exit code is 128 + signal number, SIGINT is 2, so 130 + # for some reason this becomes -2 here + if res is not None and (res.exited == -2 or res.exited == 130): + raise KeyboardInterrupt() + return res target_path = Path(module_path) / target result, time_result = TimedOperationResult.run( diff --git a/tasks/gotest.py b/tasks/gotest.py index 4aa9a4b3e151f..af889e8b6dcae 100644 --- a/tasks/gotest.py +++ b/tasks/gotest.py @@ -145,6 +145,9 @@ def command(test_results, module, module_result): out_stream=test_profiler, warn=True, ) + # early stop on SIGINT: exit code is 128 + signal number, SIGINT is 2, so 130 + if res is not None and res.exited == 130: + raise KeyboardInterrupt() module_result.result_json_path = os.path.join(module_path, GO_TEST_RESULT_TMP_JSON) From d598917c9843de41c0178306f41a91705f2ed6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Wed, 18 Dec 2024 12:09:48 +0100 Subject: [PATCH 23/43] Fix `pulumi.DependsOn` in `test-infra-definitions` (#31393) --- .gitlab/common/test_infra_version.yml | 2 +- test/new-e2e/go.mod | 2 +- test/new-e2e/go.sum | 4 +- test/new-e2e/tests/containers/k8s_test.go | 105 +--------------------- 4 files changed, 8 insertions(+), 105 deletions(-) diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index 71934be35d15a..9008ad3e857b1 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -4,4 +4,4 @@ variables: # and check the job creating the image to make sure you have the right SHA prefix TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 4b4112f5f64d + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 6459608ed9fa diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index 2bd2138c3ea69..92faae31be87d 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -58,7 +58,7 @@ require ( // `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version // Example: github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB // => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB - github.com/DataDog/test-infra-definitions v0.0.0-20241217110507-4b4112f5f64d + github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa github.com/aws/aws-sdk-go-v2 v1.32.6 github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index b08902fb2e378..e922c7c7a56b9 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -17,8 +17,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= -github.com/DataDog/test-infra-definitions v0.0.0-20241217110507-4b4112f5f64d h1:F1yqGiWtXVsHkMiNUhs8bgaoZ1WlV3rYGRZ1CtCxpm8= -github.com/DataDog/test-infra-definitions v0.0.0-20241217110507-4b4112f5f64d/go.mod h1:1PAUwGjC25ACjfft4HrLEmHliuajlvjzcLFWpuqAIyk= +github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa h1:l8KLWgU9l2qTlMtu4ing3V6PptTO+suaU8zusc45IiM= +github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa/go.mod h1:1PAUwGjC25ACjfft4HrLEmHliuajlvjzcLFWpuqAIyk= github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= diff --git a/test/new-e2e/tests/containers/k8s_test.go b/test/new-e2e/tests/containers/k8s_test.go index ea2e14c8d3742..9561b71fb7a26 100644 --- a/test/new-e2e/tests/containers/k8s_test.go +++ b/test/new-e2e/tests/containers/k8s_test.go @@ -827,7 +827,7 @@ func (suite *k8sSuite) TestDogstatsdInAgent() { // Test with UDP + DD_ENTITY_ID suite.testDogstatsd(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDP) // Test with UDP + External Data - suite.testDogstatsdExternalData(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDPExternalData) + suite.testDogstatsd(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDPExternalData) } func (suite *k8sSuite) TestDogstatsdStandalone() { @@ -873,78 +873,6 @@ func (suite *k8sSuite) testDogstatsd(kubeNamespace, kubeDeployment string) { }) } -// testDogstatsdExternalData tests that the External Data origin resolution works for the dogstatsd. -func (suite *k8sSuite) testDogstatsdExternalData(kubeNamespace, kubeDeployment string) { - ctx := context.Background() - - // Record old pod, so we can be sure we are not looking at the incorrect one after deletion - oldPods, err := suite.K8sClient.CoreV1().Pods(kubeNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", kubeDeployment).String(), - }) - suite.Require().NoError(err) - suite.Require().Len(oldPods.Items, 1) - oldPod := oldPods.Items[0] - - // Delete the pod to ensure it is recreated after the admission controller is deployed - err = suite.K8sClient.CoreV1().Pods(kubeNamespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", kubeDeployment).String(), - }) - suite.Require().NoError(err) - - // Wait for the fresh pod to be created - var pod corev1.Pod - suite.Require().EventuallyWithTf(func(c *assert.CollectT) { - pods, err := suite.K8sClient.CoreV1().Pods(kubeNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", kubeDeployment).String(), - }) - if !assert.NoError(c, err) { - return - } - if !assert.Len(c, pods.Items, 1) { - return - } - pod = pods.Items[0] - if !assert.NotEqual(c, oldPod.Name, pod.Name) { - return - } - }, 2*time.Minute, 10*time.Second, "Failed to witness the creation of pod with name %s in namespace %s", kubeDeployment, kubeNamespace) - - suite.Require().Len(pod.Spec.Containers, 1) - - suite.testMetric(&testMetricArgs{ - Filter: testMetricFilterArgs{ - Name: "custom.metric", - Tags: []string{ - "^kube_deployment:" + regexp.QuoteMeta(kubeDeployment) + "$", - "^kube_namespace:" + regexp.QuoteMeta(kubeNamespace) + "$", - }, - }, - Expect: testMetricExpectArgs{ - Tags: &[]string{ - `^container_id:`, - `^container_name:dogstatsd$`, - `^display_container_name:dogstatsd`, - `^git.commit.sha:`, // org.opencontainers.image.revision docker image label - `^git.repository_url:https://github.com/DataDog/test-infra-definitions$`, // org.opencontainers.image.source docker image label - `^image_id:ghcr.io/datadog/apps-dogstatsd@sha256:`, - `^image_name:ghcr.io/datadog/apps-dogstatsd$`, - `^image_tag:main$`, - `^kube_container_name:dogstatsd$`, - `^kube_deployment:` + regexp.QuoteMeta(kubeDeployment) + `$`, - "^kube_namespace:" + regexp.QuoteMeta(kubeNamespace) + "$", - `^kube_ownerref_kind:replicaset$`, - `^kube_ownerref_name:` + regexp.QuoteMeta(kubeDeployment) + `-[[:alnum:]]+$`, - `^kube_qos:Burstable$`, - `^kube_replica_set:` + regexp.QuoteMeta(kubeDeployment) + `-[[:alnum:]]+$`, - `^pod_name:` + regexp.QuoteMeta(kubeDeployment) + `-[[:alnum:]]+-[[:alnum:]]+$`, - `^pod_phase:running$`, - `^series:`, - `^short_image:apps-dogstatsd$`, - }, - }, - }) -} - func (suite *k8sSuite) TestPrometheus() { // Test Prometheus check suite.testMetric(&testMetricArgs{ @@ -1020,37 +948,12 @@ func (suite *k8sSuite) testAdmissionControllerPod(namespace string, name string, }, 5*time.Minute, 10*time.Second, "The deployment with name %s in namespace %s does not exist or does not have the auto detected languages annotation", name, namespace) } - // Record old pod, so we can be sure we are not looking at the incorrect one after deletion - oldPods, err := suite.K8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", name).String(), - }) - suite.Require().NoError(err) - suite.Require().Len(oldPods.Items, 1) - oldPod := oldPods.Items[0] - - // Delete the pod to ensure it is recreated after the admission controller is deployed - err = suite.K8sClient.CoreV1().Pods(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ + pods, err := suite.K8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ LabelSelector: fields.OneTermEqualSelector("app", name).String(), }) suite.Require().NoError(err) - - // Wait for the fresh pod to be created - var pod corev1.Pod - suite.Require().EventuallyWithTf(func(c *assert.CollectT) { - pods, err := suite.K8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", name).String(), - }) - if !assert.NoError(c, err) { - return - } - if !assert.Len(c, pods.Items, 1) { - return - } - pod = pods.Items[0] - if !assert.NotEqual(c, oldPod.Name, pod.Name) { - return - } - }, 2*time.Minute, 10*time.Second, "Failed to witness the creation of pod with name %s in namespace %s", name, namespace) + suite.Require().Len(pods.Items, 1) + pod := pods.Items[0] suite.Require().Len(pod.Spec.Containers, 1) From 0bda4481f31fd183a6a2fe333b7375a26720ea91 Mon Sep 17 00:00:00 2001 From: louis-cqrl <93274433+louis-cqrl@users.noreply.github.com> Date: Wed, 18 Dec 2024 12:34:22 +0100 Subject: [PATCH 24/43] Add test to ensure LastUpdated is well populated for default scrubber rules (#32322) --- pkg/util/scrubber/default_test.go | 4 ++++ pkg/util/scrubber/scrubber_test.go | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/pkg/util/scrubber/default_test.go b/pkg/util/scrubber/default_test.go index 7c093d9e40300..0bb2d721c3983 100644 --- a/pkg/util/scrubber/default_test.go +++ b/pkg/util/scrubber/default_test.go @@ -474,6 +474,8 @@ func TestAddStrippedKeys(t *testing.T) { AddStrippedKeys([]string{"foobar"}) assertClean(t, contents, `foobar: "********"`) + + dynamicReplacers = []Replacer{} } func TestAddStrippedKeysNewReplacer(t *testing.T) { @@ -486,6 +488,8 @@ func TestAddStrippedKeysNewReplacer(t *testing.T) { cleaned, err := newScrubber.ScrubBytes([]byte(contents)) require.NoError(t, err) assert.Equal(t, strings.TrimSpace(`foobar: "********"`), strings.TrimSpace(string(cleaned))) + + dynamicReplacers = []Replacer{} } func TestCertConfig(t *testing.T) { diff --git a/pkg/util/scrubber/scrubber_test.go b/pkg/util/scrubber/scrubber_test.go index dc23ec52d1971..5a6245d81cddd 100644 --- a/pkg/util/scrubber/scrubber_test.go +++ b/pkg/util/scrubber/scrubber_test.go @@ -28,6 +28,16 @@ func TestNewWithDefaults(t *testing.T) { assert.Equal(t, len(scrubberEmpty.multiLineReplacers), len(scrubber.multiLineReplacers)) } +func TestLastUpdated(t *testing.T) { + scrubber := NewWithDefaults() + for _, replacer := range scrubber.singleLineReplacers { + assert.NotNil(t, replacer.LastUpdated, "single line replacer has no LastUpdated: %v", replacer) + } + for _, replacer := range scrubber.multiLineReplacers { + assert.NotNil(t, replacer.LastUpdated, "multi line replacer has no LastUpdated: %v", replacer) + } +} + func TestRepl(t *testing.T) { scrubber := New() scrubber.AddReplacer(SingleLine, Replacer{ From 06bdd8baed5d856fc32498533b451da20d993665 Mon Sep 17 00:00:00 2001 From: Adel Haj Hassan <41540817+adel121@users.noreply.github.com> Date: Wed, 18 Dec 2024 12:43:20 +0100 Subject: [PATCH 25/43] fix unit-test bug in leaderelection subscription unit test (#32324) --- .../apiserver/leaderelection/leaderelection_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go index af604ec84e37c..eefce053e2e22 100644 --- a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go +++ b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go @@ -211,7 +211,7 @@ func TestSubscribe(t *testing.T) { coreClient: client.CoreV1(), coordClient: client.CoordinationV1(), leaderMetric: &dummyGauge{}, - lockType: cmLock.ConfigMapsResourceLock, + lockType: tc.lockType, } notif1 := le.Subscribe() @@ -227,6 +227,9 @@ func TestSubscribe(t *testing.T) { le.EnsureLeaderElectionRuns() require.True(t, le.IsLeader()) + err = tc.getTokenFunc(client) + require.NoError(t, err) + counter1, counter2 := 0, 0 for { select { From 2470d047b2cb6a37d546c5886d2ad5577b08a4f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Wed, 18 Dec 2024 14:26:20 +0100 Subject: [PATCH 26/43] [EBPF] gpu: query k8s device -> container mapping (#32020) --- cmd/system-probe/modules/gpu.go | 1 + pkg/ebpf/uprobes/testutil.go | 21 +++- pkg/gpu/consumer.go | 2 +- pkg/gpu/consumer_test.go | 2 +- pkg/gpu/context.go | 79 +++++++++++++- pkg/gpu/context_test.go | 176 ++++++++++++++++++++++++++++++++ pkg/gpu/probe.go | 7 +- pkg/gpu/probe_stub.go | 2 + pkg/gpu/probe_test.go | 1 + pkg/gpu/stats_test.go | 2 +- pkg/gpu/stream_test.go | 30 +++--- pkg/gpu/testutil/mocks.go | 77 +++++++++++++- 12 files changed, 367 insertions(+), 33 deletions(-) create mode 100644 pkg/gpu/context_test.go diff --git a/cmd/system-probe/modules/gpu.go b/cmd/system-probe/modules/gpu.go index 7ec273e7c2869..c7dbac874dccd 100644 --- a/cmd/system-probe/modules/gpu.go +++ b/cmd/system-probe/modules/gpu.go @@ -55,6 +55,7 @@ var GPUMonitoring = module.Factory{ //(https://github.com/NVIDIA/go-nvml/blob/main/pkg/nvml/lib.go#L30) NvmlLib: nvml.New(nvml.WithLibraryPath(c.NVMLLibraryPath)), ProcessMonitor: processEventConsumer, + WorkloadMeta: deps.WMeta, } ret := probeDeps.NvmlLib.Init() diff --git a/pkg/ebpf/uprobes/testutil.go b/pkg/ebpf/uprobes/testutil.go index 93b109b499aa3..2d4e8b85d4b25 100644 --- a/pkg/ebpf/uprobes/testutil.go +++ b/pkg/ebpf/uprobes/testutil.go @@ -13,6 +13,7 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "sync" "testing" "time" @@ -132,6 +133,21 @@ type FakeProcFSEntry struct { Command string Exe string Maps string + Env map[string]string +} + +// getEnvironContents returns the formatted contents of the /proc//environ file for the entry. +func (f *FakeProcFSEntry) getEnvironContents() string { + if len(f.Env) == 0 { + return "" + } + + formattedEnvVars := make([]string, 0, len(f.Env)) + for k, v := range f.Env { + formattedEnvVars = append(formattedEnvVars, fmt.Sprintf("%s=%s", k, v)) + } + + return strings.Join(formattedEnvVars, "\x00") + "\x00" } // CreateFakeProcFS creates a fake /proc filesystem with the given entries, useful for testing attachment to processes. @@ -145,16 +161,13 @@ func CreateFakeProcFS(t *testing.T, entries []FakeProcFSEntry) string { createFile(t, filepath.Join(baseDir, "comm"), entry.Command) createFile(t, filepath.Join(baseDir, "maps"), entry.Maps) createSymlink(t, entry.Exe, filepath.Join(baseDir, "exe")) + createFile(t, filepath.Join(baseDir, "environ"), entry.getEnvironContents()) } return procRoot } func createFile(t *testing.T, path, data string) { - if data == "" { - return - } - dir := filepath.Dir(path) require.NoError(t, os.MkdirAll(dir, 0775)) require.NoError(t, os.WriteFile(path, []byte(data), 0775)) diff --git a/pkg/gpu/consumer.go b/pkg/gpu/consumer.go index f4ebb7f04b0f2..55148350b12e3 100644 --- a/pkg/gpu/consumer.go +++ b/pkg/gpu/consumer.go @@ -220,7 +220,7 @@ func (c *cudaEventConsumer) getStreamKey(header *gpuebpf.CudaEventHeader) stream // Try to get the GPU device if we can, but do not fail if we can't as we want to report // the data even if we can't get the GPU UUID - gpuDevice, err := c.sysCtx.getCurrentActiveGpuDevice(int(pid), int(tid)) + gpuDevice, err := c.sysCtx.getCurrentActiveGpuDevice(int(pid), int(tid), containerID) if err != nil { log.Warnf("Error getting GPU device for process %d: %v", pid, err) } else { diff --git a/pkg/gpu/consumer_test.go b/pkg/gpu/consumer_test.go index 6530f2bb49738..c7c7aa04837d7 100644 --- a/pkg/gpu/consumer_test.go +++ b/pkg/gpu/consumer_test.go @@ -22,7 +22,7 @@ import ( func TestConsumerCanStartAndStop(t *testing.T) { handler := ddebpf.NewRingBufferHandler(consumerChannelSize) cfg := config.New() - ctx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) + ctx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) consumer := newCudaEventConsumer(ctx, handler, cfg) diff --git a/pkg/gpu/context.go b/pkg/gpu/context.go index 77bac5185339f..59d778bfea5c1 100644 --- a/pkg/gpu/context.go +++ b/pkg/gpu/context.go @@ -16,10 +16,15 @@ import ( "github.com/NVIDIA/go-nvml/pkg/nvml" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/gpu/cuda" "github.com/DataDog/datadog-agent/pkg/util/ktime" + "github.com/DataDog/datadog-agent/pkg/util/log" ) +const nvidiaResourceName = "nvidia.com/gpu" + // systemContext holds certain attributes about the system that are used by the GPU probe. type systemContext struct { // maxGpuThreadsPerDevice maps each device index to the maximum number of threads it can run in parallel @@ -60,6 +65,9 @@ type systemContext struct { // visibleDevicesCache is a cache of visible devices for each process, to avoid // looking into the environment variables every time visibleDevicesCache map[int][]nvml.Device + + // workloadmeta is the workloadmeta component that we use to get necessary container metadata + workloadmeta workloadmeta.Component } // symbolsEntry embeds cuda.Symbols adding a field for keeping track of the last @@ -73,7 +81,7 @@ func (e *symbolsEntry) updateLastUsedTime() { e.lastUsedTime = time.Now() } -func getSystemContext(nvmlLib nvml.Interface, procRoot string) (*systemContext, error) { +func getSystemContext(nvmlLib nvml.Interface, procRoot string, wmeta workloadmeta.Component) (*systemContext, error) { ctx := &systemContext{ maxGpuThreadsPerDevice: make(map[int]int), deviceSmVersions: make(map[int]int), @@ -83,6 +91,7 @@ func getSystemContext(nvmlLib nvml.Interface, procRoot string) (*systemContext, procRoot: procRoot, selectedDeviceByPIDAndTID: make(map[int]map[int]int32), visibleDevicesCache: make(map[int][]nvml.Device), + workloadmeta: wmeta, } if err := ctx.fillDeviceInfo(); err != nil { @@ -209,13 +218,75 @@ func (ctx *systemContext) cleanupOldEntries() { } } +// filterDevicesForContainer filters the available GPU devices for the given +// container. If the ID is not empty, we check the assignment of GPU resources +// to the container and return only the devices that are available to the +// container. +func (ctx *systemContext) filterDevicesForContainer(devices []nvml.Device, containerID string) ([]nvml.Device, error) { + if containerID == "" { + // If the process is not running in a container, we assume all devices are available. + return devices, nil + } + + container, err := ctx.workloadmeta.GetContainer(containerID) + if err != nil { + // If we don't find the container, we assume all devices are available. + // This can happen sometimes, e.g. if we don't have the container in the + // store yet. Do not block metrics on that. + if errors.IsNotFound(err) { + return devices, nil + } + + // Some error occurred while retrieving the container, this could be a + // general error with the store, report it. + return nil, fmt.Errorf("cannot retrieve data for container %s: %s", containerID, err) + } + + var filteredDevices []nvml.Device + for _, resource := range container.AllocatedResources { + // Only consider NVIDIA GPUs + if resource.Name != nvidiaResourceName { + continue + } + + for _, device := range devices { + uuid, ret := device.GetUUID() + if ret != nvml.SUCCESS { + log.Warnf("Error getting GPU UUID for device %s: %s", device, nvml.ErrorString(ret)) + continue + } + + if resource.ID == uuid { + filteredDevices = append(filteredDevices, device) + break + } + } + } + + // We didn't find any devices assigned to the container, report it as an error. + if len(filteredDevices) == 0 { + return nil, fmt.Errorf("no GPU devices found for container %s that matched its allocated resources %+v", containerID, container.AllocatedResources) + } + + return filteredDevices, nil +} + // getCurrentActiveGpuDevice returns the active GPU device for a given process and thread, based on the // last selection (via cudaSetDevice) this thread made and the visible devices for the process. -func (ctx *systemContext) getCurrentActiveGpuDevice(pid int, tid int) (nvml.Device, error) { +func (ctx *systemContext) getCurrentActiveGpuDevice(pid int, tid int, containerID string) (nvml.Device, error) { visibleDevices, ok := ctx.visibleDevicesCache[pid] if !ok { - var err error - visibleDevices, err = cuda.GetVisibleDevicesForProcess(ctx.gpuDevices, pid, ctx.procRoot) + // Order is important! We need to filter the devices for the container + // first. In a container setting, the environment variable acts as a + // filter on the devices that are available to the process, not on the + // devices available on the host system. + var err error // avoid shadowing visibleDevices, declare error before so we can use = instead of := + visibleDevices, err = ctx.filterDevicesForContainer(ctx.gpuDevices, containerID) + if err != nil { + return nil, fmt.Errorf("error filtering devices for container %s: %w", containerID, err) + } + + visibleDevices, err = cuda.GetVisibleDevicesForProcess(visibleDevices, pid, ctx.procRoot) if err != nil { return nil, fmt.Errorf("error getting visible devices for process %d: %w", pid, err) } diff --git a/pkg/gpu/context_test.go b/pkg/gpu/context_test.go new file mode 100644 index 0000000000000..dbb6e1735df24 --- /dev/null +++ b/pkg/gpu/context_test.go @@ -0,0 +1,176 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux_bpf + +package gpu + +import ( + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/ebpf/uprobes" + "github.com/DataDog/datadog-agent/pkg/gpu/testutil" + "github.com/DataDog/datadog-agent/pkg/util/kernel" +) + +func TestFilterDevicesForContainer(t *testing.T) { + wmetaMock := testutil.GetWorkloadMetaMock(t) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), wmetaMock) + require.NotNil(t, sysCtx) + require.NoError(t, err) + + // Create a container with a single GPU and add it to the store + containerID := "abcdef" + deviceIndex := 2 + gpuUUID := testutil.GPUUUIDs[deviceIndex] + container := &workloadmeta.Container{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindContainer, + ID: containerID, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: containerID, + }, + AllocatedResources: []workloadmeta.ContainerAllocatedResource{ + { + Name: nvidiaResourceName, + ID: gpuUUID, + }, + }, + } + + wmetaMock.Set(container) + storeContainer, err := wmetaMock.GetContainer(containerID) + require.NoError(t, err, "container should be found in the store") + require.NotNil(t, storeContainer, "container should be found in the store") + + t.Run("NoContainer", func(t *testing.T) { + filtered, err := sysCtx.filterDevicesForContainer(sysCtx.gpuDevices, "") + require.NoError(t, err) + testutil.RequireDeviceListsEqual(t, filtered, sysCtx.gpuDevices) // With no container, all devices should be returned + }) + + t.Run("NonExistentContainer", func(t *testing.T) { + filtered, err := sysCtx.filterDevicesForContainer(sysCtx.gpuDevices, "non-existent-at-all") + require.NoError(t, err) + testutil.RequireDeviceListsEqual(t, filtered, sysCtx.gpuDevices) // If we can't find the container, all devices should be returned + }) + + t.Run("ContainerWithGPU", func(t *testing.T) { + filtered, err := sysCtx.filterDevicesForContainer(sysCtx.gpuDevices, containerID) + require.NoError(t, err) + require.Len(t, filtered, 1) + testutil.RequireDeviceListsEqual(t, filtered, sysCtx.gpuDevices[deviceIndex:deviceIndex+1]) + }) +} + +func TestGetCurrentActiveGpuDevice(t *testing.T) { + pidNoContainer := 1234 + pidNoContainerButEnv := 2235 + pidContainer := 3238 + pidContainerAndEnv := 3239 + + envVisibleDevices := []int32{1, 2, 3} + envVisibleDevicesStr := make([]string, len(envVisibleDevices)) + for i, idx := range envVisibleDevices { + envVisibleDevicesStr[i] = strconv.Itoa(int(idx)) + } + envVisibleDevicesValue := strings.Join(envVisibleDevicesStr, ",") + + procFs := uprobes.CreateFakeProcFS(t, []uprobes.FakeProcFSEntry{ + {Pid: uint32(pidNoContainer)}, + {Pid: uint32(pidContainer)}, + {Pid: uint32(pidContainerAndEnv), Env: map[string]string{"CUDA_VISIBLE_DEVICES": envVisibleDevicesValue}}, + {Pid: uint32(pidNoContainerButEnv), Env: map[string]string{"CUDA_VISIBLE_DEVICES": envVisibleDevicesValue}}, + }) + + wmetaMock := testutil.GetWorkloadMetaMock(t) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), procFs, wmetaMock) + require.NotNil(t, sysCtx) + require.NoError(t, err) + + // Create a container with a single GPU and add it to the store + containerID := "abcdef" + container := &workloadmeta.Container{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindContainer, + ID: containerID, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: containerID, + }, + } + + containerDeviceIndexes := []int32{1, 2, 3, 4} + for _, idx := range containerDeviceIndexes { + gpuUUID := testutil.GPUUUIDs[idx] + resource := workloadmeta.ContainerAllocatedResource{ + Name: nvidiaResourceName, + ID: gpuUUID, + } + container.AllocatedResources = append(container.AllocatedResources, resource) + } + + wmetaMock.Set(container) + storeContainer, err := wmetaMock.GetContainer(containerID) + require.NoError(t, err, "container should be found in the store") + require.NotNil(t, storeContainer, "container should be found in the store") + + cases := []struct { + name string + pid int + containerID string + configuredDeviceIdx []int32 + expectedDeviceIdx []int32 + }{ + { + name: "NoContainer", + containerID: "", + pid: pidNoContainer, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{1, 2}, + }, + { + name: "NoContainerButEnv", + containerID: "", + pid: pidNoContainerButEnv, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{envVisibleDevices[1], envVisibleDevices[2]}, + }, + { + name: "WithContainer", + containerID: containerID, + pid: pidContainer, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{containerDeviceIndexes[1], containerDeviceIndexes[2]}, + }, + { + name: "WithContainerAndEnv", + pid: pidContainerAndEnv, + containerID: containerID, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{containerDeviceIndexes[envVisibleDevices[1]], containerDeviceIndexes[envVisibleDevices[2]]}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + for i, idx := range c.configuredDeviceIdx { + sysCtx.setDeviceSelection(c.pid, c.pid+i, idx) + } + + for i, idx := range c.expectedDeviceIdx { + activeDevice, err := sysCtx.getCurrentActiveGpuDevice(c.pid, c.pid+i, c.containerID) + require.NoError(t, err) + testutil.RequireDevicesEqual(t, sysCtx.gpuDevices[idx], activeDevice, "invalid device at index %d (real index is %d, selected index is %d)", i, idx, c.configuredDeviceIdx[i]) + } + }) + } +} diff --git a/pkg/gpu/probe.go b/pkg/gpu/probe.go index 3ab58ede54f35..fafc90df19c28 100644 --- a/pkg/gpu/probe.go +++ b/pkg/gpu/probe.go @@ -22,6 +22,7 @@ import ( "github.com/cilium/ebpf" "github.com/DataDog/datadog-agent/comp/core/telemetry" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu/model" ddebpf "github.com/DataDog/datadog-agent/pkg/ebpf" ebpftelemetry "github.com/DataDog/datadog-agent/pkg/ebpf/telemetry" @@ -81,6 +82,10 @@ type ProbeDependencies struct { // ProcessMonitor is the process monitor interface ProcessMonitor uprobes.ProcessMonitor + + // WorkloadMeta used to retrieve data about workloads (containers, processes) running + // on the host + WorkloadMeta workloadmeta.Component } // Probe represents the GPU monitoring probe @@ -109,7 +114,7 @@ func NewProbe(cfg *config.Config, deps ProbeDependencies) (*Probe, error) { } attachCfg := getAttacherConfig(cfg) - sysCtx, err := getSystemContext(deps.NvmlLib, cfg.ProcRoot) + sysCtx, err := getSystemContext(deps.NvmlLib, cfg.ProcRoot, deps.WorkloadMeta) if err != nil { return nil, fmt.Errorf("error getting system context: %w", err) } diff --git a/pkg/gpu/probe_stub.go b/pkg/gpu/probe_stub.go index 734a11919abb9..2f4ebf544c984 100644 --- a/pkg/gpu/probe_stub.go +++ b/pkg/gpu/probe_stub.go @@ -11,6 +11,7 @@ import ( "github.com/NVIDIA/go-nvml/pkg/nvml" "github.com/DataDog/datadog-agent/comp/core/telemetry" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu/model" "github.com/DataDog/datadog-agent/pkg/ebpf" "github.com/DataDog/datadog-agent/pkg/gpu/config" @@ -21,6 +22,7 @@ type ProbeDependencies struct { Telemetry telemetry.Component NvmlLib nvml.Interface ProcessMonitor any // uprobes.ProcessMonitor is only compiled with the linux_bpf build tag, so we need to use type any here + WorkloadMeta workloadmeta.Component } // Probe is not implemented on non-linux systems diff --git a/pkg/gpu/probe_test.go b/pkg/gpu/probe_test.go index f3eda56e2d2d9..ff2834b0327bc 100644 --- a/pkg/gpu/probe_test.go +++ b/pkg/gpu/probe_test.go @@ -47,6 +47,7 @@ func (s *probeTestSuite) getProbe() *Probe { deps := ProbeDependencies{ NvmlLib: testutil.GetBasicNvmlMock(), ProcessMonitor: consumerstestutil.NewTestProcessConsumer(t), + WorkloadMeta: testutil.GetWorkloadMetaMock(t), } probe, err := NewProbe(cfg, deps) require.NoError(t, err) diff --git a/pkg/gpu/stats_test.go b/pkg/gpu/stats_test.go index c0445e6c4e869..5aa4b2304cf3a 100644 --- a/pkg/gpu/stats_test.go +++ b/pkg/gpu/stats_test.go @@ -31,7 +31,7 @@ func getMetricsEntry(key model.StatsKey, stats *model.GPUStats) *model.Utilizati } func getStatsGeneratorForTest(t *testing.T) (*statsGenerator, map[streamKey]*StreamHandler, int64) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) require.NotNil(t, sysCtx) diff --git a/pkg/gpu/stream_test.go b/pkg/gpu/stream_test.go index f33e4f342e98b..0e4f0a5b734c8 100644 --- a/pkg/gpu/stream_test.go +++ b/pkg/gpu/stream_test.go @@ -21,10 +21,16 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/kernel" ) -func TestKernelLaunchesHandled(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) +func getSystemContextForTest(t *testing.T) *systemContext { + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + require.NotNil(t, sysCtx) + + return sysCtx +} + +func TestKernelLaunchesHandled(t *testing.T) { + stream := newStreamHandler(0, "", getSystemContextForTest(t)) kernStartTime := uint64(1) launch := &gpuebpf.CudaKernelLaunch{ @@ -81,9 +87,7 @@ func TestKernelLaunchesHandled(t *testing.T) { } func TestMemoryAllocationsHandled(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime := uint64(1) memFreeTime := uint64(2) @@ -152,9 +156,7 @@ func TestMemoryAllocationsHandled(t *testing.T) { } func TestMemoryAllocationsDetectLeaks(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime := uint64(1) memAddr := uint64(42) @@ -187,9 +189,7 @@ func TestMemoryAllocationsDetectLeaks(t *testing.T) { } func TestMemoryAllocationsNoCrashOnInvalidFree(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime := uint64(1) memFreeTime := uint64(2) @@ -231,9 +231,7 @@ func TestMemoryAllocationsNoCrashOnInvalidFree(t *testing.T) { } func TestMemoryAllocationsMultipleAllocsHandled(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime1, memAllocTime2 := uint64(1), uint64(10) memFreeTime1, memFreeTime2 := uint64(15), uint64(20) @@ -324,7 +322,7 @@ func TestMemoryAllocationsMultipleAllocsHandled(t *testing.T) { func TestKernelLaunchesIncludeEnrichedKernelData(t *testing.T) { proc := kernel.ProcFSRoot() - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), proc) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), proc, testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) // Set up the caches in system context so no actual queries are done diff --git a/pkg/gpu/testutil/mocks.go b/pkg/gpu/testutil/mocks.go index a7dadd39215ad..009385125600a 100644 --- a/pkg/gpu/testutil/mocks.go +++ b/pkg/gpu/testutil/mocks.go @@ -9,8 +9,19 @@ package testutil import ( + "fmt" + "testing" + "github.com/NVIDIA/go-nvml/pkg/nvml" nvmlmock "github.com/NVIDIA/go-nvml/pkg/nvml/mock" + "github.com/stretchr/testify/require" + "go.uber.org/fx" + + "github.com/DataDog/datadog-agent/comp/core" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + workloadmetafxmock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx-mock" + workloadmetamock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/mock" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) // DefaultGpuCores is the default number of cores for a GPU device in the mock. @@ -18,13 +29,18 @@ const DefaultGpuCores = 10 // GPUUUIDs is a list of UUIDs for the devices returned by the mock var GPUUUIDs = []string{ - "GPU-12345678-1234-1234-1234-123456789012", - "GPU-99999999-1234-1234-1234-123456789013", - "GPU-00000000-1234-1234-1234-123456789014", + "GPU-00000000-1234-1234-1234-123456789012", + "GPU-11111111-1234-1234-1234-123456789013", + "GPU-22222222-1234-1234-1234-123456789014", + "GPU-33333333-1234-1234-1234-123456789015", + "GPU-44444444-1234-1234-1234-123456789016", + "GPU-55555555-1234-1234-1234-123456789017", + "GPU-66666666-1234-1234-1234-123456789018", } -// GPUCores is a list of number of cores for the devices returned by the mock, should be the same length as GPUUUIDs -var GPUCores = []int{DefaultGpuCores, 20, 30} +// GPUCores is a list of number of cores for the devices returned by the mock, +// should be the same length as GPUUUIDs. If not, GetBasicNvmlMock will panic. +var GPUCores = []int{DefaultGpuCores, 20, 30, 40, 50, 60, 70} // DefaultGpuUUID is the UUID for the default device returned by the mock var DefaultGpuUUID = GPUUUIDs[0] @@ -47,6 +63,11 @@ func GetDeviceMock(deviceIdx int) *nvmlmock.Device { // GetBasicNvmlMock returns a mock of the nvml.Interface with a single device with 10 cores, // useful for basic tests that need only the basic interaction with NVML to be working. func GetBasicNvmlMock() *nvmlmock.Interface { + if len(GPUUUIDs) != len(GPUCores) { + // Make it really easy to spot errors if we change any of the arrays. + panic("GPUUUIDs and GPUCores must have the same length, please fix it") + } + return &nvmlmock.Interface{ DeviceGetCountFunc: func() (int, nvml.Return) { return len(GPUUUIDs), nvml.SUCCESS @@ -59,3 +80,49 @@ func GetBasicNvmlMock() *nvmlmock.Interface { }, } } + +// GetWorkloadMetaMock returns a mock of the workloadmeta.Component. +func GetWorkloadMetaMock(t *testing.T) workloadmetamock.Mock { + return fxutil.Test[workloadmetamock.Mock](t, fx.Options( + core.MockBundle(), + workloadmetafxmock.MockModule(workloadmeta.NewParams()), + )) +} + +// RequireDevicesEqual checks that the two devices are equal by comparing their UUIDs, which gives a better +// output than using require.Equal on the devices themselves +func RequireDevicesEqual(t *testing.T, expected, actual nvml.Device, msgAndArgs ...interface{}) { + extraFmt := "" + if len(msgAndArgs) > 0 { + extraFmt = fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + ": " + } + + expectedUUID, ret := expected.GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%s%scannot retrieve UUID for expected device %v%s", extraFmt, expected) + + actualUUID, ret := actual.GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%scannot retrieve UUID for actual device %v%s", extraFmt, actual) + + require.Equal(t, expectedUUID, actualUUID, "%sUUIDs do not match", extraFmt) +} + +// RequireDeviceListsEqual checks that the two device lists are equal by comparing their UUIDs, which gives a better +// output than using require.ElementsMatch on the lists themselves +func RequireDeviceListsEqual(t *testing.T, expected, actual []nvml.Device, msgAndArgs ...interface{}) { + extraFmt := "" + if len(msgAndArgs) > 0 { + extraFmt = fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + ": " + } + + require.Len(t, actual, len(expected), "%sdevice lists have different lengths", extraFmt) + + for i := range expected { + expectedUUID, ret := expected[i].GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%scannot retrieve UUID for expected device index %d", extraFmt, i) + + actualUUID, ret := actual[i].GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%scannot retrieve UUID for actual device index %d", extraFmt, i) + + require.Equal(t, expectedUUID, actualUUID, "%sUUIDs do not match for element %d", extraFmt, i) + } +} From e6ab5c07c265543c31b4893891870433bd8f17f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lian=20Raimbault?= <161456554+CelianR@users.noreply.github.com> Date: Wed, 18 Dec 2024 08:29:40 -0500 Subject: [PATCH 27/43] Add ci metrics origins (#31996) --- tasks/libs/common/datadog_api.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tasks/libs/common/datadog_api.py b/tasks/libs/common/datadog_api.py index 55b888678b768..7b8b9df7f6a8a 100644 --- a/tasks/libs/common/datadog_api.py +++ b/tasks/libs/common/datadog_api.py @@ -16,8 +16,12 @@ def create_metric(metric_type, metric_name, timestamp, value, tags, unit=None, m unit = unit or unset metadata = unset - if metric_origin: - metadata = MetricMetadata(origin=MetricOrigin(**metric_origin)) + origin_metadata = metric_origin or { + "origin_product": 10, # Agent + "origin_sub_product": 54, # Agent CI + "origin_product_detail": 64, # Gitlab + } + metadata = MetricMetadata(origin=MetricOrigin(**origin_metadata)) return MetricSeries( metric=metric_name, From f5e9ae44cee33c5260a9f1c1afa493ddc236958b Mon Sep 17 00:00:00 2001 From: Adel Haj Hassan <41540817+adel121@users.noreply.github.com> Date: Wed, 18 Dec 2024 14:36:15 +0100 Subject: [PATCH 28/43] quit leader election if context was cancelled (#32331) --- .../apiserver/leaderelection/leaderelection.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go index 45ea62c6e2d26..79e8a89895beb 100644 --- a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go +++ b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go @@ -228,9 +228,15 @@ func (le *LeaderEngine) EnsureLeaderElectionRuns() error { func (le *LeaderEngine) runLeaderElection() { for { - log.Infof("Starting leader election process for %q...", le.HolderIdentity) - le.leaderElector.Run(le.ctx) - log.Info("Leader election lost") + select { + case <-le.ctx.Done(): + log.Infof("Quitting leader election process: context was cancelled") + return + default: + log.Infof("Starting leader election process for %q...", le.HolderIdentity) + le.leaderElector.Run(le.ctx) + log.Info("Leader election lost") + } } } From 188825c6b2364e4cff5e180a2163af3ef7bd164e Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Wed, 18 Dec 2024 14:38:50 +0100 Subject: [PATCH 29/43] usm: Include PathId in debug traced program list (#32329) --- pkg/network/usm/utils/debugger.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/network/usm/utils/debugger.go b/pkg/network/usm/utils/debugger.go index 4da685bf2fbc9..8094b2e02c6c3 100644 --- a/pkg/network/usm/utils/debugger.go +++ b/pkg/network/usm/utils/debugger.go @@ -31,6 +31,7 @@ type Attacher interface { type TracedProgram struct { ProgramType string FilePath string + PathID PathIdentifier PIDs []uint32 } @@ -131,6 +132,7 @@ func (d *tlsDebugger) GetTracedPrograms(moduleName string) []TracedProgram { program.ProgramType = programType program.FilePath = registration.sampleFilePath + program.PathID = pathID } registry.m.Unlock() From b6fd306a800e50d2813dc74d4bf6912483aac554 Mon Sep 17 00:00:00 2001 From: Sylvain Afchain Date: Wed, 18 Dec 2024 15:27:44 +0100 Subject: [PATCH 30/43] [CWS] move cgroup context in linux specific serializer (#32338) --- docs/cloud-workload-security/backend_linux.md | 8 +- .../backend_linux.schema.json | 6 +- pkg/security/serializers/serializers_base.go | 10 -- .../serializers_base_linux_easyjson.go | 110 ++----------- pkg/security/serializers/serializers_linux.go | 10 ++ .../serializers/serializers_linux_easyjson.go | 146 +++++++++++++----- 6 files changed, 135 insertions(+), 155 deletions(-) diff --git a/docs/cloud-workload-security/backend_linux.md b/docs/cloud-workload-security/backend_linux.md index 7b0794f82fc9d..e4e1af8c22c37 100644 --- a/docs/cloud-workload-security/backend_linux.md +++ b/docs/cloud-workload-security/backend_linux.md @@ -1718,9 +1718,6 @@ CSM Threats event for Linux systems have the following JSON schema: "container": { "$ref": "#/$defs/ContainerContext" }, - "cgroup": { - "$ref": "#/$defs/CGroupContext" - }, "network": { "$ref": "#/$defs/NetworkContext" }, @@ -1730,6 +1727,9 @@ CSM Threats event for Linux systems have the following JSON schema: "security_profile": { "$ref": "#/$defs/SecurityProfileContext" }, + "cgroup": { + "$ref": "#/$defs/CGroupContext" + }, "selinux": { "$ref": "#/$defs/SELinuxEvent" }, @@ -1802,10 +1802,10 @@ CSM Threats event for Linux systems have the following JSON schema: | `exit` | $ref | Please see [ExitEvent](#exitevent) | | `process` | $ref | Please see [ProcessContext](#processcontext) | | `container` | $ref | Please see [ContainerContext](#containercontext) | -| `cgroup` | $ref | Please see [CGroupContext](#cgroupcontext) | | `network` | $ref | Please see [NetworkContext](#networkcontext) | | `dd` | $ref | Please see [DDContext](#ddcontext) | | `security_profile` | $ref | Please see [SecurityProfileContext](#securityprofilecontext) | +| `cgroup` | $ref | Please see [CGroupContext](#cgroupcontext) | | `selinux` | $ref | Please see [SELinuxEvent](#selinuxevent) | | `bpf` | $ref | Please see [BPFEvent](#bpfevent) | | `mmap` | $ref | Please see [MMapEvent](#mmapevent) | diff --git a/docs/cloud-workload-security/backend_linux.schema.json b/docs/cloud-workload-security/backend_linux.schema.json index b6f9cddc6cbb7..b4aa6b761fbfb 100644 --- a/docs/cloud-workload-security/backend_linux.schema.json +++ b/docs/cloud-workload-security/backend_linux.schema.json @@ -1707,9 +1707,6 @@ "container": { "$ref": "#/$defs/ContainerContext" }, - "cgroup": { - "$ref": "#/$defs/CGroupContext" - }, "network": { "$ref": "#/$defs/NetworkContext" }, @@ -1719,6 +1716,9 @@ "security_profile": { "$ref": "#/$defs/SecurityProfileContext" }, + "cgroup": { + "$ref": "#/$defs/CGroupContext" + }, "selinux": { "$ref": "#/$defs/SELinuxEvent" }, diff --git a/pkg/security/serializers/serializers_base.go b/pkg/security/serializers/serializers_base.go index 32f84ace271f8..805e52f8c67cc 100644 --- a/pkg/security/serializers/serializers_base.go +++ b/pkg/security/serializers/serializers_base.go @@ -18,15 +18,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/scrubber" ) -// CGroupContextSerializer serializes a cgroup context to JSON -// easyjson:json -type CGroupContextSerializer struct { - // CGroup ID - ID string `json:"id,omitempty"` - // CGroup manager - Manager string `json:"manager,omitempty"` -} - // ContainerContextSerializer serializes a container context to JSON // easyjson:json type ContainerContextSerializer struct { @@ -213,7 +204,6 @@ type BaseEventSerializer struct { *ExitEventSerializer `json:"exit,omitempty"` *ProcessContextSerializer `json:"process,omitempty"` *ContainerContextSerializer `json:"container,omitempty"` - *CGroupContextSerializer `json:"cgroup,omitempty"` } // TLSContextSerializer defines a tls context serializer diff --git a/pkg/security/serializers/serializers_base_linux_easyjson.go b/pkg/security/serializers/serializers_base_linux_easyjson.go index 743795c140527..8502c05b45842 100644 --- a/pkg/security/serializers/serializers_base_linux_easyjson.go +++ b/pkg/security/serializers/serializers_base_linux_easyjson.go @@ -1777,72 +1777,7 @@ func (v ContainerContextSerializer) MarshalEasyJSON(w *jwriter.Writer) { func (v *ContainerContextSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers14(l, v) } -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(in *jlexer.Lexer, out *CGroupContextSerializer) { - isTopLevel := in.IsStart() - if in.IsNull() { - if isTopLevel { - in.Consumed() - } - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeFieldName(false) - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "id": - out.ID = string(in.String()) - case "manager": - out.Manager = string(in.String()) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') - if isTopLevel { - in.Consumed() - } -} -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(out *jwriter.Writer, in CGroupContextSerializer) { - out.RawByte('{') - first := true - _ = first - if in.ID != "" { - const prefix string = ",\"id\":" - first = false - out.RawString(prefix[1:]) - out.String(string(in.ID)) - } - if in.Manager != "" { - const prefix string = ",\"manager\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - out.String(string(in.Manager)) - } - out.RawByte('}') -} - -// MarshalEasyJSON supports easyjson.Marshaler interface -func (v CGroupContextSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(w, v) -} - -// UnmarshalEasyJSON supports easyjson.Unmarshaler interface -func (v *CGroupContextSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(l, v) -} -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(in *jlexer.Lexer, out *BaseEventSerializer) { +func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(in *jlexer.Lexer, out *BaseEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -1855,7 +1790,6 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( out.ExitEventSerializer = new(ExitEventSerializer) out.ProcessContextSerializer = new(ProcessContextSerializer) out.ContainerContextSerializer = new(ContainerContextSerializer) - out.CGroupContextSerializer = new(CGroupContextSerializer) in.Delim('{') for !in.IsDelim('}') { key := in.UnsafeFieldName(false) @@ -1912,16 +1846,6 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( } (*out.ContainerContextSerializer).UnmarshalEasyJSON(in) } - case "cgroup": - if in.IsNull() { - in.Skip() - out.CGroupContextSerializer = nil - } else { - if out.CGroupContextSerializer == nil { - out.CGroupContextSerializer = new(CGroupContextSerializer) - } - (*out.CGroupContextSerializer).UnmarshalEasyJSON(in) - } default: in.SkipRecursive() } @@ -1932,7 +1856,7 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( in.Consumed() } } -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(out *jwriter.Writer, in BaseEventSerializer) { +func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(out *jwriter.Writer, in BaseEventSerializer) { out.RawByte('{') first := true _ = first @@ -1992,29 +1916,19 @@ func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( } (*in.ContainerContextSerializer).MarshalEasyJSON(out) } - if in.CGroupContextSerializer != nil { - const prefix string = ",\"cgroup\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - (*in.CGroupContextSerializer).MarshalEasyJSON(out) - } out.RawByte('}') } // MarshalEasyJSON supports easyjson.Marshaler interface func (v BaseEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(w, v) + easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BaseEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(l, v) + easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(l, v) } -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(in *jlexer.Lexer, out *AWSSecurityCredentialsSerializer) { +func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(in *jlexer.Lexer, out *AWSSecurityCredentialsSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -2053,7 +1967,7 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17( in.Consumed() } } -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(out *jwriter.Writer, in AWSSecurityCredentialsSerializer) { +func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(out *jwriter.Writer, in AWSSecurityCredentialsSerializer) { out.RawByte('{') first := true _ = first @@ -2087,14 +2001,14 @@ func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17( // MarshalEasyJSON supports easyjson.Marshaler interface func (v AWSSecurityCredentialsSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(w, v) + easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *AWSSecurityCredentialsSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(l, v) + easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(l, v) } -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(in *jlexer.Lexer, out *AWSIMDSEventSerializer) { +func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(in *jlexer.Lexer, out *AWSIMDSEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -2135,7 +2049,7 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( in.Consumed() } } -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(out *jwriter.Writer, in AWSIMDSEventSerializer) { +func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(out *jwriter.Writer, in AWSIMDSEventSerializer) { out.RawByte('{') first := true _ = first @@ -2154,10 +2068,10 @@ func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( // MarshalEasyJSON supports easyjson.Marshaler interface func (v AWSIMDSEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(w, v) + easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *AWSIMDSEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(l, v) + easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(l, v) } diff --git a/pkg/security/serializers/serializers_linux.go b/pkg/security/serializers/serializers_linux.go index 69b1cf0bd225a..4f38767025b47 100644 --- a/pkg/security/serializers/serializers_linux.go +++ b/pkg/security/serializers/serializers_linux.go @@ -81,6 +81,15 @@ type FileSerializer struct { MountOrigin string `json:"mount_origin,omitempty"` } +// CGroupContextSerializer serializes a cgroup context to JSON +// easyjson:json +type CGroupContextSerializer struct { + // CGroup ID + ID string `json:"id,omitempty"` + // CGroup manager + Manager string `json:"manager,omitempty"` +} + // UserContextSerializer serializes a user context to JSON // easyjson:json type UserContextSerializer struct { @@ -622,6 +631,7 @@ type EventSerializer struct { *NetworkContextSerializer `json:"network,omitempty"` *DDContextSerializer `json:"dd,omitempty"` *SecurityProfileContextSerializer `json:"security_profile,omitempty"` + *CGroupContextSerializer `json:"cgroup,omitempty"` *SELinuxEventSerializer `json:"selinux,omitempty"` *BPFEventSerializer `json:"bpf,omitempty"` diff --git a/pkg/security/serializers/serializers_linux_easyjson.go b/pkg/security/serializers/serializers_linux_easyjson.go index ec264f2cb6de4..e61dfb697a062 100644 --- a/pkg/security/serializers/serializers_linux_easyjson.go +++ b/pkg/security/serializers/serializers_linux_easyjson.go @@ -3595,6 +3595,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( out.NetworkContextSerializer = new(NetworkContextSerializer) out.DDContextSerializer = new(DDContextSerializer) out.SecurityProfileContextSerializer = new(SecurityProfileContextSerializer) + out.CGroupContextSerializer = new(CGroupContextSerializer) out.SELinuxEventSerializer = new(SELinuxEventSerializer) out.BPFEventSerializer = new(BPFEventSerializer) out.MMapEventSerializer = new(MMapEventSerializer) @@ -3652,6 +3653,16 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*out.SecurityProfileContextSerializer).UnmarshalEasyJSON(in) } + case "cgroup": + if in.IsNull() { + in.Skip() + out.CGroupContextSerializer = nil + } else { + if out.CGroupContextSerializer == nil { + out.CGroupContextSerializer = new(CGroupContextSerializer) + } + (*out.CGroupContextSerializer).UnmarshalEasyJSON(in) + } case "selinux": if in.IsNull() { in.Skip() @@ -3889,16 +3900,6 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*out.ContainerContextSerializer).UnmarshalEasyJSON(in) } - case "cgroup": - if in.IsNull() { - in.Skip() - out.CGroupContextSerializer = nil - } else { - if out.CGroupContextSerializer == nil { - out.CGroupContextSerializer = new(CGroupContextSerializer) - } - (*out.CGroupContextSerializer).UnmarshalEasyJSON(in) - } default: in.SkipRecursive() } @@ -3939,6 +3940,16 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*in.SecurityProfileContextSerializer).MarshalEasyJSON(out) } + if in.CGroupContextSerializer != nil { + const prefix string = ",\"cgroup\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + (*in.CGroupContextSerializer).MarshalEasyJSON(out) + } if in.SELinuxEventSerializer != nil { const prefix string = ",\"selinux\":" if first { @@ -4180,16 +4191,6 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*in.ContainerContextSerializer).MarshalEasyJSON(out) } - if in.CGroupContextSerializer != nil { - const prefix string = ",\"cgroup\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - (*in.CGroupContextSerializer).MarshalEasyJSON(out) - } out.RawByte('}') } @@ -4663,7 +4664,72 @@ func (v CapsetSerializer) MarshalEasyJSON(w *jwriter.Writer) { func (v *CapsetSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers28(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(in *jlexer.Lexer, out *BindEventSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(in *jlexer.Lexer, out *CGroupContextSerializer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "id": + out.ID = string(in.String()) + case "manager": + out.Manager = string(in.String()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(out *jwriter.Writer, in CGroupContextSerializer) { + out.RawByte('{') + first := true + _ = first + if in.ID != "" { + const prefix string = ",\"id\":" + first = false + out.RawString(prefix[1:]) + out.String(string(in.ID)) + } + if in.Manager != "" { + const prefix string = ",\"manager\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.Manager)) + } + out.RawByte('}') +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v CGroupContextSerializer) MarshalEasyJSON(w *jwriter.Writer) { + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(w, v) +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *CGroupContextSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(l, v) +} +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(in *jlexer.Lexer, out *BindEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4696,7 +4762,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(out *jwriter.Writer, in BindEventSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(out *jwriter.Writer, in BindEventSerializer) { out.RawByte('{') first := true _ = first @@ -4715,14 +4781,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BindEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BindEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(in *jlexer.Lexer, out *BPFProgramSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(in *jlexer.Lexer, out *BPFProgramSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4782,7 +4848,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(out *jwriter.Writer, in BPFProgramSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(out *jwriter.Writer, in BPFProgramSerializer) { out.RawByte('{') first := true _ = first @@ -4846,14 +4912,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BPFProgramSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BPFProgramSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(in *jlexer.Lexer, out *BPFMapSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(in *jlexer.Lexer, out *BPFMapSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4886,7 +4952,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(out *jwriter.Writer, in BPFMapSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(out *jwriter.Writer, in BPFMapSerializer) { out.RawByte('{') first := true _ = first @@ -4911,14 +4977,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BPFMapSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BPFMapSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(in *jlexer.Lexer, out *BPFEventSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(in *jlexer.Lexer, out *BPFEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4969,7 +5035,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(out *jwriter.Writer, in BPFEventSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(out *jwriter.Writer, in BPFEventSerializer) { out.RawByte('{') first := true _ = first @@ -4993,14 +5059,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BPFEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BPFEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(in *jlexer.Lexer, out *AnomalyDetectionSyscallEventSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(in *jlexer.Lexer, out *AnomalyDetectionSyscallEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -5031,7 +5097,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(out *jwriter.Writer, in AnomalyDetectionSyscallEventSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(out *jwriter.Writer, in AnomalyDetectionSyscallEventSerializer) { out.RawByte('{') first := true _ = first @@ -5045,10 +5111,10 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33( // MarshalEasyJSON supports easyjson.Marshaler interface func (v AnomalyDetectionSyscallEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *AnomalyDetectionSyscallEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(l, v) } From 0171d14c90075fd883544131731d5c1c9c05be35 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Wed, 18 Dec 2024 15:31:57 +0100 Subject: [PATCH 31/43] feat(ci): Enable timestamp in logs (#32327) --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b2761698a9b16..95cb72a1c4df5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -276,6 +276,7 @@ variables: # Feature flags FF_SCRIPT_SECTIONS: 1 # Prevent multiline scripts log collapsing, see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3392 FF_KUBERNETES_HONOR_ENTRYPOINT: true # Honor the entrypoint in the Docker image when running Kubernetes jobs + FF_TIMESTAMPS: true # # Condition mixins for simplification of rules From 52f0517517169d661a94895a296c73b85424c86d Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Wed, 18 Dec 2024 15:36:54 +0100 Subject: [PATCH 32/43] Update path to trigger run of asc e2e tests (#32298) --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 95cb72a1c4df5..2038aa4649f91 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -842,7 +842,9 @@ workflow: - !reference [.on_e2e_main_release_or_rc] - changes: paths: - # TODO: Add paths that should trigger tests for ASC + - cmd/**/* + - pkg/**/* + - comp/**/* - test/new-e2e/tests/agent-shared-components/**/* compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 From 5615184aff643b2f613fdaad57b5aeabaea48d3a Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Wed, 18 Dec 2024 17:52:27 +0100 Subject: [PATCH 33/43] feat(fleet): Support integrations configuration in the Fleet Policies dir (#31263) --- cmd/agent/common/autodiscovery.go | 5 +++++ comp/core/flare/providers.go | 1 + comp/core/gui/guiimpl/checks.go | 17 ++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/cmd/agent/common/autodiscovery.go b/cmd/agent/common/autodiscovery.go index 8ff855aeca19f..67a220b706750 100644 --- a/cmd/agent/common/autodiscovery.go +++ b/cmd/agent/common/autodiscovery.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "time" "go.uber.org/atomic" @@ -41,6 +42,10 @@ var ( ) func setupAutoDiscovery(confSearchPaths []string, wmeta workloadmeta.Component, ac autodiscovery.Component) { + if pkgconfigsetup.Datadog().GetString("fleet_policies_dir") != "" { + confSearchPaths = append(confSearchPaths, filepath.Join(pkgconfigsetup.Datadog().GetString("fleet_policies_dir"), "conf.d")) + } + providers.InitConfigFilesReader(confSearchPaths) acTelemetryStore := ac.GetTelemetryStore() diff --git a/comp/core/flare/providers.go b/comp/core/flare/providers.go index edb2c7ac86520..69ceae5ccf76f 100644 --- a/comp/core/flare/providers.go +++ b/comp/core/flare/providers.go @@ -53,6 +53,7 @@ func (f *flare) collectLogsFiles(fb types.FlareBuilder) error { func (f *flare) collectConfigFiles(fb types.FlareBuilder) error { confSearchPaths := map[string]string{ "": f.config.GetString("confd_path"), + "fleet": filepath.Join(f.config.GetString("fleet_policies_dir"), "conf.d"), "dist": filepath.Join(f.params.distPath, "conf.d"), "checksd": f.params.pythonChecksPath, } diff --git a/comp/core/gui/guiimpl/checks.go b/comp/core/gui/guiimpl/checks.go index 058a75b08f05c..f54c682f72b8b 100644 --- a/comp/core/gui/guiimpl/checks.go +++ b/comp/core/gui/guiimpl/checks.go @@ -42,9 +42,19 @@ var ( filepath.Join(defaultpaths.GetDistPath(), "checks.d"), // Custom checks pkgconfigsetup.Datadog().GetString("additional_checksd"), // Custom checks defaultpaths.PyChecksPath, // Integrations-core checks + getFleetPoliciesPath(), // Fleet Policies } ) +// getFleetPoliciesPath returns the path to the fleet policies directory if it is set in the configuration +// otherwise it returns an empty string +func getFleetPoliciesPath() string { + if len(pkgconfigsetup.Datadog().GetString("fleet_policies_dir")) > 0 { + return filepath.Join(pkgconfigsetup.Datadog().GetString("fleet_policies_dir"), "conf.d") + } + return "" +} + // Adds the specific handlers for /checks/ endpoints func checkHandler(r *mux.Router, collector collector.Component, ac autodiscovery.Component) { r.HandleFunc("/running", http.HandlerFunc(sendRunningChecks)).Methods("POST") @@ -208,6 +218,9 @@ func getCheckConfigFile(w http.ResponseWriter, r *http.Request) { var file []byte var e error for _, path := range configPaths { + if len(path) == 0 { + continue + } filePath, err := securejoin.SecureJoin(path, fileName) if err != nil { log.Errorf("Error: Unable to join config path with the file name: %s", fileName) @@ -443,7 +456,9 @@ func getConfigsInPath(path string) ([]string, error) { func listConfigs(w http.ResponseWriter, _ *http.Request) { filenames := []string{} for _, path := range configPaths { - + if len(path) == 0 { + continue + } configs, e := getConfigsInPath(path) if e != nil { log.Errorf("Unable to list configurations from %s: %v", path, e) From 8f2d236ef1735bc576fb116495526de0221d1ff3 Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Wed, 18 Dec 2024 11:52:36 -0500 Subject: [PATCH 34/43] [Logs agent] Disable HTTP/2 when using a proxy (#32308) Co-authored-by: Jen Gilbert --- pkg/logs/client/http/destination.go | 7 +++++ pkg/logs/client/http/destination_test.go | 31 +++++++++++++++++++ ...-http1-for-log-agent-2cbfba763697ab42.yaml | 4 +-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/pkg/logs/client/http/destination.go b/pkg/logs/client/http/destination.go index 2a554e5476504..93bf35ec67167 100644 --- a/pkg/logs/client/http/destination.go +++ b/pkg/logs/client/http/destination.go @@ -396,6 +396,13 @@ func httpClientFactory(timeout time.Duration, cfg pkgconfigmodel.Reader) func() var transport *http.Transport transportConfig := cfg.Get("logs_config.http_protocol") + + // If any proxy is set, use http1 + // This will be removed in a future version + if cfg.GetProxies() != nil { + transportConfig = "http1" + } + // Configure transport based on user setting switch transportConfig { case "http1": diff --git a/pkg/logs/client/http/destination_test.go b/pkg/logs/client/http/destination_test.go index 92bf984a79e5a..3f65bba9b9e2c 100644 --- a/pkg/logs/client/http/destination_test.go +++ b/pkg/logs/client/http/destination_test.go @@ -494,3 +494,34 @@ func TestTransportProtocol_HTTP1FallBack(t *testing.T) { // Assert that the server automatically falls back to HTTP/1.1 assert.Equal(t, "HTTP/1.1", resp.Proto) } + +func TestTransportProtocol_HTTP1WhenUsingProxy(t *testing.T) { + c := configmock.New(t) + + // Force client to use ALNP + c.SetWithoutSource("logs_config.http_protocol", "auto") + c.SetWithoutSource("skip_ssl_validation", true) + + // The test server uses TLS, so if we set the http proxy (not https), it still makes + // a request to the test server, but disable HTTP/2 since a proxy is configured. + c.SetWithoutSource("proxy.http", "http://foo.bar") + + server := NewTestHTTPSServer(false) + defer server.Close() + + timeout := 5 * time.Second + client := httpClientFactory(timeout, c)() + + req, err := http.NewRequest("POST", server.URL, nil) + if err != nil { + t.Fatalf("Failed to create request: %v", err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to send request: %v", err) + } + defer resp.Body.Close() + + // Assert that the server chose HTTP/1.1 because a proxy was configured + assert.Equal(t, "HTTP/1.1", resp.Proto) +} diff --git a/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml b/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml index 53476dcfa0bb9..e9c1031ef5d3b 100644 --- a/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml +++ b/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml @@ -9,7 +9,7 @@ features: - | Introduced a new configuration variable `logs_config.http_protocol`, allowing users to enforce HTTP/1.1 for outgoing HTTP connections in the Datadog Agent. This provides better control over transport protocols and improves compatibility with systems that do not support HTTP/2. - By default, the log agent will now attempt to use HTTP/2 and fall back to the best available protocol if HTTP/2 is not supported. + By default, the log agent will now attempt to use HTTP/2 (unless a proxy is configured) and fall back to the best available protocol if HTTP/2 is not supported. enhancements: - | - Improved logging to add visiblity for latency and transport protocol \ No newline at end of file + Improved logging to add visibility for latency and transport protocol \ No newline at end of file From 33f08c8145777e0e0072188f6c4908ceb5dccb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9lian=20Raimbault?= <161456554+CelianR@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:52:44 -0500 Subject: [PATCH 35/43] release.create-github-release: Use current version not next one (#32337) --- tasks/libs/releasing/version.py | 78 +++++++++++++++++++++++++++++++++ tasks/release.py | 67 ++-------------------------- 2 files changed, 81 insertions(+), 64 deletions(-) diff --git a/tasks/libs/releasing/version.py b/tasks/libs/releasing/version.py index 23472d57c0cca..85e66ea283080 100644 --- a/tasks/libs/releasing/version.py +++ b/tasks/libs/releasing/version.py @@ -6,6 +6,7 @@ from invoke import Exit from tasks.libs.ciproviders.github_api import GithubAPI +from tasks.libs.common.color import Color, color_message from tasks.libs.common.constants import ( AGENT_VERSION_CACHE_NAME, ALLOWED_REPO_NIGHTLY_BRANCHES, @@ -441,3 +442,80 @@ def get_matching_pattern(ctx, major_version, release=False): ) pattern = max(tags, key=cmp_to_key(semver.compare)) return pattern + + +def deduce_version(ctx, branch, as_str: bool = True, trust: bool = False, next_version: bool = True) -> str | Version: + """Deduces the version from the release branch name. + + Args: + next_version: If True, will return the next tag version, otherwise will return the current tag version. Example: If there are 7.60.0 and 7.60.1 tags, it will return 7.60.2 if next_tag is True, 7.60.1 otherwise. + """ + release_version = get_next_version_from_branch(ctx, branch, as_str=as_str, next_version=next_version) + + print( + f'{color_message("Info", Color.BLUE)}: Version {release_version} deduced from branch {branch}', file=sys.stderr + ) + + if ( + trust + or not os.isatty(sys.stdin.fileno()) + or yes_no_question( + 'Is this the version you want to use?', + color="orange", + default=False, + ) + ): + return release_version + + raise Exit(color_message("Aborting.", "red"), code=1) + + +def get_version_major(branch: str) -> int: + """Get the major version from a branch name.""" + + return 7 if branch == 'main' else int(branch.split('.')[0]) + + +def get_all_version_tags(ctx) -> list[str]: + """Returns the tags for all the versions of the Agent in git.""" + + cmd = "bash -c 'git tag | grep -E \"^[0-9]\\.[0-9]+\\.[0-9]+$\"'" + + return ctx.run(cmd, hide=True).stdout.strip().split('\n') + + +def get_next_version_from_branch(ctx, branch: str, as_str: bool = True, next_version: bool = True) -> str | Version: + """Returns the latest version + 1 belonging to a branch. + + Args: + next_version: If True, will return the next tag version, otherwise will return the current tag version. Example: If there are 7.60.0 and 7.60.1 tags, it will return 7.60.2 if next_tag is True, 7.60.1 otherwise. + + Example: + get_latest_version_from_branch("7.55.x") -> Version(7, 55, 4) if there are 7.55.0, 7.55.1, 7.55.2, 7.55.3 tags. + get_latest_version_from_branch("6.99.x") -> Version(6, 99, 0) if there are no 6.99.* tags. + """ + + re_branch = re.compile(r"^([0-9]\.[0-9]+\.)x$") + + try: + matched = re_branch.match(branch).group(1) + except Exception as e: + raise Exit( + f'{color_message("Error:", "red")}: Branch {branch} is not a release branch (should be X.Y.x)', code=1 + ) from e + + tags = [tuple(map(int, tag.split('.'))) for tag in get_all_version_tags(ctx) if tag.startswith(matched)] + versions = sorted(Version(*tag) for tag in tags) + + minor, major = tuple(map(int, branch.split('.')[:2])) + + if next_version: + # Get version after the latest one + version = versions[-1].next_version(bump_patch=True) if versions else Version(minor, major, 0) + else: + # Get current latest version + assert versions, f"No tags found for branch {branch} (expected at least one tag)" + + version = versions[-1] + + return str(version) if as_str else version diff --git a/tasks/release.py b/tasks/release.py index 4e5805b0ffb76..c37e6e944deeb 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -70,10 +70,11 @@ VERSION_RE, _create_version_from_match, current_version, + deduce_version, + get_version_major, next_final_version, next_rc_version, ) -from tasks.libs.types.version import Version from tasks.pipeline import edit_schedule, run from tasks.release_metrics.metrics import get_prs_metrics, get_release_lead_time @@ -85,68 +86,6 @@ BACKPORT_LABEL_COLOR = "5319e7" -def deduce_version(ctx, branch, as_str=True, trust=False) -> str | Version: - release_version = get_next_version_from_branch(ctx, branch, as_str=as_str) - - print( - f'{color_message("Info", Color.BLUE)}: Version {release_version} deduced from branch {branch}', file=sys.stderr - ) - - if ( - trust - or not os.isatty(sys.stdin.fileno()) - or yes_no_question( - 'Is this the version you want to use?', - color="orange", - default=False, - ) - ): - return release_version - - raise Exit(color_message("Aborting.", "red"), code=1) - - -def get_version_major(branch: str) -> int: - """Get the major version from a branch name.""" - - return 7 if branch == 'main' else int(branch.split('.')[0]) - - -def get_all_version_tags(ctx) -> list[str]: - """Returns the tags for all the versions of the Agent in git.""" - - cmd = "bash -c 'git tag | grep -E \"^[0-9]\\.[0-9]+\\.[0-9]+$\"'" - - return ctx.run(cmd, hide=True).stdout.strip().split('\n') - - -def get_next_version_from_branch(ctx, branch: str, as_str=True) -> str | Version: - """Returns the latest version + 1 belonging to a branch. - - Example: - get_latest_version_from_branch("7.55.x") -> Version(7, 55, 4) if there are 7.55.0, 7.55.1, 7.55.2, 7.55.3 tags. - get_latest_version_from_branch("6.99.x") -> Version(6, 99, 0) if there are no 6.99.* tags. - """ - - re_branch = re.compile(r"^([0-9]\.[0-9]+\.)x$") - - try: - matched = re_branch.match(branch).group(1) - except Exception as e: - raise Exit( - f'{color_message("Error:", "red")}: Branch {branch} is not a release branch (should be X.Y.x)', code=1 - ) from e - - tags = [tuple(map(int, tag.split('.'))) for tag in get_all_version_tags(ctx) if tag.startswith(matched)] - versions = sorted(Version(*tag) for tag in tags) - - minor, major = tuple(map(int, branch.split('.')[:2])) - - latest = versions[-1].next_version(bump_patch=True) if versions else Version(minor, major, 0) - - return str(latest) if as_str else latest - - @task def list_major_change(_, milestone): """List all PR labeled "major_changed" for this release.""" @@ -1221,7 +1160,7 @@ def create_github_release(ctx, release_branch, draft=True): ) notes = [] - version = deduce_version(ctx, release_branch) + version = deduce_version(ctx, release_branch, next_version=False) with agent_context(ctx, release_branch): for section, filename in sections: From de8114135d3a0c8a1bf806ef7959858dac31c95e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 16:54:58 +0000 Subject: [PATCH 36/43] Bump actions/checkout from 4.1.4 to 4.2.2 (#30957) Co-authored-by: chouetz --- .github/workflows/add_milestone.yml | 2 +- .github/workflows/buildimages-update.yml | 4 ++-- .github/workflows/chase_release_managers.yml | 2 +- .github/workflows/code_review_complexity.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/create_rc_pr.yml | 4 ++-- .github/workflows/create_release_schedule.yml | 2 +- .github/workflows/cws-btfhub-sync.yml | 6 +++--- .github/workflows/datadog-static-analysis.yml | 2 +- .github/workflows/docs-dev.yml | 2 +- .github/workflows/external-contributor.yml | 2 +- .github/workflows/go-update-commenter.yml | 4 ++-- .github/workflows/go_mod_tidy.yml | 2 +- .github/workflows/gohai.yml | 2 +- .github/workflows/label-analysis.yml | 6 +++--- .github/workflows/markdown-lint-check.yml | 2 +- .github/workflows/report-merged-pr.yml | 2 +- .github/workflows/serverless-benchmarks.yml | 4 ++-- .github/workflows/serverless-binary-size.yml | 4 ++-- .github/workflows/serverless-integration.yml | 4 ++-- 20 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/add_milestone.yml b/.github/workflows/add_milestone.yml index 33885369f0e0c..6bcd28e967c2e 100644 --- a/.github/workflows/add_milestone.yml +++ b/.github/workflows/add_milestone.yml @@ -22,7 +22,7 @@ jobs: GH_REPO: ${{ github.repository }} steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false diff --git a/.github/workflows/buildimages-update.yml b/.github/workflows/buildimages-update.yml index 1138e8557d011..a480105bf17e8 100644 --- a/.github/workflows/buildimages-update.yml +++ b/.github/workflows/buildimages-update.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: # credentials are needed to create the PR at the end of the workflow persist-credentials: true @@ -53,7 +53,7 @@ jobs: fi - name: Checkout branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: ${{ steps.branch_fetch.outputs.RESULT == 'true' }} with: ref: ${{ inputs.branch }} diff --git a/.github/workflows/chase_release_managers.yml b/.github/workflows/chase_release_managers.yml index 436217659ae48..475068736efea 100644 --- a/.github/workflows/chase_release_managers.yml +++ b/.github/workflows/chase_release_managers.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} persist-credentials: false diff --git a/.github/workflows/code_review_complexity.yml b/.github/workflows/code_review_complexity.yml index 8c26254f6c626..3f36b387e0f34 100644 --- a/.github/workflows/code_review_complexity.yml +++ b/.github/workflows/code_review_complexity.yml @@ -21,7 +21,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Setup python diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bb6d39efa5e6d..bc52cb502bdaf 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index 2a685c3cc85df..66fbf9462a168 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -21,7 +21,7 @@ jobs: warning: ${{ steps.warning.outputs.value }} steps: - name: Checkout repository - if: ${{ env.IS_AGENT6_RELEASE == 'false' }} + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: sparse-checkout: 'tasks' @@ -69,7 +69,7 @@ jobs: fail-fast: false steps: - name: Checkout the main branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: true diff --git a/.github/workflows/create_release_schedule.yml b/.github/workflows/create_release_schedule.yml index f8dc006160e3c..4b749ba3bdc97 100644 --- a/.github/workflows/create_release_schedule.yml +++ b/.github/workflows/create_release_schedule.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} persist-credentials: false diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml index 795953b22a354..94498f5ed7525 100644 --- a/.github/workflows/cws-btfhub-sync.yml +++ b/.github/workflows/cws-btfhub-sync.yml @@ -52,13 +52,13 @@ jobs: df -h - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.base_branch || 'main' }} persist-credentials: false - name: Checkout btfhub-archive repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/btfhub-archive path: dev/dist/archive @@ -110,7 +110,7 @@ jobs: pull-requests: write steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.base_branch || 'main' }} diff --git a/.github/workflows/datadog-static-analysis.yml b/.github/workflows/datadog-static-analysis.yml index 3063c7c0db456..a77f4ba632c1d 100644 --- a/.github/workflows/datadog-static-analysis.yml +++ b/.github/workflows/datadog-static-analysis.yml @@ -11,7 +11,7 @@ jobs: name: Datadog Static Analyzer steps: - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Check code meets quality and security standards diff --git a/.github/workflows/docs-dev.yml b/.github/workflows/docs-dev.yml index 1eab6460db4b6..59e803598f6cd 100644 --- a/.github/workflows/docs-dev.yml +++ b/.github/workflows/docs-dev.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false # Fetch all history for applying timestamps to every page diff --git a/.github/workflows/external-contributor.yml b/.github/workflows/external-contributor.yml index f0beb805c2a50..03cf46fd03511 100644 --- a/.github/workflows/external-contributor.yml +++ b/.github/workflows/external-contributor.yml @@ -17,7 +17,7 @@ jobs: if: github.event.pull_request.head.repo.full_name != github.repository steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: main fetch-depth: 0 diff --git a/.github/workflows/go-update-commenter.yml b/.github/workflows/go-update-commenter.yml index 1028110bc4fda..c74a404a09c4c 100644 --- a/.github/workflows/go-update-commenter.yml +++ b/.github/workflows/go-update-commenter.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: # get the Go version of the target branch - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.base_ref }} persist-credentials: false @@ -26,7 +26,7 @@ jobs: echo version="$(cat .go-version)" >> $GITHUB_OUTPUT # get the Go version of the PR branch - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Get current Go version diff --git a/.github/workflows/go_mod_tidy.yml b/.github/workflows/go_mod_tidy.yml index e48d806c9dce9..4cc1a55690e6f 100644 --- a/.github/workflows/go_mod_tidy.yml +++ b/.github/workflows/go_mod_tidy.yml @@ -17,7 +17,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} - name: Checkout PR diff --git a/.github/workflows/gohai.yml b/.github/workflows/gohai.yml index 851c2f9d0ccd1..7ab3b5e49d2fc 100644 --- a/.github/workflows/gohai.yml +++ b/.github/workflows/gohai.yml @@ -34,7 +34,7 @@ jobs: go-file: [.go-version, pkg/gohai/go.mod] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 diff --git a/.github/workflows/label-analysis.yml b/.github/workflows/label-analysis.yml index ec53d5a695fed..7c4345e2ec323 100644 --- a/.github/workflows/label-analysis.yml +++ b/.github/workflows/label-analysis.yml @@ -23,7 +23,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Setup python @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 persist-credentials: false @@ -125,7 +125,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} - name: Setup python diff --git a/.github/workflows/markdown-lint-check.yml b/.github/workflows/markdown-lint-check.yml index 5f440614390ae..b5fe87f21f0f8 100644 --- a/.github/workflows/markdown-lint-check.yml +++ b/.github/workflows/markdown-lint-check.yml @@ -9,7 +9,7 @@ jobs: markdown-link-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - uses: gaurav-nelson/github-action-markdown-link-check@d53a906aa6b22b8979d33bc86170567e619495ec # v1.0.15 diff --git a/.github/workflows/report-merged-pr.yml b/.github/workflows/report-merged-pr.yml index feefb3c5446bd..879b5b7e13330 100644 --- a/.github/workflows/report-merged-pr.yml +++ b/.github/workflows/report-merged-pr.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false diff --git a/.github/workflows/serverless-benchmarks.yml b/.github/workflows/serverless-benchmarks.yml index 8ad83b34d36d1..a3a4ecbf609e5 100644 --- a/.github/workflows/serverless-benchmarks.yml +++ b/.github/workflows/serverless-benchmarks.yml @@ -24,7 +24,7 @@ jobs: sha: ${{ steps.prepare.outputs.sha }} steps: - name: Checkout ${{ github.base_ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.base_ref }} persist-credentials: false @@ -63,7 +63,7 @@ jobs: steps: - name: Checkout ${{ github.ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.sha }} persist-credentials: false diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml index 7be692d81d51a..fda6772398b9e 100644 --- a/.github/workflows/serverless-binary-size.yml +++ b/.github/workflows/serverless-binary-size.yml @@ -19,7 +19,7 @@ jobs: pull-requests: write # Add comment to PR steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: go/src/github.com/DataDog/datadog-agent persist-credentials: false @@ -36,7 +36,7 @@ jobs: fi - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/datadog-lambda-extension path: go/src/github.com/DataDog/datadog-lambda-extension diff --git a/.github/workflows/serverless-integration.yml b/.github/workflows/serverless-integration.yml index ace5e88fbda98..b321f2fb82182 100644 --- a/.github/workflows/serverless-integration.yml +++ b/.github/workflows/serverless-integration.yml @@ -26,7 +26,7 @@ jobs: name: ${{ matrix.suite }} on ${{ matrix.architecture }} steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: go/src/github.com/DataDog/datadog-agent persist-credentials: false @@ -40,7 +40,7 @@ jobs: run: sudo yarn global add serverless@^3.36.0 --prefix /usr/local - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/datadog-lambda-extension path: go/src/github.com/DataDog/datadog-lambda-extension From 8949577ed72baea2dd7697db60ad37b838d3999e Mon Sep 17 00:00:00 2001 From: Andrew Glaude Date: Wed, 18 Dec 2024 11:56:21 -0500 Subject: [PATCH 37/43] APM: Skip flaky trace config hostname test (#32233) --- comp/trace/config/config_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/comp/trace/config/config_test.go b/comp/trace/config/config_test.go index 9c9d45eb284ac..9706e2c784f4c 100644 --- a/comp/trace/config/config_test.go +++ b/comp/trace/config/config_test.go @@ -9,6 +9,7 @@ import ( "bufio" "bytes" "context" + _ "embed" "encoding/json" "errors" "net/http" @@ -245,6 +246,9 @@ func TestTelemetryEndpointsConfig(t *testing.T) { }) } +//go:embed testdata/stringcode.go.tmpl +var stringCodeBody string + func TestConfigHostname(t *testing.T) { t.Run("fail", func(t *testing.T) { overrides := map[string]interface{}{ @@ -360,11 +364,6 @@ func TestConfigHostname(t *testing.T) { }) t.Run("external", func(t *testing.T) { - body, err := os.ReadFile("testdata/stringcode.go.tmpl") - if err != nil { - t.Fatal(err) - } - // makeProgram creates a new binary file which returns the given response and exits to the OS // given the specified code, returning the path of the program. makeProgram := func(t *testing.T, response string, code int) string { @@ -372,7 +371,7 @@ func TestConfigHostname(t *testing.T) { if err != nil { t.Fatal(err) } - tmpl, err := template.New("program").Parse(string(body)) + tmpl, err := template.New("program").Parse(stringCodeBody) if err != nil { t.Fatal(err) } @@ -399,6 +398,7 @@ func TestConfigHostname(t *testing.T) { fallbackHostnameFunc = func() (string, error) { return "fallback.host", nil } t.Run("good", func(t *testing.T) { + t.Skip("Skip flaky test while we explore fixes.") bin := makeProgram(t, "host.name", 0) defer os.Remove(bin) From 623e2dcecf5174cf7519afe4913409ac739db9bf Mon Sep 17 00:00:00 2001 From: Romain Marcadier Date: Wed, 18 Dec 2024 18:16:00 +0100 Subject: [PATCH 38/43] [serverless/proxy]: override default ErrorHandler (#32326) --- pkg/serverless/proxy/proxy.go | 38 ++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/pkg/serverless/proxy/proxy.go b/pkg/serverless/proxy/proxy.go index 7d43dd36db257..1f8a2e3087658 100644 --- a/pkg/serverless/proxy/proxy.go +++ b/pkg/serverless/proxy/proxy.go @@ -8,6 +8,7 @@ package proxy import ( "context" + "errors" "net/http" "net/http/httputil" "net/url" @@ -68,9 +69,44 @@ func newProxy(target string, processor invocationlifecycle.InvocationProcessor) Scheme: "http", Host: target, } + proxy := httputil.NewSingleHostReverseProxy(url) + + // The default error handler logs "http: proxy error: %v" then returns an HTTP 502 (bad gateway) + // response. This is unfortunate because it lacks much any context on the original request that + // failed, and the commonly observed error today is "context deadline exceeded", which is not + // actionnable if you don't know what request it was for. It also logs to STDERR and does not + // honor the agent's log level. + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + log.Debugf( + "[serverless/proxy][%T] %s %s -- proxy error: %v", + // The dynamic type of processor informs about what kind of proxy this was (main/appsec) + processor, + // The request method and URL are useful to understand what exactly failed. We won't log + // the body (too large) or headers (risks containing sensitive data, such as API keys) + r.Method, r.URL, + // What happened that caused us to be called? + err, + ) + + // If the error is a [context.DeadlineExceeded], we return an HTTP 504 (gateway timeout) + // instead of the generic HTTP 502 (bad gateway) to give the client a better idea of what is + // going on (this may influence retry behavior, for example). + if errors.Is(err, context.DeadlineExceeded) { + w.WriteHeader(http.StatusGatewayTimeout) + } else { + // Return an HTTP 502 (bad gateway) error response; defer the retrying to the client. + w.WriteHeader(http.StatusBadGateway) + } + + // Writing the error message as best-effort, we simply debug-log any error that occur here. + if _, err := w.Write([]byte(err.Error())); err != nil { + log.Debugf("[serverless/proxy][%T] failed to write error message to response body: %v", processor, err) + } + } + return &runtimeProxy{ target: url, - proxy: httputil.NewSingleHostReverseProxy(url), + proxy: proxy, processor: processor, } } From 99d8d32638b9b302dc0566872b3df8d59e57d94d Mon Sep 17 00:00:00 2001 From: pducolin <45568537+pducolin@users.noreply.github.com> Date: Wed, 18 Dec 2024 18:16:10 +0100 Subject: [PATCH 39/43] [tasks] add back comments to test_infra_version.yaml (#32344) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Célian Raimbault <161456554+CelianR@users.noreply.github.com> --- tasks/buildimages.py | 16 +++++++++++++--- tasks/libs/ciproviders/gitlab_api.py | 5 ++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tasks/buildimages.py b/tasks/buildimages.py index e2bf5316e18c7..06ea84211ac1a 100644 --- a/tasks/buildimages.py +++ b/tasks/buildimages.py @@ -46,13 +46,23 @@ def update_test_infra_definitions(ctx: Context, commit_sha: str, go_mod_only: bo """ Update the test-infra-definition image version in the Gitlab CI as well as in the e2e go.mod """ - if not go_mod_only: - update_test_infra_def(".gitlab/common/test_infra_version.yml", commit_sha[:12], is_dev_image) - + print(f"Updating test-infra-definitions to {commit_sha}") with ctx.cd("test/new-e2e"): ctx.run(f"go get github.com/DataDog/test-infra-definitions@{commit_sha}") ctx.run("go mod tidy") + if not go_mod_only: + prefix_comment = """# File generated by inv buildimages.update-test-infra-definitions +# Please do not edit this file manually +# To update the test-infra-definitions version, run `inv buildimages.update-test-infra-definitions --commit-sha ` [--is-dev-image] +""" + update_test_infra_def( + file_path=".gitlab/common/test_infra_version.yml", + image_tag=commit_sha[:12], + is_dev_image=is_dev_image, + prefix_comment=prefix_comment, + ) + @task( help={ diff --git a/tasks/libs/ciproviders/gitlab_api.py b/tasks/libs/ciproviders/gitlab_api.py index adfd8aacfb569..f1c0b4784d2f6 100644 --- a/tasks/libs/ciproviders/gitlab_api.py +++ b/tasks/libs/ciproviders/gitlab_api.py @@ -1258,12 +1258,10 @@ def full_config_get_all_stages(full_config: dict) -> set[str]: return all_stages -def update_test_infra_def(file_path, image_tag, is_dev_image=False): +def update_test_infra_def(file_path, image_tag, is_dev_image=False, prefix_comment=""): """ Updates TEST_INFRA_DEFINITIONS_BUILDIMAGES in `.gitlab/common/test_infra_version.yml` file """ - import yaml - test_infra_def = {} with open(file_path) as test_infra_version_file: try: @@ -1276,6 +1274,7 @@ def update_test_infra_def(file_path, image_tag, is_dev_image=False): except yaml.YAMLError as e: raise Exit(f"Error while loading {file_path}: {e}") from e with open(file_path, "w") as test_infra_version_file: + test_infra_version_file.write(prefix_comment + ('\n\n' if prefix_comment else '')) # Add explicit_start=True to keep the document start marker --- # See "Document Start" in https://www.yaml.info/learn/document.html for more details yaml.dump(test_infra_def, test_infra_version_file, explicit_start=True) From 0d05fdd396532c8cbde7e43355681c6133a4b9f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Wed, 18 Dec 2024 18:16:21 +0100 Subject: [PATCH 40/43] omnibus: python: probe the build environment for CC/CXX (#32340) --- omnibus/config/software/python3.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 2fe37a032fc24..1cd3c096363f2 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -54,11 +54,11 @@ # Don't forward CC and CXX to python extensions Makefile, it's quite unlikely that any non default # compiler we use would end up being available in the system/docker image used by customers - if linux_target? && ENV["CC"] + if linux_target? && env["CC"] command "sed -i \"s/^CC=[[:space:]]*${CC}/CC=gcc/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CC}/gcc/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env end - if linux_target? && ENV["CXX"] + if linux_target? && env["CXX"] command "sed -i \"s/^CXX=[[:space:]]*${CXX}/CC=g++/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CXX}/g++/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env end From 3b2649bfd399d2d93e3de64b91f23b05164d7df4 Mon Sep 17 00:00:00 2001 From: Sylvain Baubeau Date: Wed, 18 Dec 2024 18:22:22 +0100 Subject: [PATCH 41/43] [CWS] Allow specifying cgroup managers for dumps generation (#32287) --- pkg/config/setup/system_probe_cws.go | 1 + pkg/security/config/config.go | 4 ++ .../ebpf/c/include/constants/custom.h | 4 ++ .../ebpf/c/include/helpers/activity_dump.h | 22 ++++++---- pkg/security/ebpf/c/include/hooks/cgroup.h | 14 +++---- pkg/security/ebpf/c/include/maps.h | 2 +- pkg/security/probe/probe_ebpf.go | 8 +--- pkg/security/resolvers/cgroup/resolver.go | 18 +++++++- .../resolvers/process/resolver_ebpf.go | 2 +- pkg/security/resolvers/resolvers_ebpf.go | 4 ++ pkg/security/secl/containerutils/cgroup.go | 6 +++ pkg/security/secl/containerutils/helpers.go | 33 ++++++++------- .../secl/containerutils/helpers_test.go | 2 +- .../security_profile/dump/load_controller.go | 42 ++++++++++++++++--- pkg/security/security_profile/dump/manager.go | 21 ++++++++-- 15 files changed, 133 insertions(+), 50 deletions(-) diff --git a/pkg/config/setup/system_probe_cws.go b/pkg/config/setup/system_probe_cws.go index d2cc6276907d3..c689e1fd68e28 100644 --- a/pkg/config/setup/system_probe_cws.go +++ b/pkg/config/setup/system_probe_cws.go @@ -57,6 +57,7 @@ func initCWSSystemProbeConfig(cfg pkgconfigmodel.Config) { cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.min_timeout", "10m") cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.max_dump_size", 1750) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_cgroups_count", 5) + cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.cgroup_managers", []string{"docker", "podman", "containerd", "cri-o"}) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_event_types", []string{"exec", "open", "dns", "imds"}) cfg.BindEnv("runtime_security_config.activity_dump.cgroup_dump_timeout") // deprecated in favor of dump_duration cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.dump_duration", "900s") diff --git a/pkg/security/config/config.go b/pkg/security/config/config.go index ab6b0b6b591e1..9d165a4ae1900 100644 --- a/pkg/security/config/config.go +++ b/pkg/security/config/config.go @@ -101,6 +101,9 @@ type RuntimeSecurityConfig struct { // ActivityDumpTracedCgroupsCount defines the maximum count of cgroups that should be monitored concurrently. Leave this parameter to 0 to prevent the generation // of activity dumps based on cgroups. ActivityDumpTracedCgroupsCount int + // ActivityDumpCgroupsManagers defines the cgroup managers we generate dumps for. + ActivityDumpCgroupsManagers []string + // ActivityDumpTracedEventTypes defines the list of events that should be captured in an activity dump. Leave this // parameter empty to monitor all event types. If not already present, the `exec` event will automatically be added // to this list. @@ -368,6 +371,7 @@ func NewRuntimeSecurityConfig() (*RuntimeSecurityConfig, error) { ActivityDumpLoadControlPeriod: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.load_controller_period"), ActivityDumpLoadControlMinDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.min_timeout"), ActivityDumpTracedCgroupsCount: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.traced_cgroups_count"), + ActivityDumpCgroupsManagers: pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.cgroup_managers"), ActivityDumpTracedEventTypes: parseEventTypeStringSlice(pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.traced_event_types")), ActivityDumpCgroupDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.dump_duration"), ActivityDumpRateLimiter: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.rate_limiter"), diff --git a/pkg/security/ebpf/c/include/constants/custom.h b/pkg/security/ebpf/c/include/constants/custom.h index 88be17fa3c80b..1e2311bf8fc7e 100644 --- a/pkg/security/ebpf/c/include/constants/custom.h +++ b/pkg/security/ebpf/c/include/constants/custom.h @@ -196,4 +196,8 @@ static __attribute__((always_inline)) u64 get_imds_ip() { #define CGROUP_MANAGER_CRI 4 #define CGROUP_MANAGER_SYSTEMD 5 +#define CGROUP_MANAGER_MASK 0b111 +#define CGROUP_SYSTEMD_SERVICE (0 << 8) +#define CGROUP_SYSTEMD_SCOPE (1 << 8) + #endif diff --git a/pkg/security/ebpf/c/include/helpers/activity_dump.h b/pkg/security/ebpf/c/include/helpers/activity_dump.h index c0f8d246006bf..70e776cca8161 100644 --- a/pkg/security/ebpf/c/include/helpers/activity_dump.h +++ b/pkg/security/ebpf/c/include/helpers/activity_dump.h @@ -53,10 +53,16 @@ __attribute__((always_inline)) struct cgroup_tracing_event_t *get_cgroup_tracing return evt; } +__attribute__((always_inline)) u32 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { + u32 cgroup_manager = cgroup->cgroup_flags & CGROUP_MANAGER_MASK; + u32 supported = (cgroup->cgroup_flags != 0) && (bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_manager) != NULL); + return supported; +} + __attribute__((always_inline)) bool reserve_traced_cgroup_spot(struct cgroup_context_t *cgroup, u64 now, u64 cookie, struct activity_dump_config *config) { // insert dump config defaults - u32 defaults_key = 0; - struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &defaults_key); + u32 cgroup_flags = cgroup->cgroup_flags; + struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_flags); if (defaults == NULL) { // should never happen, ignore return false; @@ -102,11 +108,15 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return 0; } - if ((container->cgroup_context.cgroup_flags & 0b111) == CGROUP_MANAGER_SYSTEMD) { + if (!is_cgroup_activity_dumps_supported(&container->cgroup_context)) { return 0; } - copy_container_id(container->container_id, evt->container.container_id); + if ((container->cgroup_context.cgroup_flags&CGROUP_MANAGER_MASK) != CGROUP_MANAGER_SYSTEMD) { + copy_container_id(container->container_id, evt->container.container_id); + } else { + evt->container.container_id[0] = '\0'; + } evt->container.cgroup_context = container->cgroup_context; evt->cookie = cookie; evt->config = config; @@ -115,10 +125,6 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return cookie; } -__attribute__((always_inline)) u64 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { - return (cgroup->cgroup_flags != 0) && ((cgroup->cgroup_flags&0b111) != CGROUP_MANAGER_SYSTEMD); -} - __attribute__((always_inline)) u64 should_trace_new_process_cgroup(void *ctx, u64 now, u32 pid, struct container_context_t *container) { // should we start tracing this cgroup ? struct cgroup_context_t cgroup_context; diff --git a/pkg/security/ebpf/c/include/hooks/cgroup.h b/pkg/security/ebpf/c/include/hooks/cgroup.h index a231142b8f90a..b7ce66a870b92 100644 --- a/pkg/security/ebpf/c/include/hooks/cgroup.h +++ b/pkg/security/ebpf/c/include/hooks/cgroup.h @@ -189,13 +189,13 @@ static __attribute__((always_inline)) int trace__cgroup_write(ctx_t *ctx) { #endif int length = bpf_probe_read_str(prefix, sizeof(cgroup_prefix_t), container_id) & 0xff; - if (cgroup_flags == 0 && ( - (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') - || - (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') - )) { - cgroup_flags = CGROUP_MANAGER_SYSTEMD; - } else if (cgroup_flags != 0) { + if (cgroup_flags == 0) { + if (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SERVICE; + } else if (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SCOPE; + } + } else { bpf_probe_read(&new_entry.container.container_id, sizeof(new_entry.container.container_id), container_id); } diff --git a/pkg/security/ebpf/c/include/maps.h b/pkg/security/ebpf/c/include/maps.h index d6ab3ceb74dfb..3b821b19fe5cf 100644 --- a/pkg/security/ebpf/c/include/maps.h +++ b/pkg/security/ebpf/c/include/maps.h @@ -28,7 +28,7 @@ BPF_ARRAY_MAP(syscall_ctx_gen_id, u32, 1) BPF_ARRAY_MAP(syscall_ctx, char[MAX_SYSCALL_CTX_SIZE], MAX_SYSCALL_CTX_ENTRIES) BPF_HASH_MAP(activity_dumps_config, u64, struct activity_dump_config, 1) // max entries will be overridden at runtime -BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 1) +BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 5) BPF_HASH_MAP(traced_cgroups, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(cgroup_wait_list, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(traced_pids, u32, u64, 8192) // max entries will be overridden at runtime diff --git a/pkg/security/probe/probe_ebpf.go b/pkg/security/probe/probe_ebpf.go index f06cbe8fa45ae..5fd26b4f63f8a 100644 --- a/pkg/security/probe/probe_ebpf.go +++ b/pkg/security/probe/probe_ebpf.go @@ -818,15 +818,11 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { return } - if cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)); err != nil { + cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)) + if err != nil { seclog.Debugf("Failed to resolve cgroup: %s", err) } else { event.CgroupTracing.CGroupContext = *cgroupContext - if cgroupContext.CGroupFlags.IsContainer() { - containerID, _ := containerutils.FindContainerID(cgroupContext.CGroupID) - event.CgroupTracing.ContainerContext.ContainerID = containerID - } - p.profileManagers.activityDumpManager.HandleCGroupTracingEvent(&event.CgroupTracing) } diff --git a/pkg/security/resolvers/cgroup/resolver.go b/pkg/security/resolvers/cgroup/resolver.go index 2137cccf14506..dcb1de1e6d378 100644 --- a/pkg/security/resolvers/cgroup/resolver.go +++ b/pkg/security/resolvers/cgroup/resolver.go @@ -48,6 +48,7 @@ type ResolverInterface interface { type Resolver struct { *utils.Notifier[Event, *cgroupModel.CacheEntry] sync.Mutex + cgroups *simplelru.LRU[model.PathKey, *model.CGroupContext] hostWorkloads *simplelru.LRU[containerutils.CGroupID, *cgroupModel.CacheEntry] containerWorkloads *simplelru.LRU[containerutils.ContainerID, *cgroupModel.CacheEntry] } @@ -80,6 +81,11 @@ func NewResolver() (*Resolver, error) { return nil, err } + cr.cgroups, err = simplelru.NewLRU(2048, func(_ model.PathKey, _ *model.CGroupContext) {}) + if err != nil { + return nil, err + } + return cr, nil } @@ -121,10 +127,19 @@ func (cr *Resolver) AddPID(process *model.ProcessCacheEntry) { } else { cr.hostWorkloads.Add(process.CGroup.CGroupID, newCGroup) } + cr.cgroups.Add(process.CGroup.CGroupFile, &process.CGroup) cr.NotifyListeners(CGroupCreated, newCGroup) } +// GetCGroupContext returns the cgroup context with the specified path key +func (cr *Resolver) GetCGroupContext(cgroupPath model.PathKey) (*model.CGroupContext, bool) { + cr.Lock() + defer cr.Unlock() + + return cr.cgroups.Get(cgroupPath) +} + // GetWorkload returns the workload referenced by the provided ID func (cr *Resolver) GetWorkload(id containerutils.ContainerID) (*cgroupModel.CacheEntry, bool) { if id == "" { @@ -171,6 +186,7 @@ func (cr *Resolver) deleteWorkloadPID(pid uint32, workload *cgroupModel.CacheEnt // check if the workload should be deleted if len(workload.PIDs) <= 0 { + cr.cgroups.Remove(workload.CGroupFile) cr.hostWorkloads.Remove(workload.CGroupID) if workload.ContainerID != "" { cr.containerWorkloads.Remove(workload.ContainerID) @@ -183,5 +199,5 @@ func (cr *Resolver) Len() int { cr.Lock() defer cr.Unlock() - return cr.hostWorkloads.Len() + cr.containerWorkloads.Len() + return cr.cgroups.Len() } diff --git a/pkg/security/resolvers/process/resolver_ebpf.go b/pkg/security/resolvers/process/resolver_ebpf.go index 5bcccdb52c540..409ab86fc2da7 100644 --- a/pkg/security/resolvers/process/resolver_ebpf.go +++ b/pkg/security/resolvers/process/resolver_ebpf.go @@ -340,7 +340,7 @@ func (p *EBPFResolver) enrichEventFromProc(entry *model.ProcessCacheEntry, proc // Retrieve the container ID of the process from /proc containerID, cgroup, err := p.containerResolver.GetContainerContext(pid) if err != nil { - return fmt.Errorf("snapshot failed for %d: couldn't parse container ID: %w", proc.Pid, err) + return fmt.Errorf("snapshot failed for %d: couldn't parse container and cgroup context: %w", proc.Pid, err) } entry.ContainerID = containerID diff --git a/pkg/security/resolvers/resolvers_ebpf.go b/pkg/security/resolvers/resolvers_ebpf.go index b8899bbd99adf..c9adeb82e557b 100644 --- a/pkg/security/resolvers/resolvers_ebpf.go +++ b/pkg/security/resolvers/resolvers_ebpf.go @@ -219,6 +219,10 @@ func (r *EBPFResolvers) Start(ctx context.Context) error { // ResolveCGroupContext resolves the cgroup context from a cgroup path key func (r *EBPFResolvers) ResolveCGroupContext(pathKey model.PathKey, cgroupFlags containerutils.CGroupFlags) (*model.CGroupContext, error) { + if cgroupContext, found := r.CGroupResolver.GetCGroupContext(pathKey); found { + return cgroupContext, nil + } + path, err := r.DentryResolver.Resolve(pathKey, true) if err != nil { return nil, fmt.Errorf("failed to resolve cgroup file %v: %w", pathKey, err) diff --git a/pkg/security/secl/containerutils/cgroup.go b/pkg/security/secl/containerutils/cgroup.go index 74e7a64540c84..9cc6cca12d884 100644 --- a/pkg/security/secl/containerutils/cgroup.go +++ b/pkg/security/secl/containerutils/cgroup.go @@ -24,6 +24,12 @@ const ( CGroupManagerSystemd // systemd ) +// CGroup flags +const ( + SystemdService CGroupFlags = (0 << 8) + SystemdScope CGroupFlags = (1 << 8) +) + const ( // ContainerRuntimeDocker is used to specify that a container is managed by Docker ContainerRuntimeDocker = "docker" diff --git a/pkg/security/secl/containerutils/helpers.go b/pkg/security/secl/containerutils/helpers.go index 0e46e87af7f2a..311322f94ffc6 100644 --- a/pkg/security/secl/containerutils/helpers.go +++ b/pkg/security/secl/containerutils/helpers.go @@ -29,39 +29,42 @@ func init() { containerIDPattern = regexp.MustCompile(ContainerIDPatternStr) } -func isSystemdCgroup(cgroup CGroupID) bool { - return strings.HasSuffix(string(cgroup), ".service") || strings.HasSuffix(string(cgroup), ".scope") +func isSystemdScope(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".scope") +} + +func isSystemdService(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".service") +} + +func getSystemdCGroupFlags(cgroup CGroupID) uint64 { + if isSystemdScope(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdScope) + } else if isSystemdService(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdService) + } + return 0 } // FindContainerID extracts the first sub string that matches the pattern of a container ID along with the container flags induced from the container runtime prefix func FindContainerID(s CGroupID) (ContainerID, uint64) { match := containerIDPattern.FindIndex([]byte(s)) if match == nil { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - - return "", 0 + return "", getSystemdCGroupFlags(s) } // first, check what's before if match[0] != 0 { previousChar := string(s[match[0]-1]) if strings.ContainsAny(previousChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } // then, check what's after if match[1] < len(s) { nextChar := string(s[match[1]]) if strings.ContainsAny(nextChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } diff --git a/pkg/security/secl/containerutils/helpers_test.go b/pkg/security/secl/containerutils/helpers_test.go index b5474b0df5016..5a2066e0295f2 100644 --- a/pkg/security/secl/containerutils/helpers_test.go +++ b/pkg/security/secl/containerutils/helpers_test.go @@ -63,7 +63,7 @@ func TestFindContainerID(t *testing.T) { { // Some random path which could match garden format input: "/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-f9176c6a-2a34-4ce2-86af-60d16888ed8e.scope", output: "", - flags: CGroupManagerSystemd, + flags: CGroupManagerSystemd | CGroupManager(SystemdScope), }, { // GARDEN with prefix / suffix input: "prefix01234567-0123-4567-890a-bcdesuffix", diff --git a/pkg/security/security_profile/dump/load_controller.go b/pkg/security/security_profile/dump/load_controller.go index ae8767bcfd153..f9bc80cdd3b4f 100644 --- a/pkg/security/security_profile/dump/load_controller.go +++ b/pkg/security/security_profile/dump/load_controller.go @@ -16,6 +16,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/metrics" + "github.com/DataDog/datadog-agent/pkg/security/secl/containerutils" "github.com/DataDog/datadog-agent/pkg/security/secl/model" "github.com/DataDog/datadog-agent/pkg/security/seclog" ) @@ -34,6 +35,7 @@ type ActivityDumpLoadController struct { // eBPF maps activityDumpConfigDefaults *ebpf.Map + activityDumpLoadConfig map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig } // NewActivityDumpLoadController returns a new activity dump load controller @@ -58,7 +60,11 @@ func NewActivityDumpLoadController(adm *ActivityDumpManager) (*ActivityDumpLoadC }, nil } -func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDumpLoadConfig { +func (lc *ActivityDumpLoadController) getDefaultLoadConfigs() (map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig, error) { + if lc.activityDumpLoadConfig != nil { + return lc.activityDumpLoadConfig, nil + } + defaults := NewActivityDumpLoadConfig( lc.adm.config.RuntimeSecurity.ActivityDumpTracedEventTypes, lc.adm.config.RuntimeSecurity.ActivityDumpCgroupDumpTimeout, @@ -68,14 +74,38 @@ func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDump lc.adm.resolvers.TimeResolver, ) defaults.WaitListTimestampRaw = uint64(lc.adm.config.RuntimeSecurity.ActivityDumpCgroupWaitListTimeout) - return defaults + + allDefaultConfigs := map[string]containerutils.CGroupManager{ + containerutils.CGroupManagerDocker.String(): containerutils.CGroupManagerDocker, + containerutils.CGroupManagerPodman.String(): containerutils.CGroupManagerPodman, + containerutils.CGroupManagerCRI.String(): containerutils.CGroupManagerCRI, + containerutils.CGroupManagerCRIO.String(): containerutils.CGroupManagerCRIO, + containerutils.CGroupManagerSystemd.String(): containerutils.CGroupManagerSystemd, + } + defaultConfigs := make(map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig) + for _, cgroupManager := range lc.adm.config.RuntimeSecurity.ActivityDumpCgroupsManagers { + cgroupManager, found := allDefaultConfigs[cgroupManager] + if !found { + return nil, fmt.Errorf("unsupported cgroup manager '%s'", cgroupManager) + } + defaultConfigs[cgroupManager] = defaults + } + lc.activityDumpLoadConfig = defaultConfigs + return defaultConfigs, nil } -// PushCurrentConfig pushes the current load controller config to kernel space -func (lc *ActivityDumpLoadController) PushCurrentConfig() error { +// PushDefaultCurrentConfigs pushes the current load controller configs to kernel space +func (lc *ActivityDumpLoadController) PushDefaultCurrentConfigs() error { + defaultConfigs, err := lc.getDefaultLoadConfigs() + if err != nil { + return err + } + // push default load config values - if err := lc.activityDumpConfigDefaults.Put(uint32(0), lc.getDefaultLoadConfig()); err != nil { - return fmt.Errorf("couldn't update default activity dump load config: %w", err) + for cgroupManager, defaultConfig := range defaultConfigs { + if err := lc.activityDumpConfigDefaults.Put(uint32(cgroupManager), defaultConfig); err != nil { + return fmt.Errorf("couldn't update default activity dump load config: %w", err) + } } return nil } diff --git a/pkg/security/security_profile/dump/manager.go b/pkg/security/security_profile/dump/manager.go index 079f45c13bc21..be81a28ca5c4d 100644 --- a/pkg/security/security_profile/dump/manager.go +++ b/pkg/security/security_profile/dump/manager.go @@ -321,7 +321,8 @@ func NewActivityDumpManager(config *config.Config, statsdClient statsd.ClientInt if err != nil { return nil, fmt.Errorf("couldn't instantiate the activity dump load controller: %w", err) } - if err = loadController.PushCurrentConfig(); err != nil { + + if err = loadController.PushDefaultCurrentConfigs(); err != nil { return nil, fmt.Errorf("failed to push load controller config settings to kernel space: %w", err) } adm.loadController = loadController @@ -449,8 +450,8 @@ func (adm *ActivityDumpManager) HandleCGroupTracingEvent(event *model.CgroupTrac adm.Lock() defer adm.Unlock() - if len(event.ContainerContext.ContainerID) == 0 { - seclog.Warnf("received a cgroup tracing event with an empty container ID") + if len(event.CGroupContext.CGroupID) == 0 { + seclog.Warnf("received a cgroup tracing event with an empty cgroup ID") return } @@ -514,7 +515,19 @@ workloadLoop: } // if we're still here, we can start tracing this workload - if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *adm.loadController.getDefaultLoadConfig()); err != nil { + defaultConfigs, err := adm.loadController.getDefaultLoadConfigs() + if err != nil { + seclog.Errorf("%v", err) + continue + } + + defaultConfig, found := defaultConfigs[containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags)] + if !found { + seclog.Errorf("Failed to find default activity dump config for %s", containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags).String()) + continue + } + + if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *defaultConfig); err != nil { if !errors.Is(err, unix.E2BIG) { seclog.Debugf("%v", err) break From 83c3dbd9266341686f223b4c336f41b12b9f9f79 Mon Sep 17 00:00:00 2001 From: "agent-platform-auto-pr[bot]" <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:25:41 +0000 Subject: [PATCH 42/43] [test-infra-definitions][automated] Bump test-infra-definitions to 221bbc806266eb15b90cb875deb79180e7591fbc (#32345) Co-authored-by: agent-platform-auto-pr[bot] <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> --- .gitlab/common/test_infra_version.yml | 7 ++----- test/new-e2e/go.mod | 12 ++++++------ test/new-e2e/go.sum | 24 ++++++++++++------------ 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index 9008ad3e857b1..944343f20da2f 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -1,7 +1,4 @@ --- variables: - # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev - # and check the job creating the image to make sure you have the right SHA prefix - TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" - # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 6459608ed9fa + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 221bbc806266 + TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: '' diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index 92faae31be87d..be219da1e3d81 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -58,7 +58,7 @@ require ( // `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version // Example: github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB // => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB - github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa + github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 github.com/aws/aws-sdk-go-v2 v1.32.6 github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 @@ -72,10 +72,10 @@ require ( github.com/kr/pretty v0.3.1 github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c github.com/pkg/sftp v1.13.7 - github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 - github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 - github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 - github.com/pulumi/pulumi/sdk/v3 v3.140.0 + github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 + github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 + github.com/pulumi/pulumi/sdk/v3 v3.142.0 github.com/samber/lo v1.47.0 github.com/stretchr/testify v1.10.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -171,7 +171,7 @@ require ( github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl/v2 v2.20.1 // indirect + github.com/hashicorp/hcl/v2 v2.22.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index e922c7c7a56b9..7284ab1424a36 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -17,8 +17,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= -github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa h1:l8KLWgU9l2qTlMtu4ing3V6PptTO+suaU8zusc45IiM= -github.com/DataDog/test-infra-definitions v0.0.0-20241218082354-6459608ed9fa/go.mod h1:1PAUwGjC25ACjfft4HrLEmHliuajlvjzcLFWpuqAIyk= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 h1:w+uoG7RdtPiW6YvJvWrzWOC78eNzCemcH8ZM6tJoUBw= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266/go.mod h1:+13pRFKChJo9VZ0WcHFm5GTPCNYyim3hxFjy6cpcLG8= github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= @@ -268,8 +268,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.22.0 h1:hkZ3nCtqeJsDhPRFz5EA9iwcG1hNWGePOTw6oyul12M= +github.com/hashicorp/hcl/v2 v2.22.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -403,10 +403,10 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435 github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE= github.com/pulumi/esc v0.10.0 h1:jzBKzkLVW0mePeanDRfqSQoCJ5yrkux0jIwAkUxpRKE= github.com/pulumi/esc v0.10.0/go.mod h1:2Bfa+FWj/xl8CKqRTWbWgDX0SOD4opdQgvYSURTGK2c= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 h1:wA38Ep4sEphX+3YGwFfaxRHs7NQv8dNObFepX6jaRa4= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1/go.mod h1:m/ejZ2INurqq/ncDjJfgC1Ff/lnbt0J/uO33BnPVots= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 h1:6082hB+ILpPB/0V5F+LTmHbX1BO54tCVOQCVOL/FYI4= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1/go.mod h1:z2bnBPHNYfk72IW1P01H9qikBtBSBhCwi3QpH6Y/38Q= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 h1:OvCLqUueOja9YE2WEGPYAw+lKHFRbLQ7QjwX55+uNsA= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0/go.mod h1:FFzye44v9E0BgaFXVB/9X7KH0S0MapoXEy2YonrQfz4= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 h1:jil2EBzZnKsRDrLfvx2gnAaq17HQLrTbpPsIb3h+98U= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0/go.mod h1:r+K4M7jnLqvvQDeR/0mBRq2EPZaqsDg24Ciy3ml/thA= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1 h1:miIJy4njnFYw7VxMLvEztoMPr9zYC2kqBTwRlaFAf48= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1/go.mod h1:LR1QBq0C1NIhmD9E0uKozCAu32j5qsamhrIsTSNVMS8= github.com/pulumi/pulumi-azure-native-sdk/compute/v2 v2.73.1 h1:79HTKSE1uJQolCRUHRFnIbSPNSIhxekIhznHnjpLi6s= @@ -427,16 +427,16 @@ github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 h1:s2Cpu6E2lmADNUbutbJGm6O+O9j0mBLlrh github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 h1:21oSj+TKlKTzQcxN9Hik7iSNNHPUQXN4s3itOnahy/w= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0/go.mod h1:YaEZms1NgXFqGhObKVofcAeWXu2V+3t/BAXdHQZq7fU= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 h1:quqoGsLbF7lpGpGU4mi5WfVLIAo4gfvoQeYYmemx1Dg= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3/go.mod h1:9dBA6+rtpKmyZB3k1XryUOHDOuNdoTODFKEEZZCtrz8= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 h1:7AjJpUyW6YHHpZr0bI6Fy1A3/b7ERxq1LAo5mlyNN1Y= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0/go.mod h1:ATS+UN8pguMxypQAK+SaPewesU+UN5dpf93PNqVuHzs= github.com/pulumi/pulumi-libvirt/sdk v0.5.3 h1:CiUGTweLLIxbAbADxxnwPv4BK8pxXfU8urokJvK1ihM= github.com/pulumi/pulumi-libvirt/sdk v0.5.3/go.mod h1:gAhyIZKtzs4rknrl8fu8BQnyqijAmViFbaUkyuHt4xY= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7 h1:39rhOe/PTUGMYia8pR5T2wbxxMt2pwrlonf0ncYKSzE= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7/go.mod h1:cxxDhJzUPt/YElfvlWa15Q4NGF6XXS8kUs4OQsCxSBk= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 h1:tXemWrzeVTqG8zq6hBdv1TdPFXjgZ+dob63a/6GlF1o= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1/go.mod h1:hODo3iEmmXDFOXqPK+V+vwI0a3Ww7BLjs5Tgamp86Ng= -github.com/pulumi/pulumi/sdk/v3 v3.140.0 h1:+Z/RBvdYg7tBNkBwk4p/FzlV7niBT3TbLAICq/Y0LDU= -github.com/pulumi/pulumi/sdk/v3 v3.140.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= +github.com/pulumi/pulumi/sdk/v3 v3.142.0 h1:SmcVddGuvwAh3g3XUVQQ5gVRQUKH1yZ6iETpDNHIHlw= +github.com/pulumi/pulumi/sdk/v3 v3.142.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= github.com/pulumiverse/pulumi-time/sdk v0.1.0 h1:xfi9HKDgV+GgDxQ23oSv9KxC3DQqViGTcMrJICRgJv0= github.com/pulumiverse/pulumi-time/sdk v0.1.0/go.mod h1:NUa1zA74DF002WrM6iF111A6UjX9knPpXufVRvBwNyg= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= From c29edf5b7aaad704dd85a89e72f42efee9e7556d Mon Sep 17 00:00:00 2001 From: Wassim Dhif Date: Wed, 18 Dec 2024 18:50:32 +0100 Subject: [PATCH 43/43] feat(apm): implement external data resolution (#32295) Signed-off-by: Wassim DHIF --- cmd/trace-agent/config/remote/config.go | 2 +- comp/otelcol/ddflareextension/impl/go.mod | 2 + .../exporter/datadogexporter/go.mod | 2 + .../otlp/components/statsprocessor/go.mod | 2 + comp/trace/config/setup.go | 6 ++- go.mod | 2 +- modules.yml | 3 +- pkg/proto/pbgo/core/model.pb.go | 4 +- pkg/trace/api/api.go | 2 +- pkg/trace/api/api_test.go | 5 ++- pkg/trace/api/container.go | 3 +- pkg/trace/api/container_linux.go | 43 ++++++++++++++++--- pkg/trace/api/debugger.go | 2 +- pkg/trace/api/evp_proxy.go | 2 +- pkg/trace/api/internal/header/headers.go | 10 +++++ pkg/trace/api/otlp.go | 4 +- pkg/trace/api/pipeline_stats.go | 2 +- pkg/trace/api/profiles.go | 2 +- pkg/trace/api/symdb.go | 2 +- pkg/trace/config/config.go | 4 ++ pkg/trace/go.mod | 2 + pkg/trace/stats/oteltest/go.mod | 2 + .../external_data_apm-6538531e3f34f305.yaml | 13 ++++++ test/otel/go.mod | 2 + 24 files changed, 100 insertions(+), 23 deletions(-) create mode 100644 releasenotes/notes/external_data_apm-6538531e3f34f305.yaml diff --git a/cmd/trace-agent/config/remote/config.go b/cmd/trace-agent/config/remote/config.go index d2c511a4586e7..6b650364743a7 100644 --- a/cmd/trace-agent/config/remote/config.go +++ b/cmd/trace-agent/config/remote/config.go @@ -43,7 +43,7 @@ func putBuffer(buffer *bytes.Buffer) { // ConfigHandler is the HTTP handler for configs func ConfigHandler(r *api.HTTPReceiver, cf rcclient.ConfigFetcher, cfg *config.AgentConfig, statsd statsd.ClientInterface, timing timing.Reporter) http.Handler { - cidProvider := api.NewIDProvider(cfg.ContainerProcRoot) + cidProvider := api.NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo) return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer timing.Since("datadog.trace_agent.receiver.config_process_ms", time.Now()) tags := r.TagStats(api.V07, req.Header, "").AsTags() diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index 7af5d796d92fb..21d3f17b9066a 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -13,6 +13,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/tagger/tags => ../../../core/tagger/tags github.com/DataDog/datadog-agent/comp/core/tagger/types => ../../../core/tagger/types github.com/DataDog/datadog-agent/comp/core/tagger/utils => ../../../core/tagger/utils @@ -151,6 +152,7 @@ require ( require go.opentelemetry.io/collector/extension/extensiontest v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index c3e28a412d2e6..301ef0f3b1f48 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ../../../../../core/telemetry github.com/DataDog/datadog-agent/comp/def => ../../../../../def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../../../../forwarder/defaultforwarder @@ -138,6 +139,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/status v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder v0.56.0-rc.3 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index 4ad268c10dfcb..18606500a04d2 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -3,6 +3,7 @@ module github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsproces go 1.22.0 replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../metricsclient github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../../../../comp/trace/compression/impl-gzip @@ -33,6 +34,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.56.0-rc.3 // indirect diff --git a/comp/trace/config/setup.go b/comp/trace/config/setup.go index f19c5eeaf8a4f..06a157dc6418b 100644 --- a/comp/trace/config/setup.go +++ b/comp/trace/config/setup.go @@ -19,13 +19,14 @@ import ( "go.opentelemetry.io/collector/component/componenttest" - apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" corecompcfg "github.com/DataDog/datadog-agent/comp/core/config" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/configcheck" + apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/config/env" "github.com/DataDog/datadog-agent/pkg/config/model" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -121,6 +122,9 @@ func prepareConfig(c corecompcfg.Component, tagger tagger.Component) (*config.Ag cfg.ContainerTags = func(cid string) ([]string, error) { return tagger.Tag(types.NewEntityID(types.ContainerID, cid), types.HighCardinality) } + cfg.ContainerIDFromOriginInfo = func(originInfo origindetection.OriginInfo) (string, error) { + return tagger.GenerateContainerIDFromOriginInfo(originInfo) + } cfg.ContainerProcRoot = coreConfigObject.GetString("container_proc_root") cfg.GetAgentAuthToken = apiutil.GetAuthToken return cfg, nil diff --git a/go.mod b/go.mod index 30981dbec9ce2..c8de7db9cc5f0 100644 --- a/go.mod +++ b/go.mod @@ -654,7 +654,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 github.com/DataDog/datadog-agent/comp/core/status v0.59.0-rc.6 github.com/DataDog/datadog-agent/comp/core/status/statusimpl v0.56.0-rc.3 - github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-00010101000000-000000000000 + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/DataDog/datadog-agent/comp/core/tagger/tags v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/core/tagger/types v0.59.0 github.com/DataDog/datadog-agent/comp/core/telemetry v0.59.0 diff --git a/modules.yml b/modules.yml index 291db46bc53cc..5d5e5cb2057ff 100644 --- a/modules.yml +++ b/modules.yml @@ -35,7 +35,8 @@ modules: comp/core/status: used_by_otel: true comp/core/status/statusimpl: default - comp/core/tagger/origindetection: default + comp/core/tagger/origindetection: + used_by_otel: true comp/core/tagger/tags: used_by_otel: true comp/core/tagger/types: diff --git a/pkg/proto/pbgo/core/model.pb.go b/pkg/proto/pbgo/core/model.pb.go index 016a2cf53e260..9262b4dc43a5a 100644 --- a/pkg/proto/pbgo/core/model.pb.go +++ b/pkg/proto/pbgo/core/model.pb.go @@ -1155,8 +1155,8 @@ type GenerateContainerIDFromOriginInfoRequest_ExternalData struct { unknownFields protoimpl.UnknownFields Init *bool `protobuf:"varint,1,opt,name=init,proto3,oneof" json:"init,omitempty"` // Init is true if the container is an init container. - ContainerName *string `protobuf:"bytes,2,opt,name=containerName,proto3,oneof" json:"containerName,omitempty"` // Container name as seen by the Admission Controller. - PodUID *string `protobuf:"bytes,3,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID as seen by the Admission Controller. + ContainerName *string `protobuf:"bytes,2,opt,name=containerName,proto3,oneof" json:"containerName,omitempty"` // Container name in the Kubernetes Pod spec. + PodUID *string `protobuf:"bytes,3,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID in the Kubernetes Pod spec. } func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) Reset() { diff --git a/pkg/trace/api/api.go b/pkg/trace/api/api.go index c74ecbb137fb9..5cb57d2478a27 100644 --- a/pkg/trace/api/api.go +++ b/pkg/trace/api/api.go @@ -141,7 +141,7 @@ func NewHTTPReceiver( } } log.Infof("Receiver configured with %d decoders and a timeout of %dms", semcount, conf.DecoderTimeout) - containerIDProvider := NewIDProvider(conf.ContainerProcRoot) + containerIDProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) telemetryForwarder := NewTelemetryForwarder(conf, containerIDProvider, statsd) return &HTTPReceiver{ Stats: info.NewReceiverStats(), diff --git a/pkg/trace/api/api_test.go b/pkg/trace/api/api_test.go index 9a468dc164487..2a1bdafe71307 100644 --- a/pkg/trace/api/api_test.go +++ b/pkg/trace/api/api_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/trace/config" @@ -576,7 +577,9 @@ func TestDecodeV05(t *testing.T) { req, err := http.NewRequest("POST", "/v0.5/traces", bytes.NewReader(b)) assert.NoError(err) req.Header.Set(header.ContainerID, "abcdef123789456") - tp, err := decodeTracerPayload(v05, req, NewIDProvider(""), "python", "3.8.1", "1.2.3") + tp, err := decodeTracerPayload(v05, req, NewIDProvider("", func(_ origindetection.OriginInfo) (string, error) { + return "abcdef123789456", nil + }), "python", "3.8.1", "1.2.3") assert.NoError(err) assert.EqualValues(tp, &pb.TracerPayload{ ContainerID: "abcdef123789456", diff --git a/pkg/trace/api/container.go b/pkg/trace/api/container.go index 508277c5abaf9..0c789f9d06ca7 100644 --- a/pkg/trace/api/container.go +++ b/pkg/trace/api/container.go @@ -12,6 +12,7 @@ import ( "net" "net/http" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" ) @@ -28,7 +29,7 @@ type IDProvider interface { type idProvider struct{} // NewIDProvider initializes an IDProvider instance, in non-linux environments the procRoot arg is unused. -func NewIDProvider(_ string) IDProvider { +func NewIDProvider(_ string, _ func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { return &idProvider{} } diff --git a/pkg/trace/api/container_linux.go b/pkg/trace/api/container_linux.go index 2129a6377c97f..6dd12d227894d 100644 --- a/pkg/trace/api/container_linux.go +++ b/pkg/trace/api/container_linux.go @@ -18,6 +18,7 @@ import ( "syscall" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/util/cgroups" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -96,7 +97,7 @@ func (i *noCgroupsProvider) GetContainerID(_ context.Context, h http.Header) str } // NewIDProvider initializes an IDProvider instance using the provided procRoot to perform cgroups lookups in linux environments. -func NewIDProvider(procRoot string) IDProvider { +func NewIDProvider(procRoot string, containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { // taken from pkg/util/containers/metrics/system.collector_linux.go var hostPrefix string if strings.HasPrefix(procRoot, "/host") { @@ -120,10 +121,11 @@ func NewIDProvider(procRoot string) IDProvider { } c := NewCache(1 * time.Minute) return &cgroupIDProvider{ - procRoot: procRoot, - controller: cgroupController, - cache: c, - reader: reader, + procRoot: procRoot, + controller: cgroupController, + cache: c, + reader: reader, + containerIDFromOriginInfo: containerIDFromOriginInfo, } } @@ -131,8 +133,9 @@ type cgroupIDProvider struct { procRoot string controller string // reader is used to retrieve the container ID from its cgroup v2 inode. - reader *cgroups.Reader - cache *Cache + reader *cgroups.Reader + cache *Cache + containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) } // GetContainerID returns the container ID. @@ -157,6 +160,11 @@ func (c *cgroupIDProvider) GetContainerID(ctx context.Context, h http.Header) st return containerID } + // Retrieve container ID from External Data header + if externalData := h.Get(header.ExternalData); externalData != "" { + return c.resolveContainerIDFromExternalData(externalData) + } + return "" } @@ -296,6 +304,27 @@ func (c *cgroupIDProvider) getCachedContainerID(key string, retrievalFunc func() return val, nil } +// resolveContainerIDFromExternalData returns the container ID for the given External Data. +func (c *cgroupIDProvider) resolveContainerIDFromExternalData(rawExternalData string) string { + var generatedContainerID string + + externalData, err := origindetection.ParseExternalData(rawExternalData) + if err != nil { + log.Errorf("Could not parse external data (%s): %v", rawExternalData, err) + return "" + } + generatedContainerID, err = c.containerIDFromOriginInfo(origindetection.OriginInfo{ + ExternalData: externalData, + ProductOrigin: origindetection.ProductOriginAPM, + }) + if err != nil { + log.Errorf("Could not generate container ID from external data (%s): %v", rawExternalData, err) + return "" + } + + return generatedContainerID +} + // The below cache is copied from /pkg/util/containers/v2/metrics/provider/cache.go. It is not // imported to avoid making the datadog-agent module a dependency of the pkg/trace module. The // datadog-agent module contains replace directives which are not inherited by packages that diff --git a/pkg/trace/api/debugger.go b/pkg/trace/api/debugger.go index 61164f1d3ed02..51816276a3f3b 100644 --- a/pkg/trace/api/debugger.go +++ b/pkg/trace/api/debugger.go @@ -81,7 +81,7 @@ func debuggerErrorHandler(err error) http.Handler { // newDebuggerProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newDebuggerProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/api/evp_proxy.go b/pkg/trace/api/evp_proxy.go index 452d73017aa8d..2795e6304b605 100644 --- a/pkg/trace/api/evp_proxy.go +++ b/pkg/trace/api/evp_proxy.go @@ -86,7 +86,7 @@ func evpProxyForwarder(conf *config.AgentConfig, statsd statsd.ClientInterface) req.Header["X-Forwarded-For"] = nil }, ErrorLog: logger, - Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot), statsd}, + Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo), statsd}, } } diff --git a/pkg/trace/api/internal/header/headers.go b/pkg/trace/api/internal/header/headers.go index 8ae458be834f5..88cb3549b2836 100644 --- a/pkg/trace/api/internal/header/headers.go +++ b/pkg/trace/api/internal/header/headers.go @@ -25,6 +25,16 @@ const ( // * "ci-,in-" LocalData = "Datadog-Entity-ID" + // ExternalData is a list that contain prefixed-items, split by a ','. Current items are: + // * "it-" if the container is an init container. + // * "cn-" for the container name. + // * "pu-" for the pod UID. + // Order does not matter. + // Possible values: + // * "it-false,cn-nginx,pu-3413883c-ac60-44ab-96e0-9e52e4e173e2" + // * "cn-init,pu-cb4aba1d-0129-44f1-9f1b-b4dc5d29a3b3,it-true" + ExternalData = "Datadog-External-Env" + // Lang specifies the name of the header which contains the language from // which the traces originate. Lang = "Datadog-Meta-Lang" diff --git a/pkg/trace/api/otlp.go b/pkg/trace/api/otlp.go index 88491ea9052ec..eb136d267e090 100644 --- a/pkg/trace/api/otlp.go +++ b/pkg/trace/api/otlp.go @@ -9,7 +9,6 @@ import ( "context" "encoding/hex" "fmt" - "github.com/DataDog/datadog-agent/pkg/trace/transform" "math" "net" "net/http" @@ -26,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/trace/sampler" "github.com/DataDog/datadog-agent/pkg/trace/timing" "github.com/DataDog/datadog-agent/pkg/trace/traceutil" + "github.com/DataDog/datadog-agent/pkg/trace/transform" "github.com/DataDog/datadog-go/v5/statsd" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" @@ -102,7 +102,7 @@ func NewOTLPReceiver(out chan<- *Payload, cfg *config.AgentConfig, statsd statsd enableReceiveResourceSpansV2Val = 1.0 } _ = statsd.Gauge("datadog.trace_agent.otlp.enable_receive_resource_spans_v2", enableReceiveResourceSpansV2Val, nil, 1) - return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} + return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} } // Start starts the OTLPReceiver, if any of the servers were configured as active. diff --git a/pkg/trace/api/pipeline_stats.go b/pkg/trace/api/pipeline_stats.go index b6fe3f2cd161c..b8daf12590bf7 100644 --- a/pkg/trace/api/pipeline_stats.go +++ b/pkg/trace/api/pipeline_stats.go @@ -71,7 +71,7 @@ func pipelineStatsErrorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. func newPipelineStatsProxy(conf *config.AgentConfig, urls []*url.URL, apiKeys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { log.Debug("[pipeline_stats] Creating reverse proxy") - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/profiles.go b/pkg/trace/api/profiles.go index 9b43efc94de74..19add3e307daa 100644 --- a/pkg/trace/api/profiles.go +++ b/pkg/trace/api/profiles.go @@ -109,7 +109,7 @@ func errorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. // For more details please see multiTransport. func newProfileProxy(conf *config.AgentConfig, targets []*url.URL, keys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/symdb.go b/pkg/trace/api/symdb.go index 31b96c494ce5f..b238b478eb65c 100644 --- a/pkg/trace/api/symdb.go +++ b/pkg/trace/api/symdb.go @@ -62,7 +62,7 @@ func symDBErrorHandler(err error) http.Handler { // newSymDBProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newSymDBProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getSymDBDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/config/config.go b/pkg/trace/config/config.go index 5d0e59f059e39..cbf0b0ea5c3a3 100644 --- a/pkg/trace/config/config.go +++ b/pkg/trace/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/obfuscate" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/trace/log" @@ -445,6 +446,9 @@ type AgentConfig struct { // ContainerTags ... ContainerTags func(cid string) ([]string, error) `json:"-"` + // ContainerIDFromOriginInfo ... + ContainerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) `json:"-"` + // ContainerProcRoot is the root dir for `proc` info ContainerProcRoot string diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index 51564f328f09e..29bf241afd7b8 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -52,6 +52,7 @@ require ( ) require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/shirou/gopsutil/v4 v4.24.11 go.opentelemetry.io/collector/component/componenttest v0.115.0 ) @@ -115,6 +116,7 @@ require ( replace ( github.com/DataDog/datadog-agent => ../../ + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../comp/trace/compression/impl-gzip github.com/DataDog/datadog-agent/comp/trace/compression/impl-zstd => ../../comp/trace/compression/impl-zstd diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index c3759db9cc74b..071e7c2c2eba0 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -20,6 +20,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect @@ -87,6 +88,7 @@ require ( ) replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../../../../comp/otelcol/otlp/components/metricsclient github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor => ../../../../comp/otelcol/otlp/components/statsprocessor github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../comp/trace/compression/def diff --git a/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml new file mode 100644 index 0000000000000..4aa3e9ef23e70 --- /dev/null +++ b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Implement External Data resolution for APM. This is needed to support the + latest Origin Detection spec and resolution with nested virtualization. + diff --git a/test/otel/go.mod b/test/otel/go.mod index a4523c0b33b0b..88d5f3a127be5 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ./../../comp/core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ./../../comp/core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../comp/core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ./../../comp/core/telemetry github.com/DataDog/datadog-agent/comp/def => ./../../comp/def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../comp/forwarder/defaultforwarder @@ -129,6 +130,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/flare/builder v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/flare/types v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 // indirect