From 0b8caf84eac83bf73d49f60176ba680cf6f9672c Mon Sep 17 00:00:00 2001 From: Yang Song Date: Wed, 23 Oct 2024 14:12:48 -0400 Subject: [PATCH] [OTEL-2063] Extent OTel Agent flare validation in E2E tests (#30362) --- test/new-e2e/go.mod | 17 + test/new-e2e/go.sum | 36 +++ .../tests/otel/otel-agent/minimal_test.go | 11 +- .../testdata/minimal-full-config.yml | 290 ++++++++++++++++++ .../testdata/minimal-provided-config.yml | 195 ++++++++++++ .../otel-agent/testdata/minimal-sources.json | 23 ++ test/new-e2e/tests/otel/utils/config_utils.go | 46 ++- 7 files changed, 612 insertions(+), 6 deletions(-) create mode 100644 test/new-e2e/tests/otel/otel-agent/testdata/minimal-full-config.yml create mode 100644 test/new-e2e/tests/otel/otel-agent/testdata/minimal-provided-config.yml create mode 100644 test/new-e2e/tests/otel/otel-agent/testdata/minimal-sources.json diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index e6508eda2978a..ed2b6c5cf0484 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -19,6 +19,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/telemetry => ../../comp/core/telemetry github.com/DataDog/datadog-agent/comp/def => ../../comp/def github.com/DataDog/datadog-agent/comp/netflow/payload => ../../comp/netflow/payload + github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/def => ../../comp/otelcol/ddflareextension/def github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../pkg/collector/check/defaults github.com/DataDog/datadog-agent/pkg/config/env => ../../pkg/config/env github.com/DataDog/datadog-agent/pkg/config/model => ../../pkg/config/model @@ -45,6 +46,7 @@ replace ( require ( github.com/DataDog/agent-payload/v5 v5.0.122 + github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/def v0.56.2 github.com/DataDog/datadog-agent/pkg/util/optional v0.56.2 github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.2 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.2 @@ -297,6 +299,13 @@ require ( require ( github.com/DataDog/datadog-agent/comp/core/tagger/utils v0.56.2 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.67.0 // indirect github.com/pulumi/pulumi-azure-native-sdk/compute/v2 v2.56.0 // indirect github.com/pulumi/pulumi-azure-native-sdk/containerservice/v2 v2.67.0 // indirect @@ -304,4 +313,12 @@ require ( github.com/pulumi/pulumi-azure-native-sdk/v2 v2.67.0 // indirect github.com/pulumi/pulumi-gcp/sdk/v6 v6.67.1 // indirect github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 // indirect + go.opentelemetry.io/collector/component v0.104.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.104.0 // indirect + go.opentelemetry.io/collector/confmap v0.104.0 // indirect + go.opentelemetry.io/collector/extension v0.104.0 // indirect + go.opentelemetry.io/collector/featuregate v1.11.0 // indirect + go.opentelemetry.io/collector/pdata v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect ) diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index 50dbc8c6c2db5..fdebd2e0da797 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -205,6 +205,8 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -269,6 +271,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= @@ -292,6 +296,12 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1 h1:/R8eXqasSTsmDCsAyYj+81Wteg8AqrV9CP6gvsTsOmM= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -322,12 +332,16 @@ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -512,6 +526,18 @@ github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8 github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= github.com/zorkian/go-datadog-api v2.30.0+incompatible h1:R4ryGocppDqZZbnNc5EDR8xGWF/z/MxzWnqTUijDQes= github.com/zorkian/go-datadog-api v2.30.0+incompatible/go.mod h1:PkXwHX9CUQa/FpB9ZwAD45N1uhCW4MT/Wj7m36PbKss= +go.opentelemetry.io/collector/component v0.104.0 h1:jqu/X9rnv8ha0RNZ1a9+x7OU49KwSMsPbOuIEykHuQE= +go.opentelemetry.io/collector/component v0.104.0/go.mod h1:1C7C0hMVSbXyY1ycCmaMUAR9fVwpgyiNQqxXtEWhVpw= +go.opentelemetry.io/collector/config/configtelemetry v0.104.0 h1:eHv98XIhapZA8MgTiipvi+FDOXoFhCYOwyKReOt+E4E= +go.opentelemetry.io/collector/config/configtelemetry v0.104.0/go.mod h1:WxWKNVAQJg/Io1nA3xLgn/DWLE/W1QOB2+/Js3ACi40= +go.opentelemetry.io/collector/confmap v0.104.0 h1:d3yuwX+CHpoyCh0iMv3rqb/vwAekjSm4ZDL6UK1nZSA= +go.opentelemetry.io/collector/confmap v0.104.0/go.mod h1:F8Lue+tPPn2oldXcfqI75PPMJoyzgUsKVtM/uHZLA4w= +go.opentelemetry.io/collector/extension v0.104.0 h1:bftkgFMKya/QIwK+bOxEAPVs/TvTez+s1mlaiUznJkA= +go.opentelemetry.io/collector/extension v0.104.0/go.mod h1:x7K0KyM1JGrtLbafEbRoVp0VpGBHpyx9hu87bsja6S4= +go.opentelemetry.io/collector/featuregate v1.11.0 h1:Z7puIymKoQRm3oNM/NH8reWc2zRPz2PNaJvuokh0lQY= +go.opentelemetry.io/collector/featuregate v1.11.0/go.mod h1:PsOINaGgTiFc+Tzu2K/X2jP+Ngmlp7YKGV1XrnBkH7U= +go.opentelemetry.io/collector/pdata v1.11.0 h1:rzYyV1zfTQQz1DI9hCiaKyyaczqawN75XO9mdXmR/hE= +go.opentelemetry.io/collector/pdata v1.11.0/go.mod h1:IHxHsp+Jq/xfjORQMDJjSH6jvedOSTOyu3nbxqhWSYE= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= @@ -520,10 +546,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9RO go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= +go.opentelemetry.io/otel/exporters/prometheus v0.49.0 h1:Er5I1g/YhfYv9Affk9nJLfH/+qCCVVg1f2R9AbJfqDQ= +go.opentelemetry.io/otel/exporters/prometheus v0.49.0/go.mod h1:KfQ1wpjf3zsHjzP149P4LyAwWRupc6c7t1ZJ9eXpKQM= go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= +go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= @@ -532,6 +562,12 @@ go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93V go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/test/new-e2e/tests/otel/otel-agent/minimal_test.go b/test/new-e2e/tests/otel/otel-agent/minimal_test.go index b527fa29fdf47..e3ccb0faaf971 100644 --- a/test/new-e2e/tests/otel/otel-agent/minimal_test.go +++ b/test/new-e2e/tests/otel/otel-agent/minimal_test.go @@ -26,6 +26,15 @@ type minimalTestSuite struct { //go:embed config/minimal.yml var minimalConfig string +//go:embed testdata/minimal-provided-config.yml +var minimalProvidedConfig string + +//go:embed testdata/minimal-full-config.yml +var minimalFullConfig string + +//go:embed testdata/minimal-sources.json +var minimalSources string + func TestOTelAgentMinimal(t *testing.T) { values := ` datadog: @@ -73,5 +82,5 @@ func (s *minimalTestSuite) TestOTelAgentInstalled() { } func (s *minimalTestSuite) TestOTelFlare() { - utils.TestOTelFlare(s) + utils.TestOTelFlare(s, minimalProvidedConfig, minimalFullConfig, minimalSources) } diff --git a/test/new-e2e/tests/otel/otel-agent/testdata/minimal-full-config.yml b/test/new-e2e/tests/otel/otel-agent/testdata/minimal-full-config.yml new file mode 100644 index 0000000000000..13581a739d845 --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/testdata/minimal-full-config.yml @@ -0,0 +1,290 @@ +connectors: + datadog/connector: + traces: + compute_stats_by_span_kind: true + compute_top_level_by_span_kind: true + ignore_resources: [] + peer_tags: [] + peer_tags_aggregation: true + resource_attributes_as_container_tags: [] + span_name_as_resource_name: true + span_name_remappings: {} + trace_buffer: 1000 +exporters: + datadog: + api: + fail_on_invalid_key: false + key: '[REDACTED]' + site: datadoghq.com + auth: null + compression: "" + cookies: null + disable_keep_alives: false + endpoint: "" + headers: {} + host_metadata: + enabled: true + hostname_source: config_or_system + tags: [] + hostname: otelcol-docker + http2_ping_timeout: 0s + http2_read_idle_timeout: 0s + idle_conn_timeout: null + logs: + batch_wait: 5 + compression_level: 6 + dialer: + timeout: 0s + dump_payloads: false + use_compression: true + max_conns_per_host: null + max_idle_conns: null + max_idle_conns_per_host: null + metrics: + apm_stats_receiver_addr: "" + delta_ttl: 3600 + dialer: + timeout: 0s + enabled: false + histograms: + mode: distributions + send_aggregation_metrics: false + send_count_sum_metrics: false + instrumentation_library_metadata_as_tags: false + instrumentation_scope_metadata_as_tags: false + resource_attributes_as_tags: true + summaries: + mode: gauges + sums: + cumulative_monotonic_mode: to_delta + initial_cumulative_monotonic_value: auto + tag_cardinality: "" + tags: "" + only_metadata: false + proxy_url: "" + read_buffer_size: 0 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 1000 + storage: null + timeout: 15s + tls: + ca_file: "" + ca_pem: '[REDACTED]' + cert_file: "" + cert_pem: '[REDACTED]' + cipher_suites: [] + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + key_pem: '[REDACTED]' + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + traces: + compute_top_level_by_span_kind: true + dialer: + timeout: 0s + ignore_resources: [] + span_name_as_resource_name: true + span_name_remappings: {} + trace_buffer: 0 + write_buffer_size: 0 +extensions: + ddflare/dd-autoconfigured: + auth: null + compression_algorithms: [] + cors: null + endpoint: localhost:7777 + include_metadata: false + max_request_body_size: 0 + response_headers: {} + tls: null + health_check/dd-autoconfigured: + auth: null + check_collector_pipeline: + enabled: false + exporter_failure_threshold: 5 + interval: 5m + compression_algorithms: [] + cors: null + endpoint: localhost:13133 + include_metadata: false + max_request_body_size: 0 + path: / + response_body: null + response_headers: {} + tls: null + pprof/dd-autoconfigured: + block_profile_fraction: 0 + dialer: + timeout: 0s + endpoint: localhost:1777 + mutex_profile_fraction: 0 + save_to_file: "" + zpages/dd-autoconfigured: + auth: null + compression_algorithms: [] + cors: null + endpoint: localhost:55679 + include_metadata: false + max_request_body_size: 0 + response_headers: {} + tls: null +processors: + batch: + metadata_cardinality_limit: 1000 + metadata_keys: [] + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 200ms + infraattributes/dd-autoconfigured: + cardinality: 0 + logs: + log: [] + metrics: + metric: [] + traces: + span: [] +receivers: + otlp: + protocols: + grpc: + auth: null + dialer: + timeout: 0s + endpoint: 0.0.0.0:4317 + include_metadata: false + keepalive: null + max_concurrent_streams: 0 + max_recv_msg_size_mib: 0 + read_buffer_size: 524288 + tls: null + transport: tcp + write_buffer_size: 0 + http: + auth: null + compression_algorithms: [] + cors: null + endpoint: 0.0.0.0:4318 + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + response_headers: {} + tls: null + traces_url_path: /v1/traces + prometheus/dd-autoconfigured: + config: + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_protocols: + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText0.0.4 + scrape_timeout: 10s + scrape_configs: + - enable_compression: true + enable_http2: true + follow_redirects: true + honor_timestamps: true + job_name: datadog-agent + metrics_path: /metrics + scheme: http + scrape_interval: 10s + scrape_protocols: + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText0.0.4 + scrape_timeout: 10s + static_configs: + - targets: + - 0.0.0.0:8888 + track_timestamps_staleness: false + report_extra_scrape_metrics: false + start_time_metric_regex: "" + target_allocator: null + trim_metric_suffixes: false + use_start_time_metric: false +service: + extensions: + - pprof/dd-autoconfigured + - zpages/dd-autoconfigured + - health_check/dd-autoconfigured + - ddflare/dd-autoconfigured + pipelines: + logs: + exporters: + - datadog + processors: + - batch + - infraattributes/dd-autoconfigured + receivers: + - otlp + metrics: + exporters: + - datadog + processors: + - batch + - infraattributes/dd-autoconfigured + receivers: + - otlp + - datadog/connector + metrics/dd-autoconfigured/datadog: + exporters: + - datadog + processors: [] + receivers: + - prometheus/dd-autoconfigured + traces: + exporters: + - datadog/connector + processors: + - batch + - infraattributes/dd-autoconfigured + receivers: + - otlp + traces/send: + exporters: + - datadog + processors: + - batch + - infraattributes/dd-autoconfigured + receivers: + - otlp + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + error_output_paths: + - stderr + initial_fields: {} + level: info + output_paths: + - stderr + sampling: + enabled: true + initial: 10 + thereafter: 100 + tick: 10s + metrics: + address: :8888 + level: Normal + readers: [] + resource: {} + traces: + processors: [] + propagators: [] \ No newline at end of file diff --git a/test/new-e2e/tests/otel/otel-agent/testdata/minimal-provided-config.yml b/test/new-e2e/tests/otel/otel-agent/testdata/minimal-provided-config.yml new file mode 100644 index 0000000000000..7ac2ab6efa33c --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/testdata/minimal-provided-config.yml @@ -0,0 +1,195 @@ +connectors: + datadog/connector: + traces: + compute_stats_by_span_kind: true + compute_top_level_by_span_kind: true + ignore_resources: [] + peer_tags: [] + peer_tags_aggregation: true + resource_attributes_as_container_tags: [] + span_name_as_resource_name: true + span_name_remappings: {} + trace_buffer: 1000 +exporters: + datadog: + api: + fail_on_invalid_key: false + key: '[REDACTED]' + site: datadoghq.com + auth: null + compression: "" + cookies: null + disable_keep_alives: false + endpoint: "" + headers: {} + host_metadata: + enabled: true + hostname_source: config_or_system + tags: [] + hostname: otelcol-docker + http2_ping_timeout: 0s + http2_read_idle_timeout: 0s + idle_conn_timeout: null + logs: + batch_wait: 5 + compression_level: 6 + dialer: + timeout: 0s + dump_payloads: false + use_compression: true + max_conns_per_host: null + max_idle_conns: null + max_idle_conns_per_host: null + metrics: + apm_stats_receiver_addr: "" + delta_ttl: 3600 + dialer: + timeout: 0s + enabled: false + histograms: + mode: distributions + send_aggregation_metrics: false + send_count_sum_metrics: false + instrumentation_library_metadata_as_tags: false + instrumentation_scope_metadata_as_tags: false + resource_attributes_as_tags: true + summaries: + mode: gauges + sums: + cumulative_monotonic_mode: to_delta + initial_cumulative_monotonic_value: auto + tag_cardinality: "" + tags: "" + only_metadata: false + proxy_url: "" + read_buffer_size: 0 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 1000 + storage: null + timeout: 15s + tls: + ca_file: "" + ca_pem: '[REDACTED]' + cert_file: "" + cert_pem: '[REDACTED]' + cipher_suites: [] + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + key_pem: '[REDACTED]' + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + traces: + compute_top_level_by_span_kind: true + dialer: + timeout: 0s + ignore_resources: [] + span_name_as_resource_name: true + span_name_remappings: {} + trace_buffer: 0 + write_buffer_size: 0 +extensions: {} +processors: + batch: + metadata_cardinality_limit: 1000 + metadata_keys: [] + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 200ms +receivers: + otlp: + protocols: + grpc: + auth: null + dialer: + timeout: 0s + endpoint: 0.0.0.0:4317 + include_metadata: false + keepalive: null + max_concurrent_streams: 0 + max_recv_msg_size_mib: 0 + read_buffer_size: 524288 + tls: null + transport: tcp + write_buffer_size: 0 + http: + auth: null + compression_algorithms: [] + cors: null + endpoint: 0.0.0.0:4318 + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + response_headers: {} + tls: null + traces_url_path: /v1/traces +service: + extensions: [] + pipelines: + logs: + exporters: + - datadog + processors: + - batch + receivers: + - otlp + metrics: + exporters: + - datadog + processors: + - batch + receivers: + - otlp + - datadog/connector + traces: + exporters: + - datadog/connector + processors: + - batch + receivers: + - otlp + traces/send: + exporters: + - datadog + processors: + - batch + receivers: + - otlp + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + error_output_paths: + - stderr + initial_fields: {} + level: info + output_paths: + - stderr + sampling: + enabled: true + initial: 10 + thereafter: 100 + tick: 10s + metrics: + address: :8888 + level: Normal + readers: [] + resource: {} + traces: + processors: [] + propagators: [] \ No newline at end of file diff --git a/test/new-e2e/tests/otel/otel-agent/testdata/minimal-sources.json b/test/new-e2e/tests/otel/otel-agent/testdata/minimal-sources.json new file mode 100644 index 0000000000000..8928af8a6f2d6 --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/testdata/minimal-sources.json @@ -0,0 +1,23 @@ +{ + "health_check/dd-autoconfigured": { + "url": [ + "localhost:13133" + ] + }, + "pprof/dd-autoconfigured": { + "url": [ + "localhost:1777/debug/pprof/heap", + "localhost:1777/debug/pprof/allocs", + "localhost:1777/debug/pprof/profile" + ] + }, + "zpages/dd-autoconfigured": { + "url": [ + "localhost:55679/debug/servicez", + "localhost:55679/debug/pipelinez", + "localhost:55679/debug/extensionz", + "localhost:55679/debug/featurez", + "localhost:55679/debug/tracez" + ] + } +} \ No newline at end of file diff --git a/test/new-e2e/tests/otel/utils/config_utils.go b/test/new-e2e/tests/otel/utils/config_utils.go index 8ef614c8fcebd..8f69ace313d17 100644 --- a/test/new-e2e/tests/otel/utils/config_utils.go +++ b/test/new-e2e/tests/otel/utils/config_utils.go @@ -8,13 +8,18 @@ package utils import ( "context" + "encoding/json" "strings" + "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + + extension "github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/def" ) // TestOTelAgentInstalled checks that the OTel Agent is installed in the test suite @@ -24,7 +29,7 @@ func TestOTelAgentInstalled(s OTelTestSuite) { } // TestOTelFlare tests that the OTel Agent flare functionality works as expected -func TestOTelFlare(s OTelTestSuite) { +func TestOTelFlare(s OTelTestSuite, providedCfg string, fullCfg string, sources string) { err := s.Env().FakeIntake.Client().FlushServerAndResetAggregators() require.NoError(s.T(), err) @@ -56,10 +61,19 @@ func TestOTelFlare(s OTelTestSuite) { otelResponseContent, err := flare.GetFileContent(otelResponse) s.T().Log("Got flare otel-response.json", otelResponseContent) require.NoError(s.T(), err) - expectedContents := []string{"otel-agent", "ddflare/dd-autoconfigured:", "health_check/dd-autoconfigured:", "pprof/dd-autoconfigured:", "zpages/dd-autoconfigured:", "infraattributes/dd-autoconfigured:", "prometheus/dd-autoconfigured:", "key: '[REDACTED]'"} - for _, expected := range expectedContents { - assert.Contains(s.T(), otelResponseContent, expected) - } + var resp extension.Response + require.NoError(s.T(), json.Unmarshal([]byte(otelResponseContent), &resp)) + + assert.Equal(s.T(), "otel-agent", resp.AgentCommand) + assert.Equal(s.T(), "Datadog Agent OpenTelemetry Collector", resp.AgentDesc) + assert.Equal(s.T(), "", resp.RuntimeOverrideConfig) + + validateConfigs(s.T(), providedCfg, resp.CustomerConfig) + validateConfigs(s.T(), fullCfg, resp.RuntimeConfig) + + srcJSONStr, err := json.Marshal(resp.Sources) + require.NoError(s.T(), err) + assert.JSONEq(s.T(), sources, string(srcJSONStr)) } func getAgentPod(s OTelTestSuite) corev1.Pod { @@ -70,3 +84,25 @@ func getAgentPod(s OTelTestSuite) corev1.Pod { require.NotEmpty(s.T(), res.Items) return res.Items[0] } + +func validateConfigs(t *testing.T, expectedCfg string, actualCfg string) { + var actualConfRaw map[string]any + require.NoError(t, yaml.Unmarshal([]byte(actualCfg), &actualConfRaw)) + + // Traces, metrics and logs endpoints are set dynamically to the fake intake address in the config + // These endpoints vary from test to test and should be ignored in the comparison + exps, _ := actualConfRaw["exporters"].(map[string]any) + ddExp, _ := exps["datadog"].(map[string]any) + tcfg := ddExp["traces"].(map[string]any) + delete(tcfg, "endpoint") + mcfg := ddExp["metrics"].(map[string]any) + delete(mcfg, "endpoint") + lcfg := ddExp["logs"].(map[string]any) + delete(lcfg, "endpoint") + + actualCfgBytes, err := yaml.Marshal(actualConfRaw) + require.NoError(t, err) + actualCfg = string(actualCfgBytes) + + assert.YAMLEq(t, expectedCfg, actualCfg) +}