Configure e2e tests for OTel collector (#5270)

* Add a job that publish otel-collector on agent qa registry

* Change tag

* Try to run e2e tests

* remove pipeline id and commit sha

* Remove things specific to datadog-agent

* Add stage

* Set variables

* Rename job

* Deploy tha agent with Helm

* Setup opentelemetry-collector-contrib to install collector and run e2e tests

* Configure to use image built in the CI

* Check if metrics and logs are found

* try with pulumi logs

* Fix go test flag

* Update dependency

* Fix use pipeline ID

* Exclude e2e-tests from collector CI

* Address suggestions

* Fix tests

* Fix tests

* Update deps

* Add more tests

* Fix endpoint

* Finish tests and simplify config

* Fix mod version

* Fix ttl

* Use string instead of map

---------

Co-authored-by: Stanley Liu <[email protected]>

.github/ALLOWLIST

@@ -26,3 +26,4 @@ internal/common
 ## UNMAINTAINED components
 exporter/skywalkingexporter/
 receiver/googlecloudspannerreceiver/
+e2e-tests/

.github/CODEOWNERS

@@ -307,3 +307,4 @@ reports/distributions/contrib.yaml @open-telemetry/collector-contrib-approvers
 
 exporter/skywalkingexporter/                                        @open-telemetry/collector-contrib-approvers 
 receiver/googlecloudspannerreceiver/                                @open-telemetry/collector-contrib-approvers 
+e2e-tests/

.gitlab-ci.yml

@@ -9,13 +9,19 @@ variables:
   SANDBOX_CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel"
   DEMO_CLUSTER_ARN: "arn:aws:eks:us-east-1:172597598159:cluster/prod-eks-otel-a-us-east-1"
   DEPLOY_SCRIPT: "./ci/scripts/ci-deploy.sh"
+  # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
+  # and check the job creating the image to make sure you have the right SHA prefix
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
+  # Make sure to update test-infra-definitions version in go.mod as well
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: b02a714d7de7
 
 
 # BUILD STAGES
 stages:
   - build-and-push-ci-image
   - build-collector-image
   - push-collector-image
+  - e2e
   - staging-deploy
   - prod-deploy
 
@@ -64,8 +70,19 @@ push-collector-image-staging:
   !!merge <<: *push-collector-image
   variables:
     BUILD_REGISTRY: $BUILD_SANDBOX_REGISTRY
+push-collector-image-agent-qa:
+  stage: push-collector-image
+  needs: ['build-collector-image-main']
+  trigger:
+    project: DataDog/public-images
+    branch: main
+    strategy: depend
+  variables:
+    IMG_SOURCES: $BUILD_DD_REGISTRY:otelcolcontrib-v${CI_COMMIT_SHORT_SHA}
+    IMG_DESTINATIONS: otel-collector-contrib:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
+    IMG_REGISTRIES: agent-qa
 
-# DEPLOY COLLECTOR STAGING 
+# DEPLOY COLLECTOR STAGING
 .staging-deploy: &staging-deploy
   stage: staging-deploy
   tags: ["runner:docker", "size:large"]
@@ -121,7 +138,7 @@ staging-deploy-ds-gateway:
     CLUSTER_ROLE: opentelemetry-collector-ds-gateway
     REGISTRY: $BUILD_SANDBOX_REGISTRY
 
-# DEPLOY COLLECTOR PROD 
+# DEPLOY COLLECTOR PROD
 .prod-deploy: &prod-deploy-demo-eks
   stage: prod-deploy
   tags: ["runner:docker", "size:large"]
@@ -146,8 +163,43 @@ prod-deploy:
     CLUSTER_NAME: $DEMO_CLUSTER_NAME
     CLUSTER_ARN: $DEMO_CLUSTER_ARN
     NAMESPACE: otel
-    NODE_GROUP: 
+    NODE_GROUP:
     MODE: daemonset
     REPLICA_COUNT: 1
     CLUSTER_ROLE: opentelemetry-collector
-    REGISTRY: $BUILD_DEMO_REGISTRY
+    REGISTRY: $BUILD_DEMO_REGISTRY
+
+.new_e2e_template:
+  stage: e2e
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  tags: ["arch:amd64"]
+  before_script:
+    # Setup AWS Credentials
+    - mkdir -p ~/.aws
+    - aws ssm get-parameter --region us-east-1 --name "ci.opentelemetry-collector-contrib.agent-qa-profile" --with-decryption --query "Parameter.Value" --output text >> ~/.aws/config
+    - export AWS_PROFILE=agent-qa-ci
+    # Now all `aws` commands target the agent-qa profile
+    - aws ssm get-parameter --region us-east-1 --name "ci.opentelemetry-collector-contrib.ssh_public_key" --with-decryption --query "Parameter.Value" --output text > $E2E_PUBLIC_KEY_PATH
+    - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && aws ssm get-parameter --region us-east-1 --name "ci.opentelemetry-collector-contrib.ssh_private_key" --with-decryption --query "Parameter.Value" --output text > $E2E_PRIVATE_KEY_PATH
+    # Use S3 backend
+    - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
+  variables:
+    KUBERNETES_MEMORY_REQUEST: 12Gi
+    KUBERNETES_MEMORY_LIMIT: 16Gi
+    KUBERNETES_CPU_REQUEST: 6
+    E2E_PIPELINE_ID: $CI_PIPELINE_ID
+    E2E_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
+    E2E_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub
+    E2E_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key
+    E2E_KEY_PAIR_NAME: ci.open-telemetry-collector-contrib
+    E2E_OUTPUT_DIR: $CI_PROJECT_DIR/e2e-output
+    E2E_PULUMI_VERBOSE_PROGRESS_STREAMS: true
+  script:
+    - cd e2e-tests && go test $TARGETS/... -v -timeout 0 --args -docker_secret $(aws ecr get-login-password)
+
+new-e2e-otel-collector:
+  extends: .new_e2e_template
+  needs: ['push-collector-image-agent-qa']
+  variables:
+    TARGETS: ./tests
+    TEAM: otel

Makefile

@@ -22,6 +22,7 @@ EX_COMPONENTS=-not -path "./receiver/*" -not -path "./processor/*" -not -path ".
 EX_INTERNAL=-not -path "./internal/*"
 EX_PKG=-not -path "./pkg/*"
 EX_CMD=-not -path "./cmd/*"
+EX_E2E=-not -path "./e2e-tests/*"
 
 # NONROOT_MODS includes ./* dirs (excludes . dir)
 NONROOT_MODS := $(shell find . $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) )
@@ -49,7 +50,7 @@ PKG_MODS := $(shell find ./pkg/* $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) )
 CMD_MODS_0 := $(shell find ./cmd/[a-m]* $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) )
 CMD_MODS_1 := $(shell find ./cmd/[n-z]* $(FIND_MOD_ARGS) -not -path "./cmd/otelcontribcol/*" -exec $(TO_MOD_DIR) )
 CMD_MODS := $(CMD_MODS_0) $(CMD_MODS_1)
-OTHER_MODS := $(shell find . $(EX_COMPONENTS) $(EX_INTERNAL) $(EX_PKG) $(EX_CMD) $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) ) $(PWD)
+OTHER_MODS := $(shell find . $(EX_COMPONENTS) $(EX_INTERNAL) $(EX_PKG) $(EX_CMD) $(EX_E2E) $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) ) $(PWD)
 ALL_MODS := $(RECEIVER_MODS) $(PROCESSOR_MODS) $(EXPORTER_MODS) $(EXTENSION_MODS) $(CONNECTOR_MODS) $(INTERNAL_MODS) $(PKG_MODS) $(CMD_MODS) $(OTHER_MODS)
 
 FIND_INTEGRATION_TEST_MODS={ find . -type f -name "*integration_test.go" & find . -type f -name "*e2e_test.go" -not -path "./testbed/*"; }

e2e-tests/go.mod

@@ -0,0 +1,198 @@
+module github.com/DataDog/opentelemetry-collector-contrib/e2e-tests
+
+go 1.22.4
+
+require (
+	github.com/DataDog/datadog-agent/test/fakeintake v0.57.0-devel.0.20240809183524-3ecb6c03fbb8
+	github.com/DataDog/datadog-agent/test/new-e2e v0.57.0-devel.0.20240809183524-3ecb6c03fbb8
+	github.com/DataDog/test-infra-definitions v0.0.0-20240808172947-94712f9a273b
+	github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1
+	github.com/pulumi/pulumi/sdk/v3 v3.126.0
+	github.com/stretchr/testify v1.9.0
+	gopkg.in/yaml.v3 v3.0.1
+	k8s.io/api v0.30.2
+	k8s.io/apimachinery v0.30.2
+)
+
+require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/BurntSushi/toml v1.2.1 // indirect
+	github.com/DataDog/agent-payload/v5 v5.0.122 // indirect
+	github.com/DataDog/datadog-agent/comp/netflow/payload v0.56.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/proto v0.56.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/optional v0.56.0-rc.3 // indirect
+	github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect
+	github.com/DataDog/datadog-api-client-go/v2 v2.27.0 // indirect
+	github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a // indirect
+	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
+	github.com/agext/levenshtein v1.2.3 // indirect
+	github.com/alessio/shellescape v1.4.2 // indirect
+	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
+	github.com/atotto/clipboard v0.1.4 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.19 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.19 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ecr v1.30.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ecs v1.42.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.14 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.56.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssm v1.50.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.12 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.13 // indirect
+	github.com/aws/smithy-go v1.20.3 // indirect
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/charmbracelet/bubbles v0.18.0 // indirect
+	github.com/charmbracelet/bubbletea v0.25.0 // indirect
+	github.com/charmbracelet/lipgloss v0.10.0 // indirect
+	github.com/cheggaaa/pb v1.0.29 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/containerd/console v1.0.4 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/djherbis/times v1.6.0 // indirect
+	github.com/docker/cli v24.0.7+incompatible // indirect
+	github.com/docker/docker v25.0.6+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.5.0 // indirect
+	github.com/go-git/go-git/v5 v5.12.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.2.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/hcl/v2 v2.20.1 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-localereader v0.0.1 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mitchellh/go-ps v1.0.0 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/muesli/reflow v0.3.0 // indirect
+	github.com/muesli/termenv v0.15.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/nxadm/tail v1.4.11 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opentracing/basictracer-go v1.1.0 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pgavlin/fx v0.1.6 // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/sftp v1.13.6 // indirect
+	github.com/pkg/term v1.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect
+	github.com/pulumi/esc v0.9.1 // indirect
+	github.com/pulumi/pulumi-aws/sdk/v6 v6.44.0 // indirect
+	github.com/pulumi/pulumi-awsx/sdk/v2 v2.13.0 // indirect
+	github.com/pulumi/pulumi-command/sdk v0.9.2 // indirect
+	github.com/pulumi/pulumi-docker/sdk/v4 v4.5.1 // indirect
+	github.com/pulumi/pulumi-eks/sdk/v2 v2.7.6 // indirect
+	github.com/pulumi/pulumi-random/sdk/v4 v4.16.3 // indirect
+	github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 // indirect
+	github.com/pulumiverse/pulumi-time/sdk v0.0.17 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
+	github.com/samber/lo v1.39.0 // indirect
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/skeema/knownhosts v1.2.2 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/texttheater/golang-levenshtein v1.0.1 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/tweekmonster/luser v0.0.0-20161003172636-3fa38070dbd7 // indirect
+	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
+	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/zclconf/go-cty v1.14.4 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/oauth2 v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/term v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/time v0.6.0 // indirect
+	golang.org/x/tools v0.23.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
+	google.golang.org/grpc v1.65.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/client-go v0.30.2 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	lukechampine.com/frand v1.4.2 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)