OTEL-2238 update integration test to build ddflare and datadog conver…

…ter (DataDog#31189)

.gitlab/JOBOWNERS

@@ -133,7 +133,7 @@ trigger_auto_staging_release           @DataDog/agent-delivery
 integration_tests_windows*    @DataDog/windows-agent
 integration_tests_otel        @DataDog/opentelemetry
 docker_image_build_otel       @DataDog/opentelemetry
-ddflare_extension_ocb_build   @DataDog/opentelemetry
+datadog_otel_components_ocb_build  @DataDog/opentelemetry
 agent_integration_tests       @DataDog/container-integrations
 docker_integration_tests      @DataDog/container-integrations
 

.gitlab/integration_test/otel.yml

@@ -57,35 +57,25 @@ docker_image_build_otel:
     - when: always
 
 
-ddflare_extension_ocb_build:
+datadog_otel_components_ocb_build:
   stage: integration_test
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   needs: ["go_deps"]
+  artifacts:
+    paths:
+      - ocb-output.log
+      - otelcol-custom.log
+      - flare-info.log
   before_script:
     - !reference [.retrieve_linux_go_deps]
-    - mkdir -p /tmp/otel-ci
-    - cp test/otel/testdata/* /tmp/otel-ci/
-    - wget -O /tmp/otel-ci/ocb https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fbuilder%2Fv0.114.0/ocb_0.114.0_linux_amd64
-    - chmod +x /tmp/otel-ci/ocb
   script:
-    - echo 'Building collector with OCB and test ddflare extension'
-    - /tmp/otel-ci/ocb --config=/tmp/otel-ci/builder-config.yaml > ocb-output.log 2>&1
-    - grep -q 'Compiled' ocb-output.log || (echo "OCB failed to compile" && exit 1)
-    - "grep -q '{\"binary\": \"/tmp/otel-ci/otelcol-custom/otelcol-custom\"}' ocb-output.log || (echo \"OCB failed to compile\" && exit 1)"
-    - /tmp/otel-ci/otelcol-custom/otelcol-custom --config /tmp/otel-ci/collector-config.yaml > otelcol-custom.log 2>&1 &
-    - OTELCOL_PID=$!  # Capture the process ID
-    - sleep 10  # Wait for the process to start
-    - grep -q 'Everything is ready. Begin running and processing data.' otelcol-custom.log || (echo "custom collector failed to start" && kill $OTELCOL_PID && exit 1)
-    - curl -k https://localhost:7777 > flare-info.log 2>&1 # get the dd flare info
-    - "grep -q '\"provided_configuration\": \"\"' flare-info.log || (echo \"provided config should not be supported with ocb\" && kill $OTELCOL_PID && exit 1)"
-    - grep -q 'extensions:\\n  - ddflare\\n' flare-info.log || (echo "ddflare extension should be enabled" && kill $OTELCOL_PID && exit 1)
-    - kill $OTELCOL_PID  # Kill the process
+    - echo "Building custom collector with datadog components"
+    - test/otel/testdata/ocb_build_script.sh
+    - echo "see artifacts for job logs"
   rules:
     - if: $CI_PIPELINE_SOURCE =~ /^schedule.*$/
       when: never
-    - if: $CI_COMMIT_TAG
-      when: never
     - if: $CI_COMMIT_MESSAGE =~ /.*\[skip cancel\].*/
       when: never
     - if: $CI_COMMIT_REF_NAME =~ /.*-skip-cancel$/

test/otel/testdata/builder-config.yaml

@@ -1,10 +1,11 @@
-connectors:
-- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector
-    v0.114.0
 dist:
   description: Basic OTel Collector distribution for Developers
   name: otelcol-custom
   output_path: /tmp/otel-ci/otelcol-custom
+connectors:
+- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector
+    v0.114.0
+- gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector v0.114.0
 exporters:
 - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.114.0
 - gomod: go.opentelemetry.io/collector/exporter/nopexporter v0.114.0
@@ -15,7 +16,7 @@ exporters:
 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter
     v0.114.0
 extensions:
-- gomod: github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/impl v0.60.0
+- gomod: github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/impl v0.61.0
   path: ./comp/otelcol/ddflareextension/impl
 - gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.114.0
 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension
@@ -80,3 +81,6 @@ receivers:
     v0.114.0
 - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver
     v0.114.0
+converters:
+- gomod: github.com/DataDog/datadog-agent/comp/otelcol/converter/impl v0.61.0
+  path: ./comp/otelcol/converter/impl

test/otel/testdata/collector-config.yaml

@@ -11,41 +11,25 @@ processors:
 
 exporters:
   debug:
-  otlp:
-    endpoint: "localhost:4317"
-    tls:
-      insecure: true  # Set to true for testing; adjust for production.
-  datadog/exporter:
-    api:
-      site: datadoghq.com
-      key: "12345"
 
 extensions:
   ddflare:
     include_metadata: true  # Set to true for testing; adjust for production.
-  health_check:
-  zpages:
-  pprof:
 
 service:
-  extensions:
-    - ddflare
-    - health_check
-    - pprof
-    - zpages
   pipelines:
     traces:
       receivers: [otlp]
       processors: [batch]
-      exporters: [debug, otlp]
+      exporters: [debug]
     metrics:
       receivers: [otlp]
       processors: [batch]
-      exporters: [debug, otlp]
+      exporters: [debug]
     logs:
       receivers: [otlp]
       processors: [batch]
-      exporters: [datadog/exporter]
+      exporters: [debug]
 
 # TODO: enable Additional configuration providers
 # providers:

test/otel/testdata/ocb_build_script.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+OTELCOL_PID=0
+
+mkdir -p /tmp/otel-ci
+trap 'rm -rf /tmp/otel-ci && kill $OTELCOL_PID' EXIT
+cp ./test/otel/testdata/builder-config.yaml /tmp/otel-ci/
+cp ./test/otel/testdata/collector-config.yaml /tmp/otel-ci/
+cp ./tools/ci/retry.sh /tmp/otel-ci/
+chmod +x /tmp/otel-ci/retry.sh
+
+OCB_VERSION="0.114.0"
+CGO_ENABLED=0 go install -trimpath -ldflags="-s -w" go.opentelemetry.io/collector/cmd/builder@v${OCB_VERSION}
+mv "$(go env GOPATH)/bin/builder" /tmp/otel-ci/ocb
+
+chmod +x /tmp/otel-ci/ocb
+
+/tmp/otel-ci/ocb --config=/tmp/otel-ci/builder-config.yaml > ocb-output.log 2>&1
+grep -q 'Compiled' ocb-output.log || (echo "OCB failed to build custom collector" && exit 1)
+grep -q '{"binary": "/tmp/otel-ci/otelcol-custom/otelcol-custom"}' ocb-output.log || (echo "OCB failed to build custom collector" && exit 1)
+
+/tmp/otel-ci/otelcol-custom/otelcol-custom --config /tmp/otel-ci/collector-config.yaml > otelcol-custom.log 2>&1 &
+OTELCOL_PID=$!
+/tmp/otel-ci/retry.sh grep -q 'Everything is ready. Begin running and processing data.' otelcol-custom.log || (echo "Failed to start otelcol-custom" && exit 1)
+
+/tmp/otel-ci/retry.sh curl -k https://localhost:7777 > flare-info.log 2>&1 
+grep -q '"provided_configuration": ""' flare-info.log || (echo "provided config is not empty" && exit 1)
+grep -q 'ddflare/dd-autoconfigured' flare-info.log || (echo "ddflare extension should be enabled" && exit 1)
+grep -q 'health_check/dd-autoconfigured' flare-info.log || (echo "health_check extension should be enabled" && exit 1)
+grep -q 'pprof/dd-autoconfigured' flare-info.log || (echo "pprof extension should be enabled" && exit 1)
+grep -q 'zpages/dd-autoconfigured' flare-info.log || (echo "zpages extension should be enabled" && exit 1)
+
+echo "OCB build script completed successfully"