Merge branch 'main' into use-telemetry-component-for-autodiscovery-co…

…mponent

.github/CODEOWNERS

@@ -165,6 +165,7 @@
 /cmd/agent/dist/conf.d/jetson.d/        @DataDog/agent-devx-infra
 /cmd/agent/dist/conf.d/oracle.d/    @DataDog/database-monitoring
 /cmd/agent/dist/conf.d/oracle-dbm.d/    @DataDog/database-monitoring
+/cmd/agent/dist/conf.d/network_path.d/  @DataDog/Networks @DataDog/network-device-monitoring
 /cmd/agent/dist/conf.d/sbom.d/          @DataDog/container-integrations
 /cmd/agent/dist/conf.d/snmp.d/          @DataDog/network-device-monitoring
 /cmd/agent/dist/conf.d/win32_event_log.d/ @DataDog/windows-agent
@@ -207,6 +208,7 @@
 /Dockerfiles/agent/entrypoint.d.windows/ @DataDog/container-integrations @DataDog/windows-agent
 /Dockerfiles/agent/entrypoint.ps1        @DataDog/container-integrations @DataDog/windows-agent
 /Dockerfiles/agent/windows/              @DataDog/container-integrations @DataDog/windows-agent
+/Dockerfiles/agent-ot/Dockerfile.byoc   @DataDog/opentelemetry
 
 /docs/                                  @DataDog/documentation @DataDog/agent-devx-loops
 /docs/dev/checks/                       @DataDog/documentation @DataDog/agent-metrics-logs
@@ -603,6 +605,7 @@
 /test/workload-checks/                        @DataDog/single-machine-performance
 
 /tools/                                 @DataDog/agent-devx-loops
+/tools/ci                               @DataDog/agent-devx-infra
 /tools/ebpf/                            @DataDog/ebpf-platform
 /tools/gdb/                             @DataDog/agent-shared-components
 /tools/go-update/                       @DataDog/agent-shared-components

.github/dependabot.yaml

@@ -7,7 +7,6 @@ updates:
       - dependencies-go
       - team/triage
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # These dependencies are replaced in the main go.mod file.
       # They are ignored since bumping them would have no effect on the build.
@@ -45,7 +44,6 @@ updates:
       - dependencies-go
       - team/agent-apm
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -61,7 +59,6 @@ updates:
       - dependencies-go
       - team/agent-shared-components
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -77,7 +74,6 @@ updates:
       - dependencies-go
       - team/agent-apm
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -93,7 +89,6 @@ updates:
       - dependencies-go
       - team/agent-security
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -113,7 +108,6 @@ updates:
       - team/agent-delivery
       - changelog/no-changelog
       - qa/no-code-change
-    milestone: 22
     schedule:
       interval: monthly
     open-pull-requests-limit: 100
@@ -124,7 +118,6 @@ updates:
       - dependencies-go
       - team/network-device-monitoring
       - changelog/no-changelog
-    milestone: 22
     ignore:
       # Ignore internal modules
       - dependency-name: github.com/DataDog/datadog-agent/*
@@ -140,7 +133,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/testing
-    milestone: 22
     ignore:
       # Ignore test-infra-definitions because bumping the GO package inside `go.mod`
       # requires to also bump `TEST_INFRA_DEFINITIONS_BUILDIMAGES` inside `.gitlab/common/test_infra_version.yml`
@@ -162,7 +154,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/testing
-    milestone: 22
     schedule:
       interval: weekly
     open-pull-requests-limit: 100
@@ -175,7 +166,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/tooling
-    milestone: 22
     schedule:
       interval: monthly
     open-pull-requests-limit: 100
@@ -187,7 +177,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/testing
-    milestone: 22
     schedule:
       interval: weekly
     open-pull-requests-limit: 100
@@ -200,7 +189,6 @@ updates:
       - changelog/no-changelog
       - qa/no-code-change
       - dev/tooling
-    milestone: 22
     schedule:
       interval: weekly
     open-pull-requests-limit: 100

.github/workflows/create_rc_pr.yml

@@ -48,6 +48,9 @@ jobs:
 
             - name: Check for changes since last RC
               id: check_for_changes
+              env:
+                ATLASSIAN_USERNAME: ${{ secrets.ATLASSIAN_USERNAME }}
+                ATLASSIAN_PASSWORD: ${{ secrets.ATLASSIAN_PASSWORD }}
               run: |
                 echo "CHANGES=$(inv -e release.check-for-changes -r ${{ env.RELEASE_BRANCH }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT
 

.gitlab-ci.yml

@@ -219,6 +219,9 @@ variables:
   GITLAB_FULL_API_TOKEN_SSM_NAME: ci.datadog-agent.gitlab_full_api_token  # ci-cd
   INSTALL_SCRIPT_API_KEY_SSM_NAME: ci.agent-linux-install-script.datadog_api_key_2  # agent-delivery
   JIRA_READ_API_TOKEN_SSM_NAME: ci.datadog-agent.jira_read_api_token  # agent-devx-infra
+  AGENT_GITHUB_APP_ID_SSM_NAME: ci.datadog-agent.platform-github-app-id  # agent-devx-infra
+  AGENT_GITHUB_INSTALLATION_ID_SSM_NAME: ci.datadog-agent.platform-github-app-installation-id  # agent-devx-infra
+  AGENT_GITHUB_KEY_SSM_NAME: ci.datadog-agent.platform-github-app-key  # agent-devx-infra
   MACOS_GITHUB_APP_ID_SSM_NAME: ci.datadog-agent.macos_github_app_id  # agent-devx-infra
   MACOS_GITHUB_INSTALLATION_ID_SSM_NAME: ci.datadog-agent.macos_github_installation_id  # agent-devx-infra
   MACOS_GITHUB_KEY_SSM_NAME: ci.datadog-agent.macos_github_key_b64  # agent-devx-infra
@@ -359,14 +362,12 @@ variables:
 .if_mergequeue: &if_mergequeue
   if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
 
-.fakeintake_changes: &fakeintake_changes
-  changes:
-    paths:
-      - "test/fakeintake/**/*"
-      - .gitlab/binary_build/fakeintake.yml
-      - .gitlab/container_build/fakeintake.yml
-      - .gitlab/dev_container_deploy/fakeintake.yml
-    compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+.fakeintake_paths: &fakeintake_paths
+  paths:
+    - "test/fakeintake/**/*"
+    - .gitlab/binary_build/fakeintake.yml
+    - .gitlab/container_build/fakeintake.yml
+    - .gitlab/dev_container_deploy/fakeintake.yml
 
 #
 # Workflow rules
@@ -685,8 +686,8 @@ workflow:
   - .gitlab/kernel_matrix_testing/common.yml
   - test/new-e2e/system-probe/**/*
   - test/new-e2e/scenarios/system-probe/**/*
-  - test/new-e2e/runner/**/*
-  - test/new-e2e-utils/**/*
+  - test/new-e2e/pkg/runner/**/*
+  - test/new-e2e/pkg/utils/**/*
   - test/new-e2e/go.mod
   - tasks/security_agent.py
   - tasks/kmt.py
@@ -707,7 +708,7 @@ workflow:
   changes:
       paths:
         - tools/windows/DatadogAgentInstaller/**/*
-        - .gitlab/new-e2e_testing/windows.yml
+        - .gitlab/kitchen_testing/new-e2e_testing/windows.yml
         - test/new-e2e/tests/windows/install-test/**/*
         - test/new-e2e/tests/windows/domain-test/**/*
         - tasks/msi.py
@@ -740,8 +741,8 @@ workflow:
   - .gitlab/kernel_matrix_testing/common.yml
   - test/new-e2e/system-probe/**/*
   - test/new-e2e/scenarios/system-probe/**/*
-  - test/new-e2e/runner/**/*
-  - test/new-e2e-utils/**/*
+  - test/new-e2e/pkg/runner/**/*
+  - test/new-e2e/pkg/utils/**/*
   - test/new-e2e/go.mod
   - tasks/system_probe.py
   - tasks/kmt.py
@@ -790,7 +791,9 @@ workflow:
 
 .on_e2e_or_fakeintake_changes_or_manual:
   - !reference [.on_e2e_main_release_or_rc]
-  - <<: *fakeintake_changes
+  - changes:
+      <<: *fakeintake_paths
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
     variables:
       FAKEINTAKE_IMAGE_OVERRIDE: "public.ecr.aws/datadog/fakeintake:v$CI_COMMIT_SHORT_SHA"
     when: on_success
@@ -815,7 +818,6 @@ workflow:
         - pkg/collector/corechecks/containers/**/*
         - pkg/collector/corechecks/containerimage/**/*
         - pkg/collector/corechecks/containerlifecycle/**/*
-        - pkg/collector/corechecks/kubernetes/**/*
         - pkg/collector/corechecks/sbom/**/*
         - pkg/sbom/**/*
         - pkg/util/clusteragent/**/*
@@ -888,13 +890,13 @@ workflow:
         - comp/core/agenttelemetry/**/*
         - comp/core/autodiscovery/providers/config_reader*.go
         - comp/core/autodiscovery/providers/file*.go
+        - comp/dogstatsd/**/*
         - comp/forwarder/**/*
         - comp/logs/**/*
         - comp/serializer/compression/**/*
         - pkg/aggregator/**/*
         - pkg/collector/**/*
         - pkg/commonchecks/**/*
-        - pkg/dogstatsd/**/*
         - pkg/jmxfetch/**/*
         - pkg/logs/**/*
         - pkg/metrics/**/*
@@ -1040,7 +1042,6 @@ workflow:
       paths:
         - omnibus/**/*
         - .gitlab-ci.yml
-        - .gitlab/package_build.yml
         - release.json
         - .gitlab/package_build/**/*
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
@@ -1053,10 +1054,13 @@ workflow:
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
 .on_fakeintake_changes:
-  - <<: *fakeintake_changes
+  - changes:
+      <<: *fakeintake_paths
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
 
 .on_fakeintake_changes_on_main:
-  - <<: *fakeintake_changes
+  - changes:
+      <<: *fakeintake_paths
     <<: *if_main_branch
 
 .fast_on_dev_branch_only:

.gitlab/.pre/test_gitlab_configuration.yml

@@ -7,4 +7,19 @@ test_gitlab_configuration:
   script:
     - source /root/.bashrc
     - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_FULL_API_TOKEN_SSM_NAME)
-    - inv linter.gitlab-ci
+    - inv -e linter.gitlab-ci
+    - inv -e linter.job-change-path
+    - inv -e linter.gitlab-change-paths
+
+test_gitlab_compare_to:
+  stage: .pre
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  rules:
+    - !reference [.on_gitlab_changes]
+  script:
+    - source /root/.bashrc
+    - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_FULL_API_TOKEN_SSM_NAME)
+    - !reference [.setup_agent_github_app]
+    - pip install -r tasks/requirements.txt
+    - inv pipeline.compare-to-itself

.gitlab/common/shared.yml

@@ -41,6 +41,12 @@
       echo "Using GitHub App instance 2"
     fi
 
+.setup_agent_github_app:
+  - export GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $AGENT_GITHUB_KEY_SSM_NAME)
+  - export GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $AGENT_GITHUB_APP_ID_SSM_NAME)
+  - export GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $AGENT_GITHUB_INSTALLATION_ID_SSM_NAME)
+  - echo "Using agent GitHub App"
+
 # Install `dd-pkg` and lint packages produced by Omnibus, supports only deb and rpm packages
 .lint_linux_packages:
   - curl -sSL "https://dd-package-tools.s3.amazonaws.com/dd-pkg/${DD_PKG_VERSION}/dd-pkg_Linux_${DD_PKG_ARCH}.tar.gz" | tar -xz -C /usr/local/bin dd-pkg

.gitlab/container_build/docker_linux.yml

@@ -23,6 +23,7 @@
     - crane flatten -t ${TARGET_TAG} ${TARGET_TAG}
   # Workaround for temporary network failures
   retry: 2
+  timeout: 30m
 
 .docker_build_job_definition_amd64:
   extends: .docker_build_job_definition
@@ -306,6 +307,7 @@ docker_build_dogstatsd_amd64:
   variables:
     IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/dogstatsd
     BUILD_CONTEXT: Dockerfiles/dogstatsd/alpine
+  timeout: 20m
 
 # build the dogstatsd image
 docker_build_dogstatsd_arm64:
@@ -319,3 +321,4 @@ docker_build_dogstatsd_arm64:
   variables:
     IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent/dogstatsd
     BUILD_CONTEXT: Dockerfiles/dogstatsd/alpine
+  timeout: 20m

.gitlab/container_build/fakeintake.yml

@@ -4,6 +4,7 @@ docker_build_fakeintake:
   rules:
     - !reference [.except_mergequeue]
     - !reference [.on_fakeintake_changes]
+    - !reference [.on_fakeintake_changes_on_main]
   needs: []
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10-py3
   tags: ["arch:amd64"]

.gitlab/deploy_containers/deploy_containers_a7.yml

@@ -93,10 +93,13 @@ deploy_containers-ot:
   rules:
     - when: manual
       allow_failure: true
+  variables:
+    AGENT_REPOSITORY: agent
+    IMG_REGISTRIES: public
+    VERSION: 7
   dependencies: []
   before_script:
     - source /root/.bashrc
-    - export VERSION="$(inv agent.version --major-version 7 --url-safe --pipeline-id $PARENT_PIPELINE_ID)"
   parallel:
     matrix:
       - IMG_SOURCES: ${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta-amd64,${SRC_AGENT}:v${PARENT_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-ot-beta-arm64

.gitlab/deploy_packages/oci.yml

@@ -21,6 +21,9 @@ include:
     - export VERSION=$(inv agent.version --url-safe)-1
     - git config --global credential.helper '!f() { echo username=x-access-token; echo "password=$GH_TOKEN"; };f'
     - ${CI_PROJECT_DIR}/tools/ci/retry.sh git clone -v --depth=1 https://github.com/DataDog/datadog-packages /tmp/datadog-packages
+    - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
+    - export GOPROXY="binaries.ddbuild.io,proxy.golang.org,direct"
+    - export GONOSUMDB="github.com/DataDog,go.ddbuild.io"
     - cd /tmp/datadog-packages/cmd/datadog-package
     - go build .
     - ./datadog-package push registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:${VERSION} ${OMNIBUS_PACKAGE_DIR}/${OCI_PRODUCT}-${VERSION}.oci.tar

.gitlab/deps_fetch/deps_fetch.yml

@@ -11,6 +11,10 @@
   - mkdir -p $GOPATH/pkg/mod && tar xJf modcache_tools.tar.xz -C $GOPATH/pkg/mod
   - rm -f modcache_tools.tar.xz
 
+.retrieve_linux_go_e2e_deps:
+  - mkdir -p $GOPATH/pkg/mod && tar xJf modcache_e2e.tar.xz -C $GOPATH/pkg/mod
+  - rm -f modcache_e2e.tar.xz
+
 .cache_policy:
   rules:
   - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -76,3 +80,29 @@ go_tools_deps:
       paths:
         - modcache_tools.tar.xz
   retry: 1
+
+go_e2e_deps:
+  stage: deps_fetch
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  needs: ["setup_agent_version"]
+  extends: .cache_policy
+  variables:
+    KUBERNETES_CPU_REQUEST: 16
+  script:
+    - if [ -f modcache_e2e.tar.xz  ]; then exit 0; fi
+    - source /root/.bashrc
+    - inv -e new-e2e-tests.deps
+    - cd $GOPATH/pkg/mod/ && tar c -I "pxz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_e2e.tar.xz .
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - $CI_PROJECT_DIR/modcache_e2e.tar.xz
+  cache:
+    - key:
+        files:
+          - ./test/new-e2e/go.mod
+        prefix: "go_e2e_deps"
+      paths:
+        - modcache_e2e.tar.xz
+  retry: 1

.gitlab/dev_container_deploy/e2e.yml

@@ -21,9 +21,7 @@ qa_agent_ot:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy
   rules:
-    # incident-28463: this job is currently broken
-    # - !reference [.on_container_or_e2e_changes]
-    - !reference [.manual]
+    - !reference [.on_container_or_e2e_changes]
   needs:
     - docker_build_ot_agent7
     - docker_build_ot_agent7_arm64