Merge branch 'main' into AMLII-1408-remove-start-method-in-favor-of-o…

…n-start-hook-in-dogstatsd-server-component
DataDog · Feb 14, 2024 · ba45285 · ba45285
2 parents 2538b87 + c0941c6
commit ba45285
Show file tree

Hide file tree

Showing 225 changed files with 2,623 additions and 1,180 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -526,6 +526,7 @@
 /test/new-e2e/tests/agent-metric-logs   @DataDog/agent-metrics-logs
 /test/new-e2e/tests/windows                   @DataDog/windows-agent @DataDog/windows-kernel-integrations
 /test/new-e2e/tests/apm                       @DataDog/agent-apm
+/test/new-e2e/tests/remote-config             @DataDog/remote-config
 /test/system/                                 @DataDog/agent-shared-components
 /test/system/dogstatsd/                       @DataDog/agent-metrics-logs
 /test/benchmarks/apm_scripts/                 @DataDog/agent-apm

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,12 +1,12 @@
 <!--
 * New contributors are highly encouraged to read our
   [CONTRIBUTING](/CONTRIBUTING.md) documentation.
+* Both Contributor and Reviewer Checklists are available at https://github.com/DataDog/datadog-agent/blob/main/docs/dev/contributing.md#pull-requests.
 * The pull request:
   * Should only fix one issue or add one feature at a time.
   * Must update the test suite for the relevant functionality.
   * Should pass all status checks before being reviewed or merged.
 * Commit titles should be prefixed with general area of pull request's change.
-
 -->
 ### What does this PR do?
 
@@ -44,22 +44,3 @@
 * Write here in detail or link to detailed instructions on how this change can
   be tested/QAd/validated, including any environment setup.
 -->
-
-### Reviewer's Checklist
-<!--
-* Authors can use this list as a reference to ensure that there are no problems
-  during the review but the signing off is to be done by the reviewer(s).
-
-Note: Adding GitHub labels is only possible for contributors with write access.
--->
-
-- [ ] If known, an appropriate milestone has been selected; otherwise the `Triage` milestone is set.
-- [ ] Use the `major_change` label if your change either has a major impact on the code base, is impacting multiple teams or is changing important well-established internals of the Agent. This label will be use during QA to make sure each team pay extra attention to the changed behavior. For any customer facing change use a releasenote.
-- [ ] A [release note](https://github.com/DataDog/datadog-agent/blob/main/docs/dev/contributing.md#reno) has been added or the `changelog/no-changelog` label has been applied.
-- [ ] Changed code has automated tests for its functionality.
-- [ ] Adequate QA/testing plan information is provided. Except if the `qa/skip-qa` label, with required either `qa/done` or `qa/no-code-change` labels, are applied.
-- [ ] At least one `team/..` label has been applied, indicating the team(s) that should QA this change.
-- [ ] If applicable, docs team has been notified or [an issue has been opened on the documentation repo](https://github.com/DataDog/documentation/issues/new).
-- [ ] If applicable, the `need-change/operator` and `need-change/helm` labels have been applied.
-- [ ] If applicable, the `k8s/<min-version>` label, indicating the lowest Kubernetes version compatible with this feature.
-- [ ] If applicable, the [config template](https://github.com/DataDog/datadog-agent/blob/main/pkg/config/config_template.yaml) has been updated.
diff --git a/.github/workflows/label-analysis.yml b/.github/workflows/label-analysis.yml
@@ -13,14 +13,26 @@ env:
   GH_REPO: ${{ github.repository }}
 
 jobs:
+  fetch-labels:
+    if: github.triggering_actor != 'dd-devflow[bot]'
+    runs-on: ubuntu-latest
+    outputs:
+      LABELS: ${{ steps.pr-labels.outputs.LABELS }}
+    steps:
+      - name: Get PR labels
+        id: pr-labels
+        run: |
+          labels=$(gh pr view ${{github.event.number}} --json labels --jq '.labels[].name' | tr '\n' ' ')
+          echo "Fetched labels for PR ${{github.event.number}}: $labels"
+          echo "LABELS=$labels" >> "$GITHUB_OUTPUT"
   team-label:
+    needs: fetch-labels
     if: github.triggering_actor != 'dd-devflow[bot]'
     runs-on: ubuntu-latest
     steps:
       - name: Check team assignment
         run: |
-          labels=$(gh pr view ${{github.event.number}} --json labels --jq '.labels[].name')
-          for label in $labels; do
+          for label in $LABELS; do
             if [[ "$label" =~ ^qa/ ]]; then
               echo "A label to skip QA is set -- no need for team assignment"
               exit 0
@@ -32,15 +44,29 @@ jobs:
           done
           echo "PR ${{github.event.number}} requires at least one non-triage team assignment label (label starting by 'team/')"
           exit 1
+        env:
+          LABELS: ${{ needs.fetch-labels.outputs.LABELS}}
   skip-qa:
+    needs: fetch-labels
     if: github.triggering_actor != 'dd-devflow[bot]'
     runs-on: ubuntu-latest
     steps:
       - name: Check qa/[done|no-code-change] labels are not set together
         run: |
-          labels=$(gh pr view ${{github.event.number}} --json labels --jq '.labels[].name')
-          if [[ "$labels" =~ ^qa/done && "$labels" =~ ^qa/no-code-change ]]; then
+          is_qa_done=1
+          is_qa_no_code_change=1
+          for label in $LABELS; do
+            if [[ "$label" == "qa/done" ]]; then
+              is_qa_done=0
+            fi
+            if [[ "$label" == "qa/no-code-change" ]]; then
+              is_qa_no_code_change=0
+            fi
+          done
+          if [ $is_qa_done -eq 0 ] && [ $is_qa_no_code_change -eq 0 ]; then
             echo "Both 'qa/done' and 'qa/no-code-change' labels are set -- only one of them should be set"
             exit 1
           fi
           echo "No issue with 'qa/done' and 'qa/no-code-change' labels"
+        env:
+          LABELS: ${{ needs.fetch-labels.outputs.LABELS}}
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -185,7 +185,7 @@ variables:
   # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 9f1c04270e18
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 6ba25fcf6a61
   DATADOG_AGENT_BUILDERS: v22276738-b36b132
 
   DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
@@ -967,21 +967,28 @@ workflow:
   - when: manual
     allow_failure: true
 
-.on_windows_installer_changes_or_manual:
-  - <<: *if_main_branch
-  - <<: *if_mergequeue
-    when: never
-  - changes:
+.if_windows_installer_changes: &if_windows_installer_changes
+  changes:
       paths:
         - tools/windows/DatadogAgentInstaller/**/*
         - .gitlab/new-e2e_testing/windows.yml
         - test/new-e2e/tests/windows/install-test/**/*
         - tasks/msi.py
       compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+
+.on_windows_installer_changes_or_manual:
+  - <<: *if_main_branch
+  - <<: *if_mergequeue
+    when: never
+  - <<: *if_windows_installer_changes
     when: on_success
   - when: manual
     allow_failure: true
 
+.except_windows_installer_changes:
+  - <<: *if_windows_installer_changes
+    when: never
+
 .on_system_probe_or_e2e_changes_or_manual:
   - <<: *if_main_branch
   - <<: *if_mergequeue
@@ -1083,6 +1090,25 @@ workflow:
     when: manual
     allow_failure: true
 
+.on_apm_or_e2e_changes_or_manual:
+  - <<: *if_disable_e2e
+    when: never
+  - <<: *if_main_branch
+    when: on_success
+  - <<: *if_mergequeue
+    when: never
+  - changes:
+      paths:
+        - pkg/trace/**/*
+        - cmd/trace-agent/**/*
+        - comp/trace/**/*
+        - test/new-e2e/tests/apm/**/*
+        - test/new-e2e/go.mod
+      compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916
+    when: on_success
+  - when: manual
+    allow_failure: true
+
 .on_trace_agent_changes_or_manual:
   - changes:
       - pkg/trace/**/*

diff --git a/.gitlab/e2e.yml b/.gitlab/e2e.yml
@@ -97,9 +97,10 @@ k8s-e2e-otlp-main:
     E2E_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key
     E2E_KEY_PAIR_NAME: datadog-agent-ci-rsa
     E2E_PIPELINE_ID: $CI_PIPELINE_ID
+    E2E_COMMIT_SHA: $CI_COMMIT_SHORT_SHA
     E2E_OUTPUT_DIR: $CI_PROJECT_DIR/e2e-output
   script:
-    - inv -e new-e2e-tests.run --targets $TARGETS -c ddagent:fullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} -c ddagent:clusterAgentFullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/cluster-agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} -c dddogstatsd:fullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/dogstatsd:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} -c ddagent:imagePullRegistry=669783387624.dkr.ecr.us-east-1.amazonaws.com -c ddagent:imagePullUsername=AWS -c ddagent:imagePullPassword=$(aws ecr get-login-password) --junit-tar "junit-${CI_JOB_NAME}.tgz" ${EXTRA_PARAMS}
+    - inv -e new-e2e-tests.run --targets $TARGETS -c ddagent:imagePullRegistry=669783387624.dkr.ecr.us-east-1.amazonaws.com -c ddagent:imagePullUsername=AWS -c ddagent:imagePullPassword=$(aws ecr get-login-password) --junit-tar "junit-${CI_JOB_NAME}.tgz" ${EXTRA_PARAMS}
   artifacts:
     expire_in: 2 weeks
     when: always
@@ -217,7 +218,7 @@ new-e2e-npm-dev:
   rules:
     - !reference [.if_run_e2e_tests]
     - !reference [.on_dev_branch_manual]
-  needs: ["deploy_deb_testing-a7_x64", "deploy_windows_testing-a7"]
+  needs: ["deploy_deb_testing-a7_x64", "deploy_windows_testing-a7", "qa_agent"]
   variables:
     TARGETS: ./tests/npm
 
@@ -316,28 +317,12 @@ new-e2e-orchestrator-main:
 
   allow_failure: true #TODO: Remove when https://github.com/DataDog/datadog-agent/pull/22113 is merged
 
-new-e2e-apm-dev:
+new-e2e-apm:
   extends: .new_e2e_template
   rules:
-    - !reference [.if_run_e2e_tests]
-    - !reference [.on_dev_branch_manual]
+    !reference [.on_apm_or_e2e_changes_or_manual]
   needs:
     - qa_agent
-    - qa_dca
-    - qa_dogstatsd
-  variables:
-    TARGETS: ./tests/apm
-    TEAM: apm-agent
-  parallel:
-    matrix:
-      - EXTRA_PARAMS: --run TestDockerFakeintakeSuiteUDS
-      - EXTRA_PARAMS: --run TestDockerFakeintakeSuiteTCP
-      - EXTRA_PARAMS: --run TestVMFakeintakeSuiteUDS
-      - EXTRA_PARAMS: --run TestVMFakeintakeSuiteTCP
-
-new-e2e-apm-main:
-  extends: .new_e2e_template
-  rules: !reference [.on_main_or_rc_and_no_skip_e2e]
   variables:
     TARGETS: ./tests/apm
     TEAM: apm-agent

diff --git a/.gitlab/e2e_test_junit_upload.yml b/.gitlab/e2e_test_junit_upload.yml
@@ -120,5 +120,5 @@ e2e_test_junit_upload:
     - new-e2e-process-main
     - new-e2e-cws-main
     - new-e2e-orchestrator-main
-    - new-e2e-apm-main
+    - new-e2e-apm
     - new-e2e-remote-config
diff --git a/.gitlab/kitchen_testing/windows.yml b/.gitlab/kitchen_testing/windows.yml
@@ -176,21 +176,11 @@ kitchen_windows_upgrade5_agent-a7:
     - .kitchen_scenario_windows_a7
     - .kitchen_test_upgrade5_agent
 
-kitchen_windows_upgrade6_agent-a6:
-  extends:
-    - .kitchen_scenario_windows_a6
-    - .kitchen_test_upgrade6_agent
-
 kitchen_windows_upgrade6_agent-a7:
   extends:
     - .kitchen_scenario_windows_a7
     - .kitchen_test_upgrade6_agent
 
-kitchen_windows_upgrade7_agent-a7:
-  extends:
-    - .kitchen_scenario_windows_a7
-    - .kitchen_test_upgrade7_agent
-
 kitchen_windows_process_agent-a7:
   variables:
     KITCHEN_OSVERS: "win2022"

diff --git a/.gitlab/kitchen_tests_upload.yml b/.gitlab/kitchen_tests_upload.yml
@@ -18,9 +18,7 @@ kitchen_tests_upload_common:
     - kitchen_windows_process_agent-a7
     - kitchen_windows_upgrade5_agent-a6
     - kitchen_windows_upgrade5_agent-a7
-    - kitchen_windows_upgrade6_agent-a6
     - kitchen_windows_upgrade6_agent-a7
-    - kitchen_windows_upgrade7_agent-a7
   variables:
     DD_ENV: ci
   script:

diff --git a/.gitlab/maintenance_jobs/kitchen.yml b/.gitlab/maintenance_jobs/kitchen.yml
@@ -7,12 +7,12 @@ periodic_kitchen_cleanup_s3:
   tags: ["arch:amd64"]
   rules: !reference [.on_testing_cleanup]
   script:
-    - MAX_AGE_HOURS=24 BUCKET_NAME=$DEB_TESTING_S3_BUCKET BUCKET_PREFIX=pool python3 /deploy_scripts/cleanup_s3.py
-    - MAX_AGE_HOURS=24 BUCKET_NAME=$DEB_TESTING_S3_BUCKET BUCKET_PREFIX=dists python3 /deploy_scripts/cleanup_s3.py
-    - MAX_AGE_HOURS=24 BUCKET_NAME=$RPM_TESTING_S3_BUCKET BUCKET_PREFIX=testing/ python3 /deploy_scripts/cleanup_s3.py
-    - MAX_AGE_HOURS=24 BUCKET_NAME=$RPM_TESTING_S3_BUCKET BUCKET_PREFIX=suse/testing/ python3 /deploy_scripts/cleanup_s3.py
-    - MAX_AGE_HOURS=24 BUCKET_NAME=$WIN_S3_BUCKET BUCKET_PREFIX=pipelines/A6/ python3 /deploy_scripts/cleanup_s3.py
-    - MAX_AGE_HOURS=24 BUCKET_NAME=$WIN_S3_BUCKET BUCKET_PREFIX=pipelines/A7/ python3 /deploy_scripts/cleanup_s3.py
+    - MAX_AGE_HOURS=72 BUCKET_NAME=$DEB_TESTING_S3_BUCKET BUCKET_PREFIX=pool python3 /deploy_scripts/cleanup_s3.py
+    - MAX_AGE_HOURS=72 BUCKET_NAME=$DEB_TESTING_S3_BUCKET BUCKET_PREFIX=dists python3 /deploy_scripts/cleanup_s3.py
+    - MAX_AGE_HOURS=72 BUCKET_NAME=$RPM_TESTING_S3_BUCKET BUCKET_PREFIX=testing/ python3 /deploy_scripts/cleanup_s3.py
+    - MAX_AGE_HOURS=72 BUCKET_NAME=$RPM_TESTING_S3_BUCKET BUCKET_PREFIX=suse/testing/ python3 /deploy_scripts/cleanup_s3.py
+    - MAX_AGE_HOURS=72 BUCKET_NAME=$WIN_S3_BUCKET BUCKET_PREFIX=pipelines/A6/ python3 /deploy_scripts/cleanup_s3.py
+    - MAX_AGE_HOURS=72 BUCKET_NAME=$WIN_S3_BUCKET BUCKET_PREFIX=pipelines/A7/ python3 /deploy_scripts/cleanup_s3.py
 
 # Kills any VMs that might have been left over by kitchen
 # The script only deletes VMs that have been there for >= 4 hours, which is more than the time limit

diff --git a/.gitlab/new-e2e_testing/windows.yml b/.gitlab/new-e2e_testing/windows.yml
@@ -4,7 +4,9 @@
   variables:
     TARGETS: ./tests/windows/install-test
     TEAM: windows-agent
-    EXTRA_PARAMS: --run 'TestMSI/Agent*/TestUpgrade$'
+    EXTRA_PARAMS: --run "TestMSI/Agent/$E2E_MSI_TEST$"
+  extends:
+    - .new_e2e_template
   before_script:
     # WINDOWS_AGENT_VERSION is used to verify the installed agent version
     # Must run before new_e2e_template changes the aws profile
@@ -15,19 +17,29 @@
     - export LAST_STABLE_VERSION=$(invoke release.get-release-json-value "last_stable::$AGENT_MAJOR_VERSION")
     - !reference [.new_e2e_template, script]
 
+.new-e2e_windows_installer_tests:
+  parallel:
+    matrix:
+      - E2E_MSI_TEST: TestInstall
+      - E2E_MSI_TEST: TestRepair
+      - E2E_MSI_TEST: TestUpgrade
+      - E2E_MSI_TEST: TestUpgradeRollback
+
 # Agent 6
 .new-e2e_windows_a6_x86_64:
   variables:
     WINDOWS_AGENT_ARCH: "x86_64"
+  extends:
+    - .new-e2e_windows_msi
+    - .new-e2e_agent_a6
   needs: ["deploy_windows_testing-a6"]
 
+## full tests
 new-e2e-windows-agent-msi-windows-server-a6-x86_64:
   stage: kitchen_testing
   extends:
-    - .new_e2e_template
-    - .new-e2e_windows_msi
     - .new-e2e_windows_a6_x86_64
-    - .new-e2e_agent_a6
+    - .new-e2e_windows_installer_tests
   rules:
     - !reference [.on_deploy_a6]
     - !reference [.on_windows_installer_changes_or_manual]
@@ -36,15 +48,33 @@ new-e2e-windows-agent-msi-windows-server-a6-x86_64:
 .new-e2e_windows_a7_x86_64:
   variables:
     WINDOWS_AGENT_ARCH: "x86_64"
+  extends:
+    - .new-e2e_windows_msi
+    - .new-e2e_agent_a7
   needs: ["deploy_windows_testing-a7"]
 
+## full tests
 new-e2e-windows-agent-msi-windows-server-a7-x86_64:
   stage: kitchen_testing
   extends:
-    - .new_e2e_template
+    - .new-e2e_windows_a7_x86_64
+    - .new-e2e_windows_installer_tests
+  rules:
+    - !reference [.on_deploy_a7]
+    - !reference [.on_windows_installer_changes_or_manual]
+
+## single test for PRs
+## skipped if the full tests are running
+new-e2e-windows-agent-msi-upgrade-windows-server-a7-x86_64:
+  stage: kitchen_testing
+  extends:
     - .new-e2e_windows_msi
     - .new-e2e_windows_a7_x86_64
-    - .new-e2e_agent_a7
   rules:
+    - !reference [.except_main_or_release_branch]
+    - !reference [.except_windows_installer_changes]
     - !reference [.on_default_new-e2e_tests_a7]
-    - !reference [.on_windows_installer_changes_or_manual]
+    # must be last since it ends with when: on_success
+    - !reference [.except_deploy]
+  variables:
+    E2E_MSI_TEST: TestUpgradeRollback
diff --git a/.gitlab/package_build/dmg.yml b/.gitlab/package_build/dmg.yml
@@ -31,6 +31,7 @@ agent_dmg-x64-a7:
   variables:
     AGENT_MAJOR_VERSION: 7
     PYTHON_RUNTIMES: "3"
+  timeout: 6h
   before_script:
     - source /root/.bashrc
     - export RELEASE_VERSION=$RELEASE_VERSION_7
diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml
@@ -19,6 +19,7 @@ tests_macos:
     - inv -e github.trigger-macos-test --datadog-agent-ref "$CI_COMMIT_SHA" --python-runtimes "$PYTHON_RUNTIMES" --version-cache "$VERSION_CACHE_CONTENT"
   after_script:
     - inv -e junit-macos-repack --infile junit-tests_macos.tgz --outfile junit-tests_macos-repacked.tgz
+  timeout: 6h
   artifacts:
     expire_in: 2 weeks
     when: always
@@ -36,6 +37,7 @@ lint_macos:
   tags: ["arch:amd64"]
   variables:
     PYTHON_RUNTIMES: '3'
+  timeout: 6h
   script:
     - source /root/.bashrc
     - export GITHUB_KEY_B64=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.macos_github_key_b64 --with-decryption --query "Parameter.Value" --out text)

diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv
@@ -1013,7 +1013,7 @@ core,github.com/googleapis/gax-go/v2/apierror,BSD-3-Clause,Copyright 2017 Google
 core,github.com/googleapis/gax-go/v2/apierror/internal/proto,BSD-3-Clause,Copyright 2017 Google Inc.
 core,github.com/googleapis/gax-go/v2/callctx,BSD-3-Clause,Copyright 2017 Google Inc.
 core,github.com/googleapis/gax-go/v2/internal,BSD-3-Clause,Copyright 2017 Google Inc.
-core,github.com/gorilla/mux,BSD-3-Clause,Copyright (c) 2012-2018 The Gorilla Authors. All rights reserved | Google LLC (https://opensource.google.com/) | Kamil Kisielk <[email protected]> | Matt Silverlock <[email protected]> | Rodrigo Moraes (https://github.com/moraes)
+core,github.com/gorilla/mux,BSD-3-Clause,Copyright (c) 2023 The Gorilla Authors. All rights reserved
 core,github.com/gosnmp/gosnmp,BSD-2-Clause,Copyright 2012-2020 The GoSNMP Authors. All rights reserved.
 core,github.com/grpc-ecosystem/go-grpc-middleware,Apache-2.0,Copyright 2016 Michal Witkowski. All Rights Reserved.
 core,github.com/grpc-ecosystem/go-grpc-middleware/auth,Apache-2.0,Copyright 2016 Michal Witkowski. All Rights Reserved.