Merge branch 'main' into igoragoli/update-trace-agent-macrobenchmark

.ddqa/config.toml

@@ -160,7 +160,6 @@ jira_statuses = [
 ]
 github_team = "container-integrations"
 github_labels = ["team/containers"]
-exclude_members = ["vboulineau"]
 
 [teams."Container Ecosystems"]
 jira_project = "CECO"
@@ -242,7 +241,7 @@ github_team = "database-monitoring"
 github_labels = ["team/database-monitoring"]
 
 [teams."Container App"]
-jira_project = "CAP2"
+jira_project = "CAP"
 jira_issue_type = "Task"
 jira_statuses = [
   "💼 To Do",
@@ -254,7 +253,7 @@ github_labels = ["team/container-app"]
 
 [teams."APM Onboarding"]
 jira_project = "APMON"
-jira_issue_type = "Task"
+jira_issue_type = "Story"
 jira_statuses = [
   "Selected For Development",
   "In Progress",

.github/CODEOWNERS

@@ -197,17 +197,20 @@
 # BEGIN COMPONENTS
 /comp @DataDog/agent-shared-components
 /comp/aggregator @DataDog/agent-shared-components
+/comp/checks @DataDog/agent-shared-components
 /comp/core @DataDog/agent-shared-components
 /comp/dogstatsd @DataDog/agent-metrics-logs
 /comp/forwarder @DataDog/agent-shared-components
 /comp/logs @DataDog/agent-metrics-logs
 /comp/metadata @DataDog/agent-shared-components
 /comp/ndmtmp @DataDog/network-device-monitoring
+/comp/netflow @DataDog/network-device-monitoring
 /comp/otelcol @DataDog/opentelemetry
 /comp/process @DataDog/processes
 /comp/remote-config @DataDog/remote-config
 /comp/systray @DataDog/windows-agent
 /comp/trace @DataDog/agent-apm
+/comp/checks/agentcrashdetect @DataDog/windows-kernel-integrations
 /comp/core/sysprobeconfig @DataDog/ebpf-platform
 # END COMPONENTS
 
@@ -262,6 +265,7 @@
 /pkg/collector/corechecks/embed/jmx/    @Datadog/agent-metrics-logs
 /pkg/collector/corechecks/embed/apm*.go            @Datadog/agent-platform @DataDog/agent-apm
 /pkg/collector/corechecks/embed/process_agent*.go  @Datadog/agent-platform @DataDog/processes
+/pkg/collector/corechecks/orchestrator/   @DataDog/container-app
 /pkg/collector/corechecks/kubernetes/       @DataDog/container-integrations
 /pkg/collector/corechecks/net/          @DataDog/agent-platform
 /pkg/collector/corechecks/oracle-dbm        @DataDog/database-monitoring
@@ -270,6 +274,7 @@
 /pkg/collector/corechecks/system/                 @DataDog/agent-platform
 /pkg/collector/corechecks/system/wincrashdetect/  @DataDog/windows-kernel-integrations
 /pkg/collector/corechecks/system/**/*_windows*.go @DataDog/agent-platform @DataDog/windows-agent
+/pkg/collector/corechecks/system/wincrashdetect/         @DataDog/windows-kernel-integrations
 /pkg/collector/corechecks/system/winkmem/         @DataDog/windows-agent
 /pkg/collector/corechecks/system/winproc/         @DataDog/windows-agent
 /pkg/collector/corechecks/systemd/      @DataDog/agent-integrations
@@ -281,8 +286,10 @@
 /pkg/config/environment*.go             @DataDog/container-integrations @DataDog/container-app
 /pkg/config/log*.go                     @Datadog/agent-platform
 /pkg/config/process*.go                 @DataDog/processes
-/pkg/config/system_probe.go             @DataDog/ebpf-platform
-/pkg/config/system_probe_cws.go         @DataDog/agent-security
+/pkg/config/system_probe.go               @DataDog/ebpf-platform
+/pkg/config/system_probe_cws.go           @DataDog/agent-security
+/pkg/config/system_probe_cws_notwin.go    @DataDog/agent-security
+/pkg/config/system_probe_cws_windows.go   @DataDog/windows-kernel-integrations
 /pkg/config/remote/                     @DataDog/remote-config
 /pkg/config/remote/meta/                @DataDog/remote-config @DataDog/software-integrity-and-trust
 /pkg/containerlifecycle/                @Datadog/container-integrations
@@ -320,6 +327,7 @@
 /pkg/util/kubernetes/                   @DataDog/container-integrations @DataDog/container-app
 /pkg/util/orchestrator/                 @DataDog/container-app
 /pkg/util/podman/                       @DataDog/container-integrations
+/pkg/util/prometheus                    @DataDog/container-integrations
 /pkg/util/trivy/                        @DataDog/container-integrations @DataDog/agent-security
 /pkg/util/cgroups/                      @DataDog/container-integrations
 /pkg/util/retry/                        @DataDog/container-integrations
@@ -383,7 +391,7 @@
 /pkg/workloadmeta/                      @DataDog/container-integrations
 /pkg/workloadmeta/collectors/internal/cloudfoundry @DataDog/platform-integrations
 /pkg/sbom/                              @DataDog/container-integrations @DataDog/agent-security
-
+/pkg/internaltelemetry                  @DataDog/windows-kernel-integrations
 /pkg-config/                            @DataDog/agent-platform
 
 /releasenotes/                          @DataDog/documentation

.github/workflows/windows-unittests.yml

@@ -54,7 +54,7 @@ jobs:
           powershell.exe -Command ./tasks/winbuildscripts/pre-go-build.ps1
           # FIXME: skipping rtloader tests because they fail with a DLL-not-found error
           # inv -e rtloader.test
-          inv -e test --skip-linters --rerun-fails=2 --python-runtimes 3 --coverage --profile --python-home-3=$pythonLocation
+          inv -e test --skip-linters --rerun-fails=2 --python-runtimes 3 --coverage --profile --python-home-3=$pythonLocation --timeout=600
 
       - name: Upload Codecov results
         uses: codecov/codecov-action@v3

.gitignore

@@ -39,6 +39,7 @@ system-probe.yaml
 security-agent.yaml
 dogstatsd.yaml
 cloudfoundry.yaml
+apm-inject.yaml
 Dockerfiles/cluster-agent/datadog-cluster.yaml
 Dockerfiles/cluster-agent/dist
 Dockerfiles/cluster-agent/security-agent-policies

.gitlab-ci.yml

@@ -294,6 +294,8 @@ variables:
 # Rules used to define whether a pipeline should run, and with which variables
 #
 
+# WARNING: Do not change below if you want to globally disable the caching proxy (all branches would need to be rebased for it to be effectively disabled).
+# Instead use the break-glass mechanism by looking for the "Remotely disable Artifactory" doc in Confluence.
 workflow:
   rules:
     - <<: *if_triggered_pipeline
@@ -310,8 +312,8 @@ workflow:
         USE_CACHING_PROXY_RUBY: "true"
     - <<: *if_deploy
       variables:
-        USE_CACHING_PROXY_PYTHON: "false"
-        USE_CACHING_PROXY_RUBY: "false"
+        USE_CACHING_PROXY_PYTHON: "true"
+        USE_CACHING_PROXY_RUBY: "true"
     - if: $CI_COMMIT_TAG == null
       variables:
         USE_CACHING_PROXY_PYTHON: "false"

.gitlab/dev_container_deploy/docker_linux.yml

@@ -247,11 +247,9 @@ dev_nightly-dogstatsd:
     IMG_DESTINATIONS: dogstatsd-dev:nightly-${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA},dogstatsd-dev:nightly-${CI_COMMIT_REF_SLUG}
 
 # push images to `datadog-agent-qa` ECR for the end-to-end tests defined in `e2e.yml`
-qa_agent:
+.qa_agent:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy
-  rules:
-    !reference [.on_main_a7]
   needs:
     - docker_build_agent7
     - docker_build_agent7_arm64
@@ -262,27 +260,53 @@ qa_agent:
     IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-arm64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-win1809-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-winltsc2022-amd64
     IMG_DESTINATIONS: agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
 
-qa_dca:
+.qa_dca:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy
-  rules:
-    !reference [.on_main_a7]
   needs:
     - docker_build_cluster_agent_amd64
   variables:
     IMG_REGISTRIES: agent-qa
     IMG_SOURCES: ${SRC_DCA}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64
     IMG_DESTINATIONS: cluster-agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
 
-qa_dogstatsd:
+.qa_dogstatsd:
   extends: .docker_publish_job_definition
   stage: dev_container_deploy
-  rules:
-    !reference [.on_main_a7]
   needs:
     - docker_build_dogstatsd_amd64
     - docker_build_dogstatsd_arm64
   variables:
     IMG_REGISTRIES: agent-qa
     IMG_SOURCES: ${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-amd64,${SRC_DSD}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-arm64
     IMG_DESTINATIONS: dogstatsd:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
+
+qa_main_agent:
+  extends: .qa_agent
+  rules:
+    !reference [.on_main_and_no_skip_e2e]
+
+qa_branch_agent:
+  extends: .qa_agent
+  rules:
+    !reference [.on_dev_branch_manual]
+
+qa_main_dca:
+  extends: .qa_dca
+  rules:
+    !reference [.on_main_and_no_skip_e2e]
+
+qa_branch_dca:
+  extends: .qa_dca
+  rules:
+    !reference [.on_dev_branch_manual]
+
+qa_main_dogstatsd:
+  extends: .qa_dogstatsd
+  rules:
+    !reference [.on_main_and_no_skip_e2e]
+
+qa_branch_dogstatsd:
+  extends: .qa_dogstatsd
+  rules:
+    !reference [.on_dev_branch_manual]

.gitlab/e2e.yml

@@ -137,12 +137,15 @@ k8s-e2e-otlp-main:
     # Use S3 backend
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
   variables:
+    KUBERNETES_MEMORY_REQUEST: 12Gi
+    KUBERNETES_MEMORY_LIMIT: 16Gi
     E2E_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub
     E2E_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key
     E2E_KEY_PAIR_NAME: datadog-agent-ci
     E2E_PIPELINE_ID: $CI_PIPELINE_ID
+    IMAGE_PATH_CONFIG: "-c ddagent:fullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} -c ddagent:clusterAgentFullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/cluster-agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
   script:
-    - inv -e new-e2e-tests.run --targets $TARGETS $CONFIGPARAMS --junit-tar "junit-${CI_JOB_NAME}.tgz" ${EXTRA_PARAMS}
+    - inv -e new-e2e-tests.run --targets $TARGETS $IMAGE_PATH_CONFIG --junit-tar "junit-${CI_JOB_NAME}.tgz" ${EXTRA_PARAMS}
   artifacts:
     expire_in: 2 weeks
     when: always
@@ -162,7 +165,6 @@ new-e2e-containers-dev:
   needs: []
   variables:
     TARGETS: ./tests/containers
-    CONFIGPARAMS: "-c ddagent:fullImagePath=datadog/agent-dev:${CI_COMMIT_REF_SLUG}-py3 -c ddagent:clusterAgentFullImagePath=datadog/cluster-agent-dev:${CI_COMMIT_REF_SLUG}"
     TEAM: container-integrations
   parallel:
     matrix:
@@ -175,12 +177,11 @@ new-e2e-containers-main:
   extends: .new_e2e_template
   rules: !reference [.on_main_and_no_skip_e2e]
   needs:
-    - qa_agent
-    - qa_dca
-    - qa_dogstatsd
+    - qa_main_agent
+    - qa_main_dca
+    - qa_main_dogstatsd
   variables:
     TARGETS: ./tests/containers
-    CONFIGPARAMS: "-c ddagent:fullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} -c ddagent:clusterAgentFullImagePath=669783387624.dkr.ecr.us-east-1.amazonaws.com/cluster-agent:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}"
     TEAM: container-integrations
   # Temporary, until we manage to stabilize those tests.
   allow_failure: true

.gitlab/internal_kubernetes_deploy.yml

@@ -22,11 +22,10 @@ internal_kubernetes_deploy_experimental:
   tags: ["arch:amd64"]
   variables:
     OPTION_AUTOMATIC_ROLLOUT: "true"
-    OPTION_PRE_SCRIPT: "patch-cluster-images-operator.sh snowver ${CI_COMMIT_REF_SLUG}-jmx-${CI_COMMIT_SHORT_SHA} ${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}"
+    OPTION_PRE_SCRIPT: "patch-cluster-images-operator.sh env=all-staging ${CI_COMMIT_REF_SLUG}-jmx-${CI_COMMIT_SHORT_SHA} ${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHORT_SHA}"
     SKIP_PLAN_CHECK: "true"
-    WORKFLOW: "agents"
-    FILTER: "cluster.env == 'experimental' and cluster.shortName == 'snowver'"
+    EXPLICIT_WORKFLOWS: "//workflows:deploy_nightly.agents_nightly"
   script:
     - source /root/.bashrc
     - export GITLAB_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.gitlab_pipelines_scheduler_token --with-decryption --query "Parameter.Value" --out text)
-    - inv pipeline.trigger-child-pipeline --project-name "DataDog/k8s-datadog-agent-ops" --git-ref "main" --variables "OPTION_AUTOMATIC_ROLLOUT,WORKFLOW,OPTION_PRE_SCRIPT,FILTER,SKIP_PLAN_CHECK"
+    - inv pipeline.trigger-child-pipeline --project-name "DataDog/k8s-datadog-agent-ops" --git-ref "main" --variables "OPTION_AUTOMATIC_ROLLOUT,EXPLICIT_WORKFLOWS,OPTION_PRE_SCRIPT,FILTER,SKIP_PLAN_CHECK" --follow false

.gitlab/kitchen_tests_upload.yml

@@ -92,8 +92,9 @@ kitchen_tests_upload_common:
   variables:
     DD_ENV: ci
   script:
-    - set +x   
+    - set +x
     - find . -name "kitchen-rspec-common-*.tar.gz"
     - export DATADOG_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.datadog_api_key_org2 --with-decryption --query "Parameter.Value" --out text)
+    - export JIRA_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.datadog-agent.jira_read_api_token --with-decryption --query "Parameter.Value" --out text)
     - set -x
     - find . -maxdepth 1 -type f -name "kitchen-rspec-common-*.tar.gz" -exec inv -e junit-upload --tgz-path {} \;

.gitlab/notify.yml

@@ -3,7 +3,7 @@
 # Contains jobs which send slack notifications depending on pipeline status.
 
 include:
-  - https://gitlab-templates.ddbuild.io/slack-notifier/v1/template.yml
+  - https://gitlab-templates.ddbuild.io/slack-notifier/sdm/template.yml
 
 notify-on-tagged-success:
   extends: .slack-notifier-base
@@ -42,7 +42,7 @@ notify:
 
 send_pipeline_stats:
   stage: notify
-  # Using the buildimages image and not slack-notifier because we need Python 3.7+
+  # Using a buildimage image with python 3.7+, datadog-api-client and invoke installed
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/datadog-ci-uploader$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
   tags: ["arch:amd64"]
   when: always

cmd/agent/dist/conf.d/agentcrashdetect.d/conf.yaml.default

@@ -0,0 +1,2 @@
+instances:
+  - {}

cmd/agent/dist/conf.d/containerd.d/conf.yaml.default

@@ -30,3 +30,8 @@ instances:
     ## Set to `false` if you want to deactivate the event collection for the containerd check
     #
     collect_events: true
+
+    ## @param openmetrics_endpoint - openmetrics endpoint on which containerd runtime exposes metrics
+    #  Learn more about exposing containerd openmetrics endpoint: https://github.com/containerd/containerd/blob/main/docs/ops.md
+    #
+    openmetrics_endpoint: http://127.0.0.1:1338

cmd/agent/dist/conf.d/oracle-dbm.d/conf.yaml.example

@@ -6,6 +6,18 @@ init_config:
   ## Active session sampling interval in seconds
   # min_collection_interval: <interval_in_seconds>
 
+  ## @param global_custom_queries - list of mappings - optional
+  ## See `custom_queries` defined below.
+  ##
+  ## Global custom queries can be applied to all instances using the
+  ## `use_global_custom_queries` setting at the instance level.
+  #
+  # global_custom_queries:
+  #   - query: <QUERY> 
+  #     columns: <COLUMNS>
+  #     tags: <TAGS>
+
+
 ## Every instance is scheduled independent of the others.
 #
 instances:
@@ -183,6 +195,15 @@ instances:
         #
         # enabled: true
 
+    ## @param use_global_custom_queries - string - optional - default: 'true'
+    ## How `global_custom_queries` should be used for this instance. There are 3 options:
+    ##
+    ## 1. true - `global_custom_queries` override `custom_queries`.
+    ## 2. false - `custom_queries` override `global_custom_queries`.
+    ## 3. extend - `global_custom_queries` are used in addition to any `custom_queries`.
+    #
+    # use_global_custom_queries: 'true'
+
     ## @param tags - list of strings - optional
     ## A list of tags to attach to every metric and service check emitted by this instance.
     ##

cmd/agent/dist/conf.d/wincrashdetect.d/conf.yaml.example

@@ -0,0 +1,28 @@
+init_config:
+
+instances:
+
+    ## the Windows crash detection check will send an event if a system crash is detected
+    ## since the last time the agent is run.  It will send a single event for a given crash;
+    ## if the system is rebooted and the crash file is still present it will not send another event.
+    ##
+    ## The Windows crash detection check uses data from the crash dump file to notify when the 
+    ## crash occurred, and attempts to display which module caused the crash.
+    -
+
+    ## @param enabled - boolean - optional - default: true
+    ## Specify if the check should report Windows system crashes
+    ## This requires system-probe.
+    ## And this requires the enabled flag in the windows_crash_detection section
+    ## of system-probe.yaml to be set to true.
+    #
+    # enabled: true
+
+    ## @param tags - list of strings following the pattern: "key:value" - optional
+    ## List of tags to attach to every metric, event, and service check emitted by this integration.
+    ##
+    ## Learn more about tagging: https://docs.datadoghq.com/tagging/
+    #
+    # tags:
+    #   - <KEY_1>:<VALUE_1>
+    #   - <KEY_2>:<VALUE_2>

cmd/agent/gui/views/templates/generalStatus.tmpl

@@ -183,9 +183,12 @@
               {{- range $check,$instances := .checks.initialized_checks }}
                 {{ $check -}}<br>
                 {{- range $instance := $instances }}
-                  {{- range $k,$v := $instance }}
-                      {{ $k }} : {{ $v }}<br>
-                  {{- end -}}
+                    instance_name: {{ .instance_name }}<br>
+                    metric_count: {{ .metric_count }}<br>
+                    service_check_count: {{ .service_check_count }}<br>
+                    message: {{ .message }}<br>
+                    status: {{ .status }}<br>
+                    <br>
                 {{- end -}}
               {{- end }}
             {{- end -}}