merge main

.gitlab/common/test_infra_version.yml

@@ -4,4 +4,4 @@ variables:
   # and check the job creating the image to make sure you have the right SHA prefix
   TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
   # Make sure to update test-infra-definitions version in go.mod as well
-  TEST_INFRA_DEFINITIONS_BUILDIMAGES: c757089e5a23
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: 98a37ef8c8a9

CHANGELOG-DCA.rst

@@ -2,6 +2,44 @@
 Release Notes
 =============
 
+.. _Release Notes_7.53.0:
+
+7.53.0 / 6.53.0
+================
+
+.. _Release Notes_7.53.0_Prelude:
+
+Prelude
+-------
+
+Released on: 2024-04-30
+Pinned to datadog-agent v7.53.0: `CHANGELOG <https://github.com/DataDog/datadog-agent/blob/main/CHANGELOG.rst#7530>`_.
+
+
+.. _Release Notes_7.53.0_New Features:
+
+New Features
+------------
+
+- APM library injection now works on EKS Fargate when the admission controller
+  is configured to add an Agent sidecar in EKS Fargate.
+
+- Cluster Agent now supports activating Application Security Management, Code Vulnerabilities, and
+  Software Composition Analysis via Helm charts.
+
+
+.. _Release Notes_7.53.0_Enhancement Notes:
+
+Enhancement Notes
+-----------------
+
+- Add the `mutation_webhook` tag to `admission_webhooks.webhooks_received` and `admission_webhooks.response_duration` Cluster Agent telemetry.
+
+- When using the admission controller to inject an Agent sidecar on EKS
+  Fargate, `shareProcessNamespace` is now set to `true` automatically. This is
+  to ensure that the process collection feature works.
+
+
 .. _Release Notes_7.52.0:
 
 7.52.0 / 6.52.0

CHANGELOG.rst

@@ -2,6 +2,135 @@
 Release Notes
 =============
 
+.. _Release Notes_7.53.0:
+
+7.53.0 / 6.53.0
+================
+
+.. _Release Notes_7.53.0_Prelude:
+
+Prelude
+-------
+
+Release on: 2024-04-30
+
+- Please refer to the `7.53.0 tag on integrations-core <https://github.com/DataDog/integrations-core/blob/master/AGENT_CHANGELOG.md#datadog-agent-version-7530>`_ for the list of changes on the Core Checks
+
+
+.. _Release Notes_7.53.0_New Features:
+
+New Features
+------------
+
+- Support database-monitoring autodiscovery for Aurora cluster instances. Adds a new configuration listener to poll
+  for a specific set of Aurora cluster IDs and then create a new database-monitoring supported check
+  configuration for each endpoint. This allows for monitoring of endpoints that scale dynamically.
+
+- Add new core check orchestrator_ecs to collect running ECS tasks
+
+- APM stats now include an is_trace_root field to indicate if the stats are from the root span of a trace.
+
+- The cluster-agent now collects network policies from the cluster.
+
+- Enable 'host_benchmarks' by default when running the security-agent compliance module.
+
+- OTLP ingest now has a feature flag to identify top-level spans by span kind. This new logic can be enabled by adding `enable_otlp_compute_top_level_by_span_kind` in DD_APM_FEATURES.
+  - With this new logic, root spans and spans with a server or consumer `span.kind` will be marked as top-level. Additionally, spans with a client or producer `span.kind` will have stats computed.
+  - Enabling this feature flag may increase the number of spans that generate trace metrics, and may change which spans appear as top-level in Datadog.
+
+- Experimental: The process-agent checks (process, container, and process-discovery) can be run from the Core Agent in
+  Linux. This feature can be toggled on by setting the `process_config.run_in_core_agent.enabled` flag to `true` in
+  the `datadog.yaml` file. This feature is disabled by default.
+
+
+.. _Release Notes_7.53.0_Enhancement Notes:
+
+Enhancement Notes
+-----------------
+
+- Add the container image and container lifecycle checks to the output of the Agent status command.
+
+- Add `kubelet_core_check_enabled` flag to Agent config to control  
+  whether the kubelet core check should be loaded.
+
+- Added LastSuccessfulTime to cronjob status payload.
+
+- Add a retry mechanism to Software Bill of Materials (SBOM) collection for container images.
+  This will help to avoid intermittent failures during the collection process.
+
+- Add startup timestamp to the Agent metadata payload.
+
+- Agents are now built with Go ``1.21.9``.
+
+- Adds image repo digest string to the container payload when present
+
+- CWS: Add selftests report on Windows and platforms with no eBPF support.
+
+- CWS: Add visibility for cross container program executions on platforms with no eBPF support.
+
+- APM: Enable credit card obfuscation by default. There is a small chance that numbers that are similar to valid credit cards may be redacted, this feature can be disabled by using `apm_config.obfuscation.credit_cards.enabled`. Alternatively, it can be made more accurate through luhn checksum verification by using `apm_config.obfuscation.credit_cards.luhn`, however, this increases the performance penalty of this check.
+
+- ``logs_config.expected_tags_duration`` now works for ``journald`` logs.
+
+- [oracle] Adds `oracle.can_query` service check.
+
+- [oracle] Automatically fall back to deprecated Oracle integration mode if privileges are missing.
+
+- [oracle] Add ``service`` configuration parameter.
+
+- The connections check no longer relies on the process/container check as it can now
+  fetch container data independently.
+
+- The performance of Remote Config has been significantly improved when large amounts of configurations are received.
+
+- Send ECS task lifecycle events in the container lifecycle check.
+
+- dbm: add new SQL obfuscation mode ``normalize_only`` to support normalizing SQL without obfuscating it. 
+  This mode is useful for customers who want to view unobfuscated SQL statements.
+  By default, ``ObfuscationMode`` is set to ``obfuscate_and_normalize`` and every SQL statement is obfuscated and normalized.
+
+- USM: Handle the HTTP TRACE method.
+
+
+.. _Release Notes_7.53.0_Deprecation Notes:
+
+Deprecation Notes
+-----------------
+
+- [oracle] Deprecating Oracle integration code. The functionality is fully implemented in the ``oracle-dbm`` check which is now renamed to ``oracle``.
+
+
+.. _Release Notes_7.53.0_Bug Fixes:
+
+Bug Fixes
+---------
+
+- The `windows_registry` check can be run with the `check` sub-command.
+
+- CWS: Fix very rare event corruption.
+
+- Fixes issue where processes for ECS Fargate containers would sometimes not be associated
+  with the correct container.
+
+- Fixed a bug in the Dual Shipping feature where events were not being
+  emitted on endpoint recovery.
+
+- Fix issue with ``display_container_name`` being tagged as ``N/A`` 
+  when ``container_name`` information is available.
+
+- Fix a Windows process handle leak in the Process Agent, which was introduced in 7.52.0 when `process_collection` is enabled.
+
+- Fixes a bug where the tagger server did not properly handle a closed channel.
+
+- [oracle] Set the default for ``metric_prefix`` in ``custom_queries`` to ``oracle``.
+
+- [oracle] Fix ``global_custom_queries`` bug.
+
+- [oracle] Adds the ``oracle.process.pga_maximum_memory`` metric for backward compatibility.
+
+- Stop sending ``systemd`` metrics when they are not set
+
+
 .. _Release Notes_7.52.1:
 
 7.52.1 / 6.52.1

cmd/cluster-agent/api/v1/languagedetection/handler.go

@@ -57,11 +57,13 @@ func (handler *languageDetectionHandler) startCleanupInBackground(ctx context.Co
 	go func() {
 		cleanupTicker := time.NewTicker(handler.cfg.cleanupPeriod)
 		defer cleanupTicker.Stop()
-		select {
-		case <-cleanupTicker.C:
-			handler.ownersLanguages.cleanExpiredLanguages(handler.wlm)
-		case <-ctx.Done():
-			break
+		for {
+			select {
+			case <-cleanupTicker.C:
+				handler.ownersLanguages.cleanExpiredLanguages(handler.wlm)
+			case <-ctx.Done():
+				break
+			}
 		}
 	}()
 

omnibus/config/projects/agent.rb

@@ -223,6 +223,9 @@
   dependency 'datadog-agent-integrations-py3'
 end
 
+# Used for memory profiling with the `status py` agent subcommand
+dependency 'pympler'
+
 if linux_target?
   dependency 'datadog-security-agent-policies'
 end

omnibus/config/software/agent-dependencies.rb

@@ -12,7 +12,7 @@
 # External agents
 dependency 'jmxfetch'
 
-if linux_target? || osx_target?
+if linux_target?
   dependency 'sds'
 end
 

omnibus/config/software/datadog-agent.rb

@@ -82,7 +82,13 @@
     command "inv -e rtloader.make --python-runtimes #{py_runtimes_arg} --install-prefix \"#{install_dir}/embedded\" --cmake-options '-DCMAKE_CXX_FLAGS:=\"-D_GLIBCXX_USE_CXX11_ABI=0 -I#{install_dir}/embedded/include\" -DCMAKE_C_FLAGS:=\"-I#{install_dir}/embedded/include\" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_FIND_FRAMEWORK:STRING=NEVER'", :env => env
     command "inv -e rtloader.install"
     bundle_arg = bundled_agents ? bundled_agents.map { |k| "--bundle #{k}" }.join(" ") : "--bundle agent"
-    command "inv -e agent.build --exclude-rtloader --include-sds --python-runtimes #{py_runtimes_arg} --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded --python-home-2=#{install_dir}/embedded --python-home-3=#{install_dir}/embedded --flavor #{flavor_arg} #{bundle_arg}", env: env
+
+    include_sds = ""
+    if linux_target?
+        include_sds = "--include-sds" # we only support SDS on Linux targets for now
+    end
+    command "inv -e agent.build --exclude-rtloader #{include_sds} --python-runtimes #{py_runtimes_arg} --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded --python-home-2=#{install_dir}/embedded --python-home-3=#{install_dir}/embedded --flavor #{flavor_arg} #{bundle_arg}", env: env
+
     if heroku_target?
       command "inv -e agent.build --exclude-rtloader --python-runtimes #{py_runtimes_arg} --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded --python-home-2=#{install_dir}/embedded --python-home-3=#{install_dir}/embedded --flavor #{flavor_arg} --agent-bin=bin/agent/core-agent --bundle agent", env: env
     end

omnibus/config/software/pympler.rb

@@ -0,0 +1,42 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https:#www.datadoghq.com/).
+# Copyright 2016-present Datadog, Inc.
+
+# Even though this is a dependency that we install with `pip`, it makes sense to keep it
+# separate from the integrations-related definitions since it's not defined anywhere as
+# a dependency for integrations.
+name 'pympler'
+default_version "0.7"
+
+if with_python_runtime? "3"
+  dependency 'pip3'
+  dependency 'setuptools3'
+end
+
+if with_python_runtime? "3"
+  dependency 'pip2'
+end
+
+pympler_requirement = "pympler==#{default_version}"
+
+build do
+  if with_python_runtime? "3"
+    if windows_target?
+      python = "#{windows_safe_path(python_3_embedded)}\\python.exe"
+    else
+      python = "#{install_dir}/embedded/bin/python3"
+    end
+    command "#{python} -m pip install #{pympler_requirement}"
+  end
+
+  if with_python_runtime? "2"
+    if windows_target?
+      python = "#{windows_safe_path(python_2_embedded)}\\python.exe"
+    else
+      python = "#{install_dir}/embedded/bin/python2"
+    end
+    command "#{python} -m pip install #{pympler_requirement}"
+  end
+
+end

pkg/collector/corechecks/network-devices/cisco-sdwan/report/metrics.go

@@ -189,7 +189,7 @@ func (ms *SDWanSender) SendOMPPeerMetrics(ompPeers []client.OMPPeer) {
 		remoteTags := ms.getRemoteDeviceTags(entry.Peer)
 
 		tags := append(deviceTags, remoteTags...)
-		tags = append(tags, "legit:"+entry.Legit, "refresh:"+entry.Refresh, "type:"+entry.Type, "state:"+entry.State)
+		tags = append(tags, "legit:"+entry.Legit, "refresh:"+entry.Refresh, "state:"+entry.State)
 
 		status := 0
 		if entry.State == "up" {

pkg/collector/corechecks/network-devices/cisco-sdwan/report/metrics_test.go

@@ -589,6 +589,7 @@ func TestSendOMPPeerMetrics(t *testing.T) {
 					"device_vendor:cisco",
 					"hostname:test-vsmart",
 					"system_ip:10.0.0.2",
+					"type:vsmart",
 					"site_id:102",
 				},
 				"10.0.0.3": {
@@ -597,6 +598,7 @@ func TestSendOMPPeerMetrics(t *testing.T) {
 					"device_vendor:cisco",
 					"hostname:test-device2",
 					"system_ip:10.0.0.3",
+					"type:vedge",
 					"site_id:110",
 				},
 			},
@@ -618,7 +620,7 @@ func TestSendOMPPeerMetrics(t *testing.T) {
 						"remote_site_id:102",
 						"legit:yes",
 						"refresh:supported",
-						"type:vsmart",
+						"remote_type:vsmart",
 						"state:up",
 					},
 				},
@@ -639,7 +641,7 @@ func TestSendOMPPeerMetrics(t *testing.T) {
 						"remote_site_id:110",
 						"legit:yes",
 						"refresh:unsupported",
-						"type:vedge",
+						"remote_type:vedge",
 						"state:down",
 					},
 				},
@@ -664,6 +666,7 @@ func TestSendOMPPeerMetrics(t *testing.T) {
 					"device_vendor:cisco",
 					"hostname:test-device",
 					"system_ip:10.0.0.1",
+					"type:vsmart",
 					"site_id:100",
 				},
 			},

pkg/collector/corechecks/network-devices/cisco-sdwan/sdwan_test.go

@@ -155,7 +155,7 @@ collect_bfd_session_status: true
 	})
 
 	// Assert OMP Peer metrics
-	sender.AssertMetric(t, "Gauge", "cisco_sdwan.omp_peer.status", 1, "", []string{"system_ip:10.10.1.5", "remote_system_ip:10.10.1.13", "legit:yes", "refresh:supported", "type:vedge", "state:up"})
+	sender.AssertMetric(t, "Gauge", "cisco_sdwan.omp_peer.status", 1, "", []string{"system_ip:10.10.1.5", "remote_system_ip:10.10.1.13", "legit:yes", "refresh:supported", "state:up"})
 
 	// Assert BFD Session metrics
 	sender.AssertMetric(t, "Gauge", "cisco_sdwan.bfd_session.status", 1, "", []string{"system_ip:10.10.1.11", "remote_system_ip:10.10.1.13", "local_color:public-internet", "remote_color:public-internet", "proto:ipsec", "state:up"})

pkg/config/setup/system_probe_cws.go

@@ -105,6 +105,7 @@ func initCWSSystemProbeConfig(cfg pkgconfigmodel.Config) {
 	cfg.BindEnvAndSetDefault("runtime_security_config.hash_resolver.max_hash_burst", 1000)
 	cfg.BindEnvAndSetDefault("runtime_security_config.hash_resolver.hash_algorithms", []string{"sha1", "sha256", "ssdeep"})
 	cfg.BindEnvAndSetDefault("runtime_security_config.hash_resolver.cache_size", 500)
+	cfg.BindEnvAndSetDefault("runtime_security_config.hash_resolver.replace", map[string]string{})
 
 	// CWS - UserSessions
 	cfg.BindEnvAndSetDefault("runtime_security_config.user_sessions.cache_size", 1024)