From 125a2924073e4232b9d3806f08b268971c8293e6 Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Mon, 19 Aug 2024 09:43:22 +0200 Subject: [PATCH 001/245] [ASCII-2155] add custom go linter to check usage of pkgconfig inside component folder (#28382) Co-authored-by: FlorentClarret --- .custom-gcl.yml | 7 + .github/CODEOWNERS | 3 + .github/workflows/codeql-analysis.yml | 2 +- .golangci.yml | 512 +++++++++++++++++- .../subcommands/dogstatsdstats/command.go | 3 +- cmd/agent/subcommands/integrations/command.go | 2 +- cmd/agent/subcommands/launchgui/command.go | 4 +- .../subcommands/status/command.go | 3 +- .../subcommands/runtime/activity_dump.go | 2 +- .../subcommands/status/command.go | 3 +- cmd/system-probe/subcommands/debug/command.go | 3 +- .../subcommands/runtime/activity_dump.go | 2 +- comp/autoscaling/datadogclient/impl/none.go | 5 +- .../providers/kube_endpoints_test.go | 3 +- .../providers/kube_services_test.go | 3 +- comp/core/flare/helpers/send_flare.go | 3 +- comp/core/gui/guiimpl/agent.go | 12 +- comp/core/gui/guiimpl/checks.go | 10 +- comp/core/tagger/taggerimpl/tagger.go | 4 +- .../internal/containerd/containerd.go | 4 +- .../collectors/internal/docker/docker.go | 8 +- comp/dogstatsd/server/server.go | 2 +- .../collector/impl-pipeline/flare_filler.go | 6 +- comp/otelcol/otlp/collector.go | 5 +- comp/rdnsquerier/impl/rdnsquerier_test.go | 2 +- .../systray/systrayimpl/doconfigure.go | 4 +- pkg/api/security/security.go | 2 +- pkg/api/util/doget.go | 6 +- pkg/cli/subcommands/clusterchecks/command.go | 5 +- pkg/cli/subcommands/health/command.go | 3 +- .../kubelet/common/testing/utils.go | 6 +- pkg/collector/corechecks/loader.go | 4 +- .../portlist/portlist_linux.go | 2 +- .../snmp/internal/discovery/discovery.go | 2 +- pkg/collector/corechecks/systemd/systemd.go | 4 +- pkg/collector/python/init.go | 3 +- pkg/compliance/evaluator_rego.go | 3 +- pkg/config/settings/http/client.go | 5 +- pkg/config/setup/config.go | 2 +- pkg/jmxfetch/jmxfetch.go | 2 +- pkg/linters/components/pkgconfigusage/go.mod | 17 + pkg/linters/components/pkgconfigusage/go.sum | 20 + .../pkgconfigusage/pkgconfigusage.go | 72 +++ .../pkgconfigusage/pkgconfigusage_test.go | 41 ++ pkg/linters/testdata/src/comp/a.go | 9 + .../decoder/legacy_auto_multiline_handler.go | 4 +- pkg/logs/launchers/listener/tcp_test.go | 8 +- pkg/logs/launchers/listener/udp_nix_test.go | 8 +- pkg/logs/launchers/listener/udp_test.go | 2 +- pkg/logs/tailers/windowsevent/tailer_test.go | 12 +- pkg/metrics/metric_test.go | 3 +- pkg/network/tracer/utils_linux.go | 2 +- pkg/security/module/server_linux.go | 10 +- pkg/security/probe/field_handlers_ebpf.go | 2 +- pkg/security/resolvers/netns/resolver.go | 12 +- pkg/security/resolvers/sbom/resolver.go | 2 +- .../security_profile/dump/remote_storage.go | 6 +- pkg/snmp/gosnmplib/gosnmp_log.go | 2 +- pkg/trace/api/api_test.go | 6 +- pkg/trace/api/telemetry_test.go | 6 +- pkg/util/containers/filter.go | 2 +- pkg/util/kubernetes/autoscalers/processor.go | 2 +- pkg/util/pdhutil/pdhhelper.go | 9 +- tasks/install_tasks.py | 39 +- tasks/libs/common/check_tools_version.py | 12 +- tasks/linter.py | 17 +- tasks/modules.py | 1 + .../agent-platform/common/agent_behaviour.go | 2 +- test/new-e2e/tests/npm/test_helpers.go | 2 +- 69 files changed, 873 insertions(+), 123 deletions(-) create mode 100644 .custom-gcl.yml create mode 100644 pkg/linters/components/pkgconfigusage/go.mod create mode 100644 pkg/linters/components/pkgconfigusage/go.sum create mode 100644 pkg/linters/components/pkgconfigusage/pkgconfigusage.go create mode 100644 pkg/linters/components/pkgconfigusage/pkgconfigusage_test.go create mode 100644 pkg/linters/testdata/src/comp/a.go diff --git a/.custom-gcl.yml b/.custom-gcl.yml new file mode 100644 index 0000000000000..45466d9aef0ce --- /dev/null +++ b/.custom-gcl.yml @@ -0,0 +1,7 @@ +version: v1.59.1 + +name: golangci-lint + +plugins: + - module: 'github.com/DataDog/datadog-agent/pkg/linters/components/pkgconfigusage' + path: ./pkg/linters/components/pkgconfigusage diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5667225e22ed8..98b377a033069 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -19,6 +19,7 @@ /.go-version @DataDog/agent-shared-components @DataDog/agent-delivery # Go linters and pre-commit config /.golangci.yml @DataDog/agent-devx-loops +/.custom-gcl.yml @DataDog/agent-devx-loops /.pre-commit-config.yaml @DataDog/agent-devx-loops /CHANGELOG.rst @DataDog/agent-delivery @@ -426,6 +427,8 @@ /pkg/util/testutil/flake @DataDog/agent-devx-loops /pkg/util/trie @DataDog/container-integrations /pkg/languagedetection @DataDog/processes @DataDog/universal-service-monitoring +/pkg/linters/ @DataDog/agent-devx-loops +/pkg/linters/components/ @DataDog/agent-shared-components /pkg/logs/ @DataDog/agent-metrics-logs /pkg/logs/launchers/windowsevent @DataDog/agent-metrics-logs @DataDog/windows-agent /pkg/logs/tailers/windowsevent @DataDog/agent-metrics-logs @DataDog/windows-agent diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5ac98f7c6cd52..cb1d0be9db095 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -54,7 +54,7 @@ jobs: - name: Build DataDog agent run: | - invoke install-tools + invoke install-tools --no-custom-golangci-lint invoke deps invoke agent.build --build-exclude=systemd diff --git a/.golangci.yml b/.golangci.yml index b74881792e806..47d6301319f54 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -59,7 +59,512 @@ issues: # We are using it and it's not clear how to replace it. - text: "Temporary has been deprecated since Go 1.18" linters: [staticcheck] - + # Treat this list as a TODO for fixing issues with pkgconfigusage custom linter + # DO NOT ADD NEW ENTRIES + - path: comp/api/api/apiimpl/internal/config/endpoint.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/component.go + linters: + - pkgconfigusage + - path: comp/process/forwarders/forwardersimpl/forwarders.go + linters: + - pkgconfigusage + - path: comp/process/profiler/profilerimpl/profiler.go + linters: + - pkgconfigusage + - path: comp/logs/agent/agentimpl/agent.go + linters: + - pkgconfigusage + - path: comp/logs/agent/agentimpl/agent_core_init.go + linters: + - pkgconfigusage + - path: comp/logs/agent/agentimpl/serverless.go + linters: + - pkgconfigusage + - path: comp/logs/agent/agentimpl/agent_test.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/cloudfoundry/vm/cf_vm.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/kubemetadata/kubemetadata.go + linters: + - pkgconfigusage + - path: comp/trace/config/component.go + linters: + - pkgconfigusage + - path: comp/trace/config/config.go + linters: + - pkgconfigusage + - path: comp/trace/config/config_mock.go + linters: + - pkgconfigusage + - path: comp/trace/config/hostname.go + linters: + - pkgconfigusage + - path: comp/trace/config/remote.go + linters: + - pkgconfigusage + - path: comp/trace/config/setup.go + linters: + - pkgconfigusage + - path: comp/trace/config/config_test.go + linters: + - pkgconfigusage + - path: comp/remote-config/rcclient/rcclientimpl/rcclient.go + linters: + - pkgconfigusage + - path: comp/remote-config/rcclient/rcclientimpl/rcclient_test.go + linters: + - pkgconfigusage + - path: comp/agent/cloudfoundrycontainer/cloudfoundrycontainerimpl/cloudfoundrycontainer.go + linters: + - pkgconfigusage + - path: comp/metadata/internal/util/inventory_enabled.go + linters: + - pkgconfigusage + - path: comp/process/expvars/expvarsimpl/expvars.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/cloudfoundry/container/cf_container.go + linters: + - pkgconfigusage + - path: comp/api/api/apiimpl/listener.go + linters: + - pkgconfigusage + - path: comp/api/api/apiimpl/server.go + linters: + - pkgconfigusage + - path: comp/api/api/apiimpl/server_cmd.go + linters: + - pkgconfigusage + - path: comp/api/api/apiimpl/server_ipc.go + linters: + - pkgconfigusage + - path: comp/metadata/host/hostimpl/utils/common.go + linters: + - pkgconfigusage + - path: comp/metadata/host/hostimpl/utils/host.go + linters: + - pkgconfigusage + - path: comp/metadata/host/hostimpl/utils/meta.go + linters: + - pkgconfigusage + - path: comp/metadata/host/hostimpl/utils/host_test.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector_test.go + linters: + - pkgconfigusage + - path: comp/aggregator/demultiplexer/demultiplexerimpl/test_agent_demultiplexer.go + linters: + - pkgconfigusage + - path: comp/metadata/host/hostimpl/host.go + linters: + - pkgconfigusage + - path: comp/metadata/host/hostimpl/payload.go + linters: + - pkgconfigusage + - path: comp/core/tagger/taggerimpl/collectors/ecs_common.go + linters: + - pkgconfigusage + - path: comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go + linters: + - pkgconfigusage + - path: comp/core/tagger/taggerimpl/collectors/workloadmeta_main.go + linters: + - pkgconfigusage + - path: comp/core/tagger/taggerimpl/collectors/ecs_common_test.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/kubelet/kubelet.go + linters: + - pkgconfigusage + - path: comp/metadata/inventoryotel/inventoryotelimpl/inventoryotel_test.go + linters: + - pkgconfigusage + - path: comp/metadata/host/hostimpl/hosttags/tags.go + linters: + - pkgconfigusage + - path: comp/metadata/host/hostimpl/hosttags/tags_test.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/listeners/ratelimit/mem_based_rate_limiter.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/cloudfoundry.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/clusterchecks.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/config_reader.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/consul.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/container.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/endpointschecks.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/etcd.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_endpoints.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_endpoints_file.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_services.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_services_file.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_common.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_pods.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_services.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/providers.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/remote_config.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/utils.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/zookeeper.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/file_test.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_endpoints_test.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_services_test.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_common_test.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_services_test.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/utils_test.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/listeners/named_pipe_nowindows.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/listeners/udp.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/listeners/uds_common.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/listeners/uds_datagram.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/listeners/uds_stream.go + linters: + - pkgconfigusage + - path: comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent_test.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/common/utils/container_collect_all.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/common/utils/prometheus.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/common/utils/prometheus_apiserver_test.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/common/utils/prometheus_kubelet_test.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/serverDebug/serverdebugimpl/debug.go + linters: + - pkgconfigusage + - path: comp/otelcol/otlp/collector_test.go + linters: + - pkgconfigusage + - path: comp/core/hostname/remotehostnameimpl/hostname.go + linters: + - pkgconfigusage + - path: comp/trace/agent/impl/agent.go + linters: + - pkgconfigusage + - path: comp/trace/agent/impl/run.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/server/batch.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/server/float64_list_pool.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/server/parse.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/server/server.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/server/serverless.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/server/server_bench_test.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/server/server_test.go + linters: + - pkgconfigusage + - path: comp/process/agent/agentimpl/agent.go + linters: + - pkgconfigusage + - path: comp/process/apiserver/apiserver.go + linters: + - pkgconfigusage + - path: comp/remote-config/rcservice/rcserviceimpl/rcservice.go + linters: + - pkgconfigusage + - path: comp/netflow/config/config_test.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/listeners/common.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/listeners/container.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/listeners/environment.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/listeners/kubelet.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/listeners/service.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/listeners/staticconfig.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/listeners/snmp_test.go + linters: + - pkgconfigusage + - path: comp/core/tagger/params.go + linters: + - pkgconfigusage + - path: comp/remote-config/rcservicemrf/rcservicemrfimpl/rcservicemrf.go + linters: + - pkgconfigusage + - path: comp/process/status/statusimpl/status.go + linters: + - pkgconfigusage + - path: comp/agent/jmxlogger/jmxloggerimpl/jmxlogger.go + linters: + - pkgconfigusage + - path: comp/core/tagger/taggerimpl/remote/tagger.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/mapper/mapper.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/mapper/mapper_test.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/remote/workloadmeta/workloadmeta_test.go + linters: + - pkgconfigusage + - path: comp/forwarder/eventplatform/eventplatformimpl/epforwarder.go + linters: + - pkgconfigusage + - path: comp/core/gui/guiimpl/checks.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/replay/impl/capture.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/autodiscoveryimpl/autoconfig.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/autodiscoveryimpl/secrets.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/autodiscoveryimpl/autoconfig_test.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/packets/pool.go + linters: + - pkgconfigusage + - path: comp/process/agent/status.go + linters: + - pkgconfigusage + - path: comp/core/sysprobeconfig/component.go + linters: + - pkgconfigusage + - path: comp/core/tagger/taglist/taglist.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/remote/generic.go + linters: + - pkgconfigusage + - path: comp/core/hostname/hostnameimpl/service_test.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/status/status.go + linters: + - pkgconfigusage + - path: comp/core/sysprobeconfig/sysprobeconfigimpl/config.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/docker/docker.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/docker/image_sbom_trivy.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/containerd/containerd.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/containerd/image_sbom_trivy.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/containerd/network_linux.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/ecs/ecs.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/ecsfargate/ecsfargate.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/podman/podman.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/internal/process/process_collector.go + linters: + - pkgconfigusage + - path: comp/core/workloadmeta/collectors/util/process_util_linux.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/listeners/ratelimit/cgroup_memory_usage_linux.g + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/cloudfoundry_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/consul_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/endpointschecks_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/etcd_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_endpoints_file_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_endpoints_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_services_file_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_services_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_pods_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_services_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/zookeeper_nop.go + linters: + - pkgconfigusage + - path: comp/otelcol/otlp/no_otlp.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/cloudfoundry_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/consul_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/etcd_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_endpoints_file_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_endpoints_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_services_file_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_services_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_services_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/zookeeper_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/cloudfoundry_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/endpointschecks_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_endpoints_file_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_endpoints_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_services_file_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/kube_services_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_pods_nop.go + linters: + - pkgconfigusage + - path: comp/core/autodiscovery/providers/prometheus_services_nop.go + linters: + - pkgconfigusage + - path: comp/systray/systray/systrayimpl/doflare.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/listeners/named_pipe_windows.go + linters: + - pkgconfigusage + - path: comp/dogstatsd/packets/packet_manager_windows.go + linters: + - pkgconfigusage linters: disable-all: true enable: @@ -76,6 +581,7 @@ linters: - bodyclose # checks whether HTTP response body is closed successfully - gosimple # Linter for Go source code that specializes in simplifying code. - gocheckcompilerdirectives # Checks Go compiler directives syntax + - pkgconfigusage # Linter for checking usage of pkgconfig inside components folder linters-settings: depguard: @@ -156,3 +662,7 @@ linters-settings: - name: var-naming # non-default rules: - name: duplicated-imports + custom: + pkgconfigusage: + type: "module" + description: "Check usage of pkgconfig in components folder" \ No newline at end of file diff --git a/cmd/agent/subcommands/dogstatsdstats/command.go b/cmd/agent/subcommands/dogstatsdstats/command.go index 5b5cfdbbd43ff..c8c12ce29f9da 100644 --- a/cmd/agent/subcommands/dogstatsdstats/command.go +++ b/cmd/agent/subcommands/dogstatsdstats/command.go @@ -9,6 +9,7 @@ package dogstatsdstats import ( "bytes" "encoding/json" + "errors" "fmt" "os" @@ -88,7 +89,7 @@ func requestDogstatsdStats(_ log.Component, config config.Component, cliParams * json.Unmarshal(r, &errMap) //nolint:errcheck // If the error has been marshalled into a json object, check it and return it properly if err, found := errMap["error"]; found { - e = fmt.Errorf(err) + e = errors.New(err) } if len(errMap["error_type"]) > 0 { diff --git a/cmd/agent/subcommands/integrations/command.go b/cmd/agent/subcommands/integrations/command.go index 9bcf4b7dc63a1..867f85bfa945b 100644 --- a/cmd/agent/subcommands/integrations/command.go +++ b/cmd/agent/subcommands/integrations/command.go @@ -860,7 +860,7 @@ func moveConfigurationFiles(srcFolder string, dstFolder string) error { ))) } if errorMsg != "" { - return fmt.Errorf(errorMsg) + return errors.New(errorMsg) } return nil } diff --git a/cmd/agent/subcommands/launchgui/command.go b/cmd/agent/subcommands/launchgui/command.go index c88c53f5a98a0..dda6794fb7c2a 100644 --- a/cmd/agent/subcommands/launchgui/command.go +++ b/cmd/agent/subcommands/launchgui/command.go @@ -66,9 +66,9 @@ func launchGui(config config.Component, _ *cliParams, _ log.Component) error { // Open the GUI in a browser, passing the authorization tokens as parameters err = open("http://127.0.0.1:" + guiPort + "/auth?intent=" + string(intentToken)) if err != nil { - return fmt.Errorf("error opening GUI: " + err.Error()) + return fmt.Errorf("error opening GUI: %s", err.Error()) } - fmt.Printf("GUI opened at 127.0.0.1:" + guiPort + "\n") + fmt.Printf("GUI opened at 127.0.0.1:%s\n", guiPort) return nil } diff --git a/cmd/cluster-agent/subcommands/status/command.go b/cmd/cluster-agent/subcommands/status/command.go index 5d547bec4c6fd..7f4e3084cb6f8 100644 --- a/cmd/cluster-agent/subcommands/status/command.go +++ b/cmd/cluster-agent/subcommands/status/command.go @@ -11,6 +11,7 @@ package status import ( "bytes" "encoding/json" + "errors" "fmt" "net/url" "os" @@ -96,7 +97,7 @@ func run(log log.Component, config config.Component, cliParams *cliParams) error json.Unmarshal(r, &errMap) //nolint:errcheck // If the error has been marshalled into a json object, check it and return it properly if err, found := errMap["error"]; found { - e = fmt.Errorf(err) + e = errors.New(err) } fmt.Printf(` diff --git a/cmd/security-agent/subcommands/runtime/activity_dump.go b/cmd/security-agent/subcommands/runtime/activity_dump.go index 158a42648cf6b..46465a0986dd2 100644 --- a/cmd/security-agent/subcommands/runtime/activity_dump.go +++ b/cmd/security-agent/subcommands/runtime/activity_dump.go @@ -134,7 +134,7 @@ func generateDumpCommands(globalParams *command.GlobalParams) []*cobra.Command { activityDumpGenerateDumpCmd := &cobra.Command{ Use: "dump", Short: "generate an activity dump", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { return fxutil.OneShot(generateActivityDump, fx.Supply(cliParams), fx.Supply(core.BundleParams{ diff --git a/cmd/security-agent/subcommands/status/command.go b/cmd/security-agent/subcommands/status/command.go index d64a016f6b0af..5b03ffac1913d 100644 --- a/cmd/security-agent/subcommands/status/command.go +++ b/cmd/security-agent/subcommands/status/command.go @@ -9,6 +9,7 @@ package status import ( "bytes" "encoding/json" + "errors" "fmt" "net/url" "os" @@ -94,7 +95,7 @@ func runStatus(_ log.Component, config config.Component, _ secrets.Component, pa json.Unmarshal(r, &errMap) //nolint:errcheck // If the error has been marshalled into a json object, check it and return it properly if err, found := errMap["error"]; found { - e = fmt.Errorf(err) + e = errors.New(err) } fmt.Printf(` diff --git a/cmd/system-probe/subcommands/debug/command.go b/cmd/system-probe/subcommands/debug/command.go index ec220f144cd2c..454749bfcf2cd 100644 --- a/cmd/system-probe/subcommands/debug/command.go +++ b/cmd/system-probe/subcommands/debug/command.go @@ -8,6 +8,7 @@ package debug import ( "encoding/json" + "errors" "fmt" "strconv" @@ -79,7 +80,7 @@ func debugRuntime(sysprobeconfig sysprobeconfig.Component, cliParams *cliParams) _ = json.Unmarshal(r, &errMap) // If the error has been marshalled into a json object, check it and return it properly if e, found := errMap["error"]; found { - return fmt.Errorf(e) + return errors.New(e) } return fmt.Errorf("Could not reach system-probe: %s\nMake sure system-probe is running before running this command and contact support if you continue having issues", err) diff --git a/cmd/system-probe/subcommands/runtime/activity_dump.go b/cmd/system-probe/subcommands/runtime/activity_dump.go index 60ba970cdfa6a..72096d1160bbd 100644 --- a/cmd/system-probe/subcommands/runtime/activity_dump.go +++ b/cmd/system-probe/subcommands/runtime/activity_dump.go @@ -133,7 +133,7 @@ func generateDumpCommands(globalParams *command.GlobalParams) []*cobra.Command { activityDumpGenerateDumpCmd := &cobra.Command{ Use: "dump", Short: "generate an activity dump", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { return fxutil.OneShot(generateActivityDump, fx.Supply(cliParams), fx.Supply(core.BundleParams{ diff --git a/comp/autoscaling/datadogclient/impl/none.go b/comp/autoscaling/datadogclient/impl/none.go index 18181f8d6961b..cfd669d4eb155 100644 --- a/comp/autoscaling/datadogclient/impl/none.go +++ b/comp/autoscaling/datadogclient/impl/none.go @@ -7,8 +7,9 @@ package datadogclientimpl import ( - datadogclient "github.com/DataDog/datadog-agent/comp/autoscaling/datadogclient/def" "gopkg.in/zorkian/go-datadog-api.v2" + + datadogclient "github.com/DataDog/datadog-agent/comp/autoscaling/datadogclient/def" ) // ImplNone is a noop datadogclient implementation @@ -22,7 +23,7 @@ func NewNone() datadogclient.Component { } // QueryMetrics does nothing for the noop datadogclient implementation -func (d *ImplNone) QueryMetrics(from, to int64, query string) ([]datadog.Series, error) { +func (d *ImplNone) QueryMetrics(_, _ int64, _ string) ([]datadog.Series, error) { // noop return nil, nil } diff --git a/comp/core/autodiscovery/providers/kube_endpoints_test.go b/comp/core/autodiscovery/providers/kube_endpoints_test.go index 36b78d68d8add..3ea35201e26a0 100644 --- a/comp/core/autodiscovery/providers/kube_endpoints_test.go +++ b/comp/core/autodiscovery/providers/kube_endpoints_test.go @@ -9,7 +9,6 @@ package providers import ( "context" - "fmt" "strings" "testing" "time" @@ -463,7 +462,7 @@ func TestGenerateConfigs(t *testing.T) { }, }, } { - t.Run(fmt.Sprintf(tc.name), func(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { cfgs := generateConfigs(tc.template, tc.resolveMode, tc.endpoints) assert.EqualValues(t, tc.expectedOut, cfgs) }) diff --git a/comp/core/autodiscovery/providers/kube_services_test.go b/comp/core/autodiscovery/providers/kube_services_test.go index fcecd31581219..57da4676c08f1 100644 --- a/comp/core/autodiscovery/providers/kube_services_test.go +++ b/comp/core/autodiscovery/providers/kube_services_test.go @@ -9,7 +9,6 @@ package providers import ( "context" - "fmt" "strings" "testing" "time" @@ -241,7 +240,7 @@ func TestParseKubeServiceAnnotations(t *testing.T) { }, }, } { - t.Run(fmt.Sprintf(tc.name), func(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { cfg := config.NewConfig("datadog", "DD", strings.NewReplacer(".", "_")) if tc.hybrid { cfg.SetWithoutSource("cluster_checks.support_hybrid_ignore_ad_tags", true) diff --git a/comp/core/flare/helpers/send_flare.go b/comp/core/flare/helpers/send_flare.go index be2d06b157a32..5246ee5306469 100644 --- a/comp/core/flare/helpers/send_flare.go +++ b/comp/core/flare/helpers/send_flare.go @@ -8,6 +8,7 @@ package helpers import ( "context" "encoding/json" + "errors" "fmt" "io" "mime/multipart" @@ -184,7 +185,7 @@ func analyzeResponse(r *http.Response, apiKey string) (string, error) { if res.Error != "" { response := fmt.Sprintf("An error occurred while uploading the flare: %s. Please contact support by email.", res.Error) - return response, fmt.Errorf("%s", res.Error) + return response, errors.New(res.Error) } return fmt.Sprintf("Your logs were successfully uploaded. For future reference, your internal case id is %d", res.CaseID), nil diff --git a/comp/core/gui/guiimpl/agent.go b/comp/core/gui/guiimpl/agent.go index 8604467c25e18..e75b92b66946c 100644 --- a/comp/core/gui/guiimpl/agent.go +++ b/comp/core/gui/guiimpl/agent.go @@ -67,7 +67,7 @@ func getStatus(w http.ResponseWriter, r *http.Request, statusComponent status.Co } if err != nil { - log.Errorf("Error getting status: " + err.Error()) + log.Errorf("Error getting status: %s", err.Error()) w.Write([]byte("Error getting status: " + err.Error())) return } @@ -81,7 +81,7 @@ func getStatus(w http.ResponseWriter, r *http.Request, statusComponent status.Co func getVersion(w http.ResponseWriter, _ *http.Request) { version, e := version.Agent() if e != nil { - log.Errorf("Error getting version: " + e.Error()) + log.Errorf("Error getting version: %s", e.Error()) w.Write([]byte("Error: " + e.Error())) return } @@ -95,7 +95,7 @@ func getVersion(w http.ResponseWriter, _ *http.Request) { func getHostname(w http.ResponseWriter, r *http.Request) { hname, e := hostname.Get(r.Context()) if e != nil { - log.Errorf("Error getting hostname: " + e.Error()) + log.Errorf("Error getting hostname: %s", e.Error()) w.Write([]byte("Error: " + e.Error())) return } @@ -153,19 +153,19 @@ func makeFlare(w http.ResponseWriter, r *http.Request, flare flare.Component) { filePath, e := flare.Create(nil, nil) if e != nil { w.Write([]byte("Error creating flare zipfile: " + e.Error())) - log.Errorf("Error creating flare zipfile: " + e.Error()) + log.Errorf("Error creating flare zipfile: %s", e.Error()) return } res, e := flare.Send(filePath, payload.CaseID, payload.Email, helpers.NewLocalFlareSource()) if e != nil { w.Write([]byte("Flare zipfile successfully created: " + filePath + "

" + e.Error())) - log.Errorf("Flare zipfile successfully created: " + filePath + "\n" + e.Error()) + log.Errorf("Flare zipfile successfully created: %s\n%s", filePath, e.Error()) return } w.Write([]byte("Flare zipfile successfully created: " + filePath + "

" + res)) - log.Errorf("Flare zipfile successfully created: " + filePath + "\n" + res) + log.Infof("Flare zipfile successfully created: %s\n%s", filePath, res) } // Restarts the agent using the appropriate (platform-specific) restart function diff --git a/comp/core/gui/guiimpl/checks.go b/comp/core/gui/guiimpl/checks.go index a66ba66a93d45..c72a4e8f39c64 100644 --- a/comp/core/gui/guiimpl/checks.go +++ b/comp/core/gui/guiimpl/checks.go @@ -83,7 +83,7 @@ func runCheckHandler(collector collector.Component, ac autodiscovery.Component) for _, ch := range instances { collector.RunCheck(ch) //nolint:errcheck } - log.Infof("Scheduled new check: " + name) + log.Infof("Scheduled new check: %s", name) } } @@ -156,14 +156,14 @@ func reloadCheckHandler(collector collector.Component, ac autodiscovery.Componen name := html.EscapeString(mux.Vars(r)["name"]) instances := pkgcollector.GetChecksByNameForConfigs(name, ac.GetAllConfigs()) if len(instances) == 0 { - log.Errorf("Can't reload " + name + ": check has no new instances.") + log.Errorf("Can't reload %s: check has no new instances.", name) w.Write([]byte("Can't reload " + name + ": check has no new instances")) return } killed, e := collector.ReloadAllCheckInstances(name, instances) if e != nil { - log.Errorf("Error reloading check: " + e.Error()) + log.Errorf("Error reloading check: %s", e.Error()) w.Write([]byte("Error reloading check: " + e.Error())) return } @@ -309,7 +309,7 @@ func setCheckConfigFile(w http.ResponseWriter, r *http.Request) { return } - log.Infof("Successfully wrote new " + fileName + " config file.") + log.Infof("Successfully wrote new %s config file.", fileName) w.Write([]byte("Success")) } else if r.Method == "DELETE" { // Attempt to write new configs to custom checks directory @@ -336,7 +336,7 @@ func setCheckConfigFile(w http.ResponseWriter, r *http.Request) { return } - log.Infof("Successfully disabled integration " + fileName + " config file.") + log.Infof("Successfully disabled integration %s config file.", fileName) w.Write([]byte("Success")) } } diff --git a/comp/core/tagger/taggerimpl/tagger.go b/comp/core/tagger/taggerimpl/tagger.go index 0292acddf03b4..65e75abe51ddd 100644 --- a/comp/core/tagger/taggerimpl/tagger.go +++ b/comp/core/tagger/taggerimpl/tagger.go @@ -437,7 +437,7 @@ func (t *TaggerClient) EnrichTags(tb tagset.TagsAccumulator, originInfo taggerty if originInfo.FromUDS != packets.NoOrigin && (originInfo.FromTag == "" || !t.datadogConfig.dogstatsdEntityIDPrecedenceEnabled) { if err := t.AccumulateTagsFor(originInfo.FromUDS, cardinality, tb); err != nil { - t.log.Errorf(err.Error()) + t.log.Errorf("%s", err.Error()) } } @@ -462,7 +462,7 @@ func (t *TaggerClient) EnrichTags(tb tagset.TagsAccumulator, originInfo taggerty // Tag using Local Data if originInfo.FromUDS != packets.NoOrigin { if err := t.AccumulateTagsFor(originInfo.FromUDS, cardinality, tb); err != nil { - t.log.Errorf(err.Error()) + t.log.Errorf("%s", err.Error()) } } diff --git a/comp/core/workloadmeta/collectors/internal/containerd/containerd.go b/comp/core/workloadmeta/collectors/internal/containerd/containerd.go index 553f414551041..1cf223bcbc1a8 100644 --- a/comp/core/workloadmeta/collectors/internal/containerd/containerd.go +++ b/comp/core/workloadmeta/collectors/internal/containerd/containerd.go @@ -189,7 +189,7 @@ func (c *collector) stream(ctx context.Context) { case ev := <-c.eventsChan: if err := c.handleEvent(ctx, ev); err != nil { - log.Warnf(err.Error()) + log.Warnf("%s", err.Error()) } case err := <-c.errorsChan: @@ -261,7 +261,7 @@ func (c *collector) generateInitialContainerEvents(namespace string) ([]workload ev, err := createSetEvent(container, namespace, c.containerdClient, c.store) if err != nil { - log.Warnf(err.Error()) + log.Warnf("%s", err.Error()) continue } diff --git a/comp/core/workloadmeta/collectors/internal/docker/docker.go b/comp/core/workloadmeta/collectors/internal/docker/docker.go index f0d2de3527e9a..ef2af0107c0dd 100644 --- a/comp/core/workloadmeta/collectors/internal/docker/docker.go +++ b/comp/core/workloadmeta/collectors/internal/docker/docker.go @@ -149,13 +149,13 @@ func (c *collector) stream(ctx context.Context) { case ev := <-c.containerEventsCh: err := c.handleContainerEvent(ctx, ev) if err != nil { - log.Warnf(err.Error()) + log.Warnf("%s", err.Error()) } case ev := <-c.imageEventsCh: err := c.handleImageEvent(ctx, ev, nil) if err != nil { - log.Warnf(err.Error()) + log.Warnf("%s", err.Error()) } case <-ctx.Done(): @@ -195,7 +195,7 @@ func (c *collector) generateEventsFromContainerList(ctx context.Context, filter Action: events.ActionStart, }) if err != nil { - log.Warnf(err.Error()) + log.Warnf("%s", err.Error()) continue } @@ -220,7 +220,7 @@ func (c *collector) generateEventsFromImageList(ctx context.Context) error { for _, img := range images { imgMetadata, err := c.getImageMetadata(ctx, img.ID, nil) if err != nil { - log.Warnf(err.Error()) + log.Warnf("%s", err.Error()) continue } diff --git a/comp/dogstatsd/server/server.go b/comp/dogstatsd/server/server.go index b1f97ef2f34db..c187855c648db 100644 --- a/comp/dogstatsd/server/server.go +++ b/comp/dogstatsd/server/server.go @@ -393,7 +393,7 @@ func (s *server) start(context.Context) error { if s.config.GetString("dogstatsd_port") == listeners.RandomPortName || s.config.GetInt("dogstatsd_port") > 0 { udpListener, err := listeners.NewUDPListener(packetsChannel, sharedPacketPoolManager, s.config, s.tCapture, s.listernersTelemetry, s.packetsTelemetry) if err != nil { - s.log.Errorf(err.Error()) + s.log.Errorf("%s", err.Error()) } else { tmpListeners = append(tmpListeners, udpListener) s.udpLocalAddr = udpListener.LocalAddr() diff --git a/comp/otelcol/collector/impl-pipeline/flare_filler.go b/comp/otelcol/collector/impl-pipeline/flare_filler.go index faee05d972332..e4ea9a8a011d9 100644 --- a/comp/otelcol/collector/impl-pipeline/flare_filler.go +++ b/comp/otelcol/collector/impl-pipeline/flare_filler.go @@ -12,6 +12,7 @@ import ( "context" "crypto/tls" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -19,10 +20,11 @@ import ( "strings" "time" + "github.com/gocolly/colly/v2" + flaretypes "github.com/DataDog/datadog-agent/comp/core/flare/types" extension "github.com/DataDog/datadog-agent/comp/otelcol/extension/def" "github.com/DataDog/datadog-agent/pkg/util/log" - "github.com/gocolly/colly/v2" ) func (c *collectorImpl) fillFlare(fb flaretypes.FlareBuilder) error { @@ -160,7 +162,7 @@ func (c *collectorImpl) requestOtelConfigInfo(endpointURL string) ([]byte, error return nil, err } if res.StatusCode >= 400 { - return nil, fmt.Errorf("%s", body) + return nil, errors.New(string(body)) } return body, nil } diff --git a/comp/otelcol/otlp/collector.go b/comp/otelcol/otlp/collector.go index ae3f84e587243..e3978764031e5 100644 --- a/comp/otelcol/otlp/collector.go +++ b/comp/otelcol/otlp/collector.go @@ -27,6 +27,8 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" + otlpmetrics "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/tagger" "github.com/DataDog/datadog-agent/comp/core/tagger/types" @@ -42,7 +44,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/log" zapAgent "github.com/DataDog/datadog-agent/pkg/util/log/zap" "github.com/DataDog/datadog-agent/pkg/version" - otlpmetrics "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics" ) var pipelineError = atomic.NewError(nil) @@ -247,7 +248,7 @@ func recoverAndStoreError() { if r := recover(); r != nil { err := fmt.Errorf("OTLP pipeline had a panic: %v", r) pipelineError.Store(err) - log.Errorf(err.Error()) + log.Errorf("%s", err.Error()) } } diff --git a/comp/rdnsquerier/impl/rdnsquerier_test.go b/comp/rdnsquerier/impl/rdnsquerier_test.go index 54ecbdc8d8fc4..fb5102e511815 100644 --- a/comp/rdnsquerier/impl/rdnsquerier_test.go +++ b/comp/rdnsquerier/impl/rdnsquerier_test.go @@ -678,7 +678,7 @@ func TestRetriesExceeded(t *testing.T) { func(_ string) { assert.FailNow(t, "Sync callback should not be called") }, - func(hostname string, err error) { + func(_ string, err error) { assert.Error(t, err) wg.Done() }, diff --git a/comp/systray/systray/systrayimpl/doconfigure.go b/comp/systray/systray/systrayimpl/doconfigure.go index 76917b51df7f0..253217c13a550 100644 --- a/comp/systray/systray/systrayimpl/doconfigure.go +++ b/comp/systray/systray/systrayimpl/doconfigure.go @@ -41,9 +41,9 @@ func doConfigure(s *systrayImpl) error { // Open the GUI in a browser, passing the authorization tokens as parameters err = open("http://127.0.0.1:" + guiPort + "/auth?intent=" + string(intentToken)) if err != nil { - return fmt.Errorf("error opening GUI: " + err.Error()) + return fmt.Errorf("error opening GUI: %s", err.Error()) } - s.log.Debugf("GUI opened at 127.0.0.1:" + guiPort + "\n") + s.log.Debugf("GUI opened at 127.0.0.1:%s\n", guiPort) return nil } diff --git a/pkg/api/security/security.go b/pkg/api/security/security.go index d4aefbc38a81c..40b1310c2f79c 100644 --- a/pkg/api/security/security.go +++ b/pkg/api/security/security.go @@ -157,7 +157,7 @@ func fetchAuthToken(config configModel.Reader, tokenCreationAllowed bool) (strin // Read the token authTokenRaw, e := os.ReadFile(authTokenFile) if e != nil { - return "", fmt.Errorf("unable to read authentication token file: " + e.Error()) + return "", fmt.Errorf("unable to read authentication token file: %s", e.Error()) } // Do some basic validation diff --git a/pkg/api/util/doget.go b/pkg/api/util/doget.go index f7a9bd84593a4..9304932124b7f 100644 --- a/pkg/api/util/doget.go +++ b/pkg/api/util/doget.go @@ -8,7 +8,7 @@ package util import ( "context" "crypto/tls" - "fmt" + "errors" "io" "net/http" ) @@ -81,7 +81,7 @@ func DoGetWithOptions(c *http.Client, url string, options *ReqOptions) (body []b return body, e } if r.StatusCode >= 400 { - return body, fmt.Errorf("%s", body) + return body, errors.New(string(body)) } return body, nil } @@ -105,7 +105,7 @@ func DoPost(c *http.Client, url string, contentType string, body io.Reader) (res return resp, e } if r.StatusCode >= 400 { - return resp, fmt.Errorf("%s", resp) + return resp, errors.New(string(resp)) } return resp, nil } diff --git a/pkg/cli/subcommands/clusterchecks/command.go b/pkg/cli/subcommands/clusterchecks/command.go index 061b62c0d67c3..6a29ac12c7cd3 100644 --- a/pkg/cli/subcommands/clusterchecks/command.go +++ b/pkg/cli/subcommands/clusterchecks/command.go @@ -9,6 +9,7 @@ package clusterchecks import ( "bytes" "encoding/json" + "errors" "fmt" "github.com/fatih/color" @@ -153,7 +154,7 @@ func rebalance(_ log.Component, config config.Component, cliParams *cliParams) e json.Unmarshal(r, &errMap) //nolint:errcheck // If the error has been marshalled into a json object, check it and return it properly if e, found := errMap["error"]; found { - err = fmt.Errorf(e) + err = errors.New(e) } fmt.Printf(` @@ -196,7 +197,7 @@ func isolate(_ log.Component, config config.Component, cliParams *cliParams) err json.Unmarshal(r, &errMap) //nolint:errcheck // If the error has been marshalled into a json object, check it and return it properly if e, found := errMap["error"]; found { - err = fmt.Errorf(e) + err = errors.New(e) } fmt.Printf(` diff --git a/pkg/cli/subcommands/health/command.go b/pkg/cli/subcommands/health/command.go index e921ae6031293..f4d1233d61d99 100644 --- a/pkg/cli/subcommands/health/command.go +++ b/pkg/cli/subcommands/health/command.go @@ -9,6 +9,7 @@ package health import ( "context" "encoding/json" + "errors" "fmt" "sort" "strconv" @@ -99,7 +100,7 @@ func requestHealth(_ log.Component, config config.Component, cliParams *cliParam json.Unmarshal(r, &errMap) //nolint:errcheck // If the error has been marshalled into a json object, check it and return it properly if e, found := errMap["error"]; found { - err = fmt.Errorf(e) + err = errors.New(e) } return fmt.Errorf("could not reach agent: %v \nMake sure the agent is running before requesting the status and contact support if you continue having issues", err) diff --git a/pkg/collector/corechecks/containers/kubelet/common/testing/utils.go b/pkg/collector/corechecks/containers/kubelet/common/testing/utils.go index e62a694ea6ac2..5054431177d18 100644 --- a/pkg/collector/corechecks/containers/kubelet/common/testing/utils.go +++ b/pkg/collector/corechecks/containers/kubelet/common/testing/utils.go @@ -147,7 +147,7 @@ func CreateKubeletMock(response EndpointResponse, endpoint string) (*mock.Kubele if response.filename != "" { content, err = os.ReadFile(response.filename) if err != nil { - return nil, fmt.Errorf(fmt.Sprintf("unable to read test file at: %s, Err: %v", response.filename, err)) + return nil, fmt.Errorf("unable to read test file at: %s, Err: %w", response.filename, err) } } kubeletMock.MockReplies[endpoint] = &mock.HTTPReplyMock{ @@ -166,12 +166,12 @@ func StorePopulatedFromFile(store workloadmetamock.Mock, filename string, podUti podList, err := os.ReadFile(filename) if err != nil { - return fmt.Errorf(fmt.Sprintf("unable to load pod list, Err: %v", err)) + return fmt.Errorf("unable to load pod list, Err: %w", err) } var pods *kubelet.PodList err = json.Unmarshal(podList, &pods) if err != nil { - return fmt.Errorf(fmt.Sprintf("unable to load pod list, Err: %v", err)) + return fmt.Errorf("unable to load pod list, Err: %w", err) } for _, pod := range pods.Items { diff --git a/pkg/collector/corechecks/loader.go b/pkg/collector/corechecks/loader.go index eb09c0cefb49c..31eab283f1fe5 100644 --- a/pkg/collector/corechecks/loader.go +++ b/pkg/collector/corechecks/loader.go @@ -61,7 +61,7 @@ func (gl *GoCheckLoader) Load(senderManger sender.SenderManager, config integrat factory, found := catalog[config.Name] if !found { msg := fmt.Sprintf("Check %s not found in Catalog", config.Name) - return c, fmt.Errorf(msg) + return c, errors.New(msg) } c = factory() @@ -71,7 +71,7 @@ func (gl *GoCheckLoader) Load(senderManger sender.SenderManager, config integrat } log.Errorf("core.loader: could not configure check %s: %s", c, err) msg := fmt.Sprintf("Could not configure check %s: %s", c, err) - return c, fmt.Errorf(msg) + return c, errors.New(msg) } return c, nil diff --git a/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux.go b/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux.go index 918d9e6fc7a36..dffe0a5f83110 100644 --- a/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux.go +++ b/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux.go @@ -308,7 +308,7 @@ func (li *linuxImpl) findProcessNames(need map[string]*portMeta) error { targetBuf := make([]byte, 64) // plenty big for "socket:[165614651]" linkPath := li.readlinkPathBuf[:0] - linkPath = fmt.Appendf(linkPath, li.procMountPath+"/") + linkPath = fmt.Append(linkPath, li.procMountPath+"/") linkPath = mem.Append(linkPath, pid) linkPath = append(linkPath, "/fd/"...) linkPath = mem.Append(linkPath, fd) diff --git a/pkg/collector/corechecks/snmp/internal/discovery/discovery.go b/pkg/collector/corechecks/snmp/internal/discovery/discovery.go index 6347d190b1244..f8c3337308ea8 100644 --- a/pkg/collector/corechecks/snmp/internal/discovery/discovery.go +++ b/pkg/collector/corechecks/snmp/internal/discovery/discovery.go @@ -101,7 +101,7 @@ func (d *Discovery) runWorker(w int, jobs <-chan checkDeviceJob) { log.Debugf("subnet %s: Handling IP %s", d.config.Network, job.currentIP.String()) err := d.checkDevice(job) if err != nil { - log.Errorf(err.Error()) + log.Errorf("%s", err.Error()) } } } diff --git a/pkg/collector/corechecks/systemd/systemd.go b/pkg/collector/corechecks/systemd/systemd.go index e1596d6653e5d..3db37ea9f9751 100644 --- a/pkg/collector/corechecks/systemd/systemd.go +++ b/pkg/collector/corechecks/systemd/systemd.go @@ -411,9 +411,9 @@ func (c *SystemdCheck) submitPropertyMetricsAsGauge(sender sender.Sender, conn * if err != nil { msg := fmt.Sprintf("Cannot send property '%s' for unit '%s': %v", service.propertyName, unit.Name, err) if service.optional { - log.Debugf(msg) + log.Debugf("%s", msg) } else { - log.Warnf(msg) + log.Warnf("%s", msg) } } } diff --git a/pkg/collector/python/init.go b/pkg/collector/python/init.go index 3cc6c140ae432..5b905debba7ba 100644 --- a/pkg/collector/python/init.go +++ b/pkg/collector/python/init.go @@ -8,6 +8,7 @@ package python import ( + "errors" "expvar" "fmt" "os" @@ -258,7 +259,7 @@ func addExpvarPythonInitErrors(msg string) error { defer pyInitLock.Unlock() pyInitErrors = append(pyInitErrors, msg) - return fmt.Errorf(msg) + return errors.New(msg) } func sendTelemetry(pythonVersion string) { diff --git a/pkg/compliance/evaluator_rego.go b/pkg/compliance/evaluator_rego.go index 2e2106e88e234..a6456ad0f2242 100644 --- a/pkg/compliance/evaluator_rego.go +++ b/pkg/compliance/evaluator_rego.go @@ -8,6 +8,7 @@ package compliance import ( "context" "encoding/json" + "errors" "fmt" "os" "strconv" @@ -117,7 +118,7 @@ func newCheckEventFromRegoResult(data interface{}, rule *Rule, resolvedInputs Re if errMsg == "" { errMsg = "unknown" } - errReason = fmt.Errorf(errMsg) + errReason = errors.New(errMsg) default: errReason = fmt.Errorf("rego result invalid: bad status %q", status) } diff --git a/pkg/config/settings/http/client.go b/pkg/config/settings/http/client.go index 748ad5834fbd4..6d9a8ff42236b 100644 --- a/pkg/config/settings/http/client.go +++ b/pkg/config/settings/http/client.go @@ -9,6 +9,7 @@ package http import ( "bytes" "encoding/json" + "errors" "fmt" "html" "net/http" @@ -37,7 +38,7 @@ func (rc *runtimeSettingsHTTPClient) doGet(url string, formatError bool) (string _ = json.Unmarshal(r, &errMap) // If the error has been marshalled into a json object, check it and return it properly if e, found := errMap["error"]; found { - return "", fmt.Errorf(e) + return "", errors.New(e) } if formatError { return "", fmt.Errorf("Could not reach %s: %v \nMake sure the %s is running before requesting the runtime configuration and contact support if you continue having issues", rc.targetProcessName, err, rc.targetProcessName) @@ -130,7 +131,7 @@ func (rc *runtimeSettingsHTTPClient) Set(key string, value string) (bool, error) _ = json.Unmarshal(r, &errMap) // If the error has been marshalled into a json object, check it and return it properly if e, found := errMap["error"]; found { - return false, fmt.Errorf(e) + return false, errors.New(e) } return false, err } diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index ba058fe9715a9..0c6fa60a9afca 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -1949,7 +1949,7 @@ func LoadCustom(config pkgconfigmodel.Config, additionalKnownEnvVars []string) e } for _, warningMsg := range findUnexpectedUnicode(config) { - log.Warnf(warningMsg) + log.Warnf("%s", warningMsg) } return nil diff --git a/pkg/jmxfetch/jmxfetch.go b/pkg/jmxfetch/jmxfetch.go index 7c39684e06a8e..2f0fb0537beca 100644 --- a/pkg/jmxfetch/jmxfetch.go +++ b/pkg/jmxfetch/jmxfetch.go @@ -138,7 +138,7 @@ func (j *JMXFetch) Monitor() { if !limiter.canRestart(time.Now()) { msg := fmt.Sprintf("Too many JMXFetch restarts (%v) in time interval (%vs) - giving up", limiter.maxRestarts, limiter.interval) - log.Errorf(msg) + log.Errorf("%s", msg) s := jmxStatus.StartupError{LastError: msg, Timestamp: time.Now().Unix()} jmxStatus.SetStartupError(s) return diff --git a/pkg/linters/components/pkgconfigusage/go.mod b/pkg/linters/components/pkgconfigusage/go.mod new file mode 100644 index 0000000000000..94737011484cb --- /dev/null +++ b/pkg/linters/components/pkgconfigusage/go.mod @@ -0,0 +1,17 @@ +module github.com/DataDog/datadog-agent/pkg/linters/components/pkgconfigusage + +go 1.22.0 + +require ( + github.com/golangci/plugin-module-register v0.1.1 + github.com/stretchr/testify v1.9.0 + golang.org/x/tools v0.24.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/mod v0.20.0 // indirect + golang.org/x/sync v0.8.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/pkg/linters/components/pkgconfigusage/go.sum b/pkg/linters/components/pkgconfigusage/go.sum new file mode 100644 index 0000000000000..e4381e0e55b8d --- /dev/null +++ b/pkg/linters/components/pkgconfigusage/go.sum @@ -0,0 +1,20 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golangci/plugin-module-register v0.1.1 h1:TCmesur25LnyJkpsVrupv1Cdzo+2f7zX0H6Jkw1Ol6c= +github.com/golangci/plugin-module-register v0.1.1/go.mod h1:TTpqoB6KkwOJMV8u7+NyXMrkwwESJLOkfl9TxR1DGFc= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/linters/components/pkgconfigusage/pkgconfigusage.go b/pkg/linters/components/pkgconfigusage/pkgconfigusage.go new file mode 100644 index 0000000000000..9699c50dd87fb --- /dev/null +++ b/pkg/linters/components/pkgconfigusage/pkgconfigusage.go @@ -0,0 +1,72 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package pkgconfigusage provides a linter for ensuring pkg/config is not used inside comp folder +package pkgconfigusage + +import ( + "strings" + + "github.com/golangci/plugin-module-register/register" + "golang.org/x/tools/go/analysis" +) + +// We can replace it during tests +var componentPath = "github.com/DataDog/datadog-agent/comp/" + +func init() { + register.Plugin("pkgconfigusage", New) +} + +type pkgconfigUsagePlugin struct { +} + +// New returns a new config linter plugin +func New(any) (register.LinterPlugin, error) { + return &pkgconfigUsagePlugin{}, nil +} + +// BuildAnalyzers returns the analyzers for the plugin +func (f *pkgconfigUsagePlugin) BuildAnalyzers() ([]*analysis.Analyzer, error) { + return []*analysis.Analyzer{ + { + Name: "pkgconfigusage", + Doc: "ensure github.com/DataDog/datadog-agent/pkg/config is not used inside the components folder", + Run: f.run, + }, + }, nil +} + +// GetLoadMode returns the load mode for the plugin +func (f *pkgconfigUsagePlugin) GetLoadMode() string { + return register.LoadModeSyntax +} + +func (f *pkgconfigUsagePlugin) run(pass *analysis.Pass) (interface{}, error) { + for _, file := range pass.Files { + + if !strings.HasPrefix(pass.Pkg.Path(), componentPath) { + continue + } + + for _, imp := range file.Imports { + if imp.Path.Value == `"github.com/DataDog/datadog-agent/pkg/config"` { + pass.Report(analysis.Diagnostic{ + Pos: imp.Pos(), + End: imp.End(), + Category: "components", + Message: "github.com/DataDog/datadog-agent/pkg/config should not be used inside comp folder", + SuggestedFixes: []analysis.SuggestedFix{ + { + Message: "Use the config component instead, by declaring it as part of your component's dependencies.", + }, + }, + }) + } + } + } + + return nil, nil +} diff --git a/pkg/linters/components/pkgconfigusage/pkgconfigusage_test.go b/pkg/linters/components/pkgconfigusage/pkgconfigusage_test.go new file mode 100644 index 0000000000000..03d2c2afe160e --- /dev/null +++ b/pkg/linters/components/pkgconfigusage/pkgconfigusage_test.go @@ -0,0 +1,41 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package pkgconfigusage + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/tools/go/analysis/analysistest" +) + +func TestAll(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get wd: %s", err) + } + + originalComponentPath := componentPath + componentPath = "comp" + + t.Cleanup(func() { + componentPath = originalComponentPath + }) + + testdata := filepath.Join(filepath.Dir(filepath.Dir(wd)), "testdata") + plugin := &pkgconfigUsagePlugin{} + analyzers, err := plugin.BuildAnalyzers() + assert.NoError(t, err) + + analyzer := analyzers[0] + // We do this to skip issues with import or other errors. + // We only care about parsing the test file and run the analyzer. + analyzer.RunDespiteErrors = true + + analysistest.Run(t, testdata, analyzer, "comp/...") +} diff --git a/pkg/linters/testdata/src/comp/a.go b/pkg/linters/testdata/src/comp/a.go new file mode 100644 index 0000000000000..6a7cdf211fb57 --- /dev/null +++ b/pkg/linters/testdata/src/comp/a.go @@ -0,0 +1,9 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package a is a test package to test the linter +package a + +import _ "github.com/DataDog/datadog-agent/pkg/config" // want "pkg/config should not be used inside comp folder" diff --git a/pkg/logs/internal/decoder/legacy_auto_multiline_handler.go b/pkg/logs/internal/decoder/legacy_auto_multiline_handler.go index 61d9011103463..8e7b0a1321764 100644 --- a/pkg/logs/internal/decoder/legacy_auto_multiline_handler.go +++ b/pkg/logs/internal/decoder/legacy_auto_multiline_handler.go @@ -185,7 +185,7 @@ func (h *LegacyAutoMultilineHandler) processAndTry(message *message.Message) { if matchRatio >= h.matchThreshold { h.autoMultiLineStatus.SetMessage("state", "State: Using multi-line handler") h.autoMultiLineStatus.SetMessage("message", fmt.Sprintf("Pattern %v matched %d lines with a ratio of %f", topMatch.regexp.String(), topMatch.score, matchRatio)) - log.Debug(fmt.Sprintf("Pattern %v matched %d lines with a ratio of %f - using multi-line handler", topMatch.regexp.String(), topMatch.score, matchRatio)) + log.Debugf("Pattern %v matched %d lines with a ratio of %f - using multi-line handler", topMatch.regexp.String(), topMatch.score, matchRatio) telemetry.GetStatsTelemetryProvider().Count(autoMultiLineTelemetryMetricName, 1, []string{"success:true"}) h.detectedPattern.Set(topMatch.regexp) @@ -193,7 +193,7 @@ func (h *LegacyAutoMultilineHandler) processAndTry(message *message.Message) { } else { h.autoMultiLineStatus.SetMessage("state", "State: Using single-line handler") h.autoMultiLineStatus.SetMessage("message", fmt.Sprintf("No pattern met the line match threshold: %f during multiline auto detection. Top match was %v with a match ratio of: %f", h.matchThreshold, topMatch.regexp.String(), matchRatio)) - log.Debugf(fmt.Sprintf("No pattern met the line match threshold: %f during multiline auto detection. Top match was %v with a match ratio of: %f - using single-line handler", h.matchThreshold, topMatch.regexp.String(), matchRatio)) + log.Debugf("No pattern met the line match threshold: %f during multiline auto detection. Top match was %v with a match ratio of: %f - using single-line handler", h.matchThreshold, topMatch.regexp.String(), matchRatio) telemetry.GetStatsTelemetryProvider().Count(autoMultiLineTelemetryMetricName, 1, []string{"success:false"}) // Stay with the single line handler and no longer attempt to detect multiline matches. diff --git a/pkg/logs/launchers/listener/tcp_test.go b/pkg/logs/launchers/listener/tcp_test.go index 031369bab63f9..202d203b8174d 100644 --- a/pkg/logs/launchers/listener/tcp_test.go +++ b/pkg/logs/launchers/listener/tcp_test.go @@ -32,7 +32,7 @@ func TestTCPShouldReceivesMessages(t *testing.T) { defer conn.Close() var msg *message.Message - fmt.Fprintf(conn, "hello world\n") + fmt.Fprint(conn, "hello world\n") msg = <-msgChan assert.Equal(t, "hello world", string(msg.GetContent())) assert.Equal(t, 1, len(listener.tailers)) @@ -50,15 +50,15 @@ func TestTCPDoesNotTruncateMessagesThatAreBiggerThanTheReadBufferSize(t *testing assert.Nil(t, err) var msg *message.Message - fmt.Fprintf(conn, strings.Repeat("a", 80)+"\n") + fmt.Fprint(conn, strings.Repeat("a", 80)+"\n") msg = <-msgChan assert.Equal(t, strings.Repeat("a", 80), string(msg.GetContent())) - fmt.Fprintf(conn, strings.Repeat("a", 200)+"\n") + fmt.Fprint(conn, strings.Repeat("a", 200)+"\n") msg = <-msgChan assert.Equal(t, strings.Repeat("a", 200), string(msg.GetContent())) - fmt.Fprintf(conn, strings.Repeat("a", 70)+"\n") + fmt.Fprint(conn, strings.Repeat("a", 70)+"\n") msg = <-msgChan assert.Equal(t, strings.Repeat("a", 70), string(msg.GetContent())) diff --git a/pkg/logs/launchers/listener/udp_nix_test.go b/pkg/logs/launchers/listener/udp_nix_test.go index bce53e04fcfbc..7423ad918042d 100644 --- a/pkg/logs/launchers/listener/udp_nix_test.go +++ b/pkg/logs/launchers/listener/udp_nix_test.go @@ -47,7 +47,7 @@ func TestUDPShoulProperlyCollectLogSplitPerDatadgram(t *testing.T) { msg = <-msgChan assert.Equal(t, strings.Repeat("a", 10), string(msg.GetContent())) - fmt.Fprintf(conn, strings.Repeat("a", 10)+"\n"+strings.Repeat("a", 10)+"\n") + fmt.Fprint(conn, strings.Repeat("a", 10)+"\n"+strings.Repeat("a", 10)+"\n") msg = <-msgChan assert.Equal(t, strings.Repeat("a", 10), string(msg.GetContent())) msg = <-msgChan @@ -99,13 +99,13 @@ func TestUDPShoulDropTooBigMessages(t *testing.T) { var msg *message.Message - fmt.Fprintf(conn, strings.Repeat("a", maxUDPFrameLen-100)+"\n") + fmt.Fprint(conn, strings.Repeat("a", maxUDPFrameLen-100)+"\n") msg = <-msgChan assert.Equal(t, strings.Repeat("a", maxUDPFrameLen-100), string(msg.GetContent())) // the first frame should be dropped as it's too big compare to the limit. - fmt.Fprintf(conn, strings.Repeat("a", maxUDPFrameLen+100)+"\n") - fmt.Fprintf(conn, strings.Repeat("a", maxUDPFrameLen-200)+"\n") + fmt.Fprint(conn, strings.Repeat("a", maxUDPFrameLen+100)+"\n") + fmt.Fprint(conn, strings.Repeat("a", maxUDPFrameLen-200)+"\n") msg = <-msgChan assert.Equal(t, strings.Repeat("a", maxUDPFrameLen-200), string(msg.GetContent())) diff --git a/pkg/logs/launchers/listener/udp_test.go b/pkg/logs/launchers/listener/udp_test.go index 07a0db2ed3850..638574571548b 100644 --- a/pkg/logs/launchers/listener/udp_test.go +++ b/pkg/logs/launchers/listener/udp_test.go @@ -32,7 +32,7 @@ func TestUDPShouldReceiveMessage(t *testing.T) { var msg *message.Message - fmt.Fprintf(conn, "hello world\n") + fmt.Fprint(conn, "hello world\n") msg = <-msgChan assert.Equal(t, "hello world", string(msg.GetContent())) diff --git a/pkg/logs/tailers/windowsevent/tailer_test.go b/pkg/logs/tailers/windowsevent/tailer_test.go index 2d842877acd12..5c8c635314edb 100644 --- a/pkg/logs/tailers/windowsevent/tailer_test.go +++ b/pkg/logs/tailers/windowsevent/tailer_test.go @@ -8,13 +8,15 @@ package windowsevent import ( + "errors" "fmt" "testing" "time" - pkglog "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/cihub/seelog" + pkglog "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/cenkalti/backoff" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -23,8 +25,8 @@ import ( "github.com/DataDog/datadog-agent/pkg/logs/message" "github.com/DataDog/datadog-agent/pkg/logs/sources" "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" - "github.com/DataDog/datadog-agent/pkg/util/winutil/eventlog/api" - "github.com/DataDog/datadog-agent/pkg/util/winutil/eventlog/test" + evtapi "github.com/DataDog/datadog-agent/pkg/util/winutil/eventlog/api" + eventlog_test "github.com/DataDog/datadog-agent/pkg/util/winutil/eventlog/test" ) type ReadEventsSuite struct { @@ -89,7 +91,7 @@ func newtailer(evtapi evtapi.API, tailerconfig *Config, bookmark string, msgChan if source.Status.IsSuccess() { return nil } else if source.Status.IsError() { - return fmt.Errorf(source.Status.GetError()) + return errors.New(source.Status.GetError()) } return fmt.Errorf("start pending") }, backoff.NewConstantBackOff(50*time.Millisecond)) @@ -198,7 +200,7 @@ func (s *ReadEventsSuite) TestRecoverFromBrokenSubscription() { if tailer.source.Status.IsSuccess() { return nil } else if tailer.source.Status.IsError() { - return fmt.Errorf(tailer.source.Status.GetError()) + return errors.New(tailer.source.Status.GetError()) } return fmt.Errorf("start pending") }, backoff.NewConstantBackOff(50*time.Millisecond)) diff --git a/pkg/metrics/metric_test.go b/pkg/metrics/metric_test.go index bced04754329e..530062011e52e 100644 --- a/pkg/metrics/metric_test.go +++ b/pkg/metrics/metric_test.go @@ -7,7 +7,6 @@ package metrics import ( "encoding/json" - "fmt" "testing" "github.com/stretchr/testify/assert" @@ -35,7 +34,7 @@ func TestAPIMetricTypeMarshal(t *testing.T) { `{"type":"rate"}`, }, } { - t.Run(fmt.Sprintf(tc.Out), func(t *testing.T) { + t.Run(tc.Out, func(t *testing.T) { out, err := json.Marshal(tc.In) assert.NoError(t, err) assert.Equal(t, tc.Out, string(out)) diff --git a/pkg/network/tracer/utils_linux.go b/pkg/network/tracer/utils_linux.go index 2477b60ea38a1..5315b10785a00 100644 --- a/pkg/network/tracer/utils_linux.go +++ b/pkg/network/tracer/utils_linux.go @@ -81,5 +81,5 @@ func verifyOSVersion(kernelCode kernel.Version, platform string, exclusionList [ } errMsg := fmt.Sprintf("Kernel unsupported (%s) - ", kernelCode) errMsg += fmt.Sprintf("required functions missing: %s", strings.Join(missingFuncs, ", ")) - return false, fmt.Errorf(errMsg) + return false, errors.New(errMsg) } diff --git a/pkg/security/module/server_linux.go b/pkg/security/module/server_linux.go index e7cea21a88a07..c4d1d9556cd11 100644 --- a/pkg/security/module/server_linux.go +++ b/pkg/security/module/server_linux.go @@ -56,7 +56,7 @@ func (a *APIServer) ListActivityDumps(_ context.Context, params *api.ActivityDum if managers := p.GetProfileManagers(); managers != nil { msg, err := managers.ListActivityDumps(params) if err != nil { - seclog.Errorf(err.Error()) + seclog.Errorf("%s", err.Error()) } return msg, nil } @@ -74,7 +74,7 @@ func (a *APIServer) StopActivityDump(_ context.Context, params *api.ActivityDump if managers := p.GetProfileManagers(); managers != nil { msg, err := managers.StopActivityDump(params) if err != nil { - seclog.Errorf(err.Error()) + seclog.Errorf("%s", err.Error()) } return msg, nil } @@ -92,7 +92,7 @@ func (a *APIServer) TranscodingRequest(_ context.Context, params *api.Transcodin if managers := p.GetProfileManagers(); managers != nil { msg, err := managers.GenerateTranscoding(params) if err != nil { - seclog.Errorf(err.Error()) + seclog.Errorf("%s", err.Error()) } return msg, nil } @@ -110,7 +110,7 @@ func (a *APIServer) ListSecurityProfiles(_ context.Context, params *api.Security if managers := p.GetProfileManagers(); managers != nil { msg, err := managers.ListSecurityProfiles(params) if err != nil { - seclog.Errorf(err.Error()) + seclog.Errorf("%s", err.Error()) } return msg, nil } @@ -128,7 +128,7 @@ func (a *APIServer) SaveSecurityProfile(_ context.Context, params *api.SecurityP if managers := p.GetProfileManagers(); managers != nil { msg, err := managers.SaveSecurityProfile(params) if err != nil { - seclog.Errorf(err.Error()) + seclog.Errorf("%s", err.Error()) } return msg, nil } diff --git a/pkg/security/probe/field_handlers_ebpf.go b/pkg/security/probe/field_handlers_ebpf.go index d65e55e430921..b9cb03e12cfc7 100644 --- a/pkg/security/probe/field_handlers_ebpf.go +++ b/pkg/security/probe/field_handlers_ebpf.go @@ -538,7 +538,7 @@ func (fh *EBPFFieldHandlers) ResolveCGroupID(ev *model.Event, e *model.CGroupCon } // ResolveCGroupManager resolves the manager of the cgroup -func (fh *EBPFFieldHandlers) ResolveCGroupManager(ev *model.Event, e *model.CGroupContext) string { +func (fh *EBPFFieldHandlers) ResolveCGroupManager(ev *model.Event, _ *model.CGroupContext) string { if entry, _ := fh.ResolveProcessCacheEntry(ev); entry != nil { if manager := containerutils.CGroupManager(entry.CGroup.CGroupFlags); manager != 0 { return manager.String() diff --git a/pkg/security/resolvers/netns/resolver.go b/pkg/security/resolvers/netns/resolver.go index fe0caef888a41..73a5d92f8920c 100644 --- a/pkg/security/resolvers/netns/resolver.go +++ b/pkg/security/resolvers/netns/resolver.go @@ -657,7 +657,7 @@ func (nr *Resolver) DumpNetworkNamespaces(params *api.DumpNetworkNamespaceParams dumpFile, err := newTmpFile("network-namespace-dump-*.json") if err != nil { resp.Error = fmt.Sprintf("couldn't create temporary file: %v", err) - seclog.Warnf(resp.Error) + seclog.Warnf("%s", err.Error()) return resp } defer dumpFile.Close() @@ -667,13 +667,13 @@ func (nr *Resolver) DumpNetworkNamespaces(params *api.DumpNetworkNamespaceParams encoder := json.NewEncoder(dumpFile) if err = encoder.Encode(dump); err != nil { resp.Error = fmt.Sprintf("couldn't encode list of network namespace: %v", err) - seclog.Warnf(resp.Error) + seclog.Warnf("%s", err.Error()) return resp } if err = dumpFile.Close(); err != nil { resp.Error = fmt.Sprintf("could not close file [%s]: %s", dumpFile.Name(), err) - seclog.Warnf(resp.Error) + seclog.Warnf("%s", err.Error()) return resp } @@ -681,7 +681,7 @@ func (nr *Resolver) DumpNetworkNamespaces(params *api.DumpNetworkNamespaceParams graphFile, err := newTmpFile("network-namespace-graph-*.dot") if err != nil { resp.Error = fmt.Sprintf("couldn't create temporary file: %v", err) - seclog.Warnf(resp.Error) + seclog.Warnf("%s", err.Error()) return resp } defer graphFile.Close() @@ -690,13 +690,13 @@ func (nr *Resolver) DumpNetworkNamespaces(params *api.DumpNetworkNamespaceParams // generate dot graph if err = nr.generateGraph(dump, graphFile); err != nil { resp.Error = fmt.Sprintf("couldn't generate dot graph: %v", err) - seclog.Warnf(resp.Error) + seclog.Warnf("%s", err.Error()) return resp } if err = graphFile.Close(); err != nil { resp.Error = fmt.Sprintf("could not close file [%s]: %s", graphFile.Name(), err) - seclog.Warnf(resp.Error) + seclog.Warnf("%s", err.Error()) return resp } diff --git a/pkg/security/resolvers/sbom/resolver.go b/pkg/security/resolvers/sbom/resolver.go index c31a73ac6ab04..96c2fd12c0d50 100644 --- a/pkg/security/resolvers/sbom/resolver.go +++ b/pkg/security/resolvers/sbom/resolver.go @@ -258,7 +258,7 @@ func (r *Resolver) Start(ctx context.Context) error { if err := retry.Do(func() error { return r.analyzeWorkload(sbom) }, retry.Attempts(maxSBOMGenerationRetries), retry.Delay(200*time.Millisecond)); err != nil { - seclog.Errorf(err.Error()) + seclog.Errorf("%s", err.Error()) } } } diff --git a/pkg/security/security_profile/dump/remote_storage.go b/pkg/security/security_profile/dump/remote_storage.go index b770651503cf1..9c9c8958eb303 100644 --- a/pkg/security/security_profile/dump/remote_storage.go +++ b/pkg/security/security_profile/dump/remote_storage.go @@ -12,6 +12,7 @@ import ( "bytes" "compress/gzip" "encoding/json" + "errors" "fmt" "mime/multipart" "net/http" @@ -20,6 +21,8 @@ import ( "go.uber.org/atomic" + "github.com/DataDog/datadog-go/v5/statsd" + logsconfig "github.com/DataDog/datadog-agent/comp/logs/agent/config" pkgconfig "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/security/config" @@ -27,7 +30,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/seclog" "github.com/DataDog/datadog-agent/pkg/security/utils" ddhttputil "github.com/DataDog/datadog-agent/pkg/util/http" - "github.com/DataDog/datadog-go/v5/statsd" ) type tooLargeEntityStatsEntry struct { @@ -176,7 +178,7 @@ func (storage *ActivityDumpRemoteStorage) sendToEndpoint(url string, apiKey stri } storage.tooLargeEntities[entry].Inc() } - return fmt.Errorf(resp.Status) + return errors.New(resp.Status) } // Persist saves the provided buffer to the persistent storage diff --git a/pkg/snmp/gosnmplib/gosnmp_log.go b/pkg/snmp/gosnmplib/gosnmp_log.go index ff8dec9d845d3..908c2a0d99838 100644 --- a/pkg/snmp/gosnmplib/gosnmp_log.go +++ b/pkg/snmp/gosnmplib/gosnmp_log.go @@ -52,6 +52,6 @@ func (sw *TraceLevelLogWriter) Write(logInput []byte) (n int, err error) { logInput = replacer.Regex.ReplaceAll(logInput, replacer.Repl) } } - log.Tracef(string(logInput)) + log.Trace(string(logInput)) return len(logInput), nil } diff --git a/pkg/trace/api/api_test.go b/pkg/trace/api/api_test.go index a922f2f7d57e7..6d035f12bdd6f 100644 --- a/pkg/trace/api/api_test.go +++ b/pkg/trace/api/api_test.go @@ -219,7 +219,7 @@ func TestLegacyReceiver(t *testing.T) { } for _, tc := range testCases { - t.Run(fmt.Sprintf(tc.name), func(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { // start testing server server := httptest.NewServer( tc.r.handleWithVersion(tc.apiVersion, tc.r.handleTraces), @@ -284,7 +284,7 @@ func TestReceiverJSONDecoder(t *testing.T) { } for _, tc := range testCases { - t.Run(fmt.Sprintf(tc.name), func(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { // start testing server server := httptest.NewServer( tc.r.handleWithVersion(tc.apiVersion, tc.r.handleTraces), @@ -344,7 +344,7 @@ func TestReceiverMsgpackDecoder(t *testing.T) { } for _, tc := range testCases { - t.Run(fmt.Sprintf(tc.name), func(t *testing.T) { + t.Run(tc.name, func(t *testing.T) { // start testing server server := httptest.NewServer( tc.r.handleWithVersion(tc.apiVersion, tc.r.handleTraces), diff --git a/pkg/trace/api/telemetry_test.go b/pkg/trace/api/telemetry_test.go index e5c76bc14524f..5c37737da22ae 100644 --- a/pkg/trace/api/telemetry_test.go +++ b/pkg/trace/api/telemetry_test.go @@ -295,7 +295,7 @@ func TestMaxInflightBytes(t *testing.T) { done := make(chan struct{}) - srv := assertingServer(t, func(req *http.Request, body []byte) error { + srv := assertingServer(t, func(req *http.Request, _ []byte) error { assert.Equal("test_apikey", req.Header.Get("DD-API-KEY")) assert.Equal("test_hostname", req.Header.Get("DD-Agent-Hostname")) assert.Equal("test_env", req.Header.Get("DD-Agent-Env")) @@ -339,7 +339,7 @@ func TestInflightBytesReset(t *testing.T) { done := make(chan struct{}) - srv := assertingServer(t, func(req *http.Request, body []byte) error { + srv := assertingServer(t, func(req *http.Request, _ []byte) error { assert.Equal("test_apikey", req.Header.Get("DD-API-KEY")) assert.Equal("test_hostname", req.Header.Get("DD-Agent-Hostname")) assert.Equal("test_env", req.Header.Get("DD-Agent-Env")) @@ -397,7 +397,7 @@ func TestActualServer(t *testing.T) { done := make(chan struct{}) - intakeMockServer := assertingServer(t, func(req *http.Request, body []byte) error { + intakeMockServer := assertingServer(t, func(req *http.Request, _ []byte) error { assert.Equal("test_apikey", req.Header.Get("DD-API-KEY")) assert.Equal("test_hostname", req.Header.Get("DD-Agent-Hostname")) assert.Equal("test_env", req.Header.Get("DD-Agent-Env")) diff --git a/pkg/util/containers/filter.go b/pkg/util/containers/filter.go index b9994718702e4..a0c389dfdde63 100644 --- a/pkg/util/containers/filter.go +++ b/pkg/util/containers/filter.go @@ -134,7 +134,7 @@ func parseFilters(filters []string) (imageFilters, nameFilters, namespaceFilters namespaceFilters = append(namespaceFilters, r) default: warnmsg := fmt.Sprintf("Container filter %q is unknown, ignoring it. The supported filters are 'image', 'name' and 'kube_namespace'", filter) - log.Warnf(warnmsg) + log.Warn(warnmsg) filterWarnings = append(filterWarnings, warnmsg) } diff --git a/pkg/util/kubernetes/autoscalers/processor.go b/pkg/util/kubernetes/autoscalers/processor.go index 3e575e1a7eac4..3440c6ba01e2d 100644 --- a/pkg/util/kubernetes/autoscalers/processor.go +++ b/pkg/util/kubernetes/autoscalers/processor.go @@ -237,7 +237,7 @@ func makeChunks(batch []string) (chunks [][]string) { uriLength = uriLength + tempSize beyond, err := isURLBeyondLimits(uriLength, len(tempBucket)) if err != nil { - log.Errorf(fmt.Sprintf("%s: %s", err.Error(), val)) + log.Errorf("%v: %s", err, val) continue } if beyond { diff --git a/pkg/util/pdhutil/pdhhelper.go b/pkg/util/pdhutil/pdhhelper.go index a5e41503862bd..48a43f4487b58 100644 --- a/pkg/util/pdhutil/pdhhelper.go +++ b/pkg/util/pdhutil/pdhhelper.go @@ -7,6 +7,7 @@ package pdhutil import ( + "errors" "fmt" "reflect" "strconv" @@ -98,8 +99,8 @@ func refreshPdhObjectCache(forceRefresh bool) (didrefresh bool, err error) { } else if refreshInterval < 0 { // invalid value e := "windows_counter_refresh_interval cannot be a negative number" - log.Errorf(e) - return false, fmt.Errorf(e) + log.Errorf("%s", e) + return false, errors.New(e) } // Only refresh at most every refresh_interval seconds @@ -132,8 +133,8 @@ func refreshPdhObjectCache(forceRefresh bool) (didrefresh bool, err error) { uintptr(1)) // do refresh if r != PDH_MORE_DATA { e := fmt.Sprintf("Failed to refresh performance counters (%#x)", r) - log.Errorf(e) - return false, fmt.Errorf(e) + log.Errorf("%s", e) + return false, errors.New(e) } // refresh successful diff --git a/tasks/install_tasks.py b/tasks/install_tasks.py index fae395b911fd7..b5e2630202786 100644 --- a/tasks/install_tasks.py +++ b/tasks/install_tasks.py @@ -1,5 +1,6 @@ -import os.path as ospath +import os import platform +import shutil import sys import zipfile from pathlib import Path @@ -9,7 +10,7 @@ from tasks.libs.ciproviders.github_api import GithubAPI from tasks.libs.common.go import download_go_dependencies from tasks.libs.common.retry import run_command_with_retry -from tasks.libs.common.utils import environ, gitlab_section +from tasks.libs.common.utils import bin_name, environ, gitlab_section TOOL_LIST = [ 'github.com/frapposelli/wwhrd', @@ -45,7 +46,7 @@ def download_tools(ctx): @task -def install_tools(ctx: Context, max_retry: int = 3): +def install_tools(ctx: Context, max_retry: int = 3, custom_golangci_lint=True): """Install all Go tools for testing.""" with gitlab_section("Installing Go tools", collapsed=True): with environ({'GO111MODULE': 'on'}): @@ -53,6 +54,34 @@ def install_tools(ctx: Context, max_retry: int = 3): with ctx.cd(path): for tool in tools: run_command_with_retry(ctx, f"go install {tool}", max_retry=max_retry) + if custom_golangci_lint: + install_custom_golanci_lint(ctx) + + +def install_custom_golanci_lint(ctx): + res = ctx.run("golangci-lint custom -v") + if res.ok: + gopath = os.getenv('GOPATH') + gobin = os.getenv('GOBIN') + + golintci_binary = bin_name('golangci-lint') + golintci_lint_backup_binary = bin_name('golangci-lint-backup') + + if gopath is None and gobin is None: + print("Not able to install custom golangci-lint binary. golangci-lint won't work as expected") + raise Exit(code=1) + + if gobin is not None and gopath is None: + shutil.move(os.path.join(gobin, golintci_binary), os.path.join(gobin, golintci_lint_backup_binary)) + shutil.move(golintci_binary, os.path.join(gobin, golintci_binary)) + + if gopath is not None: + shutil.move( + os.path.join(gopath, "bin", golintci_binary), os.path.join(gopath, "bin", golintci_lint_backup_binary) + ) + shutil.move(golintci_binary, os.path.join(gopath, "bin", golintci_binary)) + + print("Installed custom golangci-lint binary successfully") @task @@ -102,14 +131,14 @@ def install_protoc(ctx, version="26.1"): artifact_url = f"https://github.com/protocolbuffers/protobuf/releases/download/v{version}/protoc-{version}-{platform_os}-{platform_arch}.zip" zip_path = "/tmp" zip_name = "protoc" - zip_file = ospath.join(zip_path, f"{zip_name}.zip") + zip_file = os.path.join(zip_path, f"{zip_name}.zip") gh = GithubAPI(public_repo=True) # the download_from_url expect to have the path and the name of the file separated and without the extension gh.download_from_url(artifact_url, zip_path, zip_name) # Unzip it in the target destination - destination = ospath.join(Path.home(), ".local") + destination = os.path.join(Path.home(), ".local") with zipfile.ZipFile(zip_file, "r") as zip_ref: zip_ref.extract('bin/protoc', path=destination) ctx.run(f"chmod +x {destination}/bin/protoc") diff --git a/tasks/libs/common/check_tools_version.py b/tasks/libs/common/check_tools_version.py index 6ec8b7959d22f..915e484182ac9 100644 --- a/tasks/libs/common/check_tools_version.py +++ b/tasks/libs/common/check_tools_version.py @@ -38,6 +38,13 @@ def expected_golangci_lint_repo_v(ctx: Context) -> str: return "" +def custom_golangci_v(v: str) -> str: + """ + Returns the golangci-lint version with the custom suffix. + """ + return f"{v}-custom-gcl" + + def current_golangci_lint_v(ctx: Context) -> str: """ Returns the current user golangci-lint version by running golangci-lint --version @@ -53,7 +60,10 @@ def check_tools_version(ctx: Context, tools_list: list[str]) -> bool: is_expected_versions = True tools_versions = { 'go': {'current_v': current_go_v(ctx), 'expected_v': expected_go_repo_v()}, - 'golangci-lint': {'current_v': current_golangci_lint_v(ctx), 'expected_v': expected_golangci_lint_repo_v(ctx)}, + 'golangci-lint': { + 'current_v': current_golangci_lint_v(ctx), + 'expected_v': custom_golangci_v(expected_golangci_lint_repo_v(ctx)), + }, } for tool in tools_list: if tool not in tools_versions: diff --git a/tasks/linter.py b/tasks/linter.py index 432235a47ac3c..6706611d6bb54 100644 --- a/tasks/linter.py +++ b/tasks/linter.py @@ -137,6 +137,7 @@ def go( headless_mode=False, include_sds=False, only_modified_packages=False, + verbose=False, run_on=None, # noqa: U100, F841. Used by the run_on_devcontainer decorator ): """ @@ -156,7 +157,16 @@ def go( inv linter.go --targets=./pkg/collector/check,./pkg/aggregator inv linter.go --module=. """ - if not check_tools_version(ctx, ['go', 'golangci-lint']): + if not check_tools_version(ctx, ['golangci-lint']): + print( + color_message( + "Error: The golanci-lint version you are using is not the correct one. Please run inv -e install-tools to install the correct version.", + "red", + ) + ) + raise Exit(code=1) + + if not check_tools_version(ctx, ['go']): print("Warning: If you have linter errors it might be due to version mismatches.", file=sys.stderr) modules, flavor = process_input_args( @@ -184,6 +194,7 @@ def go( golangci_lint_kwargs=golangci_lint_kwargs, headless_mode=headless_mode, include_sds=include_sds, + verbose=verbose, ) if not headless_mode: @@ -217,6 +228,7 @@ def run_lint_go( golangci_lint_kwargs="", headless_mode=False, include_sds=False, + verbose=False, ): linter_tags = build_tags or compute_build_tags_for_flavor( flavor=flavor, @@ -236,6 +248,7 @@ def run_lint_go( timeout=timeout, golangci_lint_kwargs=golangci_lint_kwargs, headless_mode=headless_mode, + verbose=verbose, ) return lint_results, execution_times @@ -251,6 +264,7 @@ def lint_flavor( timeout=None, golangci_lint_kwargs: str = "", headless_mode: bool = False, + verbose: bool = False, ): """ Runs linters for given flavor, build tags, and modules. @@ -272,6 +286,7 @@ def command(module_results, module: GoModule, module_result): timeout=timeout, golangci_lint_kwargs=golangci_lint_kwargs, headless_mode=headless_mode, + verbose=verbose, ) execution_times.extend(time_results) for lint_result in lint_results: diff --git a/tasks/modules.py b/tasks/modules.py index 50ce7f5669078..38acef956278e 100644 --- a/tasks/modules.py +++ b/tasks/modules.py @@ -215,6 +215,7 @@ def dependency_path(self, agent_version): "pkg/config/utils": GoModule("pkg/config/utils", independent=True, used_by_otel=True), "pkg/errors": GoModule("pkg/errors", independent=True), "pkg/gohai": GoModule("pkg/gohai", independent=True, importable=False), + "pkg/linters/components/pkgconfigusage": GoModule("pkg/linters/components/pkgconfigusage", should_tag=False), "pkg/logs/auditor": GoModule("pkg/logs/auditor", independent=True, used_by_otel=True), "pkg/logs/client": GoModule("pkg/logs/client", independent=True, used_by_otel=True), "pkg/logs/diagnostic": GoModule("pkg/logs/diagnostic", independent=True, used_by_otel=True), diff --git a/test/new-e2e/tests/agent-platform/common/agent_behaviour.go b/test/new-e2e/tests/agent-platform/common/agent_behaviour.go index 8b220783821f6..e747cc7bf96c6 100644 --- a/test/new-e2e/tests/agent-platform/common/agent_behaviour.go +++ b/test/new-e2e/tests/agent-platform/common/agent_behaviour.go @@ -256,7 +256,7 @@ func CheckApmEnabled(t *testing.T, client *TestClient) { if err != nil && client.Host.OSFamily == componentos.LinuxFamily { err = fmt.Errorf("%w\n%s", err, ReadJournalCtl(t, client, "trace-agent\\|datadog-agent-trace")) } - t.Fatalf(err.Error()) + t.Fatalf("%s", err.Error()) } require.EqualValues(t, "127.0.0.1", boundPort.LocalAddress(), "trace-agent should only be listening locally") diff --git a/test/new-e2e/tests/npm/test_helpers.go b/test/new-e2e/tests/npm/test_helpers.go index ca501b48258dc..2e3218e28142f 100644 --- a/test/new-e2e/tests/npm/test_helpers.go +++ b/test/new-e2e/tests/npm/test_helpers.go @@ -21,7 +21,7 @@ var helperCurrentConnection *agentmodel.Connection func helperCleanup(t *testing.T) { t.Cleanup(func() { if t.Failed() { - t.Logf(krpretty.Sprintf("test failed on host %s at connection %# v", helperCurrentHostname, helperCurrentConnection)) + t.Log(krpretty.Sprintf("test failed on host %s at connection %# v", helperCurrentHostname, helperCurrentConnection)) } }) } From c954e579fa5207bd885710583f113690af7b8974 Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Mon, 19 Aug 2024 12:07:23 +0200 Subject: [PATCH 002/245] Update Go version in public doc (#28444) --- docs/public/setup.md | 2 +- tasks/update_go.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/public/setup.md b/docs/public/setup.md index 957c2ef7c14a0..ea59f66f1c084 100644 --- a/docs/public/setup.md +++ b/docs/public/setup.md @@ -101,7 +101,7 @@ This procedure ensures you not only get the correct version of `invoke`, but als ### Golang -You must [install Golang](https://golang.org/doc/install) version `1.21.7` or higher. Make sure that `$GOPATH/bin` is in your `$PATH` otherwise `invoke` cannot use any additional tool it might need. +You must [install Golang](https://golang.org/doc/install) version `1.22.6` or higher. Make sure that `$GOPATH/bin` is in your `$PATH` otherwise `invoke` cannot use any additional tool it might need. !!! note Versions of Golang that aren't an exact match to the version specified in our build images (see e.g. [here](https://github.com/DataDog/datadog-agent-buildimages/blob/c025473ee467ee6d884d532e4c12c7d982ce8fe1/circleci/Dockerfile#L43)) may not be able to build the agent and/or the [rtloader](https://github.com/DataDog/datadog-agent/tree/main/rtloader) binary properly. diff --git a/tasks/update_go.py b/tasks/update_go.py index 2ceafa4255d04..47b16a1e8241f 100644 --- a/tasks/update_go.py +++ b/tasks/update_go.py @@ -30,6 +30,7 @@ ("./test/fakeintake/docs/README.md", "[Golang ", "]", False), ("./cmd/process-agent/README.md", "`go >= ", "`", False), ("./pkg/logs/launchers/windowsevent/README.md", "install go ", "+,", False), + ("./docs/public/setup.md", "version `", "` or higher", True), ] PATTERN_MAJOR_MINOR = r'1\.\d+' From 1378abfba7ade7bc992a957fb45c783cfcc0e3e6 Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Mon, 19 Aug 2024 13:13:21 +0200 Subject: [PATCH 003/245] Remove $ from shell commands in docs to allow copy pasting (#28446) --- docs/dev/agent_dev_env.md | 10 ++++------ docs/public/setup.md | 8 ++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/dev/agent_dev_env.md b/docs/dev/agent_dev_env.md index 0929f27d19be0..a4b41fc748379 100644 --- a/docs/dev/agent_dev_env.md +++ b/docs/dev/agent_dev_env.md @@ -52,10 +52,10 @@ To install `deva`, you'll need to: The Python environment will automatically be created on the first run. and will be reused for subsequent runs. For example: ```shell -$ cd datadog-agent -$ curl -L -o deva https://github.com/DataDog/datadog-agent-devtools/releases/download/deva-v1.0.0/deva-aarch64-unknown-linux-gnu-1.0.0 -$ chmod +x deva -$ ./deva linter.go +cd datadog-agent +curl -L -o deva https://github.com/DataDog/datadog-agent-devtools/releases/download/deva-v1.0.0/deva-aarch64-unknown-linux-gnu-1.0.0 +chmod +x deva +./deva linter.go ``` Below a live demo of how the tool works: @@ -287,5 +287,3 @@ To configure the vscode editor to use a container as remote development environm [Microsoft Visual Studio Code](https://code.visualstudio.com/download) is recommended as it's lightweight and versatile. Building on Windows requires multiple 3rd-party software to be installed. To avoid the complexity, Datadog recommends to make the code change in VS Code, and then do the build in Docker image. For complete information, see [Build the Agent packages](https://github.com/DataDog/datadog-agent/blob/main/docs/dev/agent_omnibus.md) - - diff --git a/docs/public/setup.md b/docs/public/setup.md index ea59f66f1c084..5f3b8d52f362c 100644 --- a/docs/public/setup.md +++ b/docs/public/setup.md @@ -48,10 +48,10 @@ To install `deva`, you'll need to: The Python environment will automatically be created on the first run. and will be reused for subsequent runs. For example: ```shell -$ cd datadog-agent -$ curl -L -o deva https://github.com/DataDog/datadog-agent-devtools/releases/download/deva-v1.0.0/deva-aarch64-unknown-linux-gnu-1.0.0 -$ chmod +x deva -$ ./deva linter.go +cd datadog-agent +curl -L -o deva https://github.com/DataDog/datadog-agent-devtools/releases/download/deva-v1.0.0/deva-aarch64-unknown-linux-gnu-1.0.0 +chmod +x deva +./deva linter.go ``` Below a live demo of how the tool works: From 786e0e4a07cf830caa307dab2aae5a7f8798fd5c Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Mon, 19 Aug 2024 13:13:25 +0200 Subject: [PATCH 004/245] Make `image-tag` arg of update-go invoke task optional (#28469) --- tasks/update_go.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/tasks/update_go.py b/tasks/update_go.py index 47b16a1e8241f..89a25453504fb 100644 --- a/tasks/update_go.py +++ b/tasks/update_go.py @@ -56,7 +56,7 @@ def go_version(_): def update_go( ctx: Context, version: str, - image_tag: str, + image_tag: str | None = None, test_version: bool = True, warn: bool = False, release_note: bool = True, @@ -78,21 +78,22 @@ def update_go( if is_minor_update: print(color_message("WARNING: this is a change of minor version\n", "orange")) - try: - update_gitlab_config(".gitlab-ci.yml", image_tag, test_version=test_version) - except RuntimeError as e: - if warn: - print(color_message(f"WARNING: {str(e)}", "orange")) - else: - raise - - try: - update_circleci_config(".circleci/config.yml", image_tag, test_version=test_version) - except RuntimeError as e: - if warn: - print(color_message(f"WARNING: {str(e)}", "orange")) - else: - raise + if image_tag: + try: + update_gitlab_config(".gitlab-ci.yml", image_tag, test_version=test_version) + except RuntimeError as e: + if warn: + print(color_message(f"WARNING: {str(e)}", "orange")) + else: + raise + + try: + update_circleci_config(".circleci/config.yml", image_tag, test_version=test_version) + except RuntimeError as e: + if warn: + print(color_message(f"WARNING: {str(e)}", "orange")) + else: + raise _update_references(warn, version) _update_go_mods(warn, version, include_otel_modules) From c6fd3a1ea5e4bae0c8e1e02309bb1488b68fba4a Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Mon, 19 Aug 2024 13:23:51 +0200 Subject: [PATCH 005/245] [ASCII-2145] Use relative path in mockgen source comment in `pkg/proto` (#28393) --- pkg/proto/pbgo/mocks/core/api_mockgen.pb.go | 2 +- tasks/go.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/proto/pbgo/mocks/core/api_mockgen.pb.go b/pkg/proto/pbgo/mocks/core/api_mockgen.pb.go index 68d71091db1f5..b3ecb4fc4529a 100644 --- a/pkg/proto/pbgo/mocks/core/api_mockgen.pb.go +++ b/pkg/proto/pbgo/mocks/core/api_mockgen.pb.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: /workspaces/datadog-agent/pkg/proto/pbgo/core/api.pb.go +// Source: pkg/proto/pbgo/core/api.pb.go // Package mock_core is a generated GoMock package. package mock_core diff --git a/tasks/go.py b/tasks/go.py index e1821f5f22037..a65b85bb6b7b0 100644 --- a/tasks/go.py +++ b/tasks/go.py @@ -312,13 +312,14 @@ def generate_protobuf(ctx): # mockgen pbgo_dir = os.path.join(proto_root, "pbgo") mockgen_out = os.path.join(proto_root, "pbgo", "mocks") + pbgo_rel = os.path.relpath(pbgo_dir, repo_root) try: os.mkdir(mockgen_out) except FileExistsError: print(f"{mockgen_out} folder already exists") # TODO: this should be parametrized - ctx.run(f"mockgen -source={pbgo_dir}/core/api.pb.go -destination={mockgen_out}/core/api_mockgen.pb.go") + ctx.run(f"mockgen -source={pbgo_rel}/core/api.pb.go -destination={mockgen_out}/core/api_mockgen.pb.go") # generate messagepack marshallers for pkg, files in msgp_targets.items(): From 3ce2a1734db758fbd7b62e49ac44be0ddcef0db0 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Mon, 19 Aug 2024 11:23:56 +0000 Subject: [PATCH 006/245] Do not run the `notify_ebpf_complexity_changes` job on deploy pipelines nor on main (#28505) --- .gitlab-ci.yml | 1 - .gitlab/JOBOWNERS | 1 + .gitlab/kernel_matrix_testing/common.yml | 5 +++-- .gitlab/kitchen_testing/new-e2e_testing/windows.yml | 1 - tasks/unit_tests/testdata/fake_gitlab-ci.yml | 1 - 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f62f2756f3c43..c6328a6722166 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -590,7 +590,6 @@ workflow: .except_deploy: - <<: *if_deploy when: never - - when: on_success .except_no_tests_no_deploy: - if: $DEPLOY_AGENT == "false" && $DDR_WORKFLOW_ID == null && $RUN_E2E_TESTS == "off" diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS index 8a7d447774c5c..230727b1baa6d 100644 --- a/.gitlab/JOBOWNERS +++ b/.gitlab/JOBOWNERS @@ -158,6 +158,7 @@ upload_minimized_btfs* @DataDog/ebpf-platform kmt_* @DataDog/ebpf-platform upload_secagent_tests* @DataDog/ebpf-platform upload_sysprobe_tests* @DataDog/ebpf-platform +notify_ebpf_complexity_changes @DataDog/ebpf-platform pull_test_dockers* @DataDog/universal-service-monitoring # Single machine performance diff --git a/.gitlab/kernel_matrix_testing/common.yml b/.gitlab/kernel_matrix_testing/common.yml index 5dad208ec8007..88beacf1ac8cf 100644 --- a/.gitlab/kernel_matrix_testing/common.yml +++ b/.gitlab/kernel_matrix_testing/common.yml @@ -274,11 +274,12 @@ notify_ebpf_complexity_changes: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES tags: ["arch:amd64"] rules: - - !reference [.on_system_probe_or_e2e_changes_or_manual] - - !reference [.on_security_agent_changes_or_manual] - !reference [.except_mergequeue] - !reference [.except_main_or_release_branch] - !reference [.except_no_tests_no_deploy] + - !reference [.except_deploy] + - !reference [.on_system_probe_or_e2e_changes_or_manual] + - !reference [.on_security_agent_changes_or_manual] needs: # We need to specify the jobs that generate complexity explicitly, else we hit the limit of "needs" # Important: the list of platforms should match the one in .define_if_collect_complexity diff --git a/.gitlab/kitchen_testing/new-e2e_testing/windows.yml b/.gitlab/kitchen_testing/new-e2e_testing/windows.yml index 2b0dcbc1b6a58..f3c152e119d6c 100644 --- a/.gitlab/kitchen_testing/new-e2e_testing/windows.yml +++ b/.gitlab/kitchen_testing/new-e2e_testing/windows.yml @@ -165,7 +165,6 @@ new-e2e-windows-agent-msi-upgrade-windows-server-a7-x86_64: - !reference [.except_main_or_release_branch] - !reference [.except_windows_installer_changes] - !reference [.on_default_new_e2e_tests] - # must be last since it ends with when: on_success - !reference [.except_deploy] variables: E2E_MSI_TEST: TestUpgrade diff --git a/tasks/unit_tests/testdata/fake_gitlab-ci.yml b/tasks/unit_tests/testdata/fake_gitlab-ci.yml index 5bac67ba32e86..0ded780c29fcd 100644 --- a/tasks/unit_tests/testdata/fake_gitlab-ci.yml +++ b/tasks/unit_tests/testdata/fake_gitlab-ci.yml @@ -720,7 +720,6 @@ workflow: .except_deploy: - <<: *if_deploy when: never - - when: on_success .on_a6_except_deploy: - <<: *if_not_version_6 From 587140403965361dfebe2b56844931c6d65de7d4 Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Mon, 19 Aug 2024 13:24:00 +0200 Subject: [PATCH 007/245] discovery: Use map for environment variables (#28401) --- .../corechecks/servicediscovery/apm/detect.go | 81 ++++++++----------- .../servicediscovery/apm/detect_nix_test.go | 12 +-- .../servicediscovery/language/language.go | 18 ++--- .../language/language_nix_test.go | 28 +++---- .../language/language_processors_test.go | 12 +-- .../servicediscovery/module/impl_linux.go | 18 ++++- .../module/impl_linux_test.go | 2 + .../servicediscovery/service_detector.go | 2 +- .../servicediscovery/service_detector_test.go | 2 +- .../servicediscovery/servicediscovery.go | 2 +- .../servicediscovery/usm/jee_test.go | 4 +- .../servicediscovery/usm/service.go | 58 ++++++------- .../servicediscovery/usm/service_test.go | 28 +++---- .../corechecks/servicediscovery/usm/spring.go | 4 +- .../servicediscovery/usm/spring_test.go | 6 +- 15 files changed, 134 insertions(+), 143 deletions(-) diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect.go b/pkg/collector/corechecks/servicediscovery/apm/detect.go index d367c9fa9e0d1..957759933a316 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect.go @@ -34,7 +34,7 @@ const ( Injected Instrumentation = "injected" ) -type detector func(logger *zap.Logger, args []string, envs []string) Instrumentation +type detector func(logger *zap.Logger, args []string, envs map[string]string) Instrumentation var ( detectorMap = map[language.Language]detector{ @@ -47,7 +47,7 @@ var ( ) // Detect attempts to detect the type of APM instrumentation for the given service. -func Detect(logger *zap.Logger, args []string, envs []string, lang language.Language) Instrumentation { +func Detect(logger *zap.Logger, args []string, envs map[string]string, lang language.Language) Instrumentation { // first check to see if the DD_INJECTION_ENABLED is set to tracer if isInjected(envs) { return Injected @@ -61,12 +61,8 @@ func Detect(logger *zap.Logger, args []string, envs []string, lang language.Lang return None } -func isInjected(envs []string) bool { - for _, env := range envs { - if !strings.HasPrefix(env, "DD_INJECTION_ENABLED=") { - continue - } - _, val, _ := strings.Cut(env, "=") +func isInjected(envs map[string]string) bool { + if val, ok := envs["DD_INJECTION_ENABLED"]; ok { parts := strings.Split(val, ",") for _, v := range parts { if v == "tracer" { @@ -77,11 +73,11 @@ func isInjected(envs []string) bool { return false } -func rubyDetector(_ *zap.Logger, _ []string, _ []string) Instrumentation { +func rubyDetector(_ *zap.Logger, _ []string, _ map[string]string) Instrumentation { return None } -func pythonDetector(logger *zap.Logger, args []string, envs []string) Instrumentation { +func pythonDetector(logger *zap.Logger, args []string, envs map[string]string) Instrumentation { /* Check for VIRTUAL_ENV env var if it's there, use $VIRTUAL_ENV/lib/python{}/site-packages/ and see if ddtrace is inside @@ -95,28 +91,21 @@ func pythonDetector(logger *zap.Logger, args []string, envs []string) Instrument if so, return PROVIDED return NONE */ - for _, env := range envs { - if strings.HasPrefix(env, "VIRTUAL_ENV=") { - _, path, _ := strings.Cut(env, "=") - venv := os.DirFS(path) - libContents, err := fs.ReadDir(venv, "lib") - if err != nil { - continue - } + if path, ok := envs["VIRTUAL_ENV"]; ok { + venv := os.DirFS(path) + libContents, err := fs.ReadDir(venv, "lib") + if err == nil { for _, v := range libContents { if strings.HasPrefix(v.Name(), "python") && v.IsDir() { tracedir, err := fs.Stat(venv, "lib/"+v.Name()+"/site-packages/ddtrace") - if err != nil { - continue - } - if tracedir.IsDir() { + if err == nil && tracedir.IsDir() { return Provided } } } - // the virtual env didn't have ddtrace, can exit - return None } + // the virtual env didn't have ddtrace, can exit + return None } // slow option... results, err := exec.Command(args[0], `-c`, `"import sys; print(':'.join(sys.path))"`).Output() @@ -139,15 +128,12 @@ func pythonDetector(logger *zap.Logger, args []string, envs []string) Instrument return None } -func nodeDetector(logger *zap.Logger, _ []string, envs []string) Instrumentation { +func nodeDetector(logger *zap.Logger, _ []string, envs map[string]string) Instrumentation { // check package.json, see if it has dd-trace in it. // first find it wd := "" - for _, v := range envs { - if strings.HasPrefix(v, "PWD=") { - _, wd, _ = strings.Cut(v, "=") - break - } + if val, ok := envs["PWD"]; ok { + wd = val } if wd == "" { // don't know the working directory, just quit @@ -187,7 +173,7 @@ func nodeDetector(logger *zap.Logger, _ []string, envs []string) Instrumentation return None } -func javaDetector(_ *zap.Logger, args []string, envs []string) Instrumentation { +func javaDetector(_ *zap.Logger, args []string, envs map[string]string) Instrumentation { ignoreArgs := map[string]bool{ "-version": true, "-Xshare:dump": true, @@ -205,20 +191,19 @@ func javaDetector(_ *zap.Logger, args []string, envs []string) Instrumentation { } } // also don't instrument if the javaagent is there in the environment variable JAVA_TOOL_OPTIONS and friends - toolOptionEnvs := map[string]bool{ + toolOptionEnvs := []string{ // These are the environment variables that are used to pass options to the JVM - "JAVA_TOOL_OPTIONS": true, - "_JAVA_OPTIONS": true, - "JDK_JAVA_OPTIONS": true, + "JAVA_TOOL_OPTIONS", + "_JAVA_OPTIONS", + "JDK_JAVA_OPTIONS", // I'm pretty sure these won't be necessary, as they should be parsed before the JVM sees them // but there's no harm in including them - "JAVA_OPTIONS": true, - "CATALINA_OPTS": true, - "JDPA_OPTS": true, + "JAVA_OPTIONS", + "CATALINA_OPTS", + "JDPA_OPTS", } - for _, v := range envs { - name, val, _ := strings.Cut(v, "=") - if toolOptionEnvs[name] { + for _, name := range toolOptionEnvs { + if val, ok := envs[name]; ok { if strings.Contains(val, "-javaagent:") && strings.Contains(val, "dd-java-agent.jar") { return Provided } @@ -237,7 +222,7 @@ func findFile(fileName string) (io.ReadCloser, bool) { const datadogDotNetInstrumented = "Datadog.Trace.ClrProfiler.Native" -func dotNetDetector(_ *zap.Logger, args []string, envs []string) Instrumentation { +func dotNetDetector(_ *zap.Logger, args []string, envs map[string]string) Instrumentation { // if it's just the word `dotnet` by itself, don't instrument if len(args) == 1 && args[0] == "dotnet" { return None @@ -251,13 +236,11 @@ func dotNetDetector(_ *zap.Logger, args []string, envs []string) Instrumentation */ // don't instrument if the tracer is already installed foundFlags := 0 - for _, v := range envs { - if strings.HasPrefix(v, "CORECLR_PROFILER_PATH") { - foundFlags |= 1 - } - if v == "CORECLR_ENABLE_PROFILING=1" { - foundFlags |= 2 - } + if _, ok := envs["CORECLR_PROFILER_PATH"]; ok { + foundFlags |= 1 + } + if val, ok := envs["CORECLR_ENABLE_PROFILING"]; ok && val == "1" { + foundFlags |= 2 } if foundFlags == 3 { return Provided diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go index 20a43aad5f16c..757ec0593881a 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go @@ -19,19 +19,17 @@ func Test_javaDetector(t *testing.T) { data := []struct { name string args []string - envs []string + envs map[string]string result Instrumentation }{ { name: "not there", args: strings.Split("java -jar Foo.jar Foo", " "), - envs: nil, result: None, }, { name: "version", args: strings.Split("java -version", " "), - envs: nil, result: None, }, } @@ -59,31 +57,29 @@ func Test_pythonDetector(t *testing.T) { data := []struct { name string args []string - envs []string + envs map[string]string result Instrumentation }{ { name: "venv_provided", args: []string{"./echoer.sh", "nope"}, - envs: []string{"VIRTUAL_ENV=" + tmpDir}, + envs: map[string]string{"VIRTUAL_ENV": tmpDir}, result: Provided, }, { name: "venv_none", args: []string{"./testdata/echoer.sh", "nope"}, - envs: []string{"VIRTUAL_ENV=" + tmpDir2}, + envs: map[string]string{"VIRTUAL_ENV": tmpDir2}, result: None, }, { name: "cmd_provided", args: []string{"./testdata/cmd_works.sh"}, - envs: []string{}, result: Provided, }, { name: "cmd_none", args: []string{"./testdata/cmd_fails.sh"}, - envs: []string{}, result: None, }, } diff --git a/pkg/collector/corechecks/servicediscovery/language/language.go b/pkg/collector/corechecks/servicediscovery/language/language.go index 8f785390bc6c6..2a35c1db266c5 100644 --- a/pkg/collector/corechecks/servicediscovery/language/language.go +++ b/pkg/collector/corechecks/servicediscovery/language/language.go @@ -54,7 +54,7 @@ var ( ) // Detect attempts to detect the Language from the provided process information. -func (lf Finder) Detect(args []string, envs []string) (Language, bool) { +func (lf Finder) Detect(args []string, envs map[string]string) (Language, bool) { lang := lf.findLang(ProcessInfo{ Args: args, Envs: envs, @@ -76,7 +76,7 @@ func findFile(fileName string) (io.ReadCloser, bool) { // ProcessInfo holds information about a process. type ProcessInfo struct { Args []string - Envs []string + Envs map[string]string } // FileReader attempts to read the most representative file associated to a process. @@ -89,16 +89,16 @@ func (pi ProcessInfo) FileReader() (io.ReadCloser, bool) { if strings.HasPrefix(fileName, "/") { return findFile(fileName) } - for _, env := range pi.Envs { - if key, val, _ := strings.Cut(env, "="); key == "PATH" { - paths := strings.Split(val, ":") - for _, path := range paths { - if r, found := findFile(path + string(os.PathSeparator) + fileName); found { - return r, true - } + if val, ok := pi.Envs["PATH"]; ok { + paths := strings.Split(val, ":") + for _, path := range paths { + if r, found := findFile(path + string(os.PathSeparator) + fileName); found { + return r, true } } + } + // well, just try it as a relative path, maybe it works return findFile(fileName) } diff --git a/pkg/collector/corechecks/servicediscovery/language/language_nix_test.go b/pkg/collector/corechecks/servicediscovery/language/language_nix_test.go index 96c2b0e1fb1f0..4a28138c84e32 100644 --- a/pkg/collector/corechecks/servicediscovery/language/language_nix_test.go +++ b/pkg/collector/corechecks/servicediscovery/language/language_nix_test.go @@ -90,7 +90,7 @@ func TestFinder_findLang(t *testing.T) { name: "dotnet binary", pi: ProcessInfo{ Args: strings.Split("testdata/dotnet/linuxdotnettest a b c", " "), - Envs: []string{"PATH=/usr/bin"}, + Envs: map[string]string{"PATH": "/usr/bin"}, }, lang: DotNet, }, @@ -98,7 +98,7 @@ func TestFinder_findLang(t *testing.T) { name: "dotnet", pi: ProcessInfo{ Args: strings.Split("dotnet run mydll.dll a b c", " "), - Envs: []string{"PATH=/usr/bin"}, + Envs: map[string]string{"PATH": "/usr/bin"}, }, lang: DotNet, }, @@ -106,7 +106,7 @@ func TestFinder_findLang(t *testing.T) { name: "native", pi: ProcessInfo{ Args: strings.Split("./myproc a b c", " "), - Envs: []string{"PATH=/usr/bin"}, + Envs: map[string]string{"PATH": "/usr/bin"}, }, lang: "", }, @@ -132,43 +132,43 @@ func TestProcessInfoFileReader(t *testing.T) { data := []struct { name string args []string - envs []string + envs map[string]string success bool }{ { name: "full", args: []string{fullPath}, - envs: []string{"PATH=" + tempDir}, + envs: map[string]string{"PATH": tempDir}, success: true, }, { name: "full_missing", args: []string{tempDir + "/" + "not_my_file"}, - envs: []string{"PATH=" + tempDir}, + envs: map[string]string{"PATH": tempDir}, success: false, }, { name: "relative_in_path", args: []string{"my_file"}, - envs: []string{"PATH=" + tempDir}, + envs: map[string]string{"PATH": tempDir}, success: true, }, { name: "relative_in_path_missing", args: []string{"not_my_file"}, - envs: []string{"PATH=" + tempDir}, + envs: map[string]string{"PATH": tempDir}, success: false, }, { name: "relative_not_in_path", args: []string{"testdata/dotnet/linuxdotnettest"}, - envs: []string{"PATH=" + tempDir}, + envs: map[string]string{"PATH": tempDir}, success: true, }, { name: "relative_not_in_path_missing", args: []string{"testdata/dotnet/not_my_file"}, - envs: []string{"PATH=" + tempDir}, + envs: map[string]string{"PATH": tempDir}, success: false, }, } @@ -194,28 +194,28 @@ func TestFinderDetect(t *testing.T) { data := []struct { name string args []string - envs []string + envs map[string]string lang Language ok bool }{ { name: "dotnet binary", args: strings.Split("testdata/dotnet/linuxdotnettest a b c", " "), - envs: []string{"PATH=/usr/bin"}, + envs: map[string]string{"PATH": "/usr/bin"}, lang: DotNet, ok: true, }, { name: "dotnet", args: strings.Split("dotnet run mydll.dll a b c", " "), - envs: []string{"PATH=/usr/bin"}, + envs: map[string]string{"PATH": "/usr/bin"}, lang: DotNet, ok: true, }, { name: "native", args: strings.Split("./myproc a b c", " "), - envs: []string{"PATH=/usr/bin"}, + envs: map[string]string{"PATH": "/usr/bin"}, lang: Unknown, ok: false, }, diff --git a/pkg/collector/corechecks/servicediscovery/language/language_processors_test.go b/pkg/collector/corechecks/servicediscovery/language/language_processors_test.go index 19925b2073079..b58ac35b7e7a9 100644 --- a/pkg/collector/corechecks/servicediscovery/language/language_processors_test.go +++ b/pkg/collector/corechecks/servicediscovery/language/language_processors_test.go @@ -56,7 +56,7 @@ func TestPythonScript(t *testing.T) { name: "works", pi: ProcessInfo{ Args: []string{"testdata/python/yes_script.sh"}, - Envs: []string{"PATH=/usr/bin"}, + Envs: map[string]string{"PATH": "/usr/bin"}, }, want: true, }, @@ -64,7 +64,7 @@ func TestPythonScript(t *testing.T) { name: "fails", pi: ProcessInfo{ Args: []string{"testdata/python/not_a_script.sh"}, - Envs: []string{"PATH=/usr/bin"}, + Envs: map[string]string{"PATH": "/usr/bin"}, }, want: false, }, @@ -93,7 +93,7 @@ func TestRubyScript(t *testing.T) { name: "works", pi: ProcessInfo{ Args: []string{"testdata/ruby/yes_script.sh"}, - Envs: []string{"PATH=/usr/bin"}, + Envs: map[string]string{"PATH": "/usr/bin"}, }, want: true, }, @@ -101,7 +101,7 @@ func TestRubyScript(t *testing.T) { name: "fails", pi: ProcessInfo{ Args: []string{"testdata/ruby/not_a_script.sh"}, - Envs: []string{"PATH=/usr/bin"}, + Envs: map[string]string{"PATH": "/usr/bin"}, }, want: false, }, @@ -130,7 +130,7 @@ func TestDotNetBinary(t *testing.T) { name: "works", pi: ProcessInfo{ Args: []string{"testdata/dotnet/linuxdotnettest"}, - Envs: []string{"PATH=/usr/bin"}, + Envs: map[string]string{"PATH": "/usr/bin"}, }, want: true, }, @@ -138,7 +138,7 @@ func TestDotNetBinary(t *testing.T) { name: "fails", pi: ProcessInfo{ Args: []string{"testdata/dotnet/not_a_script.sh"}, - Envs: []string{"PATH=/usr/bin"}, + Envs: map[string]string{"PATH": "/usr/bin"}, }, want: false, }, diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go index 02570487f41c1..ebfb641727efd 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go @@ -199,6 +199,22 @@ type parsingContext struct { netNsInfo map[uint32]*namespaceInfo } +// envsToMap splits a list of strings containing environment variables of the +// format NAME=VAL to a map. +func envsToMap(envs ...string) map[string]string { + envMap := make(map[string]string, len(envs)) + for _, env := range envs { + name, val, found := strings.Cut(env, "=") + if !found { + continue + } + + envMap[name] = val + } + + return envMap +} + // getServiceName gets the service name for a process using the servicedetector // module. func (s *discovery) getServiceName(proc *process.Process) (string, error) { @@ -212,7 +228,7 @@ func (s *discovery) getServiceName(proc *process.Process) (string, error) { return "", nil } - return s.serviceDetector.GetServiceName(cmdline, env), nil + return s.serviceDetector.GetServiceName(cmdline, envsToMap(env...)), nil } // getService gets information for a single service. diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go index 8b50fb3988507..37601526954e1 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go @@ -278,7 +278,9 @@ func TestServiceName(t *testing.T) { cmd := exec.CommandContext(ctx, "sleep", "1000") cmd.Dir = "/tmp/" + cmd.Env = append(cmd.Env, "OTHER_ENV=test") cmd.Env = append(cmd.Env, "DD_SERVICE=foobar") + cmd.Env = append(cmd.Env, "YET_OTHER_ENV=test") err = cmd.Start() require.NoError(t, err) f.Close() diff --git a/pkg/collector/corechecks/servicediscovery/service_detector.go b/pkg/collector/corechecks/servicediscovery/service_detector.go index 44dd6ca95e7c4..3fbae755726bb 100644 --- a/pkg/collector/corechecks/servicediscovery/service_detector.go +++ b/pkg/collector/corechecks/servicediscovery/service_detector.go @@ -64,7 +64,7 @@ func makeFinalName(meta usm.ServiceMetadata) string { // GetServiceName gets the service name based on the command line arguments and // the list of environment variables. -func (sd *ServiceDetector) GetServiceName(cmdline []string, env []string) string { +func (sd *ServiceDetector) GetServiceName(cmdline []string, env map[string]string) string { meta, _ := usm.ExtractServiceMetadata(sd.logger, cmdline, env) return makeFinalName(meta) } diff --git a/pkg/collector/corechecks/servicediscovery/service_detector_test.go b/pkg/collector/corechecks/servicediscovery/service_detector_test.go index 703c02578dc0b..90186b1baabf8 100644 --- a/pkg/collector/corechecks/servicediscovery/service_detector_test.go +++ b/pkg/collector/corechecks/servicediscovery/service_detector_test.go @@ -18,7 +18,7 @@ func Test_serviceDetector(t *testing.T) { pInfo := processInfo{ PID: 100, CmdLine: []string{"my-service.py"}, - Env: []string{"PATH=testdata/test-bin", "DD_INJECTION_ENABLED=tracer"}, + Env: map[string]string{"PATH": "testdata/test-bin", "DD_INJECTION_ENABLED": "tracer"}, Stat: procStat{}, Ports: []uint16{5432}, } diff --git a/pkg/collector/corechecks/servicediscovery/servicediscovery.go b/pkg/collector/corechecks/servicediscovery/servicediscovery.go index fefec0cae9e1f..9578aa783ea47 100644 --- a/pkg/collector/corechecks/servicediscovery/servicediscovery.go +++ b/pkg/collector/corechecks/servicediscovery/servicediscovery.go @@ -46,7 +46,7 @@ type procStat struct { type processInfo struct { PID int CmdLine []string - Env []string + Env map[string]string Stat procStat Ports []uint16 } diff --git a/pkg/collector/corechecks/servicediscovery/usm/jee_test.go b/pkg/collector/corechecks/servicediscovery/usm/jee_test.go index 7a3e8623b1d27..6f8b02eec8cc2 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/jee_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/jee_test.go @@ -240,7 +240,9 @@ func TestWeblogicExtractServiceNamesForJEEServer(t *testing.T) { wlsHomeSysProp + "/wls", wlsServerMainClass, } - envs := []string{"PWD=wls/domain"} + envs := map[string]string{ + "PWD": "wls/domain", + } extractor := jeeExtractor{ctx: NewDetectionContext(zap.NewNop(), cmd, envs, memfs)} extractedContextRoots := extractor.extractServiceNamesForJEEServer() require.Equal(t, []string{ diff --git a/pkg/collector/corechecks/servicediscovery/usm/service.go b/pkg/collector/corechecks/servicediscovery/usm/service.go index c66d65414118e..8bbaa23543624 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/service.go +++ b/pkg/collector/corechecks/servicediscovery/usm/service.go @@ -76,12 +76,12 @@ func newDotnetDetector(ctx DetectionContext) detector { type DetectionContext struct { logger *zap.Logger args []string - envs []string + envs map[string]string fs fs.SubFS } // NewDetectionContext initializes DetectionContext. -func NewDetectionContext(logger *zap.Logger, args []string, envs []string, fs fs.SubFS) DetectionContext { +func NewDetectionContext(logger *zap.Logger, args []string, envs map[string]string, fs fs.SubFS) DetectionContext { return DetectionContext{ logger: logger, args: args, @@ -91,18 +91,14 @@ func NewDetectionContext(logger *zap.Logger, args []string, envs []string, fs fs } // workingDirFromEnvs returns the current working dir extracted from the PWD env -func workingDirFromEnvs(envs []string) (string, bool) { +func workingDirFromEnvs(envs map[string]string) (string, bool) { return extractEnvVar(envs, "PWD") } -func extractEnvVar(envs []string, name string) (string, bool) { - value := "" - prefix := name + "=" - for _, v := range envs { - if strings.HasPrefix(v, prefix) { - _, value, _ = strings.Cut(v, "=") - break - } +func extractEnvVar(envs map[string]string, name string) (string, bool) { + value, ok := envs[name] + if !ok { + return "", false } return value, len(value) > 0 } @@ -140,17 +136,14 @@ var binsWithContext = map[string]detectorCreatorFn{ "gunicorn": newGunicornDetector, } -func checkForInjectionNaming(envs []string) bool { +func checkForInjectionNaming(envs map[string]string) bool { fromDDService := true -outer: - for _, v := range envs { - if strings.HasPrefix(v, "DD_INJECTION_ENABLED=") { - values := strings.Split(v[len("DD_INJECTION_ENABLED="):], ",") - for _, v := range values { - if v == "service_name" { - fromDDService = false - break outer - } + if env, ok := envs["DD_INJECTION_ENABLED"]; ok { + values := strings.Split(env, ",") + for _, v := range values { + if v == "service_name" { + fromDDService = false + break } } } @@ -158,7 +151,7 @@ outer: } // ExtractServiceMetadata attempts to detect ServiceMetadata from the given process. -func ExtractServiceMetadata(logger *zap.Logger, args []string, envs []string) (ServiceMetadata, bool) { +func ExtractServiceMetadata(logger *zap.Logger, args []string, envs map[string]string) (ServiceMetadata, bool) { dc := DetectionContext{ logger: logger, args: args, @@ -271,20 +264,19 @@ func normalizeExeName(exe string) string { // chooseServiceNameFromEnvs extracts the service name from usual tracer env variables (DD_SERVICE, DD_TAGS). // returns the service name, true if found, otherwise "", false -func chooseServiceNameFromEnvs(envs []string) (string, bool) { - for _, env := range envs { - if strings.HasPrefix(env, "DD_SERVICE=") { - return strings.TrimPrefix(env, "DD_SERVICE="), true - } - if strings.HasPrefix(env, "DD_TAGS=") && strings.Contains(env, "service:") { - parts := strings.Split(strings.TrimPrefix(env, "DD_TAGS="), ",") - for _, p := range parts { - if strings.HasPrefix(p, "service:") { - return strings.TrimPrefix(p, "service:"), true - } +func chooseServiceNameFromEnvs(envs map[string]string) (string, bool) { + if val, ok := envs["DD_SERVICE"]; ok { + return val, true + } + if val, ok := envs["DD_TAGS"]; ok && strings.Contains(val, "service:") { + parts := strings.Split(val, ",") + for _, p := range parts { + if strings.HasPrefix(p, "service:") { + return strings.TrimPrefix(p, "service:"), true } } } + return "", false } diff --git a/pkg/collector/corechecks/servicediscovery/usm/service_test.go b/pkg/collector/corechecks/servicediscovery/usm/service_test.go index 7c56bc9adcd2a..fbe39e01bf22e 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/service_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/service_test.go @@ -31,7 +31,7 @@ func TestExtractServiceMetadata(t *testing.T) { tests := []struct { name string cmdline []string - envs []string + envs map[string]string expectedServiceTag string expectedAdditionalServices []string fromDDService bool @@ -58,7 +58,7 @@ func TestExtractServiceMetadata(t *testing.T) { cmdline: []string{ "./my-server.sh", }, - envs: []string{"DD_SERVICE=my-service"}, + envs: map[string]string{"DD_SERVICE": "my-service"}, expectedServiceTag: "my-service", fromDDService: true, }, @@ -67,7 +67,7 @@ func TestExtractServiceMetadata(t *testing.T) { cmdline: []string{ "./my-server.sh", }, - envs: []string{"DD_TAGS=service:my-service"}, + envs: map[string]string{"DD_TAGS": "service:my-service"}, expectedServiceTag: "my-service", fromDDService: true, }, @@ -91,7 +91,7 @@ func TestExtractServiceMetadata(t *testing.T) { "/opt/python/2.7.11/bin/python2.7", "flask", "run", "--host=0.0.0.0", }, expectedServiceTag: "flask", - envs: []string{"PWD=testdata/python"}, + envs: map[string]string{"PWD": "testdata/python"}, }, { name: "python - flask argument in path", @@ -105,7 +105,7 @@ func TestExtractServiceMetadata(t *testing.T) { cmdline: []string{ "/opt/python/2.7.11/bin/python2.7 flask run --host=0.0.0.0", }, - envs: []string{"PWD=testdata/python"}, + envs: map[string]string{"PWD": "testdata/python"}, expectedServiceTag: "flask", }, { @@ -201,7 +201,7 @@ func TestExtractServiceMetadata(t *testing.T) { "--", "index.js", }, - envs: []string{"PWD=testdata/deep"}, // it's relative but it's ok for testing purposes + envs: map[string]string{"PWD": "testdata/deep"}, // it's relative but it's ok for testing purposes expectedServiceTag: "my-awesome-package", }, { @@ -285,7 +285,7 @@ func TestExtractServiceMetadata(t *testing.T) { "-Dwls.home=/u01/oracle/wlserver/server", "-Dweblogic.home=/u01/oracle/wlserver/server", "weblogic.Server"}, - envs: []string{"PWD=" + weblogicTestAppRoot}, + envs: map[string]string{"PWD": weblogicTestAppRoot}, expectedServiceTag: "Server", expectedAdditionalServices: []string{"my_context", "sample4", "some_context_root"}, }, @@ -397,21 +397,21 @@ func TestExtractServiceMetadata(t *testing.T) { { name: "DD_SERVICE_set_manually", cmdline: []string{"java", "-jar", "Foo.jar"}, - envs: []string{"DD_SERVICE=howdy"}, + envs: map[string]string{"DD_SERVICE": "howdy"}, expectedServiceTag: "howdy", fromDDService: true, }, { name: "DD_SERVICE_set_manually_tags", cmdline: []string{"java", "-jar", "Foo.jar"}, - envs: []string{"DD_TAGS=service:howdy"}, + envs: map[string]string{"DD_TAGS": "service:howdy"}, expectedServiceTag: "howdy", fromDDService: true, }, { name: "DD_SERVICE_set_manually_injection", cmdline: []string{"java", "-jar", "Foo.jar"}, - envs: []string{"DD_SERVICE=howdy", "DD_INJECTION_ENABLED=tracer,service_name"}, + envs: map[string]string{"DD_SERVICE": "howdy", "DD_INJECTION_ENABLED": "tracer,service_name"}, expectedServiceTag: "howdy", fromDDService: false, }, @@ -455,7 +455,7 @@ func TestExtractServiceMetadata(t *testing.T) { "gunicorn", "test:app", }, - envs: []string{"GUNICORN_CMD_ARGS=--bind=127.0.0.1:8080 --workers=3 -n dummy"}, + envs: map[string]string{"GUNICORN_CMD_ARGS": "--bind=127.0.0.1:8080 --workers=3 -n dummy"}, expectedServiceTag: "dummy", }, { @@ -463,7 +463,7 @@ func TestExtractServiceMetadata(t *testing.T) { cmdline: []string{ "gunicorn", }, - envs: []string{"GUNICORN_CMD_ARGS=--bind=127.0.0.1:8080 --workers=3"}, + envs: map[string]string{"GUNICORN_CMD_ARGS": "--bind=127.0.0.1:8080 --workers=3"}, expectedServiceTag: "gunicorn", }, { @@ -480,7 +480,7 @@ func TestExtractServiceMetadata(t *testing.T) { "gunicorn", "my.package", }, - envs: []string{"WSGI_APP="}, + envs: map[string]string{"WSGI_APP": ""}, expectedServiceTag: "my.package", }, { @@ -488,7 +488,7 @@ func TestExtractServiceMetadata(t *testing.T) { cmdline: []string{ "gunicorn", }, - envs: []string{"WSGI_APP=test:app"}, + envs: map[string]string{"WSGI_APP": "test:app"}, expectedServiceTag: "test", }, } diff --git a/pkg/collector/corechecks/servicediscovery/usm/spring.go b/pkg/collector/corechecks/servicediscovery/usm/spring.go index 44940d8d8524c..15db47dfd7ed3 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/spring.go +++ b/pkg/collector/corechecks/servicediscovery/usm/spring.go @@ -82,8 +82,8 @@ func (y *environmentSource) Get(key string) (string, bool) { func (y *environmentSource) GetDefault(key string, defVal string) string { return y.m.GetDefault(strings.Map(normalizeEnv, key), defVal) } -func newEnvironmentSource(envs []string) props.PropertyGetter { - return &environmentSource{m: newArgumentSource(envs, "")} +func newEnvironmentSource(envs map[string]string) props.PropertyGetter { + return &environmentSource{m: &mapSource{m: envs}} } // normalizeEnv converts a rune into a suitable replacement for an environment variable name. diff --git a/pkg/collector/corechecks/servicediscovery/usm/spring_test.go b/pkg/collector/corechecks/servicediscovery/usm/spring_test.go index 2a3b7f3ee5547..48ff856a79e51 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/spring_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/spring_test.go @@ -299,7 +299,7 @@ func TestExtractServiceMetadataSpringBoot(t *testing.T) { name string jarname string cmdline []string - envs []string + envs map[string]string expected string }{ { @@ -342,8 +342,8 @@ func TestExtractServiceMetadataSpringBoot(t *testing.T) { "-jar", spFullPath, }, - envs: []string{ - "SPRING_APPLICATION_NAME=found", + envs: map[string]string{ + "SPRING_APPLICATION_NAME": "found", }, expected: "found", }, From 3489865c5b9faec5a33ce0d345892fe4c9bba482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Mon, 19 Aug 2024 13:34:22 +0200 Subject: [PATCH 008/245] [EBPF] Allow kmt.explain-ci-failure to detect latest pipeline for the branch (#28388) --- tasks/kernel_matrix_testing/requirements.txt | 2 +- tasks/kmt.py | 26 +++++++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tasks/kernel_matrix_testing/requirements.txt b/tasks/kernel_matrix_testing/requirements.txt index 1ac432503a436..8eea83b19b07e 100644 --- a/tasks/kernel_matrix_testing/requirements.txt +++ b/tasks/kernel_matrix_testing/requirements.txt @@ -2,5 +2,5 @@ libvirt-python==9.2.0 termcolor==2.3.0 thefuzz==0.19.0 python-Levenshtein==0.21.1 -tabulate==0.9.0 +tabulate[widechars]==0.9.0 jinja2==3.0.3 diff --git a/tasks/kmt.py b/tasks/kmt.py index 36cb715a7af78..808efb6827e64 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -44,6 +44,7 @@ from tasks.kernel_matrix_testing.vars import KMT_SUPPORTED_ARCHS, KMTPaths from tasks.libs.build.ninja import NinjaWriter from tasks.libs.ciproviders.gitlab_api import get_gitlab_repo +from tasks.libs.common.git import get_current_branch from tasks.libs.common.utils import get_build_flags from tasks.libs.pipeline.tools import loop_status from tasks.libs.releasing.version import VERSION_RE, check_version @@ -1565,12 +1566,25 @@ def validate_platform_info(ctx: Context): @task -def explain_ci_failure(_, pipeline: str): +def explain_ci_failure(ctx: Context, pipeline: str | None = None): """Show a summary of KMT failures in the given pipeline.""" if tabulate is None: raise Exit("tabulate module is not installed, please install it to continue") - info(f"[+] retrieving all CI jobs for pipeline {pipeline}") + gitlab = get_gitlab_repo() + + if pipeline is None: + branch = get_current_branch(ctx) + info(f"[+] searching for the latest pipeline for this branch ({branch})") + pipelines = cast(list[Any], gitlab.pipelines.list(ref=branch, per_page=1)) + if len(pipelines) != 1: + raise Exit(f"[!] Could not find a pipeline for branch {branch}") + pipeline = cast(str, pipelines[0].id) + + pipeline_data = gitlab.pipelines.get(pipeline) + info( + f"[+] retrieving all CI jobs for pipeline {pipeline} ({pipeline_data.web_url}), {pipeline_data.status}, created {pipeline_data.created_at} last updated {pipeline_data.updated_at}" + ) setup_jobs, test_jobs = get_all_jobs_for_pipeline(pipeline) failed_setup_jobs = [j for j in setup_jobs if j.status == "failed"] @@ -1609,8 +1623,8 @@ def explain_ci_failure(_, pipeline: str): failreasons[failed_job.name] = failreason # Check setup-env jobs that failed, they are infra failures for all related test jobs - for job in failed_setup_jobs: - for test_job in job.associated_test_jobs: + for setup_job in failed_setup_jobs: + for test_job in setup_job.associated_test_jobs: failreasons[test_job.name] = infrafail failed_jobs.append(test_job) @@ -1634,8 +1648,8 @@ def groupby_comp_vmset(job: KMTTestRunJob) -> tuple[str, str]: if test_job.component != component or test_job.vmset != vmset: continue - failreason = failreasons.get(job.name, ok) - distros[test_job.distro][job.arch] = failreason + failreason = failreasons.get(test_job.name, ok) + distros[test_job.distro][test_job.arch] = failreason if failreason == testfail: distro_arch_with_test_failures.append((test_job.distro, test_job.arch)) From 3ed31fcec4332fe6c4dced664a3d7a0774145438 Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Mon, 19 Aug 2024 13:35:56 +0200 Subject: [PATCH 009/245] chore(FA): Add info log when merging a fleet policy (#28485) --- pkg/config/model/viper.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/config/model/viper.go b/pkg/config/model/viper.go index a9b86a44bd7cb..92f4b41cd373b 100644 --- a/pkg/config/model/viper.go +++ b/pkg/config/model/viper.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "os" + "path" "reflect" "strconv" "strings" @@ -643,6 +644,7 @@ func (c *safeConfig) MergeFleetPolicy(configPath string) error { for _, key := range c.configSources[SourceFleetPolicies].AllKeys() { c.mergeViperInstances(key) } + log.Infof("Fleet policies configuration %s successfully merged", path.Base(configPath)) return nil } From 55e191905cda423fff8e15319ed13b7ddf491cb9 Mon Sep 17 00:00:00 2001 From: Carlos Date: Mon, 19 Aug 2024 12:45:56 +0100 Subject: [PATCH 010/245] Adding a check to make sure Fake Intake didn't lie to us about logs existing for a service (#28387) --- .../k8s-logs/file_tailing_cca_off_test.go | 15 ++++++--- .../log-agent/k8s-logs/file_tailing_test.go | 32 ++++++++++++------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/k8s-logs/file_tailing_cca_off_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/k8s-logs/file_tailing_cca_off_test.go index b03aa8df166bc..4d399910d9525 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/k8s-logs/file_tailing_cca_off_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/k8s-logs/file_tailing_cca_off_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,7 +33,8 @@ func TestK8sCCAOff(t *testing.T) { } func (v *k8sCCAOffSuite) TestADAnnotations() { - v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + err := v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + require.NoError(v.T(), err, "Could not reset the Fake Intake") var backOffLimit int32 = 4 testLogMessage := "Annotations pod" @@ -63,7 +65,7 @@ func (v *k8sCCAOffSuite) TestADAnnotations() { }, } - _, err := v.Env().KubernetesCluster.Client().BatchV1().Jobs("default").Create(context.TODO(), jobSpcec, metav1.CreateOptions{}) + _, err = v.Env().KubernetesCluster.Client().BatchV1().Jobs("default").Create(context.TODO(), jobSpcec, metav1.CreateOptions{}) assert.NoError(v.T(), err, "Could not start autodiscovery job") v.EventuallyWithT(func(c *assert.CollectT) { @@ -73,13 +75,16 @@ func (v *k8sCCAOffSuite) TestADAnnotations() { if assert.Contains(c, logsServiceNames, "ubuntu", "Ubuntu service not found") { filteredLogs, err := v.Env().FakeIntake.Client().FilterLogs("ubuntu") assert.NoError(c, err, "Error filtering logs") - assert.Equal(c, testLogMessage, filteredLogs[0].Message, "Test log doesn't match") + if assert.NotEmpty(v.T(), filteredLogs, "Fake Intake returned no logs even though log service name exists") { + assert.Equal(c, testLogMessage, filteredLogs[0].Message, "Test log doesn't match") + } } }, 1*time.Minute, 10*time.Second) } func (v *k8sCCAOffSuite) TestCCAOff() { - v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + err := v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + require.NoError(v.T(), err, "Could not reset the Fake Intake") var backOffLimit int32 = 4 testLogMessage := "Test pod" @@ -105,7 +110,7 @@ func (v *k8sCCAOffSuite) TestCCAOff() { }, } - _, err := v.Env().KubernetesCluster.Client().BatchV1().Jobs("default").Create(context.TODO(), jobSpcec, metav1.CreateOptions{}) + _, err = v.Env().KubernetesCluster.Client().BatchV1().Jobs("default").Create(context.TODO(), jobSpcec, metav1.CreateOptions{}) assert.NoError(v.T(), err, "Could not start job") v.EventuallyWithT(func(c *assert.CollectT) { diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/k8s-logs/file_tailing_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/k8s-logs/file_tailing_test.go index 94e47b10a1168..958f2988e0ea9 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/k8s-logs/file_tailing_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/k8s-logs/file_tailing_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,7 +33,8 @@ func TestK8sSuite(t *testing.T) { } func (v *k8sSuite) TestSingleLogAndMetadata() { - v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + err := v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + require.NoError(v.T(), err, "Could not reset the Fake Intake") var backOffLimit int32 = 4 testLogMessage := "Test log message" @@ -58,7 +60,7 @@ func (v *k8sSuite) TestSingleLogAndMetadata() { }, } - _, err := v.Env().KubernetesCluster.Client().BatchV1().Jobs("default").Create(context.TODO(), jobSpcec, metav1.CreateOptions{}) + _, err = v.Env().KubernetesCluster.Client().BatchV1().Jobs("default").Create(context.TODO(), jobSpcec, metav1.CreateOptions{}) assert.NoError(v.T(), err, "Could not properly start job") v.EventuallyWithT(func(c *assert.CollectT) { @@ -68,19 +70,22 @@ func (v *k8sSuite) TestSingleLogAndMetadata() { if assert.Contains(c, logsServiceNames, "ubuntu", "Ubuntu service not found") { filteredLogs, err := v.Env().FakeIntake.Client().FilterLogs("ubuntu") assert.NoError(c, err, "Error filtering logs") - assert.Equal(c, testLogMessage, filteredLogs[0].Message, "Test log doesn't match") - - // Check container metatdata - assert.Equal(c, filteredLogs[0].Service, "ubuntu", "Could not find service") - assert.NotNil(c, filteredLogs[0].HostName, "Hostname not found") - assert.NotNil(c, filteredLogs[0].Tags, "Log tags not found") + if assert.NotEmpty(v.T(), filteredLogs, "Fake Intake returned no logs even though log service name exists") { + assert.Equal(c, testLogMessage, filteredLogs[0].Message, "Test log doesn't match") + + // Check container metatdata + assert.Equal(c, filteredLogs[0].Service, "ubuntu", "Could not find service") + assert.NotNil(c, filteredLogs[0].HostName, "Hostname not found") + assert.NotNil(c, filteredLogs[0].Tags, "Log tags not found") + } } }, 1*time.Minute, 10*time.Second) } func (v *k8sSuite) TestLongLogLine() { - v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + err := v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + require.NoError(v.T(), err, "Could not reset the FakeIntake") var backOffLimit int32 = 4 file, err := os.ReadFile("long_line_log.txt") assert.NoError(v.T(), err, "Could not open long line file.") @@ -117,14 +122,17 @@ func (v *k8sSuite) TestLongLogLine() { if assert.Contains(c, logsServiceNames, "ubuntu", "Ubuntu service not found") { filteredLogs, err := v.Env().FakeIntake.Client().FilterLogs("ubuntu") assert.NoError(c, err, "Error filtering logs") - assert.Equal(c, string(file), fmt.Sprintf("%s%s", filteredLogs[0].Message, "\n"), "Test log doesn't match") + if assert.NotEmpty(v.T(), filteredLogs, "Fake Intake returned no logs even though log service name exists") { + assert.Equal(c, string(file), fmt.Sprintf("%s%s", filteredLogs[0].Message, "\n"), "Test log doesn't match") + } } }, 1*time.Minute, 10*time.Second) } func (v *k8sSuite) TestContainerExclude() { - v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + err := v.Env().FakeIntake.Client().FlushServerAndResetAggregators() + require.NoError(v.T(), err, "Could not reset the Fake Intake") // We're testing exclusion via namespace, so we have to create a new namespace namespaceName := "exclude-namespace" @@ -133,7 +141,7 @@ func (v *k8sSuite) TestContainerExclude() { Name: namespaceName, }, } - _, err := v.Env().KubernetesCluster.Client().CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{}) + _, err = v.Env().KubernetesCluster.Client().CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{}) assert.NoError(v.T(), err, "Could not create namespace") var backOffLimit int32 = 4 From 8d618af4d259be12741ba94c7fa5ec8b48ca6c7d Mon Sep 17 00:00:00 2001 From: Carlos Date: Mon, 19 Aug 2024 12:50:32 +0100 Subject: [PATCH 011/245] Adding @DataDog/agent-integrations to metrics origin files (#28510) --- .github/CODEOWNERS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 98b377a033069..b0a076a22a41e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -299,7 +299,9 @@ /pkg/gohai @DataDog/agent-shared-components /pkg/jmxfetch/ @DataDog/agent-metrics-logs /pkg/metrics/ @DataDog/agent-metrics-logs +/pkg/metrics/metricsource.go @DataDog/agent-metrics-logs @DataDog/agent-integrations /pkg/serializer/ @DataDog/agent-processing-and-routing +/pkg/serializer/internal/metrics/origin_mapping.go @DataDog/agent-processing-and-routing @DataDog/agent-integrations /pkg/serverless/ @DataDog/serverless /pkg/serverless/appsec/ @DataDog/asm-go /pkg/status/ @DataDog/agent-shared-components @@ -399,7 +401,6 @@ /pkg/proto/datadog/workloadmeta @DataDog/container-platform /pkg/remoteconfig/ @DataDog/remote-config /pkg/runtime/ @DataDog/agent-shared-components -/pkg/serializer/ @DataDog/agent-processing-and-routing /pkg/tagset/ @DataDog/agent-shared-components /pkg/util/ @DataDog/agent-shared-components /pkg/util/aggregatingqueue @DataDog/container-integrations @DataDog/container-platform From 2a0f28ae09983c05927c12516d2fb0ec39d059d2 Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Mon, 19 Aug 2024 14:50:36 +0300 Subject: [PATCH 012/245] service discovery: Remove unused package portlist (#28432) --- .../servicediscovery/portlist/port.go | 65 --- .../servicediscovery/portlist/port_test.go | 117 ----- .../servicediscovery/portlist/portlist.go | 93 ---- .../portlist/portlist_linux.go | 450 ------------------ .../portlist/portlist_linux_test.go | 154 ------ .../portlist/portlist_test.go | 39 -- .../servicediscovery/portlist/util.go | 61 --- .../servicediscovery/portlist/util_linux.go | 172 ------- .../servicediscovery/portlist/util_test.go | 138 ------ 9 files changed, 1289 deletions(-) delete mode 100644 pkg/collector/corechecks/servicediscovery/portlist/port.go delete mode 100644 pkg/collector/corechecks/servicediscovery/portlist/port_test.go delete mode 100644 pkg/collector/corechecks/servicediscovery/portlist/portlist.go delete mode 100644 pkg/collector/corechecks/servicediscovery/portlist/portlist_linux.go delete mode 100644 pkg/collector/corechecks/servicediscovery/portlist/portlist_linux_test.go delete mode 100644 pkg/collector/corechecks/servicediscovery/portlist/portlist_test.go delete mode 100644 pkg/collector/corechecks/servicediscovery/portlist/util.go delete mode 100644 pkg/collector/corechecks/servicediscovery/portlist/util_linux.go delete mode 100644 pkg/collector/corechecks/servicediscovery/portlist/util_test.go diff --git a/pkg/collector/corechecks/servicediscovery/portlist/port.go b/pkg/collector/corechecks/servicediscovery/portlist/port.go deleted file mode 100644 index 3567962826afa..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/portlist/port.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2014-present Datadog, Inc. - -package portlist - -import ( - "fmt" - "sort" - "strings" -) - -// Port is a listening port on the machine. -type Port struct { - Proto string // "tcp" or "udp" - Port uint16 // port number - Process string // optional process name, if found (requires suitable permissions) - Pid int // process ID, if known (requires suitable permissions) -} - -func (a *Port) equal(b *Port) bool { - return a.Port == b.Port && - a.Proto == b.Proto && - a.Process == b.Process -} - -func (a *Port) lessThan(b *Port) bool { - if a.Port != b.Port { - return a.Port < b.Port - } - if a.Proto != b.Proto { - return a.Proto < b.Proto - } - return a.Process < b.Process -} - -// List is a list of Ports. -type List []Port - -func (pl List) String() string { - out := make([]string, len(pl)) - for i, v := range pl { - out[i] = fmt.Sprintf("[%s]%s:%d", v.Process, v.Proto, v.Port) - } - return strings.Join(out, ",") -} - -// sortAndDedup sorts ps in place (by Port.lessThan) and then returns -// a subset of it with duplicate (Proto, Port) removed. -func sortAndDedup(ps List) List { - sort.Slice(ps, func(i, j int) bool { - return (&ps[i]).lessThan(&ps[j]) - }) - out := ps[:0] - var last Port - for _, p := range ps { - if last.Proto == p.Proto && last.Port == p.Port { - continue - } - out = append(out, p) - last = p - } - return out -} diff --git a/pkg/collector/corechecks/servicediscovery/portlist/port_test.go b/pkg/collector/corechecks/servicediscovery/portlist/port_test.go deleted file mode 100644 index 6a26c6b1858c5..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/portlist/port_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2014-present Datadog, Inc. - -package portlist - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/google/go-cmp/cmp" -) - -func TestPortEqualLessThan(t *testing.T) { - tests := []struct { - name string - a, b Port - want bool - }{ - { - "Port a < b", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 101, Process: "proc1"}, - true, - }, - { - "Port a > b", - Port{Proto: "tcp", Port: 101, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - false, - }, - { - "Proto a < b", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "udp", Port: 100, Process: "proc1"}, - true, - }, - { - "Proto a < b", - Port{Proto: "udp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - false, - }, - { - "Process a < b", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc2"}, - true, - }, - { - "Process a > b", - Port{Proto: "tcp", Port: 100, Process: "proc2"}, - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - false, - }, - { - "Port evaluated first", - Port{Proto: "udp", Port: 100, Process: "proc2"}, - Port{Proto: "tcp", Port: 101, Process: "proc1"}, - true, - }, - { - "Proto evaluated second", - Port{Proto: "tcp", Port: 100, Process: "proc2"}, - Port{Proto: "udp", Port: 100, Process: "proc1"}, - true, - }, - { - "Process evaluated fourth", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc2"}, - true, - }, - { - "equal", - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - Port{Proto: "tcp", Port: 100, Process: "proc1"}, - false, - }, - } - - for _, tt := range tests { - got := tt.a.lessThan(&tt.b) - assert.Equal(t, got, tt.want) - - lessBack := tt.b.lessThan(&tt.a) - if got && lessBack { - t.Errorf("%s: both a and b report being less than each other", tt.name) - } - - wantEqual := !got && !lessBack - gotEqual := tt.a.equal(&tt.b) - assert.Equal(t, gotEqual, wantEqual) - } -} - -func Test_sortAndDedup(t *testing.T) { - pl := List{ - {Proto: "tcp", Port: 100, Process: "proc1"}, - {Proto: "udp", Port: 100, Process: "proc2"}, - {Proto: "udp", Port: 100, Process: "proc2"}, - {Proto: "tcp", Port: 101, Process: "proc3"}, - } - - want := List{ - {Proto: "tcp", Port: 100, Process: "proc1"}, - {Proto: "udp", Port: 100, Process: "proc2"}, - {Proto: "tcp", Port: 101, Process: "proc3"}, - } - got := sortAndDedup(pl) - - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("sortAndDedup mismatch (-want +got):\n%s", diff) - } -} diff --git a/pkg/collector/corechecks/servicediscovery/portlist/portlist.go b/pkg/collector/corechecks/servicediscovery/portlist/portlist.go deleted file mode 100644 index 75f205f6123c5..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/portlist/portlist.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2014-present Datadog, Inc. - -// Package portlist provides functionality to fetch open ports in the current machine. -package portlist - -import ( - "errors" - "github.com/DataDog/datadog-agent/pkg/util/log" - "path/filepath" - "runtime" -) - -var ( - newOSImpl func(cfg *config) osImpl -) - -// Poller scans the systems for listening ports. -type Poller struct { - // os, if non-nil, is an OS-specific implementation of the portlist getting - // code. When non-nil, it's responsible for getting the complete list of - // cached ports complete with the process name. That is, when set, - // addProcesses is not used. - // A nil values means we don't have code for getting the list on the current - // operating system. - os osImpl -} - -type config struct { - includeLocalhost bool - procMountPoint string -} - -func newDefaultConfig() *config { - return &config{ - includeLocalhost: false, - procMountPoint: "/proc", - } -} - -// NewPoller initializes a new Poller. -func NewPoller(opts ...Option) (*Poller, error) { - if newOSImpl == nil { - return nil, errors.New("poller not implemented on " + runtime.GOOS) - } - cfg := newDefaultConfig() - for _, opt := range opts { - opt(cfg) - } - return &Poller{ - os: newOSImpl(cfg), - }, nil -} - -// osImpl is the OS-specific implementation of getting the open listening ports. -type osImpl interface { - Init() - Close() error - - // OpenPorts returns the list of open ports. The Port struct should be - // populated as completely as possible. - OpenPorts() ([]Port, error) -} - -// OpenPorts returns the list of currently listening ports. -func (p *Poller) OpenPorts() (List, error) { - p.os.Init() - defer func() { - if err := p.os.Close(); err != nil { - log.Warnf("failed to close port poller: %v", err) - } - }() - return p.os.OpenPorts() -} - -// Option is used to configure the Poller. -type Option func(cfg *config) - -// WithIncludeLocalhost allows to include/exclude localhost ports (false by default). -func WithIncludeLocalhost(includeLocalhost bool) Option { - return func(cfg *config) { - cfg.includeLocalhost = includeLocalhost - } -} - -// WithProcMountPoint allows to change the proc filesystem mount point (this is used mainly in tests). -func WithProcMountPoint(mountPoint string) Option { - return func(cfg *config) { - cfg.procMountPoint = filepath.Clean(mountPoint) - } -} diff --git a/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux.go b/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux.go deleted file mode 100644 index dffe0a5f83110..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux.go +++ /dev/null @@ -1,450 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2014-present Datadog, Inc. - -//go:build linux - -package portlist - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path" - "path/filepath" - "runtime" - "strings" - "syscall" - "unsafe" - - "go4.org/mem" - "golang.org/x/sys/unix" - - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -func init() { - newOSImpl = newLinuxImpl -} - -type linuxImpl struct { - procMountPath string - procNetFiles []*os.File // seeked to start & reused between calls - readlinkPathBuf []byte - - known map[string]*portMeta // inode string => metadata - br *bufio.Reader - includeLocalhost bool -} - -type portMeta struct { - port Port - pid int - keep bool - needsProcName bool -} - -func newLinuxImplBase(includeLocalhost bool) *linuxImpl { - return &linuxImpl{ - br: bufio.NewReader(eofReader), - known: map[string]*portMeta{}, - includeLocalhost: includeLocalhost, - procMountPath: "/proc", - } -} - -func newLinuxImpl(cfg *config) osImpl { - li := newLinuxImplBase(cfg.includeLocalhost) - li.procMountPath = cfg.procMountPoint - - for _, name := range []string{ - "/net/tcp", - "/net/tcp6", - "/net/udp", - "/net/udp6", - } { - f, err := os.Open(path.Join(cfg.procMountPoint, name)) - if err != nil { - if os.IsNotExist(err) { - continue - } - log.Warnf("failed to open proc file, ignoring: %v", err) - continue - } - li.procNetFiles = append(li.procNetFiles, f) - } - return li -} - -func (li *linuxImpl) Init() { - for _, name := range []string{ - "/net/tcp", - "/net/tcp6", - "/net/udp", - "/net/udp6", - } { - f, err := os.Open(path.Join(li.procMountPath, name)) - if err != nil { - if os.IsNotExist(err) { - continue - } - log.Warnf("failed to open proc file, ignoring: %v", err) - continue - } - li.procNetFiles = append(li.procNetFiles, f) - } -} - -func (li *linuxImpl) Close() error { - for _, f := range li.procNetFiles { - f.Close() - } - li.procNetFiles = nil - return nil -} - -const ( - v6Localhost = "00000000000000000000000001000000:" - v6Any = "00000000000000000000000000000000:0000" - v4Localhost = "0100007F:" - v4Any = "00000000:0000" -) - -var eofReader = bytes.NewReader(nil) - -func (li *linuxImpl) OpenPorts() ([]Port, error) { - br := li.br - defer br.Reset(eofReader) - - // Start by marking all previous known ports as gone. If this mark - // bit is still false later, we'll remove them. - for _, pm := range li.known { - pm.keep = false - } - - for _, f := range li.procNetFiles { - name := f.Name() - _, err := f.Seek(0, io.SeekStart) - if err != nil { - return nil, err - } - br.Reset(f) - err = li.parseProcNetFile(br, filepath.Base(name)) - if err != nil { - return nil, fmt.Errorf("parsing %q: %w", name, err) - } - } - - // Delete ports that aren't open any longer. - // And see if there are any process names we need to look for. - var needProc map[string]*portMeta - for inode, pm := range li.known { - if !pm.keep { - delete(li.known, inode) - continue - } - if pm.needsProcName { - setOrCreateMap(&needProc, inode, pm) - } - } - err := li.findProcessNames(needProc) - if err != nil { - return nil, err - } - - var ret []Port - for _, pm := range li.known { - ret = append(ret, pm.port) - } - return sortAndDedup(ret), nil -} - -// fileBase is one of "tcp", "tcp6", "udp", "udp6". -func (li *linuxImpl) parseProcNetFile(r *bufio.Reader, fileBase string) error { - proto := strings.TrimSuffix(fileBase, "6") - - // skip header row - _, err := r.ReadSlice('\n') - if err != nil { - return err - } - - fields := make([]mem.RO, 0, 20) // 17 current fields + some future slop - - wantRemote := mem.S(v4Any) - if strings.HasSuffix(fileBase, "6") { - wantRemote = mem.S(v6Any) - } - - // remoteIndex is the index within a line to the remote address field. - // -1 means not yet found. - remoteIndex := -1 - - // Add an upper bound on how many rows we'll attempt to read just - // to make sure this doesn't consume too much of their CPU. - const maxRows = 1e6 - rows := 0 - - // Scratch buffer for making inode strings. - inoBuf := make([]byte, 0, 50) - - for { - line, err := r.ReadSlice('\n') - if err == io.EOF { - break - } - if err != nil { - return err - } - rows++ - if rows >= maxRows { - break - } - if len(line) == 0 { - continue - } - - // On the first row of output, find the index of the 3rd field (index 2), - // the remote address. All the rows are aligned, at least until 4 billion open - // TCP connections, per the Linux get_tcp4_sock's "%4d: " on an int i. - if remoteIndex == -1 { - remoteIndex = fieldIndex(line, 2) - if remoteIndex == -1 { - break - } - } - - if len(line) < remoteIndex || !mem.HasPrefix(mem.B(line).SliceFrom(remoteIndex), wantRemote) { - // Fast path for not being a listener port. - continue - } - - // sl local rem ... inode - fields = mem.AppendFields(fields[:0], mem.B(line)) - local := fields[1] - rem := fields[2] - inode := fields[9] - - if !rem.Equal(wantRemote) { - // not a "listener" port - continue - } - - // If a port is bound to localhost, ignore it. - // TODO: localhost is bigger than 1 IP, we need to ignore - // more things. - if !li.includeLocalhost && (mem.HasPrefix(local, mem.S(v4Localhost)) || mem.HasPrefix(local, mem.S(v6Localhost))) { - continue - } - - // Don't use strings.Split here, because it causes - // allocations significant enough to show up in profiles. - i := mem.IndexByte(local, ':') - if i == -1 { - return fmt.Errorf("%q unexpectedly didn't have a colon", local.StringCopy()) - } - portv, err := mem.ParseUint(local.SliceFrom(i+1), 16, 16) - if err != nil { - return fmt.Errorf("%#v: %s", local.SliceFrom(9).StringCopy(), err) - } - inoBuf = append(inoBuf[:0], "socket:["...) - inoBuf = mem.Append(inoBuf, inode) - inoBuf = append(inoBuf, ']') - - if pm, ok := li.known[string(inoBuf)]; ok { - pm.keep = true - // Rest should be unchanged. - } else { - li.known[string(inoBuf)] = &portMeta{ - needsProcName: true, - keep: true, - port: Port{ - Proto: proto, - Port: uint16(portv), - }, - } - } - } - - return nil -} - -// errDone is an internal sentinel error that we found everything we were looking for. -var errDone = errors.New("done") - -// need is keyed by inode string. -func (li *linuxImpl) findProcessNames(need map[string]*portMeta) error { - if len(need) == 0 { - return nil - } - defer func() { - // Anything we didn't find, give up on and don't try to look for it later. - for _, pm := range need { - pm.needsProcName = false - } - }() - - err := li.foreachPID(func(pid mem.RO) error { - var procBuf [128]byte - fdPath := mem.Append(procBuf[:0], mem.S(li.procMountPath+"/")) - fdPath = mem.Append(fdPath, pid) - fdPath = mem.Append(fdPath, mem.S("/fd")) - - // Android logs a bunch of audit violations in logcat - // if we try to open things we don't have access - // to. So on Android only, ask if we have permission - // rather than just trying it to determine whether we - // have permission. - if runtime.GOOS == "android" && syscall.Access(string(fdPath), unix.R_OK) != nil { - return nil - } - - _ = dirWalkShallow(mem.B(fdPath), func(fd mem.RO, _ fs.DirEntry) error { - targetBuf := make([]byte, 64) // plenty big for "socket:[165614651]" - - linkPath := li.readlinkPathBuf[:0] - linkPath = fmt.Append(linkPath, li.procMountPath+"/") - linkPath = mem.Append(linkPath, pid) - linkPath = append(linkPath, "/fd/"...) - linkPath = mem.Append(linkPath, fd) - linkPath = append(linkPath, 0) // terminating NUL - li.readlinkPathBuf = linkPath // to reuse its buffer next time - n, ok := readlink(linkPath, targetBuf) - if !ok { - // Not a symlink or no permission. - // Skip it. - return nil - } - - pe := need[string(targetBuf[:n])] // m[string([]byte)] avoids alloc - if pe == nil { - return nil - } - bs, err := os.ReadFile(fmt.Sprintf("%s/%s/cmdline", li.procMountPath, pid.StringCopy())) - if err != nil { - // Usually shouldn't happen. One possibility is - // the process has gone away, so let's skip it. - return nil - } - - argv := strings.Split(strings.TrimSuffix(string(bs), "\x00"), "\x00") - if p, err := mem.ParseInt(pid, 10, 0); err == nil { - pe.pid = int(p) - } - pe.port.Process = argvSubject(argv...) - pid64, _ := mem.ParseInt(pid, 10, 0) - pe.port.Pid = int(pid64) - pe.needsProcName = false - delete(need, string(targetBuf[:n])) - if len(need) == 0 { - return errDone - } - return nil - }) - return nil - }) - if errors.Is(err, errDone) { - return nil - } - return err -} - -func (li *linuxImpl) foreachPID(fn func(pidStr mem.RO) error) error { - err := dirWalkShallow(mem.S(li.procMountPath), func(name mem.RO, _ fs.DirEntry) error { - if !isNumeric(name) { - return nil - } - return fn(name) - }) - if os.IsNotExist(err) { - // This can happen if the directory we're - // reading disappears during the run. No big - // deal. - return nil - } - return err -} - -func isNumeric(s mem.RO) bool { - for i, n := 0, s.Len(); i < n; i++ { - b := s.At(i) - if b < '0' || b > '9' { - return false - } - } - return s.Len() > 0 -} - -// fieldIndex returns the offset in line where the Nth field (0-based) begins, or -1 -// if there aren't that many fields. Fields are separated by 1 or more spaces. -func fieldIndex(line []byte, n int) int { - skip := 0 - for i := 0; i <= n; i++ { - // Skip spaces. - for skip < len(line) && line[skip] == ' ' { - skip++ - } - if skip == len(line) { - return -1 - } - if i == n { - break - } - // Skip non-space. - for skip < len(line) && line[skip] != ' ' { - skip++ - } - } - return skip -} - -// path must be null terminated. -func readlink(path, buf []byte) (n int, ok bool) { - if len(buf) == 0 || len(path) < 2 || path[len(path)-1] != 0 { - return 0, false - } - var dirfd int = unix.AT_FDCWD - r0, _, e1 := unix.Syscall6(unix.SYS_READLINKAT, - uintptr(dirfd), - uintptr(unsafe.Pointer(&path[0])), - uintptr(unsafe.Pointer(&buf[0])), - uintptr(len(buf)), - 0, 0) - n = int(r0) - if e1 != 0 { - return 0, false - } - return n, true -} - -// argvSubject takes a command and its flags, and returns the -// short/pretty name for the process. This is usually the basename of -// the binary being executed, but can sometimes vary (e.g. so that we -// don't report all Java programs as "java"). -func argvSubject(argv ...string) string { - if len(argv) == 0 { - return "" - } - ret := filepath.Base(argv[0]) - - // Handle special cases. - switch { - case ret == "mono" && len(argv) >= 2: - // .Net programs execute as `mono actualProgram.exe`. - ret = filepath.Base(argv[1]) - } - - // Handle space separated argv - ret, _, _ = strings.Cut(ret, " ") - - // Remove common noise. - ret = strings.TrimSpace(ret) - ret = strings.TrimSuffix(ret, ".exe") - - return ret -} diff --git a/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux_test.go b/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux_test.go deleted file mode 100644 index 5e1656d45cbcd..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/portlist/portlist_linux_test.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2014-present Datadog, Inc. - -//go:build linux - -package portlist - -import ( - "bufio" - "bytes" - "io" - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestFieldIndex(t *testing.T) { - tests := []struct { - in string - field int - want int - }{ - {"foo", 0, 0}, - {" foo", 0, 2}, - {"foo bar", 1, 5}, - {" foo bar", 1, 6}, - {" foo bar", 2, -1}, - {" foo bar ", 2, -1}, - {" foo bar x", 2, 10}, - {" 1: 00000000:0016 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 34062 1 0000000000000000 100 0 0 10 0", - 2, 19}, - } - for _, tt := range tests { - if got := fieldIndex([]byte(tt.in), tt.field); got != tt.want { - t.Errorf("fieldIndex(%q, %v) = %v; want %v", tt.in, tt.field, got, tt.want) - } - } -} - -func TestParsePorts(t *testing.T) { - tests := []struct { - name string - in string - file string - want map[string]*portMeta - }{ - { - name: "empty", - in: "header line (ignored)\n", - want: map[string]*portMeta{}, - }, - { - name: "ipv4", - file: "tcp", - in: `header line - 0: 0100007F:0277 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 22303 1 0000000000000000 100 0 0 10 0 - 1: 00000000:0016 00000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 34062 1 0000000000000000 100 0 0 10 0 - 2: 5501A8C0:ADD4 B25E9536:01BB 01 00000000:00000000 02:00000B2B 00000000 1000 0 155276677 2 0000000000000000 22 4 30 10 -1 -`, - want: map[string]*portMeta{ - "socket:[34062]": { - port: Port{Proto: "tcp", Port: 22}, - }, - }, - }, - { - name: "ipv6", - file: "tcp6", - in: ` sl local_address remote_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode - 0: 00000000000000000000000001000000:0277 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 35720 1 0000000000000000 100 0 0 10 0 - 1: 00000000000000000000000000000000:1F91 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 142240557 1 0000000000000000 100 0 0 10 0 - 2: 00000000000000000000000000000000:0016 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 34064 1 0000000000000000 100 0 0 10 0 - 3: 69050120005716BC64906EBE009ECD4D:D506 0047062600000000000000006E171268:01BB 01 00000000:00000000 02:0000009E 00000000 1000 0 151042856 2 0000000000000000 21 4 28 10 -1 -`, - want: map[string]*portMeta{ - "socket:[142240557]": { - port: Port{Proto: "tcp", Port: 8081}, - }, - "socket:[34064]": { - port: Port{Proto: "tcp", Port: 22}, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - buf := bytes.NewBufferString(tt.in) - r := bufio.NewReader(buf) - file := "tcp" - if tt.file != "" { - file = tt.file - } - li := newLinuxImplBase(false) - err := li.parseProcNetFile(r, file) - if err != nil { - t.Fatal(err) - } - for _, pm := range tt.want { - pm.keep = true - pm.needsProcName = true - } - if diff := cmp.Diff(li.known, tt.want, cmp.AllowUnexported(Port{}), cmp.AllowUnexported(portMeta{})); diff != "" { - t.Errorf("unexpected parsed ports (-got+want):\n%s", diff) - } - }) - } -} - -func BenchmarkParsePorts(b *testing.B) { - b.ReportAllocs() - - var contents bytes.Buffer - contents.WriteString(` sl local_address remote_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode - 0: 00000000000000000000000001000000:0277 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 35720 1 0000000000000000 100 0 0 10 0 - 1: 00000000000000000000000000000000:1F91 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 1000 0 142240557 1 0000000000000000 100 0 0 10 0 - 2: 00000000000000000000000000000000:0016 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 0 0 34064 1 0000000000000000 100 0 0 10 0 -`) - for i := 0; i < 50000; i++ { - contents.WriteString(" 3: 69050120005716BC64906EBE009ECD4D:D506 0047062600000000000000006E171268:01BB 01 00000000:00000000 02:0000009E 00000000 1000 0 151042856 2 0000000000000000 21 4 28 10 -1\n") - } - - li := newLinuxImplBase(false) - - r := bytes.NewReader(contents.Bytes()) - br := bufio.NewReader(&contents) - b.ResetTimer() - for i := 0; i < b.N; i++ { - r.Seek(0, io.SeekStart) - br.Reset(r) - err := li.parseProcNetFile(br, "tcp6") - if err != nil { - b.Fatal(err) - } - if len(li.known) != 2 { - b.Fatalf("wrong results; want 2 parsed got %d", len(li.known)) - } - } -} - -func BenchmarkFindProcessNames(b *testing.B) { - b.ReportAllocs() - li := &linuxImpl{} - need := map[string]*portMeta{ - "something-we'll-never-find": new(portMeta), - } - for i := 0; i < b.N; i++ { - if err := li.findProcessNames(need); err != nil { - b.Fatal(err) - } - } -} diff --git a/pkg/collector/corechecks/servicediscovery/portlist/portlist_test.go b/pkg/collector/corechecks/servicediscovery/portlist/portlist_test.go deleted file mode 100644 index 9efb7dd111139..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/portlist/portlist_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2014-present Datadog, Inc. - -package portlist - -import ( - "runtime" - "slices" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestPoller(t *testing.T) { - currentOs := runtime.GOOS - implOs := []string{ - "linux", - } - - p, err := NewPoller( - WithIncludeLocalhost(true), - WithProcMountPoint("/proc"), - ) - if !slices.Contains(implOs, currentOs) { - require.ErrorContains(t, err, "poller not implemented") - return - } - require.NoError(t, err) - - pl, err := p.OpenPorts() - require.NoError(t, err) - - for i, p := range pl { - t.Logf("[%d] %+v", i, p) - } - t.Logf("As String: %s", pl) -} diff --git a/pkg/collector/corechecks/servicediscovery/portlist/util.go b/pkg/collector/corechecks/servicediscovery/portlist/util.go deleted file mode 100644 index 68c8c11b08996..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/portlist/util.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2014-present Datadog, Inc. - -package portlist - -import ( - "io" - "io/fs" - "os" - - "go4.org/mem" -) - -func setOrCreateMap[K comparable, V any, T ~map[K]V](m *T, k K, v V) { - if *m == nil { - *m = make(map[K]V) - } - (*m)[k] = v -} - -var osWalkShallow func(name mem.RO, fn walkFunc) error - -// walkFunc is the callback type used with WalkShallow. -// -// The name and de are only valid for the duration of func's call -// and should not be retained. -type walkFunc func(name mem.RO, de fs.DirEntry) error - -// dirWalkShallow reads the entries in the named directory and calls fn for each. -// It does not recurse into subdirectories. -// -// If fn returns an error, iteration stops and WalkShallow returns that value. -// -// On Linux, dirWalkShallow does not allocate, so long as certain methods on the -// walkFunc's DirEntry are not called which necessarily allocate. -func dirWalkShallow(dirName mem.RO, fn walkFunc) error { - if f := osWalkShallow; f != nil { - return f(dirName, fn) - } - of, err := os.Open(dirName.StringCopy()) - if err != nil { - return err - } - defer of.Close() - for { - fis, err := of.ReadDir(100) - for _, de := range fis { - if err := fn(mem.S(de.Name()), de); err != nil { - return err - } - } - if err != nil { - if err == io.EOF { - return nil - } - return err - } - } -} diff --git a/pkg/collector/corechecks/servicediscovery/portlist/util_linux.go b/pkg/collector/corechecks/servicediscovery/portlist/util_linux.go deleted file mode 100644 index 0880542a9a472..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/portlist/util_linux.go +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2014-present Datadog, Inc. - -//go:build linux - -package portlist - -import ( - "fmt" - "io/fs" - "os" - "path/filepath" - "syscall" - "unsafe" - - "go4.org/mem" - "golang.org/x/sys/unix" - - ddsync "github.com/DataDog/datadog-agent/pkg/util/sync" -) - -func init() { - osWalkShallow = linuxWalkShallow -} - -var dirEntPool = ddsync.NewDefaultTypedPool[linuxDirEnt]() - -func linuxWalkShallow(dirName mem.RO, fn walkFunc) error { - const blockSize = 8 << 10 - buf := make([]byte, blockSize) // stack-allocated; doesn't escape - - nameb := mem.Append(buf[:0], dirName) - nameb = append(nameb, 0) - - fd, err := sysOpen(nameb) - if err != nil { - return err - } - defer syscall.Close(fd) - - bufp := 0 // starting read position in buf - nbuf := 0 // end valid data in buf - - de := dirEntPool.Get() - defer de.cleanAndPutInPool() - de.root = dirName - - for { - if bufp >= nbuf { - bufp = 0 - nbuf, err = readDirent(fd, buf) - if err != nil { - return err - } - if nbuf <= 0 { - return nil - } - } - consumed, name := parseDirEnt(&de.d, buf[bufp:nbuf]) - bufp += consumed - if len(name) == 0 || string(name) == "." || string(name) == ".." { - continue - } - de.name = mem.B(name) - if err := fn(de.name, de); err != nil { - return err - } - } -} - -type linuxDirEnt struct { - root mem.RO - d syscall.Dirent - name mem.RO -} - -func (de *linuxDirEnt) cleanAndPutInPool() { - de.root = mem.RO{} - de.name = mem.RO{} - dirEntPool.Put(de) -} - -func (de *linuxDirEnt) Name() string { return de.name.StringCopy() } -func (de *linuxDirEnt) Info() (fs.FileInfo, error) { - return os.Lstat(filepath.Join(de.root.StringCopy(), de.name.StringCopy())) -} -func (de *linuxDirEnt) IsDir() bool { - return de.d.Type == syscall.DT_DIR -} -func (de *linuxDirEnt) Type() fs.FileMode { - switch de.d.Type { - case syscall.DT_BLK: - return fs.ModeDevice // shrug - case syscall.DT_CHR: - return fs.ModeCharDevice - case syscall.DT_DIR: - return fs.ModeDir - case syscall.DT_FIFO: - return fs.ModeNamedPipe - case syscall.DT_LNK: - return fs.ModeSymlink - case syscall.DT_REG: - return 0 - case syscall.DT_SOCK: - return fs.ModeSocket - default: - return fs.ModeIrregular // shrug - } -} - -func direntNamlen(dirent *syscall.Dirent) int { - const fixedHdr = uint16(unsafe.Offsetof(syscall.Dirent{}.Name)) - limit := dirent.Reclen - fixedHdr - const dirNameLen = 256 // sizeof syscall.Dirent.Name - if limit > dirNameLen { - limit = dirNameLen - } - for i := uint16(0); i < limit; i++ { - if dirent.Name[i] == 0 { - return int(i) - } - } - panic("failed to find terminating 0 byte in dirent") -} - -func parseDirEnt(dirent *syscall.Dirent, buf []byte) (consumed int, name []byte) { - // golang.org/issue/37269 - copy(unsafe.Slice((*byte)(unsafe.Pointer(dirent)), unsafe.Sizeof(syscall.Dirent{})), buf) - if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v { - panic(fmt.Sprintf("buf size of %d smaller than dirent header size %d", len(buf), v)) - } - if len(buf) < int(dirent.Reclen) { - panic(fmt.Sprintf("buf size %d < record length %d", len(buf), dirent.Reclen)) - } - consumed = int(dirent.Reclen) - if dirent.Ino == 0 { // File absent in directory. - return - } - name = unsafe.Slice((*byte)(unsafe.Pointer(&dirent.Name[0])), direntNamlen(dirent)) - return -} - -func sysOpen(name []byte) (fd int, err error) { - if len(name) == 0 || name[len(name)-1] != 0 { - return 0, syscall.EINVAL - } - var dirfd int = unix.AT_FDCWD - for { - r0, _, e1 := syscall.Syscall(unix.SYS_OPENAT, uintptr(dirfd), - uintptr(unsafe.Pointer(&name[0])), 0) - if e1 == 0 { - return int(r0), nil - } - if e1 == syscall.EINTR { - // Since https://golang.org/doc/go1.14#runtime we - // need to loop on EINTR on more places. - continue - } - return 0, syscall.Errno(e1) - } -} - -func readDirent(fd int, buf []byte) (n int, err error) { - for { - nbuf, err := syscall.ReadDirent(fd, buf) - if err != syscall.EINTR { - return nbuf, err - } - } -} diff --git a/pkg/collector/corechecks/servicediscovery/portlist/util_test.go b/pkg/collector/corechecks/servicediscovery/portlist/util_test.go deleted file mode 100644 index 0fc07a1c6e20a..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/portlist/util_test.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) Tailscale Inc & AUTHORS -// SPDX-License-Identifier: BSD-3-Clause -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2014-present Datadog, Inc. - -package portlist - -import ( - "fmt" - "os" - "path/filepath" - "reflect" - "runtime" - "sort" - "testing" - - "go4.org/mem" -) - -// replaceImpl replaces the value of target with val. -// The old value is restored when the test ends. -func replaceImpl[T any](t testing.TB, target *T, val T) { - t.Helper() - if target == nil { - t.Fatalf("Replace: nil pointer") - return - } - old := *target - t.Cleanup(func() { - *target = old - }) - - *target = val -} - -func Test_setOrCreateMap(t *testing.T) { - t.Run("unnamed", func(t *testing.T) { - var m map[string]int - setOrCreateMap(&m, "foo", 42) - setOrCreateMap(&m, "bar", 1) - setOrCreateMap(&m, "bar", 2) - want := map[string]int{ - "foo": 42, - "bar": 2, - } - if got := m; !reflect.DeepEqual(got, want) { - t.Errorf("got %v; want %v", got, want) - } - }) - t.Run("named", func(t *testing.T) { - type M map[string]int - var m M - setOrCreateMap(&m, "foo", 1) - setOrCreateMap(&m, "bar", 1) - setOrCreateMap(&m, "bar", 2) - want := M{ - "foo": 1, - "bar": 2, - } - if got := m; !reflect.DeepEqual(got, want) { - t.Errorf("got %v; want %v", got, want) - } - }) -} - -func Test_dirWalkShallowOSSpecific(t *testing.T) { - if osWalkShallow == nil { - t.Skip("no OS-specific implementation") - } - testDirWalkShallow(t, false) -} - -func Test_dirWalkShallowPortable(t *testing.T) { - testDirWalkShallow(t, true) -} - -func testDirWalkShallow(t *testing.T, portable bool) { - if portable { - replaceImpl(t, &osWalkShallow, nil) - } - d := t.TempDir() - - t.Run("basics", func(t *testing.T) { - if err := os.WriteFile(filepath.Join(d, "foo"), []byte("1"), 0600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(d, "bar"), []byte("22"), 0400); err != nil { - t.Fatal(err) - } - if err := os.Mkdir(filepath.Join(d, "baz"), 0777); err != nil { - t.Fatal(err) - } - - var got []string - if err := dirWalkShallow(mem.S(d), func(name mem.RO, de os.DirEntry) error { - var size int64 - if fi, err := de.Info(); err != nil { - t.Errorf("Info stat error on %q: %v", de.Name(), err) - } else if !fi.IsDir() { - size = fi.Size() - } - got = append(got, fmt.Sprintf("%q %q dir=%v type=%d size=%v", name.StringCopy(), de.Name(), de.IsDir(), de.Type(), size)) - return nil - }); err != nil { - t.Fatal(err) - } - sort.Strings(got) - want := []string{ - `"bar" "bar" dir=false type=0 size=2`, - `"baz" "baz" dir=true type=2147483648 size=0`, - `"foo" "foo" dir=false type=0 size=1`, - } - if !reflect.DeepEqual(got, want) { - t.Errorf("mismatch:\n got %#q\nwant %#q", got, want) - } - }) - - t.Run("err_not_exist", func(t *testing.T) { - err := dirWalkShallow(mem.S(filepath.Join(d, "not_exist")), func(_ mem.RO, _ os.DirEntry) error { - return nil - }) - if !os.IsNotExist(err) { - t.Errorf("unexpected error: %v", err) - } - }) - - t.Run("allocs", func(t *testing.T) { - allocs := int(testing.AllocsPerRun(1000, func() { - if err := dirWalkShallow(mem.S(d), func(_ mem.RO, _ os.DirEntry) error { return nil }); err != nil { - t.Fatal(err) - } - })) - t.Logf("allocs = %v", allocs) - if !portable && runtime.GOOS == "linux" && allocs != 0 { - t.Errorf("unexpected allocs: got %v, want 0", allocs) - } - }) -} From 3923572cabe0f924fb81947a993dffa8a328ddd3 Mon Sep 17 00:00:00 2001 From: Alex Lopez Date: Mon, 19 Aug 2024 13:56:19 +0200 Subject: [PATCH 013/245] Add test for install script on docker containers (#28153) Co-authored-by: Fausto Rodrigues --- .../new-e2e_testing/docker.yml | 15 ++++ .../new-e2e_testing/include.yml | 1 + tasks/linter.py | 1 + .../new-e2e/pkg/environments/aws/host/host.go | 10 ++- .../agent-platform/common/test_client.go | 73 ++++++++++++++++--- .../install-script/install_script_test.go | 61 ++++++++++++++++ .../tests/agent-platform/install/install.go | 16 ++-- 7 files changed, 158 insertions(+), 19 deletions(-) create mode 100644 .gitlab/kitchen_testing/new-e2e_testing/docker.yml diff --git a/.gitlab/kitchen_testing/new-e2e_testing/docker.yml b/.gitlab/kitchen_testing/new-e2e_testing/docker.yml new file mode 100644 index 0000000000000..c7ef9c9303847 --- /dev/null +++ b/.gitlab/kitchen_testing/new-e2e_testing/docker.yml @@ -0,0 +1,15 @@ +new-e2e-agent-platform-install-script-docker: + stage: kitchen_testing + extends: + - .new_e2e_template + - .new_e2e_template_needs_deb_x64 + - .new-e2e_agent_a7 + - .new-e2e_install_script + rules: !reference [.on_default_new_e2e_tests] + variables: + E2E_ARCH: x86_64 + E2E_PLATFORM: "docker" + E2E_OSVERS: "none" + E2E_BRANCH_OSVERS: none + E2E_CWS_SUPPORTED_OSVERS: "none" + FLAVOR: "datadog-agent" diff --git a/.gitlab/kitchen_testing/new-e2e_testing/include.yml b/.gitlab/kitchen_testing/new-e2e_testing/include.yml index 30ab8be78521c..78c043739d7ee 100644 --- a/.gitlab/kitchen_testing/new-e2e_testing/include.yml +++ b/.gitlab/kitchen_testing/new-e2e_testing/include.yml @@ -9,3 +9,4 @@ include: - .gitlab/kitchen_testing/new-e2e_testing/centos.yml - .gitlab/kitchen_testing/new-e2e_testing/suse.yml - .gitlab/kitchen_testing/new-e2e_testing/windows.yml + - .gitlab/kitchen_testing/new-e2e_testing/docker.yml diff --git a/tasks/linter.py b/tasks/linter.py index 6706611d6bb54..c82b59ec9d3d2 100644 --- a/tasks/linter.py +++ b/tasks/linter.py @@ -544,6 +544,7 @@ def job_change_path(ctx, job_files=None): 'new-e2e-agent-platform-install-script-ubuntu-heroku-agent-a6-x86_64', 'new-e2e-agent-platform-install-script-ubuntu-heroku-agent-a7-x86_64', 'new-e2e-agent-platform-install-script-ubuntu-iot-agent-a7-x86_64', + 'new-e2e-agent-platform-install-script-docker', 'new-e2e-agent-platform-install-script-upgrade6-amazonlinux-x64', 'new-e2e-agent-platform-install-script-upgrade6-centos-fips-x86_64', 'new-e2e-agent-platform-install-script-upgrade6-centos-x86_64', diff --git a/test/new-e2e/pkg/environments/aws/host/host.go b/test/new-e2e/pkg/environments/aws/host/host.go index 6f1d6c0d607da..7652d9b846576 100644 --- a/test/new-e2e/pkg/environments/aws/host/host.go +++ b/test/new-e2e/pkg/environments/aws/host/host.go @@ -198,7 +198,15 @@ func Run(ctx *pulumi.Context, env *environments.Host, runParams RunParams) error } if params.installDocker { - dockerManager, err := docker.NewManager(&awsEnv, host) + // install the ECR credentials helper + // required to get pipeline agent images or other internally hosted images + installEcrCredsHelperCmd, err := ec2.InstallECRCredentialsHelper(awsEnv, host) + if err != nil { + return err + } + + dockerManager, err := docker.NewManager(&awsEnv, host, utils.PulumiDependsOn(installEcrCredsHelperCmd)) + if err != nil { return err } diff --git a/test/new-e2e/tests/agent-platform/common/test_client.go b/test/new-e2e/tests/agent-platform/common/test_client.go index 3357b6c4d8f4b..bf827544135ca 100644 --- a/test/new-e2e/tests/agent-platform/common/test_client.go +++ b/test/new-e2e/tests/agent-platform/common/test_client.go @@ -177,18 +177,7 @@ func (c *TestClient) GetAgentVersion() (string, error) { // ExecuteWithRetry execute the command with retry func (c *TestClient) ExecuteWithRetry(cmd string) (string, error) { - var err error - var output string - - for try := 0; try < 5; try++ { - output, err = c.Host.Execute(cmd) - if err == nil { - break - } - time.Sleep(time.Duration(math.Pow(2, float64(try))) * time.Second) - } - - return output, err + return execWithRetry(func(cmd string) (string, error) { return c.Host.Execute(cmd) }, cmd) } // NewWindowsTestClient create a TestClient for Windows VM @@ -281,3 +270,63 @@ func ReadJournalCtl(t *testing.T, client *TestClient, grepPattern string) string } return journalCtlOutput } + +// DockerTestClient is a helper to run commands on a docker container for tests +type DockerTestClient struct { + host *components.RemoteHost + containerName string +} + +// NewDockerTestClient creates a client to help write tests that run on a docker container +func NewDockerTestClient(host *components.RemoteHost, containerName string) *DockerTestClient { + return &DockerTestClient{ + host: host, + containerName: containerName, + } +} + +// RunContainer starts the docker container in the background based on the given image reference +func (c *DockerTestClient) RunContainer(image string) error { + // We run an infinite no-op to keep it alive + _, err := c.host.Execute( + fmt.Sprintf("docker run -d -e DD_HOSTNAME=docker-test --name '%s' '%s' tail -f /dev/null", c.containerName, image), + ) + return err +} + +// Cleanup force-removes the docker container associated to the client +func (c *DockerTestClient) Cleanup() error { + _, err := c.host.Execute(fmt.Sprintf("docker rm -f '%s'", c.containerName)) + return err +} + +// Execute runs commands on a Docker remote host +func (c *DockerTestClient) Execute(command string) (output string, err error) { + return c.host.Execute( + // Run command on container via docker exec and wrap with sh -c + // to provide a similar interface to remote host's execute + fmt.Sprintf("docker exec %s sh -c '%s'", c.containerName, command), + ) +} + +// ExecuteWithRetry execute the command with retry +func (c *DockerTestClient) ExecuteWithRetry(cmd string) (output string, err error) { + return execWithRetry(c.Execute, cmd) +} + +func execWithRetry(exec func(string) (string, error), cmd string) (string, error) { + var err error + var output string + maxTries := 5 + + for try := 0; try < maxTries; try++ { + output, err = exec(cmd) + if err == nil { + break + } + fmt.Printf("(attempt %d of %d) error while executing command in host: %v\n", try+1, maxTries, err) + time.Sleep(time.Duration(math.Pow(2, float64(try))) * time.Second) + } + + return output, err +} diff --git a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go index 5f682c1f98e10..150dc1792462c 100644 --- a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go +++ b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go @@ -11,6 +11,7 @@ import ( "os" "strings" "testing" + "unicode" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" @@ -43,11 +44,21 @@ type installScriptSuite struct { } func TestInstallScript(t *testing.T) { + if *platform == "docker" { + DockerTest(t) + return + } + platformJSON := map[string]map[string]map[string]string{} err := json.Unmarshal(platforms.Content, &platformJSON) require.NoErrorf(t, err, "failed to umarshall platform file: %v", err) + // Splitting an empty string results in a slice with a single empty string which wouldn't be useful + // and result in no tests being run; let's fail the test to make it obvious + if strings.TrimFunc(*osVersion, unicode.IsSpace) == "" { + t.Fatal("expecting some value to be passed for --osversion on test invocation, got none") + } osVersions := strings.Split(*osVersion, ",") cwsSupportedOsVersionList := strings.Split(*cwsSupportedOsVersion, ",") @@ -87,6 +98,20 @@ func TestInstallScript(t *testing.T) { ) }) } + +} + +func DockerTest(t *testing.T) { + t.Run("test install script on a docker container (using SysVInit)", func(tt *testing.T) { + e2e.Run(tt, + &installScriptSuiteSysVInit{}, + e2e.WithProvisioner( + awshost.ProvisionerNoAgentNoFakeIntake( + awshost.WithDocker(), + ), + ), + ) + }) } func (is *installScriptSuite) TestInstallAgent() { @@ -186,3 +211,39 @@ func (is *installScriptSuite) DogstatsdAgentTest() { common.CheckInstallationInstallScript(is.T(), client) is.testUninstall(client, "datadog-dogstatsd") } + +type installScriptSuiteSysVInit struct { + e2e.BaseSuite[environments.Host] +} + +func (is *installScriptSuiteSysVInit) TestInstallAgent() { + containerName := "installation-target" + host := is.Env().RemoteHost + client := common.NewDockerTestClient(host, containerName) + + err := client.RunContainer("public.ecr.aws/ubuntu/ubuntu:22.04_stable") + require.NoError(is.T(), err) + defer client.Cleanup() + + // We need `curl` to download the script, and `sudo` because all the existing test helpers run commands + // through it. + _, err = client.ExecuteWithRetry("apt-get update && apt-get install -y curl sudo") + require.NoError(is.T(), err) + + install.Unix(is.T(), client, installparams.WithArch(*architecture), installparams.WithFlavor(*flavor)) + + // We can't easily reuse the the helpers that assume everything runs directly on the host + // We run a few selected sanity checks here instead (sufficient for this platform anyway) + is.T().Run("datadog-agent service running", func(tt *testing.T) { + _, err := client.Execute("service datadog-agent status") + require.NoError(tt, err, "datadog-agent service should be running") + }) + is.T().Run("status command no errors", func(tt *testing.T) { + statusOutput, err := client.ExecuteWithRetry("datadog-agent \"status\"") + require.NoError(is.T(), err) + + // API Key is invalid we should not check for the following error + statusOutput = strings.ReplaceAll(statusOutput, "[ERROR] API Key is invalid", "API Key is invalid") + require.NotContains(tt, statusOutput, "ERROR") + }) +} diff --git a/test/new-e2e/tests/agent-platform/install/install.go b/test/new-e2e/tests/agent-platform/install/install.go index 74c678b2f4780..1c71c0070405b 100644 --- a/test/new-e2e/tests/agent-platform/install/install.go +++ b/test/new-e2e/tests/agent-platform/install/install.go @@ -13,12 +13,16 @@ import ( "github.com/stretchr/testify/require" - "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/install/installparams" ) +// ExecutorWithRetry represents a type that can execute a command and return its output +type ExecutorWithRetry interface { + ExecuteWithRetry(command string) (output string, err error) +} + // Unix install the agent from install script, by default will install the agent 7 build corresponding to the CI if running in the CI, else the latest Agent 7 version -func Unix(t *testing.T, client *common.TestClient, options ...installparams.Option) { +func Unix(t *testing.T, client ExecutorWithRetry, options ...installparams.Option) { params := installparams.NewParams(options...) commandLine := "" @@ -56,17 +60,17 @@ func Unix(t *testing.T, client *common.TestClient, options ...installparams.Opti } t.Run("Installing the agent", func(tt *testing.T) { - var downdloadCmd string + var downloadCmd string var source string if params.MajorVersion != "5" { source = "S3" - downdloadCmd = fmt.Sprintf(`curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent%v.sh > installscript.sh`, params.MajorVersion) + downloadCmd = fmt.Sprintf(`curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent%v.sh > installscript.sh`, params.MajorVersion) } else { source = "dd-agent repository" - downdloadCmd = "curl -L https://raw.githubusercontent.com/DataDog/dd-agent/master/packaging/datadog-agent/source/install_agent.sh > installscript.sh" + downloadCmd = "curl -L https://raw.githubusercontent.com/DataDog/dd-agent/master/packaging/datadog-agent/source/install_agent.sh > installscript.sh" } - _, err := client.ExecuteWithRetry(downdloadCmd) + _, err := client.ExecuteWithRetry(downloadCmd) require.NoError(tt, err, "failed to download install script from %s: ", source, err) cmd := fmt.Sprintf(`DD_API_KEY="%s" %v DD_SITE="datadoghq.eu" bash installscript.sh`, apikey, commandLine) From fbe1181f489a4874fa14bcdd116b19995da05e74 Mon Sep 17 00:00:00 2001 From: Kevin Fairise <132568982+KevinFairise2@users.noreply.github.com> Date: Mon, 19 Aug 2024 14:08:58 +0200 Subject: [PATCH 014/245] Use the global test_output.json file if exists (#28486) --- tasks/libs/common/junit_upload_core.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tasks/libs/common/junit_upload_core.py b/tasks/libs/common/junit_upload_core.py index defa15ab25606..d5d0283f93f19 100644 --- a/tasks/libs/common/junit_upload_core.py +++ b/tasks/libs/common/junit_upload_core.py @@ -111,12 +111,29 @@ def get_flaky_from_test_output(): Read the test output file generated by gotestsum which contains a list of json for each unit test. We catch all tests marked as flaky in source code: they contain a certain message in the output field. """ - TEST_OUTPUT_FILE = "module_test_output.json" + MODULE_TEST_OUTPUT_FILE = "module_test_output.json" + GLOBAL_TEST_OUTPUT_FILE = "test_output.json" FLAKE_MESSAGE = "flakytest: this is a known flaky test" test_output = [] flaky_tests = set() + + global_test_output_file = Path(GLOBAL_TEST_OUTPUT_FILE) + if global_test_output_file.is_file(): + with global_test_output_file.open(encoding="utf8") as f: + for line in f.readlines(): + test_output.append(json.loads(line)) + flaky_tests.update( + [ + "/".join([test["Package"], test["Test"]]) + for test in test_output + if FLAKE_MESSAGE in test.get("Output", "") + ] + ) + return flaky_tests + + # If the global test output file is not present, we look for module specific test output files for module in DEFAULT_MODULES: - test_file = Path(module, TEST_OUTPUT_FILE) + test_file = Path(module, MODULE_TEST_OUTPUT_FILE) if test_file.is_file(): with test_file.open(encoding="utf8") as f: for line in f.readlines(): From 445b6d7ed5c61b23c40ddb8bad389bc32c701012 Mon Sep 17 00:00:00 2001 From: Marethyu <45374460+Pythyu@users.noreply.github.com> Date: Mon, 19 Aug 2024 14:09:04 +0200 Subject: [PATCH 015/245] Edit ubuntu 24.04 LTS amis to custom ones (#28352) --- test/new-e2e/tests/agent-platform/platforms/platforms.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/new-e2e/tests/agent-platform/platforms/platforms.json b/test/new-e2e/tests/agent-platform/platforms/platforms.json index 29959651ba647..a0aee09367140 100644 --- a/test/new-e2e/tests/agent-platform/platforms/platforms.json +++ b/test/new-e2e/tests/agent-platform/platforms/platforms.json @@ -21,7 +21,7 @@ "ubuntu-22-04": "ami-02e99c6973f5a184a", "ubuntu-23-04": "ami-09c5d86a379ab69a5", "ubuntu-23-10": "ami-0949b45ef274e55a1", - "ubuntu-24-04": "ami-0bfc0c74356d9b384" + "ubuntu-24-04": "ami-0cef91cd3a5f44601" }, "arm64": { "ubuntu-18-04": "ami-002c0df0f55a14f82", @@ -31,7 +31,7 @@ "ubuntu-22-04": "ami-0f1d4680f8b775120", "ubuntu-23-04": "ami-05fab5da2d7fe0b0b", "ubuntu-23-10": "ami-0dea732dd5f1da0a8", - "ubuntu-24-04": "ami-0e879a1b306fffb22" + "ubuntu-24-04": "ami-0a4db546d7da6dc37" } }, "amazonlinux": { From b622df1e2c3e87bda1daa70806ae7b502e399aea Mon Sep 17 00:00:00 2001 From: Mackenzie <63265430+mackjmr@users.noreply.github.com> Date: Mon, 19 Aug 2024 14:12:27 +0200 Subject: [PATCH 016/245] [comp/otelcol/extension] Fix extension check (#28470) --- comp/otelcol/extension/impl/extension.go | 4 +-- comp/otelcol/extension/impl/extension_test.go | 31 ++++++++++++++++--- .../extension/impl/testdata/config.yaml | 3 ++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/comp/otelcol/extension/impl/extension.go b/comp/otelcol/extension/impl/extension.go index 83fdd827b51e1..ff7675e97eec0 100644 --- a/comp/otelcol/extension/impl/extension.go +++ b/comp/otelcol/extension/impl/extension.go @@ -64,7 +64,7 @@ func (ext *ddExtension) Start(_ context.Context, host component.Host) error { // List configured Extensions configstore := ext.cfg.ConfigStore - c, err := configstore.GetProvidedConf() + c, err := configstore.GetEnhancedConf() if err != nil { return err } @@ -76,7 +76,7 @@ func (ext *ddExtension) Start(_ context.Context, host component.Host) error { extensions := host.GetExtensions() for extension := range extensions { - extractor, ok := supportedDebugExtensions[extension.String()] + extractor, ok := supportedDebugExtensions[extension.Type().String()] if !ok { continue } diff --git a/comp/otelcol/extension/impl/extension_test.go b/comp/otelcol/extension/impl/extension_test.go index 05cfaddf270b8..8593207da9b4e 100644 --- a/comp/otelcol/extension/impl/extension_test.go +++ b/comp/otelcol/extension/impl/extension_test.go @@ -20,11 +20,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/confmap/provider/fileprovider" "go.opentelemetry.io/collector/confmap/provider/yamlprovider" "go.opentelemetry.io/collector/otelcol" + "go.uber.org/zap" ) var cpSettings = otelcol.ConfigProviderSettings{ @@ -86,6 +88,15 @@ func TestExtensionHTTPHandler(t *testing.T) { require.NoError(t, err) ddExt := ext.(*ddExtension) + ddExt.telemetry.Logger = zap.New(zap.NewNop().Core()) + + host := newHostWithExtensions( + map[component.ID]component.Component{ + component.MustNewIDWithName("pprof", "custom"): nil, + }, + ) + + ddExt.Start(context.TODO(), host) // Call the handler's ServeHTTP method ddExt.ServeHTTP(rr, req) @@ -105,6 +116,7 @@ func TestExtensionHTTPHandler(t *testing.T) { "runtime_override_configuration", "environment_variable_configuration", "environment", + "sources", } var response map[string]interface{} json.Unmarshal(rr.Body.Bytes(), &response) @@ -113,9 +125,20 @@ func TestExtensionHTTPHandler(t *testing.T) { _, ok := response[key] assert.True(t, ok) } +} + +type hostWithExtensions struct { + component.Host + exts map[component.ID]component.Component +} + +func newHostWithExtensions(exts map[component.ID]component.Component) component.Host { + return &hostWithExtensions{ + Host: componenttest.NewNopHost(), + exts: exts, + } +} - // There will be no sources configured and thus, that key - // should not be present - _, ok := response["sources"] - assert.False(t, ok) +func (h *hostWithExtensions) GetExtensions() map[component.ID]component.Component { + return h.exts } diff --git a/comp/otelcol/extension/impl/testdata/config.yaml b/comp/otelcol/extension/impl/testdata/config.yaml index cc3698a720498..eaa894ec2d049 100644 --- a/comp/otelcol/extension/impl/testdata/config.yaml +++ b/comp/otelcol/extension/impl/testdata/config.yaml @@ -5,7 +5,10 @@ receivers: http: exporters: otlp: +extensions: + pprof/custom: service: + extensions: [pprof/custom] pipelines: traces: receivers: [otlp] From d2885b9f44b7d886836337ad7aec3422939761fa Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Mon, 19 Aug 2024 15:29:22 +0300 Subject: [PATCH 017/245] usm: gotls: Added a metric to measure binaries without symbols (#28528) --- pkg/network/go/bininspect/newproc.go | 2 +- pkg/network/go/bininspect/symbols.go | 3 +++ pkg/network/usm/ebpf_gotls.go | 23 +++++++++++++++-------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/pkg/network/go/bininspect/newproc.go b/pkg/network/go/bininspect/newproc.go index 6547c8b29cc3c..6b2b24ecf3b22 100644 --- a/pkg/network/go/bininspect/newproc.go +++ b/pkg/network/go/bininspect/newproc.go @@ -53,7 +53,7 @@ func InspectNewProcessBinary(elfFile *elf.File, functions map[string]FunctionCon // This might fail if the binary was stripped. symbols, err := GetAllSymbolsByName(elfFile, symbolsSet) if err != nil { - return nil, fmt.Errorf("failed retrieving symbols: %+v", err) + return nil, err } inspector := newProcessBinaryInspector{ diff --git a/pkg/network/go/bininspect/symbols.go b/pkg/network/go/bininspect/symbols.go index eff36bc5642c2..d1f4aa450774c 100644 --- a/pkg/network/go/bininspect/symbols.go +++ b/pkg/network/go/bininspect/symbols.go @@ -226,6 +226,9 @@ func GetAllSymbolsByName(elfFile *elf.File, symbolSet common.StringSet) (map[str } // Only if we failed getting both regular and dynamic symbols - then we abort. + if regularSymbolsErr == elf.ErrNoSymbols && dynamicSymbolsErr == elf.ErrNoSymbols { + return nil, elf.ErrNoSymbols + } if regularSymbolsErr != nil && dynamicSymbolsErr != nil { return nil, fmt.Errorf("could not open symbol sections to resolve symbol offset: %v, %v", regularSymbolsErr, dynamicSymbolsErr) } diff --git a/pkg/network/usm/ebpf_gotls.go b/pkg/network/usm/ebpf_gotls.go index 0eb4a1613cec3..40835925f8389 100644 --- a/pkg/network/usm/ebpf_gotls.go +++ b/pkg/network/usm/ebpf_gotls.go @@ -119,6 +119,9 @@ type goTLSProgram struct { // analysis binAnalysisMetric *libtelemetry.Counter + // binNoSymbolsMetric counts Golang binaries without symbols. + binNoSymbolsMetric *libtelemetry.Counter + registry *utils.FileRegistry } @@ -176,12 +179,13 @@ func newGoTLSProgramProtocolFactory(m *manager.Manager) protocols.ProtocolFactor } return &goTLSProgram{ - done: make(chan struct{}), - cfg: c, - manager: m, - procRoot: c.ProcRoot, - binAnalysisMetric: libtelemetry.NewCounter("usm.go_tls.analysis_time", libtelemetry.OptPrometheus), - registry: utils.NewFileRegistry("go-tls"), + done: make(chan struct{}), + cfg: c, + manager: m, + procRoot: c.ProcRoot, + binAnalysisMetric: libtelemetry.NewCounter("usm.go_tls.analysis_time", libtelemetry.OptPrometheus), + binNoSymbolsMetric: libtelemetry.NewCounter("usm.go_tls.missing_symbols", libtelemetry.OptPrometheus), + registry: utils.NewFileRegistry("go-tls"), }, nil } } @@ -334,10 +338,10 @@ func (p *goTLSProgram) AttachPID(pid uint32) error { // Check go process probeList := make([]manager.ProbeIdentificationPair, 0) - return p.registry.Register(binPath, pid, registerCBCreator(p.manager, p.offsetsDataMap, &probeList, p.binAnalysisMetric), unregisterCBCreator(p.manager, &probeList, p.offsetsDataMap)) + return p.registry.Register(binPath, pid, registerCBCreator(p.manager, p.offsetsDataMap, &probeList, p.binAnalysisMetric, p.binNoSymbolsMetric), unregisterCBCreator(p.manager, &probeList, p.offsetsDataMap)) } -func registerCBCreator(mgr *manager.Manager, offsetsDataMap *ebpf.Map, probeIDs *[]manager.ProbeIdentificationPair, binAnalysisMetric *libtelemetry.Counter) func(path utils.FilePath) error { +func registerCBCreator(mgr *manager.Manager, offsetsDataMap *ebpf.Map, probeIDs *[]manager.ProbeIdentificationPair, binAnalysisMetric, binNoSymbolsMetric *libtelemetry.Counter) func(path utils.FilePath) error { return func(filePath utils.FilePath) error { start := time.Now() @@ -354,6 +358,9 @@ func registerCBCreator(mgr *manager.Manager, offsetsDataMap *ebpf.Map, probeIDs inspectionResult, err := bininspect.InspectNewProcessBinary(elfFile, functionsConfig, structFieldsLookupFunctions) if err != nil { + if errors.Is(err, elf.ErrNoSymbols) { + binNoSymbolsMetric.Add(1) + } return fmt.Errorf("error extracting inspectoin data from %s: %w", filePath.HostPath, err) } From 58d328223b6c18a60855a90078e08492852b9bb3 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Mon, 19 Aug 2024 12:51:38 +0000 Subject: [PATCH 018/245] Do not run the `golang_deps_commenter` job on deploy pipelines (#28514) --- .gitlab/source_test/golang_deps_diff.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/source_test/golang_deps_diff.yml b/.gitlab/source_test/golang_deps_diff.yml index f0acd1b8b9076..5a01ac2d74a13 100644 --- a/.gitlab/source_test/golang_deps_diff.yml +++ b/.gitlab/source_test/golang_deps_diff.yml @@ -28,7 +28,8 @@ golang_deps_commenter: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/pr-commenter:2 tags: ["arch:amd64"] rules: # this should only run on dev branches - - !reference [ .except_main_or_release_branch ] + - !reference [.except_main_or_release_branch] + - !reference [.except_deploy] - when: on_success needs: ["golang_deps_diff"] script: # ignore error message about no PR, because it happens for dev branches without PRs From d918d1b1136ead5af5c7f7d69cd74486ec5f6d69 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Mon, 19 Aug 2024 12:51:42 +0000 Subject: [PATCH 019/245] Read the base branch from the `release.json` file in the `compare-to-itself` task (#28488) --- tasks/libs/releasing/json.py | 10 +++++----- tasks/pipeline.py | 8 +++++++- tasks/release.py | 8 ++++---- tasks/unit_tests/pipeline_tests.py | 16 ++++++++++++---- tasks/unit_tests/release_tests.py | 8 ++++---- 5 files changed, 32 insertions(+), 18 deletions(-) diff --git a/tasks/libs/releasing/json.py b/tasks/libs/releasing/json.py index 50faefdf365e5..e0ea089e43fa1 100644 --- a/tasks/libs/releasing/json.py +++ b/tasks/libs/releasing/json.py @@ -43,7 +43,7 @@ } -def _load_release_json(): +def load_release_json(): with open("release.json") as release_json_stream: return json.load(release_json_stream, object_pairs_hook=OrderedDict) @@ -292,7 +292,7 @@ def update_release_json(new_version: Version, max_version: Version): """ Updates the release entries in release.json to prepare the next RC or final build. """ - release_json = _load_release_json() + release_json = load_release_json() release_entry = release_entry_for(new_version.major) print(f"Updating {release_entry} for {new_version}") @@ -304,7 +304,7 @@ def update_release_json(new_version: Version, max_version: Version): def _get_release_json_value(key): - release_json = _load_release_json() + release_json = load_release_json() path = key.split('::') @@ -318,7 +318,7 @@ def _get_release_json_value(key): def set_new_release_branch(branch): - rj = _load_release_json() + rj = load_release_json() rj["base_branch"] = branch @@ -349,7 +349,7 @@ def find_previous_tags(build, repos, all_keys): Finds the previous tags for the given repositories in the release.json file. """ tags = {} - release_json = _load_release_json() + release_json = load_release_json() for key in all_keys: r = key.casefold().removesuffix("_version").replace("_", "-") repo = next((repo for repo in repos if r in repo), None) diff --git a/tasks/pipeline.py b/tasks/pipeline.py index cb9d955d8882a..7105e23d52010 100644 --- a/tasks/pipeline.py +++ b/tasks/pipeline.py @@ -1029,11 +1029,17 @@ def compare_to_itself(ctx): # Push an empty commit to prevent linking this pipeline to the actual PR ctx.run("git commit -m 'Compare to itself' --allow-empty", hide=True) ctx.run(f"git push origin {new_branch}") + + from tasks.libs.releasing.json import load_release_json + + release_json = load_release_json() + for file in ['.gitlab-ci.yml', '.gitlab/notify/notify.yml']: with open(file) as f: content = f.read() with open(file, 'w') as f: - f.write(content.replace('compare_to: main', f'compare_to: {new_branch}')) + f.write(content.replace(f'compare_to: {release_json["base_branch"]}', f'compare_to: {new_branch}')) + ctx.run("git commit -am 'Compare to itself'", hide=True) ctx.run(f"git push origin {new_branch}", hide=True) max_attempts = 6 diff --git a/tasks/release.py b/tasks/release.py index 9b9c33ef055a0..86318764d6069 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -48,9 +48,9 @@ UNFREEZE_REPO_AGENT, UNFREEZE_REPOS, _get_release_json_value, - _load_release_json, _save_release_json, generate_repo_data, + load_release_json, set_new_release_branch, update_release_json, ) @@ -522,7 +522,7 @@ def build_rc(ctx, major_versions="6,7", patch_version=False, k8s_deployments=Fal @task(help={'key': "Path to an existing release.json key, separated with double colons, eg. 'last_stable::6'"}) def set_release_json(_, key, value): - release_json = _load_release_json() + release_json = load_release_json() path = key.split('::') current_node = release_json for key_idx in range(len(path)): @@ -623,7 +623,7 @@ def create_release_branches(ctx, base_directory="~/dd", major_versions="6,7", up # Step 2.0 - Create milestone update milestone_branch = f"release_milestone-{int(time.time())}" ctx.run(f"git switch -c {milestone_branch}") - rj = _load_release_json() + rj = load_release_json() rj["current_milestone"] = f"{next}" _save_release_json(rj) # Commit release.json @@ -714,7 +714,7 @@ def _update_last_stable(_, version, major_versions="7"): """ Updates the last_release field(s) of release.json """ - release_json = _load_release_json() + release_json = load_release_json() list_major_versions = parse_major_versions(major_versions) # If the release isn't a RC, update the last stable release field for major in list_major_versions: diff --git a/tasks/unit_tests/pipeline_tests.py b/tasks/unit_tests/pipeline_tests.py index b67519bd3bb43..a418bc1fbb67d 100644 --- a/tasks/unit_tests/pipeline_tests.py +++ b/tasks/unit_tests/pipeline_tests.py @@ -133,12 +133,14 @@ def setUp(self) -> None: @patch('builtins.open', new=MagicMock()) @patch.dict('os.environ', {"CI_COMMIT_REF_NAME": "Football"}) @patch('tasks.pipeline.time', new=MagicMock()) + @patch('tasks.libs.releasing.json.load_release_json') @patch('tasks.pipeline.datetime') @patch('tasks.pipeline.GithubAPI') @patch('tasks.pipeline.get_gitlab_repo') - def test_nominal(self, repo_mock, gh_mock, dt_mock): + def test_nominal(self, repo_mock, gh_mock, dt_mock, release_mock): dt_mock.now.return_value = self.now gh_mock.return_value = self.gh + release_mock.return_value = {"base_branch": "main"} pipelines = MagicMock() compare_to = MagicMock(sha="c0mm1t") compare_to.jobs.list.return_value = [1, 2, 3] @@ -154,12 +156,14 @@ def test_nominal(self, repo_mock, gh_mock, dt_mock): @patch('builtins.open', new=MagicMock()) @patch.dict('os.environ', {"CI_COMMIT_REF_NAME": "Football"}) @patch('tasks.pipeline.time', new=MagicMock()) + @patch('tasks.libs.releasing.json.load_release_json') @patch('tasks.pipeline.datetime') @patch('tasks.pipeline.GithubAPI') @patch('tasks.pipeline.get_gitlab_repo') - def test_no_pipeline_found(self, repo_mock, gh_mock, dt_mock): + def test_no_pipeline_found(self, repo_mock, gh_mock, dt_mock, release_mock): dt_mock.now.return_value = self.now gh_mock.return_value = self.gh + release_mock.return_value = {"base_branch": "main"} pipelines = MagicMock() pipelines.list.side_effect = [[], [], [], [], [], []] agent = MagicMock() @@ -173,12 +177,14 @@ def test_no_pipeline_found(self, repo_mock, gh_mock, dt_mock): @patch('builtins.open', new=MagicMock()) @patch.dict('os.environ', {"CI_COMMIT_REF_NAME": "Football"}) @patch('tasks.pipeline.time', new=MagicMock()) + @patch('tasks.libs.releasing.json.load_release_json') @patch('tasks.pipeline.datetime') @patch('tasks.pipeline.GithubAPI') @patch('tasks.pipeline.get_gitlab_repo') - def test_no_pipeline_found_again(self, repo_mock, gh_mock, dt_mock): + def test_no_pipeline_found_again(self, repo_mock, gh_mock, dt_mock, release_mock): dt_mock.now.return_value = self.now gh_mock.return_value = self.gh + release_mock.return_value = {"base_branch": "main"} pipelines = MagicMock() compare_to = MagicMock(sha="w4lo0") compare_to.jobs.list.return_value = [1, 2, 3] @@ -194,12 +200,14 @@ def test_no_pipeline_found_again(self, repo_mock, gh_mock, dt_mock): @patch('builtins.open', new=MagicMock()) @patch.dict('os.environ', {"CI_COMMIT_REF_NAME": "Football"}) @patch('tasks.pipeline.time', new=MagicMock()) + @patch('tasks.libs.releasing.json.load_release_json') @patch('tasks.pipeline.datetime') @patch('tasks.pipeline.GithubAPI') @patch('tasks.pipeline.get_gitlab_repo') - def test_pipeline_with_no_jobs(self, repo_mock, gh_mock, dt_mock): + def test_pipeline_with_no_jobs(self, repo_mock, gh_mock, dt_mock, release_mock): dt_mock.now.return_value = self.now gh_mock.return_value = self.gh + release_mock.return_value = {"base_branch": "main"} pipelines = MagicMock() compare_to = MagicMock(sha="c0mm1t") pipelines.list.side_effect = [[], [], [compare_to], [], [], []] diff --git a/tasks/unit_tests/release_tests.py b/tasks/unit_tests/release_tests.py index a809cb06b7047..0f52819ed6e4c 100644 --- a/tasks/unit_tests/release_tests.py +++ b/tasks/unit_tests/release_tests.py @@ -560,7 +560,7 @@ class TestFindPreviousTags(unittest.TestCase): keys = ["HARRY_POTTER_VERSION", "HERMIONE_GRANGER_VERSION", "WEASLEY_VERSION"] @patch( - 'tasks.libs.releasing.json._load_release_json', + 'tasks.libs.releasing.json.load_release_json', new=MagicMock( return_value={ 'hogwarts': { @@ -576,7 +576,7 @@ def test_one_repo(self): self.assertEqual({'harry-potter': '6.6.6'}, find_previous_tags("hogwarts", repos, self.keys)) @patch( - 'tasks.libs.releasing.json._load_release_json', + 'tasks.libs.releasing.json.load_release_json', new=MagicMock( return_value={ 'hogwarts': { @@ -595,7 +595,7 @@ def test_several_repos(self): ) @patch( - 'tasks.libs.releasing.json._load_release_json', + 'tasks.libs.releasing.json.load_release_json', new=MagicMock( return_value={ 'hogwarts': { @@ -611,7 +611,7 @@ def test_no_repo(self): self.assertEqual({}, find_previous_tags("hogwarts", repos, self.keys)) @patch( - 'tasks.libs.releasing.json._load_release_json', + 'tasks.libs.releasing.json.load_release_json', new=MagicMock( return_value={ 'hogwarts': { From 0cb0f9afe614fa5d5d2164b66a971d75104b8a14 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Mon, 19 Aug 2024 12:51:47 +0000 Subject: [PATCH 020/245] Automatically create the backport label when we create the release branch (#28466) --- tasks/libs/ciproviders/github_api.py | 6 ++++++ tasks/release.py | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tasks/libs/ciproviders/github_api.py b/tasks/libs/ciproviders/github_api.py index 35a795ca2a5a8..73eb66383f47a 100644 --- a/tasks/libs/ciproviders/github_api.py +++ b/tasks/libs/ciproviders/github_api.py @@ -341,6 +341,12 @@ def get_token_from_app(app_id_env='GITHUB_APP_ID', pkey_env='GITHUB_KEY_B64'): auth_token = integration.get_access_token(install_id) print(auth_token.token) + def create_label(self, name, color, description=""): + """ + Creates a label in the given GitHub repository. + """ + return self._repository.create_label(name, color, description) + def get_github_teams(users): for user in users: diff --git a/tasks/release.py b/tasks/release.py index 86318764d6069..c7e47a4fe1eb7 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -75,6 +75,8 @@ ".gitlab/notify/notify.yml", ] +BACKPORT_LABEL_COLOR = "5319e7" + @task def list_major_change(_, milestone): @@ -586,6 +588,7 @@ def create_release_branches(ctx, base_directory="~/dd", major_versions="6,7", up This also requires that there are no local uncommitted changes, that the current branch is 'main' or the release branch, and that no branch named 'release/' already exists locally or upstream. """ + github = GithubAPI(repository=GITHUB_REPO_NAME) list_major_versions = parse_major_versions(major_versions) @@ -602,7 +605,6 @@ def create_release_branches(ctx, base_directory="~/dd", major_versions="6,7", up if check_state: print(color_message("Checking repository state", "bold")) - github = GithubAPI(repository=GITHUB_REPO_NAME) check_clean_branch_state(ctx, github, release_branch) if not yes_no_question( @@ -617,6 +619,12 @@ def create_release_branches(ctx, base_directory="~/dd", major_versions="6,7", up for repo in UNFREEZE_REPOS: create_and_update_release_branch(ctx, repo, release_branch, base_directory=base_directory, upstream=upstream) + # create the backport label in the Agent repo + print(color_message("Creating backport label in the Agent repository", Color.BOLD)) + github.create_label( + f'backport/{release_branch}', BACKPORT_LABEL_COLOR, f'Automatically create a backport PR to {release_branch}' + ) + # Step 2 - Create PRs with new settings in datadog-agent repository with ctx.cd(f"{base_directory}/{UNFREEZE_REPO_AGENT}"): From 6c11543c7cf1f94b4c887d5704dfea27e662de1a Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Mon, 19 Aug 2024 12:56:21 +0000 Subject: [PATCH 021/245] Add parameters to easily override the Agent image used in e2e tests (#28091) --- tasks/new_e2e_tests.py | 24 ++++++--- test/new-e2e/README.md | 4 +- test/new-e2e/pkg/runner/configmap.go | 74 +++++++++++++--------------- test/new-e2e/pkg/runner/profile.go | 2 +- 4 files changed, 53 insertions(+), 51 deletions(-) diff --git a/tasks/new_e2e_tests.py b/tasks/new_e2e_tests.py index 3bf07af80a34e..c7d4c1a7e9ef5 100644 --- a/tasks/new_e2e_tests.py +++ b/tasks/new_e2e_tests.py @@ -34,6 +34,8 @@ 'verbose': 'Verbose output: log all tests as they are run (same as gotest -v) [default: True]', 'run': 'Only run tests matching the regular expression', 'skip': 'Only run tests not matching the regular expression', + 'agent_image': 'Full image path for the agent image (e.g. "repository:tag") to run the e2e tests with', + 'cluster_agent_image': 'Full image path for the cluster agent image (e.g. "repository:tag") to run the e2e tests with', }, ) def run( @@ -59,6 +61,8 @@ def run( junit_tar="", test_run_name="", test_washer=False, + agent_image="", + cluster_agent_image="", ): """ Run E2E Tests based on test-infra-definitions infrastructure provisioning. @@ -74,19 +78,25 @@ def run( if targets: e2e_module.targets = targets - envVars = {} + env_vars = {} if profile: - envVars["E2E_PROFILE"] = profile + env_vars["E2E_PROFILE"] = profile - parsedParams = {} + parsed_params = {} for param in configparams: parts = param.split("=", 1) if len(parts) != 2: raise Exit(message=f"wrong format given for config parameter, expects key=value, actual: {param}", code=1) - parsedParams[parts[0]] = parts[1] + parsed_params[parts[0]] = parts[1] - if parsedParams: - envVars["E2E_STACK_PARAMS"] = json.dumps(parsedParams) + if agent_image: + parsed_params["ddagent:fullImagePath"] = agent_image + + if cluster_agent_image: + parsed_params["ddagent:clusterAgentFullImagePath"] = cluster_agent_image + + if parsed_params: + env_vars["E2E_STACK_PARAMS"] = json.dumps(parsed_params) gotestsum_format = "standard-verbose" if verbose else "pkgname" @@ -128,7 +138,7 @@ def run( modules=[e2e_module], args=args, cmd=cmd, - env=envVars, + env=env_vars, junit_tar=junit_tar, save_result_json="", test_profiler=None, diff --git a/test/new-e2e/README.md b/test/new-e2e/README.md index 796d5b80f6ae7..31befab726d2f 100644 --- a/test/new-e2e/README.md +++ b/test/new-e2e/README.md @@ -17,7 +17,7 @@ go work use . ./test/new-e2e ## Use VsCode tasks to wrap aws-vault -The `agent-sandbox: test current file` can be used to launch test on a file withtout having to launch the whole VsCode wrapped by aws-vault exec. To use it copy the `.template` files in `.vscode` and remove the `.template` extension. +The `agent-sandbox: test current file` can be used to launch test on a file without having to launch the whole VsCode wrapped by aws-vault exec. To use it copy the `.template` files in `.vscode` and remove the `.template` extension. You need to open the `new-e2e` folder > **Note** > `go.work` file is currently ignored in `datadog-agent` @@ -38,6 +38,6 @@ The following are considered as installer tests: All these tests can be executed locally with the following command: -`aws-vault exec sso-agent-sandbox-account-admin -- inv new-e2e-tests.run --targets --osversion '' --platform '' --arch ` +`inv new-e2e-tests.run --targets --osversion '' --platform '' --arch ` The available os versions can be found in the file `./tests/agent-platform/platforms/platforms.json` diff --git a/test/new-e2e/pkg/runner/configmap.go b/test/new-e2e/pkg/runner/configmap.go index 0334186095f7e..278f0473b3a95 100644 --- a/test/new-e2e/pkg/runner/configmap.go +++ b/test/new-e2e/pkg/runner/configmap.go @@ -17,28 +17,28 @@ import ( ) const ( - // AgentAPIKey pulumi config paramater name + // AgentAPIKey pulumi config parameter name AgentAPIKey = commonconfig.DDAgentConfigNamespace + ":" + commonconfig.DDAgentAPIKeyParamName - // AgentAPPKey pulumi config paramater name + // AgentAPPKey pulumi config parameter name AgentAPPKey = commonconfig.DDAgentConfigNamespace + ":" + commonconfig.DDAgentAPPKeyParamName // AgentPipelineID pulumi config parameter name AgentPipelineID = commonconfig.DDAgentConfigNamespace + ":" + commonconfig.DDAgentPipelineID // AgentCommitSHA pulumi config parameter name AgentCommitSHA = commonconfig.DDAgentConfigNamespace + ":" + commonconfig.DDAgentCommitSHA - // InfraEnvironmentVariables pulumi config paramater name + // InfraEnvironmentVariables pulumi config parameter name InfraEnvironmentVariables = commonconfig.DDInfraConfigNamespace + ":" + commonconfig.DDInfraEnvironment - // InfraExtraResourcesTags pulumi config paramater name + // InfraExtraResourcesTags pulumi config parameter name InfraExtraResourcesTags = commonconfig.DDInfraConfigNamespace + ":" + commonconfig.DDInfraExtraResourcesTags - // AWSKeyPairName pulumi config paramater name + // AWSKeyPairName pulumi config parameter name AWSKeyPairName = commonconfig.DDInfraConfigNamespace + ":" + infraaws.DDInfraDefaultKeyPairParamName - // AWSPublicKeyPath pulumi config paramater name + // AWSPublicKeyPath pulumi config parameter name AWSPublicKeyPath = commonconfig.DDInfraConfigNamespace + ":" + infraaws.DDinfraDefaultPublicKeyPath - // AWSPrivateKeyPath pulumi config paramater name + // AWSPrivateKeyPath pulumi config parameter name AWSPrivateKeyPath = commonconfig.DDInfraConfigNamespace + ":" + infraaws.DDInfraDefaultPrivateKeyPath - // AWSPrivateKeyPassword pulumi config paramater name + // AWSPrivateKeyPassword pulumi config parameter name AWSPrivateKeyPassword = commonconfig.DDInfraConfigNamespace + ":" + infraaws.DDInfraDefaultPrivateKeyPassword ) @@ -94,48 +94,40 @@ func setConfigMapFromParameter(store parameters.Store, cm ConfigMap, paramName p // BuildStackParameters creates a config map from a profile, a scenario config map // and env/cli configuration parameters func BuildStackParameters(profile Profile, scenarioConfig ConfigMap) (ConfigMap, error) { + var err error // Priority order: profile configs < scenarioConfig < Env/CLI config cm := ConfigMap{} // Parameters from profile - cm.Set("ddinfra:env", profile.EnvironmentNames(), false) - err := SetConfigMapFromParameter(profile.ParamStore(), cm, parameters.KeyPairName, AWSKeyPairName) - if err != nil { - return nil, err - } - err = SetConfigMapFromParameter(profile.ParamStore(), cm, parameters.PublicKeyPath, AWSPublicKeyPath) - if err != nil { - return nil, err - } - err = SetConfigMapFromParameter(profile.ParamStore(), cm, parameters.PrivateKeyPath, AWSPrivateKeyPath) - if err != nil { - return nil, err + cm.Set(InfraEnvironmentVariables, profile.EnvironmentNames(), false) + params := map[parameters.StoreKey]string{ + parameters.KeyPairName: AWSKeyPairName, + parameters.PublicKeyPath: AWSPublicKeyPath, + parameters.PrivateKeyPath: AWSPrivateKeyPath, + parameters.ExtraResourcesTags: InfraExtraResourcesTags, + parameters.PipelineID: AgentPipelineID, + parameters.CommitSHA: AgentCommitSHA, } - err = SetConfigMapFromParameter(profile.ParamStore(), cm, parameters.ExtraResourcesTags, InfraExtraResourcesTags) - if err != nil { - return nil, err - } - err = SetConfigMapFromParameter(profile.ParamStore(), cm, parameters.PipelineID, AgentPipelineID) - if err != nil { - return nil, err - } - err = SetConfigMapFromParameter(profile.ParamStore(), cm, parameters.CommitSHA, AgentCommitSHA) - if err != nil { - return nil, err + + for storeKey, configMapKey := range params { + err = SetConfigMapFromParameter(profile.ParamStore(), cm, storeKey, configMapKey) + if err != nil { + return nil, err + } } // Secret parameters from profile store - err = SetConfigMapFromSecret(profile.SecretStore(), cm, parameters.APIKey, AgentAPIKey) - if err != nil { - return nil, err - } - err = SetConfigMapFromSecret(profile.SecretStore(), cm, parameters.APPKey, AgentAPPKey) - if err != nil { - return nil, err + secretParams := map[parameters.StoreKey]string{ + parameters.APIKey: AgentAPIKey, + parameters.APPKey: AgentAPPKey, + parameters.PrivateKeyPassword: AWSPrivateKeyPassword, } - err = SetConfigMapFromSecret(profile.SecretStore(), cm, parameters.PrivateKeyPassword, AWSPrivateKeyPassword) - if err != nil { - return nil, err + + for storeKey, configMapKey := range secretParams { + err = SetConfigMapFromSecret(profile.SecretStore(), cm, storeKey, configMapKey) + if err != nil { + return nil, err + } } // Merge with scenario variables diff --git a/test/new-e2e/pkg/runner/profile.go b/test/new-e2e/pkg/runner/profile.go index 3d707b214e145..6b0d2a8da355e 100644 --- a/test/new-e2e/pkg/runner/profile.go +++ b/test/new-e2e/pkg/runner/profile.go @@ -54,7 +54,7 @@ type Profile interface { // Since one Workspace supports one single program and we have one program per stack, // the path should be unique for each stack. GetWorkspacePath(stackName string) string - // ParamStore() returns the normal parameter store + // ParamStore returns the normal parameter store ParamStore() parameters.Store // SecretStore returns the secure parameter store SecretStore() parameters.Store From 2ed58272d8ab140a61ace658d071a1e33367e61f Mon Sep 17 00:00:00 2001 From: Dinesh Gurumurthy Date: Mon, 19 Aug 2024 18:26:26 +0530 Subject: [PATCH 022/245] Rename Dockerfile and use depth 1 (#28527) --- Dockerfiles/agent-ot/{Dockerfile.byoc => Dockerfile.agent-otel} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Dockerfiles/agent-ot/{Dockerfile.byoc => Dockerfile.agent-otel} (96%) diff --git a/Dockerfiles/agent-ot/Dockerfile.byoc b/Dockerfiles/agent-ot/Dockerfile.agent-otel similarity index 96% rename from Dockerfiles/agent-ot/Dockerfile.byoc rename to Dockerfiles/agent-ot/Dockerfile.agent-otel index 04132bd129dfd..cb0fb6109e780 100644 --- a/Dockerfiles/agent-ot/Dockerfile.byoc +++ b/Dockerfiles/agent-ot/Dockerfile.agent-otel @@ -24,7 +24,7 @@ RUN apt-get update && \ && rm -rf /var/lib/apt/lists/* # TEMP: Use github source code -RUN git clone https://github.com/DataDog/datadog-agent.git datadog-agent-${AGENT_VERSION} +RUN git clone --depth 1 https://github.com/DataDog/datadog-agent.git datadog-agent-${AGENT_VERSION} # Once we have stable releases, we can use the following code to download the source code # TODO: use released agent version once we have an agent release with the otel binary From ff3adfc3098260b70fec7f6d536201c381c18ad4 Mon Sep 17 00:00:00 2001 From: Dinesh Gurumurthy Date: Mon, 19 Aug 2024 18:42:42 +0530 Subject: [PATCH 023/245] Fix Duplicate processor issue (#28473) --- comp/otelcol/converter/impl/converter_test.go | 10 +++ .../otelcol/converter/impl/infraattributes.go | 58 +++++++++-------- .../config-result.yaml | 62 +++++++++++++++++++ .../dd-connector-multi-pipelines/config.yaml | 61 ++++++++++++++++++ .../dd-connector/config-result.yaml | 58 +++++++++++++++++ .../processors/dd-connector/config.yaml | 57 +++++++++++++++++ 6 files changed, 280 insertions(+), 26 deletions(-) create mode 100644 comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config-result.yaml create mode 100644 comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config.yaml create mode 100644 comp/otelcol/converter/impl/testdata/processors/dd-connector/config-result.yaml create mode 100644 comp/otelcol/converter/impl/testdata/processors/dd-connector/config.yaml diff --git a/comp/otelcol/converter/impl/converter_test.go b/comp/otelcol/converter/impl/converter_test.go index 912186d7f76ee..5cd73f87aa483 100644 --- a/comp/otelcol/converter/impl/converter_test.go +++ b/comp/otelcol/converter/impl/converter_test.go @@ -121,6 +121,16 @@ func TestConvert(t *testing.T) { provided: "receivers/no-receivers-defined/config.yaml", expectedResult: "receivers/no-receivers-defined/config-result.yaml", }, + { + name: "processors/dd-connector", + provided: "processors/dd-connector/config.yaml", + expectedResult: "processors/dd-connector/config-result.yaml", + }, + { + name: "processors/dd-connector-multi-pipelines", + provided: "processors/dd-connector-multi-pipelines/config.yaml", + expectedResult: "processors/dd-connector-multi-pipelines/config-result.yaml", + }, } for _, tc := range tests { diff --git a/comp/otelcol/converter/impl/infraattributes.go b/comp/otelcol/converter/impl/infraattributes.go index 07da344099f8d..ad6cb0474e687 100644 --- a/comp/otelcol/converter/impl/infraattributes.go +++ b/comp/otelcol/converter/impl/infraattributes.go @@ -55,45 +55,51 @@ func addProcessorToPipelinesWithDDExporter(conf *confmap.Conf, comp component) { if !ok { return } + infraAttrsInPipeline := false + ddExporterInPipeline := false for _, exporter := range exportersSlice { exporterString, ok := exporter.(string) if !ok { return } - if componentName(exporterString) == "datadog" { - // datadog component is an exporter in this pipeline. Need to make sure that processor is also configured. - _, ok := componentsMap[comp.Type] - if !ok { - componentsMap[comp.Type] = []any{} - } + if infraAttrsInPipeline { + break + } + if componentName(exporterString) != "datadog" { + continue + } + ddExporterInPipeline = true + // datadog component is an exporter in this pipeline. Need to make sure that processor is also configured. + _, ok = componentsMap[comp.Type] + if !ok { + componentsMap[comp.Type] = []any{} + } - infraAttrsInPipeline := false - processorsSlice, ok := componentsMap[comp.Type].([]any) + processorsSlice, ok := componentsMap[comp.Type].([]any) + if !ok { + return + } + for _, processor := range processorsSlice { + processorString, ok := processor.(string) if !ok { return } - for _, processor := range processorsSlice { - processorString, ok := processor.(string) - if !ok { - return - } - if componentName(processorString) == comp.Name { - infraAttrsInPipeline = true - } - - } - if !infraAttrsInPipeline { - // no processors are defined - if !componentAddedToConfig { - addComponentToConfig(conf, comp) - componentAddedToConfig = true - } - addComponentToPipeline(conf, comp, pipelineName) + if componentName(processorString) == comp.Name { + infraAttrsInPipeline = true } + } } - } + if !infraAttrsInPipeline && ddExporterInPipeline { + // no processors are defined + if !componentAddedToConfig { + addComponentToConfig(conf, comp) + componentAddedToConfig = true + } + addComponentToPipeline(conf, comp, pipelineName) + } + } } diff --git a/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config-result.yaml b/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config-result.yaml new file mode 100644 index 0000000000000..eca2c8bf5d14e --- /dev/null +++ b/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config-result.yaml @@ -0,0 +1,62 @@ +receivers: + prometheus: + config: + scrape_configs: + - job_name: "datadog-agent" + scrape_interval: 10s + static_configs: + - targets: ["0.0.0.0:8888"] + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +exporters: + debug: + verbosity: detailed + datadog: + api: + key: "0000" +processors: + infraattributes/dd-autoconfigured: + batch: + timeout: 10s +connectors: + datadog/connector: + traces: + compute_top_level_by_span_kind: true + peer_tags_aggregation: true + compute_stats_by_span_kind: true +extensions: + pprof/user-defined: + health_check/user-defined: + zpages/user-defined: + endpoint: "localhost:55679" + datadog/user-defined: + +service: + extensions: + [ + pprof/user-defined, + zpages/user-defined, + health_check/user-defined, + datadog/user-defined, + ] + pipelines: + traces: + receivers: [otlp] + processors: [batch, infraattributes/dd-autoconfigured] + exporters: [datadog] + traces/2: + receivers: [otlp] + processors: [batch, infraattributes/dd-autoconfigured] + exporters: [datadog/connector] + metrics: + receivers: [otlp, datadog/connector, prometheus] + processors: [batch, infraattributes/dd-autoconfigured] + exporters: [datadog] + logs: + receivers: [otlp] + processors: [batch, infraattributes/dd-autoconfigured] + exporters: [datadog] diff --git a/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config.yaml b/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config.yaml new file mode 100644 index 0000000000000..01d33ba6589dd --- /dev/null +++ b/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config.yaml @@ -0,0 +1,61 @@ +receivers: + prometheus: + config: + scrape_configs: + - job_name: "otelcol" + scrape_interval: 10s + static_configs: + - targets: ["0.0.0.0:8888"] + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +exporters: + debug: + verbosity: detailed + datadog: + api: + key: "0000" +processors: + batch: + timeout: 10s +connectors: + datadog/connector: + traces: + compute_top_level_by_span_kind: true + peer_tags_aggregation: true + compute_stats_by_span_kind: true +extensions: + pprof/user-defined: + health_check/user-defined: + zpages/user-defined: + endpoint: "localhost:55679" + datadog/user-defined: + +service: + extensions: + [ + pprof/user-defined, + zpages/user-defined, + health_check/user-defined, + datadog/user-defined, + ] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [datadog] + traces/2: + receivers: [otlp] + processors: [batch] + exporters: [datadog/connector] + metrics: + receivers: [otlp, datadog/connector, prometheus] + processors: [batch] + exporters: [datadog] + logs: + receivers: [otlp] + processors: [batch] + exporters: [datadog] diff --git a/comp/otelcol/converter/impl/testdata/processors/dd-connector/config-result.yaml b/comp/otelcol/converter/impl/testdata/processors/dd-connector/config-result.yaml new file mode 100644 index 0000000000000..894aaca27191b --- /dev/null +++ b/comp/otelcol/converter/impl/testdata/processors/dd-connector/config-result.yaml @@ -0,0 +1,58 @@ +receivers: + prometheus: + config: + scrape_configs: + - job_name: "datadog-agent" + scrape_interval: 10s + static_configs: + - targets: ["0.0.0.0:8888"] + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +exporters: + debug: + verbosity: detailed + datadog: + api: + key: "0000" +processors: + infraattributes/dd-autoconfigured: + batch: + timeout: 10s +connectors: + datadog/connector: + traces: + compute_top_level_by_span_kind: true + peer_tags_aggregation: true + compute_stats_by_span_kind: true +extensions: + pprof/user-defined: + health_check/user-defined: + zpages/user-defined: + endpoint: "localhost:55679" + datadog/user-defined: + +service: + extensions: + [ + pprof/user-defined, + zpages/user-defined, + health_check/user-defined, + datadog/user-defined, + ] + pipelines: + traces: + receivers: [otlp] + processors: [batch, infraattributes/dd-autoconfigured] + exporters: [datadog/connector, datadog] + metrics: + receivers: [otlp, datadog/connector, prometheus] + processors: [batch, infraattributes/dd-autoconfigured] + exporters: [datadog] + logs: + receivers: [otlp] + processors: [batch, infraattributes/dd-autoconfigured] + exporters: [datadog] diff --git a/comp/otelcol/converter/impl/testdata/processors/dd-connector/config.yaml b/comp/otelcol/converter/impl/testdata/processors/dd-connector/config.yaml new file mode 100644 index 0000000000000..6c236433a341a --- /dev/null +++ b/comp/otelcol/converter/impl/testdata/processors/dd-connector/config.yaml @@ -0,0 +1,57 @@ +receivers: + prometheus: + config: + scrape_configs: + - job_name: "otelcol" + scrape_interval: 10s + static_configs: + - targets: ["0.0.0.0:8888"] + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +exporters: + debug: + verbosity: detailed + datadog: + api: + key: "0000" +processors: + batch: + timeout: 10s +connectors: + datadog/connector: + traces: + compute_top_level_by_span_kind: true + peer_tags_aggregation: true + compute_stats_by_span_kind: true +extensions: + pprof/user-defined: + health_check/user-defined: + zpages/user-defined: + endpoint: "localhost:55679" + datadog/user-defined: + +service: + extensions: + [ + pprof/user-defined, + zpages/user-defined, + health_check/user-defined, + datadog/user-defined, + ] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [datadog/connector, datadog] + metrics: + receivers: [otlp, datadog/connector, prometheus] + processors: [batch] + exporters: [datadog] + logs: + receivers: [otlp] + processors: [batch] + exporters: [datadog] From cf5f5e8ec4c9310885079ec2ea56652ffc98760f Mon Sep 17 00:00:00 2001 From: Stan Rozenraukh Date: Mon, 19 Aug 2024 09:13:24 -0400 Subject: [PATCH 024/245] auto-instrumentation: propagate detected langs to injector (#28445) --- .../auto_instrumentation.go | 280 +++++++++++++----- .../auto_instrumentation_test.go | 175 ++++++++--- .../auto_instrumentation_util.go | 6 + .../auto_instrumentation_util_test.go | 1 - .../mutate/autoinstrumentation/env_vars.go | 12 - .../autoinstrumentation/language_versions.go | 11 +- .../autoinstrumentation/lib_requirement.go | 46 ++- .../mutate/autoinstrumentation/mutators.go | 29 +- 8 files changed, 412 insertions(+), 148 deletions(-) diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation.go index 0d50bc62e338e..457d45c35eec4 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation.go @@ -89,11 +89,19 @@ func NewWebhook(wmeta workloadmeta.Component, filter mutatecommon.InjectionFilte return nil, fmt.Errorf("invalid version for key apm_config.instrumentation.version: %w", err) } - containerRegistry := mutatecommon.ContainerRegistry("admission_controller.auto_instrumentation.container_registry") + var ( + isEnabled = config.Datadog().GetBool("admission_controller.auto_instrumentation.enabled") + containerRegistry = mutatecommon.ContainerRegistry("admission_controller.auto_instrumentation.container_registry") + pinnedLibraries []libInfo + ) + + if isEnabled { + pinnedLibraries = getPinnedLibraries(containerRegistry) + } return &Webhook{ name: webhookName, - isEnabled: config.Datadog().GetBool("admission_controller.auto_instrumentation.enabled"), + isEnabled: isEnabled, endpoint: config.Datadog().GetString("admission_controller.auto_instrumentation.endpoint"), resources: []string{"pods"}, operations: []admiv1.OperationType{admiv1.Create}, @@ -102,7 +110,7 @@ func NewWebhook(wmeta workloadmeta.Component, filter mutatecommon.InjectionFilte injectionFilter: filter, containerRegistry: containerRegistry, injectorImageTag: config.Datadog().GetString("apm_config.instrumentation.injector_image_tag"), - pinnedLibraries: getPinnedLibraries(containerRegistry), + pinnedLibraries: pinnedLibraries, version: v, wmeta: wmeta, }, nil @@ -155,10 +163,13 @@ func initContainerName(lang language) string { return fmt.Sprintf("datadog-lib-%s-init", lang) } +// isPodEligible checks whether we are allowed to inject in this pod. func (w *Webhook) isPodEligible(pod *corev1.Pod) bool { return w.injectionFilter.ShouldMutatePod(pod) } +// isEnabledInNamespace checks whether this namespace is opted into or out of +// single step (auto_instrumentation) outside pod-specific annotations. func (w *Webhook) isEnabledInNamespace(namespace string) bool { return w.injectionFilter.NSFilter.IsNamespaceEligible(namespace) } @@ -185,8 +196,8 @@ func (w *Webhook) inject(pod *corev1.Pod, ns string, _ dynamic.Interface) (bool, } } - libsToInject, autoDetected := w.extractLibInfo(pod) - if len(libsToInject) == 0 { + extractedLibInfo := w.extractLibInfo(pod) + if len(extractedLibInfo.libs) == 0 { return false, nil } @@ -202,19 +213,7 @@ func (w *Webhook) inject(pod *corev1.Pod, ns string, _ dynamic.Interface) (bool, } } - // Inject env variables used for Onboarding KPIs propagation - var injectionType string - if w.isEnabledInNamespace(pod.Namespace) { - // if Single Step Instrumentation is enabled, inject DD_INSTRUMENTATION_INSTALL_TYPE:k8s_single_step - _ = mutatecommon.InjectEnv(pod, singleStepInstrumentationInstallTypeEnvVar) - injectionType = singleStepInstrumentationInstallType - } else { - // if local library injection is enabled, inject DD_INSTRUMENTATION_INSTALL_TYPE:k8s_lib_injection - _ = mutatecommon.InjectEnv(pod, localLibraryInstrumentationInstallTypeEnvVar) - injectionType = localLibraryInstrumentationInstallType - } - - if err := w.injectAutoInstruConfig(pod, libsToInject, autoDetected, injectionType); err != nil { + if err := w.injectAutoInstruConfig(pod, extractedLibInfo); err != nil { log.Errorf("failed to inject auto instrumentation configurations: %v", err) return false, errors.New(metrics.ConfigInjectionError) } @@ -308,15 +307,59 @@ func getPinnedLibraries(registry string) []libInfo { return res } -// getLibrariesLanguageDetection runs process language auto-detection and returns languages to inject for APM Instrumentation. -// The languages information is available in workloadmeta-store and attached on the pod's owner. -func (w *Webhook) getLibrariesLanguageDetection(pod *corev1.Pod) []libInfo { - if config.Datadog().GetBool("admission_controller.auto_instrumentation.inject_auto_detected_libraries") { - // Use libraries returned by language detection for APM Instrumentation - return w.getAutoDetectedLibraries(pod) +type libInfoLanguageDetection struct { + libs []libInfo + injectionEnabled bool +} + +func (l *libInfoLanguageDetection) containerMutator(v version) containerMutator { + return containerMutatorFunc(func(c *corev1.Container) error { + if !v.usesInjector() || l == nil { + return nil + } + + var langs []string + for _, lib := range l.libs { + if lib.ctrName == c.Name { // strict container name matching + langs = append(langs, string(lib.lang)) + } + } + + // N.B. + // We report on the languages detected regardless + // of if it is empty or not to disambiguate the empty state + // language_detection reporting being disabled. + if err := (containerMutators{ + envVar{ + key: "DD_INSTRUMENTATION_LANGUAGES_DETECTED", + valFunc: identityValFunc(strings.Join(langs, ",")), + }, + envVar{ + key: "DD_INSTRUMENTATION_LANGUAGE_DETECTION_INJECTION_ENABLED", + valFunc: identityValFunc(strconv.FormatBool(l.injectionEnabled)), + }, + }).mutateContainer(c); err != nil { + return err + } + + return nil + }) +} + +// getLibrariesLanguageDetection returns the languages that were detected by process language detection. +// +// The languages information is available in workloadmeta-store +// and attached on the pod's owner. +func (w *Webhook) getLibrariesLanguageDetection(pod *corev1.Pod) *libInfoLanguageDetection { + if !config.Datadog().GetBool("language_detection.enabled") || + !config.Datadog().GetBool("language_detection.reporting.enabled") { + return nil } - return nil + return &libInfoLanguageDetection{ + libs: w.getAutoDetectedLibraries(pod), + injectionEnabled: config.Datadog().GetBool("admission_controller.auto_instrumentation.inject_auto_detected_libraries"), + } } // getAllLatestLibraries returns all supported by APM Instrumentation tracing libraries @@ -329,35 +372,125 @@ func (w *Webhook) getAllLatestLibraries() []libInfo { return libsToInject } -// extractLibInfo returns the language, the image, -// and a boolean indicating whether the library should be injected into the pod -func (w *Webhook) extractLibInfo(pod *corev1.Pod) ([]libInfo, bool) { - // If the pod is "injectable" and annotated with libraries to inject, use those. - if w.isPodEligible(pod) { - // The library version specified via annotation on the Pod takes precedence - // over libraries injected with Single Step Instrumentation - libs := w.extractLibrariesFromAnnotations(pod) - if len(libs) > 0 { - return libs, false - } +// libInfoSource describes where we got the libraries from for +// injection and is used to set up metrics/telemetry. See +// Webhook.injectAutoInstruConfig for usage. +type libInfoSource int + +const ( + // libInfoSourceNone is no source provided. + libInfoSourceNone libInfoSource = iota + // libInfoSourceLibInjection is when the user sets up annotations on their pods for + // library injection and single step is disabled. + libInfoSourceLibInjection + // libInfoSourceSingleStepInstrumentation is when we are using the instrumentation config + // to determine which libraries to inject. + libInfoSourceSingleStepInstrumentation + // libInfoSourceSingleStepLanguageDetection is when we use the language detection + // annotation to determine which libs to inject. + libInfoSourceSingleStepLangaugeDetection +) + +// injectionType produces a string to distinguish between if +// we're using "single step" or "lib injection" for metrics and logging. +func (s libInfoSource) injectionType() string { + switch s { + case libInfoSourceSingleStepInstrumentation, libInfoSourceSingleStepLangaugeDetection: + return singleStepInstrumentationInstallType + case libInfoSourceLibInjection: + return localLibraryInstrumentationInstallType + default: + return "unknown" } +} + +func (s libInfoSource) isSingleStep() bool { + return s.injectionType() == singleStepInstrumentationInstallType +} + +// isFromLanguageDetection tells us whether this source comes from +// the language detection reporting and annotation. +func (s libInfoSource) isFromLanguageDetection() bool { + return s == libInfoSourceSingleStepLangaugeDetection +} + +func (s libInfoSource) mutatePod(pod *corev1.Pod) error { + _ = mutatecommon.InjectEnv(pod, corev1.EnvVar{ + Name: instrumentationInstallTypeEnvVarName, + Value: s.injectionType(), + }) + return nil +} + +type extractedPodLibInfo struct { + // libs are the libraries we are going to attempt to inject into the given pod. + libs []libInfo + // languageDetection is set when we ran/used the language-detection annotation. + languageDetection *libInfoLanguageDetection + // source is where we got the data from, used for telemetry later. + source libInfoSource +} + +func (e extractedPodLibInfo) withLibs(l []libInfo) extractedPodLibInfo { + e.libs = l + return e +} + +func (e extractedPodLibInfo) useLanguageDetectionLibs() (extractedPodLibInfo, bool) { + if e.languageDetection != nil && len(e.languageDetection.libs) > 0 && e.languageDetection.injectionEnabled { + e.libs = e.languageDetection.libs + e.source = libInfoSourceSingleStepLangaugeDetection + return e, true + } + + return e, false +} + +func (w *Webhook) initExtractedLibInfo(pod *corev1.Pod) extractedPodLibInfo { + // it's possible to get here without single step being enabled, and the pod having + // annotations on it to opt it into pod mutation, we disambiguate those two cases. + var ( + source = libInfoSourceLibInjection + languageDetection *libInfoLanguageDetection + ) - // If auto-instrumentation is enabled in the namespace and nothing has been overridden, - // - // 1. We check for pinned libraries in the config - // 2. We check for language detection (if enabled) - // 3. We fall back to "latest" if w.isEnabledInNamespace(pod.Namespace) { - if len(w.pinnedLibraries) > 0 { - return w.pinnedLibraries, false - } + source = libInfoSourceSingleStepInstrumentation + languageDetection = w.getLibrariesLanguageDetection(pod) + } - detected := w.getLibrariesLanguageDetection(pod) - if len(detected) > 0 { - return detected, true - } + return extractedPodLibInfo{ + source: source, + languageDetection: languageDetection, + } +} + +// extractLibInfo metadata about what library information we should be +// injecting into the pod and where it came from. +func (w *Webhook) extractLibInfo(pod *corev1.Pod) extractedPodLibInfo { + extracted := w.initExtractedLibInfo(pod) + + libs := w.extractLibrariesFromAnnotations(pod) + if len(libs) > 0 { + return extracted.withLibs(libs) + } + + // if the user has pinned libraries for their configuration, + // we prefer to use these and not override their behavior. + // + // N.B. this is empty if auto-instrumentation is disabled. + if len(w.pinnedLibraries) > 0 { + return extracted.withLibs(w.pinnedLibraries) + } - return w.getAllLatestLibraries(), false + // if the language_detection injection is enabled + // (and we have things to filter to) we use that! + if e, usingLanguageDetection := extracted.useLanguageDetectionLibs(); usingLanguageDetection { + return e + } + + if extracted.source.isSingleStep() { + return extracted.withLibs(w.getAllLatestLibraries()) } // Get libraries to inject for Remote Instrumentation @@ -370,10 +503,11 @@ func (w *Webhook) extractLibInfo(pod *corev1.Pod) ([]libInfo, bool) { if version != "latest" { log.Warnf("Ignoring version %q. To inject all libs, the only supported version is latest for now", version) } - return w.getAllLatestLibraries(), false + + return extracted.withLibs(w.getAllLatestLibraries()) } - return nil, false + return extractedPodLibInfo{} } // getAutoDetectedLibraries constructs the libraries to be injected if the languages @@ -427,14 +561,14 @@ func (w *Webhook) extractLibrariesFromAnnotations(pod *corev1.Pod) []libInfo { return libList } -func (w *Webhook) initContainerMutators() []containerMutator { - return []containerMutator{ +func (w *Webhook) initContainerMutators() containerMutators { + return containerMutators{ containerResourceRequirements{w.initResourceRequirements}, containerSecurityContext{w.initSecurityContext}, } } -func (w *Webhook) newInjector(startTime time.Time, pod *corev1.Pod) *injector { +func (w *Webhook) newInjector(startTime time.Time, pod *corev1.Pod) podMutator { var opts []injectorOption for _, e := range []annotationExtractor[injectorOption]{ injectorVersionAnnotationExtractor, @@ -450,34 +584,46 @@ func (w *Webhook) newInjector(startTime time.Time, pod *corev1.Pod) *injector { opts = append(opts, opt) } - return newInjector(startTime, w.containerRegistry, w.injectorImageTag, opts...) + return newInjector(startTime, w.containerRegistry, w.injectorImageTag, opts...). + podMutator(w.version) } -func (w *Webhook) injectAutoInstruConfig(pod *corev1.Pod, libsToInject []libInfo, autoDetected bool, injectionType string) error { - if len(libsToInject) == 0 { +func (w *Webhook) injectAutoInstruConfig(pod *corev1.Pod, config extractedPodLibInfo) error { + if len(config.libs) == 0 { return nil } - // inject the env for the pod. - _ = mutatecommon.InjectEnv(pod, localLibraryInstrumentationInstallTypeEnvVar) - var ( - lastError error - configInjector = &libConfigInjector{} + lastError error + configInjector = &libConfigInjector{} + injectionType = config.source.injectionType() + autoDetected = config.source.isFromLanguageDetection() + + injector = w.newInjector(time.Now(), pod) initContainerMutators = w.initContainerMutators() - apmInjector = w.newInjector(time.Now(), pod) + containerMutators = containerMutators{ + config.languageDetection.containerMutator(w.version), + } ) - for _, lib := range libsToInject { + // Inject env variables used for Onboarding KPIs propagation... + // if Single Step Instrumentation is enabled, inject DD_INSTRUMENTATION_INSTALL_TYPE:k8s_single_step + // if local library injection is enabled, inject DD_INSTRUMENTATION_INSTALL_TYPE:k8s_lib_injection + if err := config.source.mutatePod(pod); err != nil { + return err + } + + for _, lib := range config.libs { injected := false langStr := string(lib.lang) defer func() { metrics.LibInjectionAttempts.Inc(langStr, strconv.FormatBool(injected), strconv.FormatBool(autoDetected), injectionType) }() - if err := lib.podMutator(w.version, initContainerMutators, []podMutator{ - configInjector.podMutator(lib.lang), - apmInjector.podMutator(w.version), + if err := lib.podMutator(w.version, libRequirementOptions{ + containerMutators: containerMutators, + initContainerMutators: initContainerMutators, + podMutators: []podMutator{configInjector.podMutator(lib.lang), injector}, }).mutatePod(pod); err != nil { metrics.LibInjectionErrors.Inc(langStr, strconv.FormatBool(autoDetected), injectionType) lastError = err diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go index 5c5425073fee6..174cd52642fb6 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go @@ -39,11 +39,14 @@ const commonRegistry = "gcr.io/datadoghq" func TestInjectAutoInstruConfigV2(t *testing.T) { c := configmock.New(t) + tests := []struct { name string pod *corev1.Pod + libInfo extractedPodLibInfo expectedInjectorImage string - libsToInject []libInfo + expectedLangsDetected string + expectedInstallType string wantErr bool config func() }{ @@ -55,17 +58,21 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { name: "nominal case: java", pod: common.FakePod("java-pod"), expectedInjectorImage: commonRegistry + "/apm-inject:0", - libsToInject: []libInfo{ - java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + libInfo: extractedPodLibInfo{ + libs: []libInfo{ + java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + }, }, }, { name: "nominal case: java & python", pod: common.FakePod("java-pod"), expectedInjectorImage: commonRegistry + "/apm-inject:0", - libsToInject: []libInfo{ - java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), - python.libInfo("", "gcr.io/datadoghq/dd-lib-python-init:v1"), + libInfo: extractedPodLibInfo{ + libs: []libInfo{ + java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + python.libInfo("", "gcr.io/datadoghq/dd-lib-python-init:v1"), + }, }, }, { @@ -76,8 +83,10 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { }, }.Create(), expectedInjectorImage: commonRegistry + "/apm-inject:v0", - libsToInject: []libInfo{ - java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + libInfo: extractedPodLibInfo{ + libs: []libInfo{ + java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + }, }, }, { @@ -88,16 +97,89 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { }, }.Create(), expectedInjectorImage: "docker.io/library/apm-inject-package:v27", - libsToInject: []libInfo{ - java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + libInfo: extractedPodLibInfo{ + libs: []libInfo{ + java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + }, }, }, { name: "config injector-image-override", pod: common.FakePod("java-pod"), expectedInjectorImage: "gcr.io/datadoghq/apm-inject:0.16-1", - libsToInject: []libInfo{ - java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + libInfo: extractedPodLibInfo{ + libs: []libInfo{ + java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + }, + }, + config: func() { + c.SetWithoutSource("apm_config.instrumentation.injector_image_tag", "0.16-1") + }, + }, + { + name: "config language detected env vars", + pod: common.FakePod("java-pod"), + expectedInjectorImage: "gcr.io/datadoghq/apm-inject:0.16-1", + expectedLangsDetected: "python", + expectedInstallType: "k8s_lib_injection", + libInfo: extractedPodLibInfo{ + languageDetection: &libInfoLanguageDetection{ + libs: []libInfo{ + python.defaultLibInfo(commonRegistry, "java-pod-container"), + }, + }, + libs: []libInfo{ + java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + }, + source: libInfoSourceLibInjection, + }, + config: func() { + c.SetWithoutSource("apm_config.instrumentation.injector_image_tag", "0.16-1") + }, + }, + { + name: "language detected for a different container", + pod: common.FakePod("java-pod"), + expectedInjectorImage: "gcr.io/datadoghq/apm-inject:0", + expectedLangsDetected: "", + libInfo: extractedPodLibInfo{ + languageDetection: &libInfoLanguageDetection{ + libs: []libInfo{ + python.defaultLibInfo(commonRegistry, "not-java-pod-container"), + }, + }, + libs: []libInfo{ + java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + }, + }, + }, + { + name: "language detected but no languages found", + pod: common.FakePod("java-pod"), + expectedInjectorImage: "gcr.io/datadoghq/apm-inject:0", + expectedLangsDetected: "", + libInfo: extractedPodLibInfo{ + languageDetection: &libInfoLanguageDetection{}, + libs: []libInfo{ + java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + }, + }, + }, + { + name: "with specified install type", + pod: common.FakePod("java-pod"), + expectedInjectorImage: "gcr.io/datadoghq/apm-inject:0.16-1", + expectedLangsDetected: "python", + libInfo: extractedPodLibInfo{ + languageDetection: &libInfoLanguageDetection{ + libs: []libInfo{ + python.defaultLibInfo(commonRegistry, "java-pod-container"), + }, + }, + libs: []libInfo{ + java.libInfo("", "gcr.io/datadoghq/dd-lib-java-init:v1"), + }, + source: libInfoSourceSingleStepLangaugeDetection, }, config: func() { c.SetWithoutSource("apm_config.instrumentation.injector_image_tag", "0.16-1") @@ -107,12 +189,7 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - wmeta := fxutil.Test[workloadmeta.Component]( - t, - core.MockBundle(), - workloadmetafxmock.MockModule(), - fx.Supply(workloadmeta.NewParams()), - ) + wmeta := common.FakeStoreWithDeployment(t, nil) c = configmock.New(t) c.SetWithoutSource("apm_config.instrumentation.version", "v2") @@ -124,7 +201,16 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { require.Equal(t, instrumentationV2, webhook.version) require.True(t, webhook.version.usesInjector()) - err := webhook.injectAutoInstruConfig(tt.pod, tt.libsToInject, false, "") + if tt.libInfo.source == libInfoSourceNone { + tt.libInfo.source = libInfoSourceSingleStepInstrumentation + } + + if tt.expectedInstallType == "" { + tt.expectedInstallType = "k8s_single_step" + } + + err := webhook.injectAutoInstruConfig(tt.pod, tt.libInfo) + if tt.wantErr { require.Error(t, err, "expected injectAutoInstruConfig to error") } else { @@ -143,7 +229,7 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { requireEnv := func(t *testing.T, key string, ok bool, value string) { t.Helper() val, exists := envsByName[key] - require.Equal(t, ok, exists, "expected env %v to be %v", key, ok) + require.Equal(t, ok, exists, "expected env %v exists to = %v", key, ok) require.Equal(t, value, val.Value, "expected env %v = %v", key, val) } @@ -151,7 +237,7 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { requireEnv(t, env.Name, false, "") } - if len(tt.libsToInject) == 0 { + if len(tt.libInfo.libs) == 0 { require.Zero(t, len(tt.pod.Spec.InitContainers), "no libs, no init containers") requireEnv(t, "LD_PRELOAD", false, "") return @@ -160,7 +246,7 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { require.Equal(t, volumeName, tt.pod.Spec.Volumes[0].Name, "expected datadog volume to be injected") - require.Equal(t, len(tt.libsToInject)+1, len(tt.pod.Spec.InitContainers), + require.Equal(t, len(tt.libInfo.libs)+1, len(tt.pod.Spec.InitContainers), "expected there to be one more than the number of libs to inject for init containers") for i, c := range tt.pod.Spec.InitContainers { @@ -174,7 +260,7 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { "expected the container image to be %s", tt.expectedInjectorImage) injectorMountPath = c.VolumeMounts[0].MountPath } else { // lib volumes for each of the rest of the containers - lib := tt.libsToInject[i-1] + lib := tt.libInfo.libs[i-1] require.Equal(t, lib.image, c.Image) require.Equal(t, "opt/datadog/apm/library/"+string(lib.lang), c.VolumeMounts[0].SubPath, "expected a language specific sub-path for the volume mount for lang %s", @@ -213,6 +299,16 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { requireEnv(t, "LD_PRELOAD", true, "/opt/datadog-packages/datadog-apm-inject/stable/inject/launcher.preload.so") requireEnv(t, "DD_INJECT_SENDER_TYPE", true, "k8s") + + requireEnv(t, "DD_INSTRUMENTATION_INSTALL_TYPE", true, tt.expectedInstallType) + + if tt.libInfo.languageDetection == nil { + requireEnv(t, "DD_INSTRUMENTATION_LANGUAGES_DETECTED", false, "") + requireEnv(t, "DD_INSTRUMENTATION_LANGUAGE_DETECTION_INJECTION_ENABLED", false, "") + } else { + requireEnv(t, "DD_INSTRUMENTATION_LANGUAGES_DETECTED", true, tt.expectedLangsDetected) + requireEnv(t, "DD_INSTRUMENTATION_LANGUAGE_DETECTION_INJECTION_ENABLED", true, strconv.FormatBool(tt.libInfo.languageDetection.injectionEnabled)) + } }) } } @@ -430,7 +526,10 @@ func TestInjectAutoInstruConfig(t *testing.T) { c.SetWithoutSource("apm_config.instrumentation.version", "v1") webhook := mustWebhook(t, wmeta) - err := webhook.injectAutoInstruConfig(tt.pod, tt.libsToInject, false, "") + err := webhook.injectAutoInstruConfig(tt.pod, extractedPodLibInfo{ + libs: tt.libsToInject, + source: libInfoSourceLibInjection, + }) if tt.wantErr { require.Error(t, err, "expected injectAutoInstruConfig to error") } else { @@ -560,18 +659,17 @@ func TestExtractLibInfo(t *testing.T) { containerRegistry: "registry", expectedPodEligible: pointer.Ptr(true), expectedLibsToInject: []libInfo{ - { - lang: "python", - image: "registry/dd-lib-python-init:v1", - }, + python.libInfo("", "registry/dd-lib-python-init:v1"), }, }, { - name: "python with unlabelled injection off", - pod: common.FakePodWithAnnotation("admission.datadoghq.com/python-lib.version", "v1"), - containerRegistry: "registry", - expectedPodEligible: pointer.Ptr(false), - expectedLibsToInject: []libInfo{}, + name: "python with unlabelled injection off", + pod: common.FakePodWithAnnotation("admission.datadoghq.com/python-lib.version", "v1"), + containerRegistry: "registry", + expectedPodEligible: pointer.Ptr(false), + expectedLibsToInject: []libInfo{ + python.libInfo("", "registry/dd-lib-python-init:v1"), + }, setupConfig: func() { mockConfig.SetWithoutSource("admission_controller.mutate_unlabelled", false) }, @@ -581,10 +679,7 @@ func TestExtractLibInfo(t *testing.T) { pod: common.FakePodWithAnnotation("admission.datadoghq.com/java-lib.custom-image", "custom/image"), containerRegistry: "registry", expectedLibsToInject: []libInfo{ - { - lang: "java", - image: "custom/image", - }, + java.libInfo("", "custom/image"), }, }, { @@ -870,8 +965,8 @@ func TestExtractLibInfo(t *testing.T) { require.Equal(t, *tt.expectedPodEligible, webhook.isPodEligible(tt.pod)) } - libsToInject, _ := webhook.extractLibInfo(tt.pod) - require.ElementsMatch(t, tt.expectedLibsToInject, libsToInject) + extracted := webhook.extractLibInfo(tt.pod) + require.ElementsMatch(t, tt.expectedLibsToInject, extracted.libs) }) } } @@ -2235,6 +2330,8 @@ func TestInjectAutoInstrumentation(t *testing.T) { wantWebhookInitErr: false, setupConfig: funcs{ wConfig("admission_controller.auto_instrumentation.inject_auto_detected_libraries", true), + wConfig("language_detection.enabled", true), + wConfig("language_detection.reporting.enabled", true), enableAPMInstrumentation, }, }, diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_util.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_util.go index 0dfd2fa786120..29d89b3702c4b 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_util.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_util.go @@ -50,6 +50,12 @@ func getLibListFromDeploymentAnnotations(store workloadmeta.Component, deploymen var libList []libInfo for container, languages := range deployment.InjectableLanguages { for lang := range languages { + // There's a mismatch between language detection and auto-instrumentation. + // The Node language is a js lib. + if lang == "node" { + lang = "js" + } + l := language(lang) libList = append(libList, l.defaultLibInfo(registry, container.Name)) } diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_util_test.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_util_test.go index 1b9b6fb51d557..3909775a8782a 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_util_test.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_util_test.go @@ -71,7 +71,6 @@ func TestGetOwnerNameAndKind(t *testing.T) { require.Equal(t, found, tt.wantFound) require.Equal(t, name, tt.expectedName) require.Equal(t, kind, tt.expectedKind) - }) } } diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/env_vars.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/env_vars.go index 78b508f030b43..f59a97b912d25 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/env_vars.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/env_vars.go @@ -60,18 +60,6 @@ const ( localLibraryInstrumentationInstallType = "k8s_lib_injection" ) -var ( - singleStepInstrumentationInstallTypeEnvVar = corev1.EnvVar{ - Name: instrumentationInstallTypeEnvVarName, - Value: singleStepInstrumentationInstallType, - } - - localLibraryInstrumentationInstallTypeEnvVar = corev1.EnvVar{ - Name: instrumentationInstallTypeEnvVarName, - Value: localLibraryInstrumentationInstallType, - } -) - type envVar struct { key string valFunc envValFunc diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/language_versions.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/language_versions.go index 539057bccd0d2..dccaf9fa8c9f3 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/language_versions.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/language_versions.go @@ -134,7 +134,7 @@ type libInfo struct { image string } -func (i libInfo) podMutator(v version, ics []containerMutator, ms []podMutator) podMutator { +func (i libInfo) podMutator(v version, opts libRequirementOptions) podMutator { return podMutatorFunc(func(pod *corev1.Pod) error { reqs, ok := i.libRequirement(v) if !ok { @@ -144,19 +144,12 @@ func (i libInfo) podMutator(v version, ics []containerMutator, ms []podMutator) ) } - // set the initContainerMutators on the requirements - reqs.initContainerMutators = ics + reqs.libRequirementOptions = opts if err := reqs.injectPod(pod, i.ctrName); err != nil { return err } - for _, m := range ms { - if err := m.mutatePod(pod); err != nil { - return err - } - } - return nil }) } diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/lib_requirement.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/lib_requirement.go index fa232aee5a60c..e3a372b70f31b 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/lib_requirement.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/lib_requirement.go @@ -11,32 +11,42 @@ import ( corev1 "k8s.io/api/core/v1" ) +type libRequirementOptions struct { + initContainerMutators containerMutators + containerMutators containerMutators + podMutators []podMutator +} + type libRequirement struct { - envVars []envVar - volumeMounts []volumeMount - initContainers []initContainer - volumes []volume - initContainerMutators []containerMutator + envVars []envVar + volumeMounts []volumeMount + initContainers []initContainer + volumes []volume + + libRequirementOptions } func (reqs libRequirement) injectPod(pod *corev1.Pod, ctrName string) error { for i, ctr := range pod.Spec.Containers { - if ctrName != "" && ctrName != ctr.Name { - continue - } - for _, v := range reqs.envVars { - if err := v.mutateContainer(&ctr); err != nil { - return err + if ctrName == "" || ctrName == ctr.Name { + for _, v := range reqs.envVars { + if err := v.mutateContainer(&ctr); err != nil { + return err + } } - } - for _, v := range reqs.volumeMounts { - if err := v.mutateContainer(&ctr); err != nil { - return err + for _, v := range reqs.volumeMounts { + if err := v.mutateContainer(&ctr); err != nil { + return err + } } } + if err := reqs.containerMutators.mutateContainer(&ctr); err != nil { + return err + } + pod.Spec.Containers[i] = ctr } @@ -54,5 +64,11 @@ func (reqs libRequirement) injectPod(pod *corev1.Pod, ctrName string) error { } } + for _, m := range reqs.podMutators { + if err := m.mutatePod(pod); err != nil { + return err + } + } + return nil } diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/mutators.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/mutators.go index ce12f12d878fb..d113de8cf95e1 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/mutators.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/mutators.go @@ -19,6 +19,26 @@ type containerMutator interface { mutateContainer(*corev1.Container) error } +// containerMutatorFunc is a containerMutator as a function. +type containerMutatorFunc func(*corev1.Container) error + +// mutateContainer implements containerMutator for containerMutatorFunc. +func (f containerMutatorFunc) mutateContainer(c *corev1.Container) error { + return f(c) +} + +type containerMutators []containerMutator + +func (mutators containerMutators) mutateContainer(c *corev1.Container) error { + for _, m := range mutators { + if err := m.mutateContainer(c); err != nil { + return err + } + } + + return nil +} + // podMutator describes something that can mutate a pod. type podMutator interface { mutatePod(*corev1.Pod) error @@ -40,7 +60,7 @@ func (f podMutatorFunc) mutatePod(pod *corev1.Pod) error { type initContainer struct { corev1.Container Prepend bool - Mutators []containerMutator + Mutators containerMutators } var _ podMutator = (*initContainer)(nil) @@ -48,10 +68,9 @@ var _ podMutator = (*initContainer)(nil) // mutatePod implements podMutator for initContainer. func (i initContainer) mutatePod(pod *corev1.Pod) error { container := i.Container - for _, m := range i.Mutators { - if err := m.mutateContainer(&container); err != nil { - return err - } + + if err := i.Mutators.mutateContainer(&container); err != nil { + return err } for idx, c := range pod.Spec.InitContainers { From 48f4c510b3ed87808f779376f161e2ef24cb322c Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Mon, 19 Aug 2024 15:13:28 +0200 Subject: [PATCH 025/245] [ASCII-2142] Pin Go license version and auto update (#28366) --- .wwhrd.yml | 3 +-- LICENSE-3rdparty.csv | 2 +- tasks/update_go.py | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.wwhrd.yml b/.wwhrd.yml index b293d0db0a899..84cd078577d5b 100644 --- a/.wwhrd.yml +++ b/.wwhrd.yml @@ -45,5 +45,4 @@ exceptions: additional: # list here paths to additional licenses - golang/go: "raw.githubusercontent.com/golang/go/master/LICENSE" - \ No newline at end of file + golang/go: "raw.githubusercontent.com/golang/go/go1.22.6/LICENSE" diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 986ba0d282973..53e28b42dbd4c 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -2692,7 +2692,7 @@ core,golang.org/x/text/unicode/bidi,BSD-3-Clause,Copyright (c) 2009 The Go Autho core,golang.org/x/text/unicode/norm,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved core,golang.org/x/text/width,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved core,golang.org/x/time/rate,BSD-3-Clause,Copyright 2009 The Go Authors -core,golang/go,BSD-Source-Code,Copyright (c) 2009 The Go Authors. All rights reserved. +core,golang/go,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved. core,gomodules.xyz/jsonpatch/v2,Apache-2.0,Copyright (c) 2015 The Authors core,gonum.org/v1/gonum/blas,BSD-3-Clause,Alexander Egurnov | Andrei Blinnikov | Andrew Brampton | Bailey Lissington | Bill Gray | Bill Noon | Brendan Tracey | Brent Pedersen | Bulat Khasanov | Chad Kunde | Chan Kwan Yin | Chih-Wei Chang | Chong-Yeol Nah | Chris Tessum | Christophe Meessen | Christopher Waldon | Clayton Northey | Copyright ©2013 The Gonum Authors. All rights reserved | Dan Kortschak | Dan Lorenc | Daniel Fireman | Dario Heinisch | David Kleiven | David Samborski | Davor Kapsa | DeepMind Technologies | Delaney Gillilan | Dezmond Goff | Dong-hee Na | Dustin Spicuzza | Egon Elbre | Ekaterina Efimova | Eng Zer Jun | Ethan Burns | Ethan Reesor | Evert Lammerts | Evgeny Savinov | Fabian Wickborn | Facundo Gaich | Fazlul Shahriar | Francesc Campoy | Google Inc | Gustaf Johansson | Hossein Zolfi | Iakov Davydov | Igor Mikushkin | Iskander Sharipov | Jalem Raj Rohit | James Bell | James Bowman | James Holmes <32bitkid@gmail.com> | Janne Snabb | Jeremy Atkinson | Jes Cok | Jinesi Yelizati | Jon Richards | Jonas Kahler | Jonas Schulze | Jonathan Bluett-Duncan | Jonathan J Lawlor | Jonathan Reiter | Jonathan Schroeder | Joost van Amersfoort | Jordan Stoker | Joseph Watson | Josh Wilson | Julien Roland | Kai Trukenmüller | Kent English | Kevin C. Zimmerman | Kirill Motkov | Konstantin Shaposhnikov | Leonid Kneller | Lyron Winderbaum | Marco Leogrande | Mark Canning | Mark Skilbeck | Martin Diz | Matthew Connelly | Matthieu Di Mercurio | Max Halford | Maxim Sergeev | Microsoft Corporation | MinJae Kwon | Nathan Edwards | Nick Potts | Nils Wogatzky | Olivier Wulveryck | Or Rikon | Patricio Whittingslow | Patrick DeVivo | Pontus Melke | Renee French | Rishi Desai | Robin Eklind | Roger Welin | Roman Werpachowski | Rondall Jones | Sam Zaydel | Samuel Kelemen | Saran Ahluwalia | Scott Holden | Scott Kiesel | Sebastien Binet | Shawn Smith | Sintela Ltd | Spencer Lyon | Steve McCoy | Taesu Pyo | Takeshi Yoneda | Tamir Hyman | The University of Adelaide | The University of Minnesota | The University of Washington | Thomas Berg | Tobin Harding | Valentin Deleplace | Vincent Thiery | Vladimír Chalupecký | Will Tekulve | Yasuhiro Matsumoto | Yevgeniy Vahlis | Yucheng Zhu | Yunomi | Zoe Juozapaitis | antichris | source{d} core,gonum.org/v1/gonum/blas/blas64,BSD-3-Clause,Alexander Egurnov | Andrei Blinnikov | Andrew Brampton | Bailey Lissington | Bill Gray | Bill Noon | Brendan Tracey | Brent Pedersen | Bulat Khasanov | Chad Kunde | Chan Kwan Yin | Chih-Wei Chang | Chong-Yeol Nah | Chris Tessum | Christophe Meessen | Christopher Waldon | Clayton Northey | Copyright ©2013 The Gonum Authors. All rights reserved | Dan Kortschak | Dan Lorenc | Daniel Fireman | Dario Heinisch | David Kleiven | David Samborski | Davor Kapsa | DeepMind Technologies | Delaney Gillilan | Dezmond Goff | Dong-hee Na | Dustin Spicuzza | Egon Elbre | Ekaterina Efimova | Eng Zer Jun | Ethan Burns | Ethan Reesor | Evert Lammerts | Evgeny Savinov | Fabian Wickborn | Facundo Gaich | Fazlul Shahriar | Francesc Campoy | Google Inc | Gustaf Johansson | Hossein Zolfi | Iakov Davydov | Igor Mikushkin | Iskander Sharipov | Jalem Raj Rohit | James Bell | James Bowman | James Holmes <32bitkid@gmail.com> | Janne Snabb | Jeremy Atkinson | Jes Cok | Jinesi Yelizati | Jon Richards | Jonas Kahler | Jonas Schulze | Jonathan Bluett-Duncan | Jonathan J Lawlor | Jonathan Reiter | Jonathan Schroeder | Joost van Amersfoort | Jordan Stoker | Joseph Watson | Josh Wilson | Julien Roland | Kai Trukenmüller | Kent English | Kevin C. Zimmerman | Kirill Motkov | Konstantin Shaposhnikov | Leonid Kneller | Lyron Winderbaum | Marco Leogrande | Mark Canning | Mark Skilbeck | Martin Diz | Matthew Connelly | Matthieu Di Mercurio | Max Halford | Maxim Sergeev | Microsoft Corporation | MinJae Kwon | Nathan Edwards | Nick Potts | Nils Wogatzky | Olivier Wulveryck | Or Rikon | Patricio Whittingslow | Patrick DeVivo | Pontus Melke | Renee French | Rishi Desai | Robin Eklind | Roger Welin | Roman Werpachowski | Rondall Jones | Sam Zaydel | Samuel Kelemen | Saran Ahluwalia | Scott Holden | Scott Kiesel | Sebastien Binet | Shawn Smith | Sintela Ltd | Spencer Lyon | Steve McCoy | Taesu Pyo | Takeshi Yoneda | Tamir Hyman | The University of Adelaide | The University of Minnesota | The University of Washington | Thomas Berg | Tobin Harding | Valentin Deleplace | Vincent Thiery | Vladimír Chalupecký | Will Tekulve | Yasuhiro Matsumoto | Yevgeniy Vahlis | Yucheng Zhu | Yunomi | Zoe Juozapaitis | antichris | source{d} diff --git a/tasks/update_go.py b/tasks/update_go.py index 89a25453504fb..548aff0374182 100644 --- a/tasks/update_go.py +++ b/tasks/update_go.py @@ -30,6 +30,7 @@ ("./test/fakeintake/docs/README.md", "[Golang ", "]", False), ("./cmd/process-agent/README.md", "`go >= ", "`", False), ("./pkg/logs/launchers/windowsevent/README.md", "install go ", "+,", False), + ("./.wwhrd.yml", "raw.githubusercontent.com/golang/go/go", "/LICENSE", True), ("./docs/public/setup.md", "version `", "` or higher", True), ] From fdcacd9aa220aeb861728de1d5c8c53265f758fd Mon Sep 17 00:00:00 2001 From: Guy Arbitman Date: Mon, 19 Aug 2024 16:19:27 +0300 Subject: [PATCH 026/245] service discovery: Remove use of zap logger (#28456) --- .../corechecks/servicediscovery/apm/detect.go | 30 +++++++++---------- .../servicediscovery/apm/detect_nix_test.go | 6 ++-- .../servicediscovery/language/language.go | 13 +++----- .../language/language_nix_test.go | 12 ++------ .../servicediscovery/service_detector.go | 17 ++++------- .../corechecks/servicediscovery/usm/jboss.go | 12 ++++---- .../servicediscovery/usm/jboss_test.go | 6 ++-- .../corechecks/servicediscovery/usm/jee.go | 11 ++++--- .../servicediscovery/usm/jee_test.go | 6 ++-- .../servicediscovery/usm/laravel_test.go | 3 +- .../corechecks/servicediscovery/usm/nodejs.go | 14 ++++----- .../servicediscovery/usm/nodejs_test.go | 4 +-- .../servicediscovery/usm/php_test.go | 4 +-- .../servicediscovery/usm/python_test.go | 3 +- .../servicediscovery/usm/service.go | 27 +++++++---------- .../servicediscovery/usm/service_test.go | 4 +-- .../corechecks/servicediscovery/usm/spring.go | 8 ++--- .../servicediscovery/usm/spring_test.go | 8 ++--- .../corechecks/servicediscovery/usm/tomcat.go | 8 ++--- .../servicediscovery/usm/tomcat_test.go | 7 ++--- .../servicediscovery/usm/weblogic.go | 8 ++--- .../servicediscovery/usm/weblogic_nix_test.go | 8 ++--- .../servicediscovery/usm/websphere.go | 4 +-- .../servicediscovery/usm/websphere_test.go | 6 ++-- 24 files changed, 90 insertions(+), 139 deletions(-) diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect.go b/pkg/collector/corechecks/servicediscovery/apm/detect.go index 957759933a316..e773dba407358 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect.go @@ -16,10 +16,9 @@ import ( "path/filepath" "strings" - "go.uber.org/zap" - "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/language" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/language/reader" + "github.com/DataDog/datadog-agent/pkg/util/log" ) // Instrumentation represents the state of APM instrumentation for a service. @@ -34,7 +33,7 @@ const ( Injected Instrumentation = "injected" ) -type detector func(logger *zap.Logger, args []string, envs map[string]string) Instrumentation +type detector func(args []string, envs map[string]string) Instrumentation var ( detectorMap = map[language.Language]detector{ @@ -47,7 +46,7 @@ var ( ) // Detect attempts to detect the type of APM instrumentation for the given service. -func Detect(logger *zap.Logger, args []string, envs map[string]string, lang language.Language) Instrumentation { +func Detect(args []string, envs map[string]string, lang language.Language) Instrumentation { // first check to see if the DD_INJECTION_ENABLED is set to tracer if isInjected(envs) { return Injected @@ -55,7 +54,7 @@ func Detect(logger *zap.Logger, args []string, envs map[string]string, lang lang // different detection for provided instrumentation for each if detect, ok := detectorMap[lang]; ok { - return detect(logger, args, envs) + return detect(args, envs) } return None @@ -73,11 +72,11 @@ func isInjected(envs map[string]string) bool { return false } -func rubyDetector(_ *zap.Logger, _ []string, _ map[string]string) Instrumentation { +func rubyDetector(_ []string, _ map[string]string) Instrumentation { return None } -func pythonDetector(logger *zap.Logger, args []string, envs map[string]string) Instrumentation { +func pythonDetector(args []string, envs map[string]string) Instrumentation { /* Check for VIRTUAL_ENV env var if it's there, use $VIRTUAL_ENV/lib/python{}/site-packages/ and see if ddtrace is inside @@ -110,13 +109,12 @@ func pythonDetector(logger *zap.Logger, args []string, envs map[string]string) I // slow option... results, err := exec.Command(args[0], `-c`, `"import sys; print(':'.join(sys.path))"`).Output() if err != nil { - logger.Warn("Failed to execute command", zap.Error(err)) + log.Warn("Failed to execute command", err) return None } results = bytes.TrimSpace(results) parts := strings.Split(string(results), ":") - logger.Debug("parts", zap.Strings("parts", parts)) for _, v := range parts { if strings.HasSuffix(v, "/site-packages") { _, err := os.Stat(v + "/ddtrace") @@ -128,7 +126,7 @@ func pythonDetector(logger *zap.Logger, args []string, envs map[string]string) I return None } -func nodeDetector(logger *zap.Logger, _ []string, envs map[string]string) Instrumentation { +func nodeDetector(_ []string, envs map[string]string) Instrumentation { // check package.json, see if it has dd-trace in it. // first find it wd := "" @@ -137,7 +135,7 @@ func nodeDetector(logger *zap.Logger, _ []string, envs map[string]string) Instru } if wd == "" { // don't know the working directory, just quit - logger.Debug("unable to determine working directory, assuming uninstrumented") + log.Debug("unable to determine working directory, assuming uninstrumented") return None } @@ -150,15 +148,15 @@ func nodeDetector(logger *zap.Logger, _ []string, envs map[string]string) Instru // this error means the file isn't there, so check parent directory if err != nil { if errors.Is(err, os.ErrNotExist) { - logger.Debug("package.json not found", zap.String("path", curPkgJSON)) + log.Debug("package.json not found", curPkgJSON) } else { - logger.Debug("error opening package.json", zap.String("path", curPkgJSON), zap.Error(err)) + log.Debug("error opening package.json", curPkgJSON, err) } continue } offset, err := reader.Index(f, `"dd-trace"`) if err != nil { - logger.Debug("error reading package.json", zap.String("path", curPkgJSON), zap.Error(err)) + log.Debug("error reading package.json", curPkgJSON, err) _ = f.Close() continue } @@ -173,7 +171,7 @@ func nodeDetector(logger *zap.Logger, _ []string, envs map[string]string) Instru return None } -func javaDetector(_ *zap.Logger, args []string, envs map[string]string) Instrumentation { +func javaDetector(args []string, envs map[string]string) Instrumentation { ignoreArgs := map[string]bool{ "-version": true, "-Xshare:dump": true, @@ -222,7 +220,7 @@ func findFile(fileName string) (io.ReadCloser, bool) { const datadogDotNetInstrumented = "Datadog.Trace.ClrProfiler.Native" -func dotNetDetector(_ *zap.Logger, args []string, envs map[string]string) Instrumentation { +func dotNetDetector(args []string, envs map[string]string) Instrumentation { // if it's just the word `dotnet` by itself, don't instrument if len(args) == 1 && args[0] == "dotnet" { return None diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go index 757ec0593881a..8fa660698140b 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go @@ -11,8 +11,6 @@ import ( "os" "strings" "testing" - - "go.uber.org/zap" ) func Test_javaDetector(t *testing.T) { @@ -35,7 +33,7 @@ func Test_javaDetector(t *testing.T) { } for _, d := range data { t.Run(d.name, func(t *testing.T) { - result := javaDetector(zap.NewNop(), d.args, d.envs) + result := javaDetector(d.args, d.envs) if result != d.result { t.Errorf("expected %s got %s", d.result, result) } @@ -85,7 +83,7 @@ func Test_pythonDetector(t *testing.T) { } for _, d := range data { t.Run(d.name, func(t *testing.T) { - result := pythonDetector(zap.NewNop(), d.args, d.envs) + result := pythonDetector(d.args, d.envs) if result != d.result { t.Errorf("expected %s got %s", d.result, result) } diff --git a/pkg/collector/corechecks/servicediscovery/language/language.go b/pkg/collector/corechecks/servicediscovery/language/language.go index 2a35c1db266c5..ac8e738fc72fc 100644 --- a/pkg/collector/corechecks/servicediscovery/language/language.go +++ b/pkg/collector/corechecks/servicediscovery/language/language.go @@ -7,13 +7,12 @@ package language import ( - "fmt" "io" "os" "path" "strings" - "go.uber.org/zap" + "github.com/DataDog/datadog-agent/pkg/util/log" ) // Language represents programming languages. @@ -110,9 +109,8 @@ type Matcher interface { } // New returns a new language Finder. -func New(l *zap.Logger) Finder { +func New() Finder { return Finder{ - Logger: l, Matchers: []Matcher{ PythonScript{}, RubyScript{}, @@ -123,20 +121,17 @@ func New(l *zap.Logger) Finder { // Finder allows to detect the language for a given process. type Finder struct { - Logger *zap.Logger Matchers []Matcher } func (lf Finder) findLang(pi ProcessInfo) Language { lang := FindInArgs(pi.Args) - lf.Logger.Debug("language found", zap.Any("lang", lang)) + log.Debugf("language found: %q", lang) // if we can't figure out a language from the command line, try alternate methods if lang == "" { - lf.Logger.Debug("trying alternate methods") for _, matcher := range lf.Matchers { if matcher.Match(pi) { - lf.Logger.Debug(fmt.Sprintf("%T matched", matcher)) lang = matcher.Language() break } @@ -145,7 +140,7 @@ func (lf Finder) findLang(pi ProcessInfo) Language { return lang } -// FindInArgs trys to detect the language only using the provided command line arguments. +// FindInArgs tries to detect the language only using the provided command line arguments. func FindInArgs(args []string) Language { // empty slice passed in if len(args) == 0 { diff --git a/pkg/collector/corechecks/servicediscovery/language/language_nix_test.go b/pkg/collector/corechecks/servicediscovery/language/language_nix_test.go index 4a28138c84e32..79f3d56873aef 100644 --- a/pkg/collector/corechecks/servicediscovery/language/language_nix_test.go +++ b/pkg/collector/corechecks/servicediscovery/language/language_nix_test.go @@ -11,16 +11,10 @@ import ( "os" "strings" "testing" - - "go.uber.org/zap" ) func TestNew(t *testing.T) { - f := New(zap.NewNop()) - // make sure all alternatives are registered - if f.Logger == nil { - t.Error("Logger is nil") - } + f := New() count := 0 for _, v := range f.Matchers { switch v.(type) { @@ -80,7 +74,7 @@ func Test_findInArgs(t *testing.T) { } func TestFinder_findLang(t *testing.T) { - f := New(zap.NewNop()) + f := New() data := []struct { name string pi ProcessInfo @@ -190,7 +184,7 @@ func TestProcessInfoFileReader(t *testing.T) { } func TestFinderDetect(t *testing.T) { - f := New(zap.NewNop()) + f := New() data := []struct { name string args []string diff --git a/pkg/collector/corechecks/servicediscovery/service_detector.go b/pkg/collector/corechecks/servicediscovery/service_detector.go index 3fbae755726bb..7988f4b35485d 100644 --- a/pkg/collector/corechecks/servicediscovery/service_detector.go +++ b/pkg/collector/corechecks/servicediscovery/service_detector.go @@ -10,26 +10,21 @@ import ( "strings" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/apm" - "go.uber.org/zap" - "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/language" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/servicetype" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/usm" - agentzap "github.com/DataDog/datadog-agent/pkg/util/log/zap" + "github.com/DataDog/datadog-agent/pkg/util/log" ) // ServiceDetector defines the service detector to get metadata about services. type ServiceDetector struct { - logger *zap.Logger langFinder language.Finder } // NewServiceDetector creates a new ServiceDetector object. func NewServiceDetector() *ServiceDetector { - logger := zap.New(agentzap.NewZapCore()) return &ServiceDetector{ - logger: logger, - langFinder: language.New(logger), + langFinder: language.New(), } } @@ -65,18 +60,18 @@ func makeFinalName(meta usm.ServiceMetadata) string { // GetServiceName gets the service name based on the command line arguments and // the list of environment variables. func (sd *ServiceDetector) GetServiceName(cmdline []string, env map[string]string) string { - meta, _ := usm.ExtractServiceMetadata(sd.logger, cmdline, env) + meta, _ := usm.ExtractServiceMetadata(cmdline, env) return makeFinalName(meta) } // Detect gets metadata for a service. func (sd *ServiceDetector) Detect(p processInfo) ServiceMetadata { - meta, _ := usm.ExtractServiceMetadata(sd.logger, p.CmdLine, p.Env) + meta, _ := usm.ExtractServiceMetadata(p.CmdLine, p.Env) lang, _ := sd.langFinder.Detect(p.CmdLine, p.Env) svcType := servicetype.Detect(meta.Name, p.Ports) - apmInstr := apm.Detect(sd.logger, p.CmdLine, p.Env, lang) + apmInstr := apm.Detect(p.CmdLine, p.Env, lang) - sd.logger.Debug("name info", zap.String("name", meta.Name), zap.Strings("additional names", meta.AdditionalNames)) + log.Debugf("name info - name: %q; additional names: %v", meta.Name, meta.AdditionalNames) nameSource := "generated" if meta.FromDDService { diff --git a/pkg/collector/corechecks/servicediscovery/usm/jboss.go b/pkg/collector/corechecks/servicediscovery/usm/jboss.go index d1dcde09f6e26..59ad052ba3e05 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/jboss.go +++ b/pkg/collector/corechecks/servicediscovery/usm/jboss.go @@ -13,7 +13,7 @@ import ( "path" "strings" - "go.uber.org/zap" + "github.com/DataDog/datadog-agent/pkg/util/log" ) const ( @@ -95,12 +95,12 @@ func newJbossExtractor(ctx DetectionContext) vendorExtractor { func (j jbossExtractor) findDeployedApps(domainHome string) ([]jeeDeployment, bool) { baseDir, ok := extractJavaPropertyFromArgs(j.cxt.args, jbossHomeDirSysProp) if !ok { - j.cxt.logger.Debug("jboss: unable to extract the home directory") + log.Debug("jboss: unable to extract the home directory") return nil, false } serverName, domainMode := jbossExtractServerName(j.cxt.args) if domainMode && len(serverName) == 0 { - j.cxt.logger.Debug("jboss: domain mode with missing server name") + log.Debug("jboss: domain mode with missing server name") return nil, false } configFile := jbossExtractConfigFileName(j.cxt.args, domainMode) @@ -111,7 +111,7 @@ func (j jbossExtractor) findDeployedApps(domainHome string) ([]jeeDeployment, bo baseDir = path.Join(baseDir, jbossDomainBase) sub, err = j.cxt.fs.Sub(baseDir) if err != nil { - j.cxt.logger.Debug("jboss: cannot open jboss home", zap.String("path", baseDir), zap.Error(err)) + log.Debugf("jboss: cannot open jboss home %q. Err: %v", baseDir, err) return nil, false } deployments, err = jbossDomainFindDeployments(sub, configFile, serverName) @@ -120,13 +120,13 @@ func (j jbossExtractor) findDeployedApps(domainHome string) ([]jeeDeployment, bo baseDir = path.Join(baseDir, jbossStandaloneBase) sub, err = j.cxt.fs.Sub(baseDir) if err != nil { - j.cxt.logger.Debug("jboss: cannot open jboss home", zap.String("path", baseDir), zap.Error(err)) + log.Debugf("jboss: cannot open jboss home %q. Err: %v", baseDir, err) return nil, false } deployments, err = jbossStandaloneFindDeployments(sub, configFile) } if err != nil || len(deployments) == 0 { - j.cxt.logger.Debug("jboss: cannot find deployments", zap.Error(err)) + log.Debugf("jboss: cannot find deployments. Err: %v", err) return nil, false } diff --git a/pkg/collector/corechecks/servicediscovery/usm/jboss_test.go b/pkg/collector/corechecks/servicediscovery/usm/jboss_test.go index 1c551985d0819..c1af292696275 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/jboss_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/jboss_test.go @@ -10,8 +10,6 @@ import ( "testing" "testing/fstest" - "go.uber.org/zap" - "github.com/stretchr/testify/require" ) @@ -209,7 +207,7 @@ func TestJbossExtractWarContextRoot(t *testing.T) { if len(tt.location) > 0 { memFs[tt.location] = &fstest.MapFile{Data: []byte(tt.jbossWebXML)} } - value, ok := newJbossExtractor(NewDetectionContext(zap.NewNop(), nil, nil, nil)).customExtractWarContextRoot(memFs) + value, ok := newJbossExtractor(NewDetectionContext(nil, nil, nil)).customExtractWarContextRoot(memFs) require.Equal(t, tt.expected, value) require.Equal(t, len(value) > 0, ok) }) @@ -394,7 +392,7 @@ func TestJbossFindDeployedApps(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - value, ok := newJbossExtractor(NewDetectionContext(zap.NewNop(), tt.args, nil, tt.fs)).findDeployedApps(tt.domainHome) + value, ok := newJbossExtractor(NewDetectionContext(tt.args, nil, tt.fs)).findDeployedApps(tt.domainHome) require.Equal(t, tt.expected, value) require.Equal(t, len(value) > 0, ok) }) diff --git a/pkg/collector/corechecks/servicediscovery/usm/jee.go b/pkg/collector/corechecks/servicediscovery/usm/jee.go index d971c2a1fd03a..48a822508a71e 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/jee.go +++ b/pkg/collector/corechecks/servicediscovery/usm/jee.go @@ -14,7 +14,7 @@ import ( "path" "strings" - "go.uber.org/zap" + "github.com/DataDog/datadog-agent/pkg/util/log" ) // appserver is an enumeration of application server types @@ -245,14 +245,13 @@ func normalizeContextRoot(contextRoots ...string) []string { // doExtractContextRoots tries to extract context roots for an app, given the vendor and the fs. func (je jeeExtractor) doExtractContextRoots(extractor vendorExtractor, app *jeeDeployment) []string { - je.ctx.logger.Debug("extracting context root for a jee application", zap.String("name", app.name), - zap.String("path", app.path)) + log.Debugf("extracting context root (%q) for a jee application (%q)", app.path, app.name) if len(app.contextRoot) > 0 { return []string{app.contextRoot} } fsCloser, dt, err := vfsAndTypeFromAppPath(app, je.ctx.fs) if err != nil { - je.ctx.logger.Debug("error locating the deployment", zap.Error(err)) + log.Debugf("error locating the deployment: %v", err) if dt == ear { return nil } @@ -268,7 +267,7 @@ func (je jeeExtractor) doExtractContextRoots(extractor vendorExtractor, app *jee if dt == ear { value, err := extractContextRootFromApplicationXML(fsCloser.fs) if err != nil { - je.ctx.logger.Debug("unable to extract context roots from application.xml", zap.Error(err)) + log.Debugf("unable to extract context roots from application.xml: %v", err) return nil } return value @@ -291,7 +290,7 @@ func (je jeeExtractor) extractServiceNamesForJEEServer() []string { if vendor == unknown { return nil } - je.ctx.logger.Debug("running java enterprise service extraction", zap.Stringer("vendor", vendor)) + log.Debugf("running java enterprise service extraction - vendor %q", vendor) // check if able to find which applications are deployed extractorCreator, ok := extractors[vendor] if !ok { diff --git a/pkg/collector/corechecks/servicediscovery/usm/jee_test.go b/pkg/collector/corechecks/servicediscovery/usm/jee_test.go index 6f8b02eec8cc2..5477cb19663c2 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/jee_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/jee_test.go @@ -13,8 +13,6 @@ import ( "testing" "testing/fstest" - "go.uber.org/zap" - "github.com/stretchr/testify/require" ) @@ -110,7 +108,7 @@ com.ibm.wsspi.bootstrap.WSPreLauncher -nosplash -application com.ibm.ws.bootstra for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cmd := strings.Split(strings.ReplaceAll(tt.rawCmd, "\n", " "), " ") - vendor, home := jeeExtractor{NewDetectionContext(zap.NewNop(), cmd, nil, fstest.MapFS{})}.resolveAppServer() + vendor, home := jeeExtractor{NewDetectionContext(cmd, nil, fstest.MapFS{})}.resolveAppServer() require.Equal(t, tt.expectedVendor, vendor) // the base dir is making sense only when the vendor has been properly understood if tt.expectedVendor != unknown { @@ -243,7 +241,7 @@ func TestWeblogicExtractServiceNamesForJEEServer(t *testing.T) { envs := map[string]string{ "PWD": "wls/domain", } - extractor := jeeExtractor{ctx: NewDetectionContext(zap.NewNop(), cmd, envs, memfs)} + extractor := jeeExtractor{ctx: NewDetectionContext(cmd, envs, memfs)} extractedContextRoots := extractor.extractServiceNamesForJEEServer() require.Equal(t, []string{ "app1_context", // taken from ear application.xml diff --git a/pkg/collector/corechecks/servicediscovery/usm/laravel_test.go b/pkg/collector/corechecks/servicediscovery/usm/laravel_test.go index 120c12f8a0158..8aa2b2af0e4b5 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/laravel_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/laravel_test.go @@ -11,7 +11,6 @@ import ( "testing/fstest" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func TestGetLaravelAppNameFromEnv(t *testing.T) { @@ -98,7 +97,7 @@ func TestGetLaravelAppNameFromEnv(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - name := newLaravelParser(NewDetectionContext(zap.NewExample(), nil, nil, tt.filesystem)).GetLaravelAppName("artisan") + name := newLaravelParser(NewDetectionContext(nil, nil, tt.filesystem)).GetLaravelAppName("artisan") require.Equal(t, tt.expected, name) }) } diff --git a/pkg/collector/corechecks/servicediscovery/usm/nodejs.go b/pkg/collector/corechecks/servicediscovery/usm/nodejs.go index 8d0fedf55ddc6..235b3571e6a1e 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/nodejs.go +++ b/pkg/collector/corechecks/servicediscovery/usm/nodejs.go @@ -11,9 +11,9 @@ import ( "path" "strings" - "go.uber.org/zap" - "github.com/valyala/fastjson" + + "github.com/DataDog/datadog-agent/pkg/util/log" ) type nodeDetector struct { @@ -84,21 +84,17 @@ func (n nodeDetector) maybeExtractServiceName(filename string) (string, bool) { return "", false } if !ok { - n.ctx.logger.Debug("skipping package.js because too large", zap.String("filename", filename)) + log.Debugf("skipping package.js (%q) because too large", filename) return "", true // stops here } bytes, err := io.ReadAll(file) if err != nil { - n.ctx.logger.Debug("unable to read a package.js file", - zap.String("filename", filename), - zap.Error(err)) + log.Debugf("unable to read a package.js file (%q). Err: %v", filename, err) return "", true } value, err := fastjson.ParseBytes(bytes) if err != nil { - n.ctx.logger.Debug("unable to parse a package.js file", - zap.String("filename", filename), - zap.Error(err)) + log.Debugf("unable to parse a package.js (%q) file. Err: %v", filename, err) return "", true } return string(value.GetStringBytes("name")), true diff --git a/pkg/collector/corechecks/servicediscovery/usm/nodejs_test.go b/pkg/collector/corechecks/servicediscovery/usm/nodejs_test.go index f75a1df9b86f6..8cbecdb5d67df 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/nodejs_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/nodejs_test.go @@ -8,8 +8,6 @@ package usm import ( "testing" - "go.uber.org/zap" - "github.com/stretchr/testify/assert" ) @@ -35,7 +33,7 @@ func TestFindNameFromNearestPackageJSON(t *testing.T) { expected: "my-awesome-package", }, } - instance := &nodeDetector{ctx: DetectionContext{logger: zap.NewNop(), fs: &RealFs{}}} + instance := &nodeDetector{ctx: DetectionContext{fs: &RealFs{}}} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { value, ok := instance.findNameFromNearestPackageJSON(tt.path) diff --git a/pkg/collector/corechecks/servicediscovery/usm/php_test.go b/pkg/collector/corechecks/servicediscovery/usm/php_test.go index cc246a3d382d4..16260dc33b139 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/php_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/php_test.go @@ -7,8 +7,6 @@ package usm import ( "testing" - - "go.uber.org/zap" ) func TestServiceNameFromCLI(t *testing.T) { @@ -38,7 +36,7 @@ func TestServiceNameFromCLI(t *testing.T) { expected: "", }, } - instance := &phpDetector{ctx: NewDetectionContext(zap.NewExample(), nil, nil, nil)} + instance := &phpDetector{ctx: NewDetectionContext(nil, nil, nil)} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { value, ok := instance.detect(tt.args) diff --git a/pkg/collector/corechecks/servicediscovery/usm/python_test.go b/pkg/collector/corechecks/servicediscovery/usm/python_test.go index a42f761fedea5..953e5071cd539 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/python_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/python_test.go @@ -11,7 +11,6 @@ import ( "testing/fstest" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func TestPythonDetect(t *testing.T) { @@ -75,7 +74,7 @@ func TestPythonDetect(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - value, ok := newPythonDetector(NewDetectionContext(zap.NewNop(), nil, nil, memFs)).detect(strings.Split(tt.cmd, " ")[1:]) + value, ok := newPythonDetector(NewDetectionContext(nil, nil, memFs)).detect(strings.Split(tt.cmd, " ")[1:]) require.Equal(t, tt.expected, value.Name) require.Equal(t, len(value.Name) > 0, ok) }) diff --git a/pkg/collector/corechecks/servicediscovery/usm/service.go b/pkg/collector/corechecks/servicediscovery/usm/service.go index 8bbaa23543624..aad1ba9506b2e 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/service.go +++ b/pkg/collector/corechecks/servicediscovery/usm/service.go @@ -14,8 +14,6 @@ import ( "slices" "strings" "unicode" - - "go.uber.org/zap" ) type detectorCreatorFn func(ctx DetectionContext) detector @@ -74,19 +72,17 @@ func newDotnetDetector(ctx DetectionContext) detector { // DetectionContext allows to detect ServiceMetadata. type DetectionContext struct { - logger *zap.Logger - args []string - envs map[string]string - fs fs.SubFS + args []string + envs map[string]string + fs fs.SubFS } // NewDetectionContext initializes DetectionContext. -func NewDetectionContext(logger *zap.Logger, args []string, envs map[string]string, fs fs.SubFS) DetectionContext { +func NewDetectionContext(args []string, envs map[string]string, fs fs.SubFS) DetectionContext { return DetectionContext{ - logger: logger, - args: args, - envs: envs, - fs: fs, + args: args, + envs: envs, + fs: fs, } } @@ -151,12 +147,11 @@ func checkForInjectionNaming(envs map[string]string) bool { } // ExtractServiceMetadata attempts to detect ServiceMetadata from the given process. -func ExtractServiceMetadata(logger *zap.Logger, args []string, envs map[string]string) (ServiceMetadata, bool) { +func ExtractServiceMetadata(args []string, envs map[string]string) (ServiceMetadata, bool) { dc := DetectionContext{ - logger: logger, - args: args, - envs: envs, - fs: RealFs{}, + args: args, + envs: envs, + fs: RealFs{}, } cmd := dc.args if len(cmd) == 0 || len(cmd[0]) == 0 { diff --git a/pkg/collector/corechecks/servicediscovery/usm/service_test.go b/pkg/collector/corechecks/servicediscovery/usm/service_test.go index fbe39e01bf22e..071392fc755a4 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/service_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/service_test.go @@ -13,8 +13,6 @@ import ( "testing" "github.com/stretchr/testify/require" - - "go.uber.org/zap" ) const ( @@ -495,7 +493,7 @@ func TestExtractServiceMetadata(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - meta, ok := ExtractServiceMetadata(zap.NewNop(), tt.cmdline, tt.envs) + meta, ok := ExtractServiceMetadata(tt.cmdline, tt.envs) if len(tt.expectedServiceTag) == 0 { require.False(t, ok) } else { diff --git a/pkg/collector/corechecks/servicediscovery/usm/spring.go b/pkg/collector/corechecks/servicediscovery/usm/spring.go index 15db47dfd7ed3..4dab88aeaf008 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/spring.go +++ b/pkg/collector/corechecks/servicediscovery/usm/spring.go @@ -14,10 +14,10 @@ import ( "path/filepath" "strings" - "go.uber.org/zap" - "github.com/rickar/props" "github.com/vibrantbyte/go-antpath/antpath" + + "github.com/DataDog/datadog-agent/pkg/util/log" ) const ( @@ -224,7 +224,7 @@ func (s springBootParser) scanSourcesFromFileSystem(profilePatterns map[string][ // a match is found value, err := s.newPropertySourceFromFile(p) if err != nil { - s.ctx.logger.Debug("cannot parse a property source", zap.String("filename", p), zap.Error(err)) + log.Debugf("cannot parse a property source (filename: %q). Err: %v", p, err) return nil } arr, ok := ret[profile] @@ -306,7 +306,7 @@ func (s springBootParser) GetSpringBootAppName(jarname string) (string, bool) { if !isSpringBootArchive(reader) { return "", false } - s.ctx.logger.Debug("parsing information from spring boot archive", zap.String("filename", jarname)) + log.Debugf("parsing information from spring boot archive: %q", jarname) combined := &props.Combined{Sources: []props.PropertyGetter{ newArgumentSource(s.ctx.args, "--"), diff --git a/pkg/collector/corechecks/servicediscovery/usm/spring_test.go b/pkg/collector/corechecks/servicediscovery/usm/spring_test.go index 48ff856a79e51..ef0e6bb59f9c9 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/spring_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/spring_test.go @@ -13,8 +13,6 @@ import ( "testing" "testing/fstest" - "go.uber.org/zap" - "github.com/stretchr/testify/require" ) @@ -117,7 +115,7 @@ func TestParseUri(t *testing.T) { expectedClassPathLocations: map[string][]string{}, }, } - parser := newSpringBootParser(NewDetectionContext(zap.NewNop(), nil, nil, fstest.MapFS(nil))) + parser := newSpringBootParser(NewDetectionContext(nil, nil, fstest.MapFS(nil))) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fsLocs, cpLocs := parser.parseURI(strings.Split(tt.locations, ";"), tt.configName, tt.profiles, tt.cwd) @@ -210,7 +208,7 @@ func TestArgumentPropertySource(t *testing.T) { func TestScanSourcesFromFileSystem(t *testing.T) { cwd, err := os.Getwd() require.NoError(t, err) - parser := newSpringBootParser(NewDetectionContext(zap.NewNop(), nil, nil, &RealFs{})) + parser := newSpringBootParser(NewDetectionContext(nil, nil, &RealFs{})) fileSources := parser.scanSourcesFromFileSystem(map[string][]string{ "fs": { @@ -395,7 +393,7 @@ func TestExtractServiceMetadataSpringBoot(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - app, ok := newSpringBootParser(NewDetectionContext(zap.NewNop(), tt.cmdline, tt.envs, RealFs{})).GetSpringBootAppName(tt.jarname) + app, ok := newSpringBootParser(NewDetectionContext(tt.cmdline, tt.envs, RealFs{})).GetSpringBootAppName(tt.jarname) require.Equal(t, tt.expected, app) require.Equal(t, len(app) > 0, ok) }) diff --git a/pkg/collector/corechecks/servicediscovery/usm/tomcat.go b/pkg/collector/corechecks/servicediscovery/usm/tomcat.go index 3d895ffcc37b2..587dbc2dc9f9a 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/tomcat.go +++ b/pkg/collector/corechecks/servicediscovery/usm/tomcat.go @@ -11,7 +11,7 @@ import ( "path" "strings" - "go.uber.org/zap" + "github.com/DataDog/datadog-agent/pkg/util/log" ) // tomcat vendor specific constants @@ -80,7 +80,7 @@ func (te tomcatExtractor) findDeployedApps(domainHome string) ([]jeeDeployment, func (te tomcatExtractor) scanDirForDeployments(path string, uniques *map[string]struct{}) []jeeDeployment { entries, err := fs.ReadDir(te.ctx.fs, path) if err != nil { - te.ctx.logger.Debug("error while scanning tomcat deployments", zap.String("appBase", path), zap.Error(err)) + log.Debugf("error while scanning tomcat deployments (app base: %q). Err: %v", path, err) return nil } var ret []jeeDeployment @@ -129,13 +129,13 @@ func (te tomcatExtractor) parseServerXML(domainHome string) *tomcatServerXML { xmlFilePath := path.Join(domainHome, serverXMLPath) file, err := te.ctx.fs.Open(xmlFilePath) if err != nil { - te.ctx.logger.Debug("Unable to locate tomcat server.xml", zap.String("filepath", xmlFilePath), zap.Error(err)) + log.Debugf("Unable to locate tomcat server.xml (%q). Err: %v", xmlFilePath, err) return nil } var serverXML tomcatServerXML err = xml.NewDecoder(file).Decode(&serverXML) if err != nil { - te.ctx.logger.Debug("Unable to parse tomcat server.xml", zap.String("filepath", xmlFilePath), zap.Error(err)) + log.Debugf("Unable to parse tomcat server.xml (%q). Err: %v", xmlFilePath, err) return nil } return &serverXML diff --git a/pkg/collector/corechecks/servicediscovery/usm/tomcat_test.go b/pkg/collector/corechecks/servicediscovery/usm/tomcat_test.go index 53b12ca0e3f45..e28d630f0bfbd 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/tomcat_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/tomcat_test.go @@ -11,7 +11,6 @@ import ( "testing/fstest" "github.com/stretchr/testify/require" - "go.uber.org/zap" ) func TestTomcatDefaultContextRootFromFile(t *testing.T) { @@ -48,7 +47,7 @@ func TestTomcatDefaultContextRootFromFile(t *testing.T) { expected: "", }, } - extractor := newTomcatExtractor(NewDetectionContext(zap.NewNop(), nil, nil, nil)) + extractor := newTomcatExtractor(NewDetectionContext(nil, nil, nil)) for _, tt := range tests { t.Run("Should parse "+tt.filename, func(t *testing.T) { value, ok := extractor.defaultContextRootFromFile(tt.filename) @@ -93,7 +92,7 @@ func TestScanDirForDeployments(t *testing.T) { }, }, } - extractor := tomcatExtractor{ctx: NewDetectionContext(zap.NewNop(), nil, nil, memfs)} + extractor := tomcatExtractor{ctx: NewDetectionContext(nil, nil, memfs)} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { deployments := extractor.scanDirForDeployments(tt.path, &map[string]struct{}{ @@ -170,7 +169,7 @@ func TestFindDeployedApps(t *testing.T) { } for _, tt := range tests { - extractor := tomcatExtractor{ctx: NewDetectionContext(zap.NewNop(), nil, nil, tt.fs)} + extractor := tomcatExtractor{ctx: NewDetectionContext(nil, nil, tt.fs)} deployments, ok := extractor.findDeployedApps(tt.domainHome) require.Equal(t, len(tt.expected) > 0, ok) require.Equal(t, tt.expected, deployments) diff --git a/pkg/collector/corechecks/servicediscovery/usm/weblogic.go b/pkg/collector/corechecks/servicediscovery/usm/weblogic.go index 86483ecfe8219..a4e75b7d0e646 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/weblogic.go +++ b/pkg/collector/corechecks/servicediscovery/usm/weblogic.go @@ -10,7 +10,7 @@ import ( "io/fs" "path" - "go.uber.org/zap" + "github.com/DataDog/datadog-agent/pkg/util/log" ) // weblogic vendor specific constants @@ -59,19 +59,19 @@ func (we weblogicExtractor) findDeployedApps(domainHome string) ([]jeeDeployment } serverConfigFile, err := we.ctx.fs.Open(path.Join(domainHome, wlsServerConfigDir, wlsServerConfigFile)) if err != nil { - we.ctx.logger.Debug("weblogic: unable to open config.xml", zap.Error(err)) + log.Debugf("weblogic: unable to open config.xml. Err: %v", err) return nil, false } defer serverConfigFile.Close() if ok, err := canSafelyParse(serverConfigFile); !ok { - we.ctx.logger.Debug("weblogic: config.xml looks too big", zap.Error(err)) + log.Debugf("weblogic: config.xml looks too big. Err: %v", err) return nil, false } var deployInfos weblogicDeploymentInfo err = xml.NewDecoder(serverConfigFile).Decode(&deployInfos) if err != nil { - we.ctx.logger.Debug("weblogic: cannot parse config.xml", zap.Error(err)) + log.Debugf("weblogic: cannot parse config.xml. Err: %v", err) return nil, false } var deployments []jeeDeployment diff --git a/pkg/collector/corechecks/servicediscovery/usm/weblogic_nix_test.go b/pkg/collector/corechecks/servicediscovery/usm/weblogic_nix_test.go index 4400fdea2f24a..0107ec9010cc5 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/weblogic_nix_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/weblogic_nix_test.go @@ -16,8 +16,6 @@ import ( "testing" "testing/fstest" - "go.uber.org/zap" - "github.com/stretchr/testify/require" ) @@ -80,7 +78,7 @@ func TestWeblogicFindDeployedApps(t *testing.T) { if len(tt.serverName) > 0 { args = append(args, wlsServerNameSysProp+tt.serverName) } - value, ok := weblogicExtractor{ctx: NewDetectionContext(zap.NewNop(), args, nil, tt.fs)}.findDeployedApps(tt.domainHome) + value, ok := weblogicExtractor{ctx: NewDetectionContext(args, nil, tt.fs)}.findDeployedApps(tt.domainHome) require.Equal(t, len(value) > 0, ok) require.Equal(t, tt.expected, value) }) @@ -138,7 +136,7 @@ http://xmlns.oracle.com/weblogic/weblogic-web-app/1.4/weblogic-web-app.xsd">inva // now create a zip reader to pass to the tested function reader, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) require.NoError(t, err) - value, ok := newWeblogicExtractor(NewDetectionContext(zap.NewNop(), nil, nil, nil)).customExtractWarContextRoot(reader) + value, ok := newWeblogicExtractor(NewDetectionContext(nil, nil, nil)).customExtractWarContextRoot(reader) require.Equal(t, len(tt.expected) > 0, ok) require.Equal(t, tt.expected, value) }) @@ -151,7 +149,7 @@ func TestWeblogicExtractExplodedWarContextRoot(t *testing.T) { cwd, err := os.Getwd() require.NoError(t, err) fs := os.DirFS(path.Join(cwd, weblogicTestAppRoot, "test.war")) - value, ok := newWeblogicExtractor(NewDetectionContext(zap.NewNop(), nil, nil, nil)).customExtractWarContextRoot(fs) + value, ok := newWeblogicExtractor(NewDetectionContext(nil, nil, nil)).customExtractWarContextRoot(fs) require.True(t, ok) require.Equal(t, "my_context", value) } diff --git a/pkg/collector/corechecks/servicediscovery/usm/websphere.go b/pkg/collector/corechecks/servicediscovery/usm/websphere.go index c2731f7dc3339..52f58931703bc 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/websphere.go +++ b/pkg/collector/corechecks/servicediscovery/usm/websphere.go @@ -11,7 +11,7 @@ import ( "io/fs" "path" - "go.uber.org/zap" + "github.com/DataDog/datadog-agent/pkg/util/log" ) type ( @@ -101,7 +101,7 @@ func (we websphereExtractor) findDeployedApps(domainHome string) ([]jeeDeploymen dt: ear, }) } else if err != nil { - we.ctx.logger.Debug("websphere: unable to know if an application is deployed", zap.String("path", m), zap.Error(err)) + log.Debugf("websphere: unable to know if an application is deployed (path %q). Err: %v", m, err) } } return apps, len(apps) > 0 diff --git a/pkg/collector/corechecks/servicediscovery/usm/websphere_test.go b/pkg/collector/corechecks/servicediscovery/usm/websphere_test.go index e99d700f3a0be..fd7a476d8eb9f 100644 --- a/pkg/collector/corechecks/servicediscovery/usm/websphere_test.go +++ b/pkg/collector/corechecks/servicediscovery/usm/websphere_test.go @@ -10,8 +10,6 @@ import ( "testing" "testing/fstest" - "go.uber.org/zap" - "github.com/stretchr/testify/require" ) @@ -112,7 +110,7 @@ func TestWebsphereFindDeployedApps(t *testing.T) { &fstest.MapFile{Data: []byte(tt.deploymentXML)} } - value, ok := newWebsphereExtractor(NewDetectionContext(zap.NewNop(), tt.args, nil, fs)).findDeployedApps("base") + value, ok := newWebsphereExtractor(NewDetectionContext(tt.args, nil, fs)).findDeployedApps("base") require.Equal(t, tt.expected, value) require.Equal(t, len(value) > 0, ok) }) @@ -120,7 +118,7 @@ func TestWebsphereFindDeployedApps(t *testing.T) { } func TestWebsphereDefaultContextRootFromFile(t *testing.T) { - value, ok := newWebsphereExtractor(NewDetectionContext(zap.NewNop(), nil, nil, nil)).defaultContextRootFromFile("myapp.war") + value, ok := newWebsphereExtractor(NewDetectionContext(nil, nil, nil)).defaultContextRootFromFile("myapp.war") require.Equal(t, "myapp", value) require.True(t, ok) } From 35a12f6c1789de72289c74a3f627ce0205bb70cd Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Mon, 19 Aug 2024 15:45:25 +0200 Subject: [PATCH 027/245] discovery: Get injected environment variables (#28433) Co-authored-by: Jon Bodner Co-authored-by: Guy Arbitman --- .../servicediscovery/module/envs.go | 115 ++++++++++++++++++ .../servicediscovery/module/envs_test.go | 109 +++++++++++++++++ .../servicediscovery/module/impl_linux.go | 20 +-- .../module/impl_linux_test.go | 19 +++ 4 files changed, 245 insertions(+), 18 deletions(-) create mode 100644 pkg/collector/corechecks/servicediscovery/module/envs.go create mode 100644 pkg/collector/corechecks/servicediscovery/module/envs_test.go diff --git a/pkg/collector/corechecks/servicediscovery/module/envs.go b/pkg/collector/corechecks/servicediscovery/module/envs.go new file mode 100644 index 0000000000000..08f5c54994565 --- /dev/null +++ b/pkg/collector/corechecks/servicediscovery/module/envs.go @@ -0,0 +1,115 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +package module + +import ( + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/DataDog/datadog-agent/pkg/util/kernel" + "github.com/shirou/gopsutil/v3/process" +) + +const ( + // injectorMemFdName is the name the injector (Datadog/auto_inject) uses. + injectorMemFdName = "dd_environ" + injectorMemFdPath = "/memfd:" + injectorMemFdName + " (deleted)" + + // memFdMaxSize is used to limit the amount of data we read from the memfd. + // This is for safety to limit our memory usage in the case of a corrupt + // file. + memFdMaxSize = 4096 +) + +// readEnvsFile reads the env file created by the auto injector. The file +// contains the variables in a format similar to /proc/$PID/environ: ENV=VAL, +// separated by \000. +func readEnvsFile(path string) ([]string, error) { + reader, err := os.Open(path) + if err != nil { + return nil, err + } + defer reader.Close() + + data, err := io.ReadAll(io.LimitReader(reader, memFdMaxSize)) + if err != nil { + return nil, err + } + + return strings.Split(string(data), "\000"), nil +} + +// getInjectedEnvs gets environment variables injected by the auto injector, if +// present. The auto injector creates a memfd file with a specific name into which +// it writes the environment variables. In order to find the correct file, we +// need to iterate the list of files (named after file descriptor numbers) in +// /proc/$PID/fd and get the name from the target of the symbolic link. +// +// ``` +// $ ls -l /proc/1750097/fd/ +// total 0 +// lrwx------ 1 foo foo 64 Aug 13 14:24 0 -> /dev/pts/6 +// lrwx------ 1 foo foo 64 Aug 13 14:24 1 -> /dev/pts/6 +// lrwx------ 1 foo foo 64 Aug 13 14:24 2 -> /dev/pts/6 +// lrwx------ 1 foo foo 64 Aug 13 14:24 3 -> '/memfd:dd_environ (deleted)' +// ``` +func getInjectedEnvs(proc *process.Process) []string { + fdsPath := kernel.HostProc(strconv.Itoa(int(proc.Pid)), "fd") + entries, err := os.ReadDir(fdsPath) + if err != nil { + return nil + } + + for _, entry := range entries { + path := filepath.Join(fdsPath, entry.Name()) + name, err := os.Readlink(path) + if err != nil { + continue + } + + if name != injectorMemFdPath { + continue + } + + envs, _ := readEnvsFile(path) + return envs + } + + return nil +} + +// envsToMap splits a list of strings containing environment variables of the +// format NAME=VAL to a map. +func envsToMap(envs ...string) map[string]string { + envMap := make(map[string]string, len(envs)) + for _, env := range envs { + name, val, found := strings.Cut(env, "=") + if !found { + continue + } + + envMap[name] = val + } + + return envMap +} + +// getEnvs gets the environment variables for the process, both the initial +// ones, and if present, the ones injected via the auto injector. +func getEnvs(proc *process.Process) (map[string]string, error) { + envs, err := proc.Environ() + if err != nil { + return nil, err + } + + envs = append(envs, getInjectedEnvs(proc)...) + return envsToMap(envs...), nil +} diff --git a/pkg/collector/corechecks/servicediscovery/module/envs_test.go b/pkg/collector/corechecks/servicediscovery/module/envs_test.go new file mode 100644 index 0000000000000..7ef168b963f67 --- /dev/null +++ b/pkg/collector/corechecks/servicediscovery/module/envs_test.go @@ -0,0 +1,109 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +package module + +import ( + "bytes" + "fmt" + "os" + "strings" + "testing" + + "github.com/shirou/gopsutil/v3/process" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestInjectedEnvBasic(t *testing.T) { + curPid := os.Getpid() + proc, err := process.NewProcess(int32(curPid)) + require.NoError(t, err) + envs := getInjectedEnvs(proc) + require.Nil(t, envs) + + // Provide an injected replacement for some already-present env variable + first := os.Environ()[0] + parts := strings.Split(first, "=") + key := parts[0] + + expected := []string{"key1=val1", "key2=val2", "key3=val3", fmt.Sprint(key, "=", "new")} + createEnvsMemfd(t, expected) + + envMap, err := getEnvs(proc) + require.NoError(t, err) + require.Subset(t, envMap, map[string]string{ + "key1": "val1", + "key2": "val2", + "key3": "val3", + key: "new", + }) +} + +func TestInjectedEnvLimit(t *testing.T) { + env := "A=" + strings.Repeat("A", memFdMaxSize*2) + full := []string{env} + createEnvsMemfd(t, full) + + expected := []string{full[0][:memFdMaxSize]} + + proc, err := process.NewProcess(int32(os.Getpid())) + require.NoError(t, err) + envs := getInjectedEnvs(proc) + require.Equal(t, expected, envs) +} + +// createEnvsMemfd creates an memfd in the current process with the specified +// environment variables in the same way as Datadog/auto_inject. +func createEnvsMemfd(t *testing.T, envs []string) { + t.Helper() + + var b bytes.Buffer + for _, env := range envs { + _, err := b.WriteString(env) + require.NoError(t, err) + + err = b.WriteByte(0) + require.NoError(t, err) + } + + memfd, err := memfile(injectorMemFdName, b.Bytes()) + require.NoError(t, err) + t.Cleanup(func() { unix.Close(memfd) }) +} + +// memfile takes a file name used, and the byte slice containing data the file +// should contain. +// +// name does not need to be unique, as it's used only for debugging purposes. +// +// It is up to the caller to close the returned descriptor. +func memfile(name string, b []byte) (int, error) { + fd, err := unix.MemfdCreate(name, 0) + if err != nil { + return 0, fmt.Errorf("MemfdCreate: %v", err) + } + + err = unix.Ftruncate(fd, int64(len(b))) + if err != nil { + return 0, fmt.Errorf("Ftruncate: %v", err) + } + + data, err := unix.Mmap(fd, 0, len(b), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) + if err != nil { + return 0, fmt.Errorf("Mmap: %v", err) + } + + copy(data, b) + + err = unix.Munmap(data) + if err != nil { + return 0, fmt.Errorf("Munmap: %v", err) + } + + return fd, nil +} diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go index ebfb641727efd..a0b90a834b798 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go @@ -199,22 +199,6 @@ type parsingContext struct { netNsInfo map[uint32]*namespaceInfo } -// envsToMap splits a list of strings containing environment variables of the -// format NAME=VAL to a map. -func envsToMap(envs ...string) map[string]string { - envMap := make(map[string]string, len(envs)) - for _, env := range envs { - name, val, found := strings.Cut(env, "=") - if !found { - continue - } - - envMap[name] = val - } - - return envMap -} - // getServiceName gets the service name for a process using the servicedetector // module. func (s *discovery) getServiceName(proc *process.Process) (string, error) { @@ -223,12 +207,12 @@ func (s *discovery) getServiceName(proc *process.Process) (string, error) { return "", nil } - env, err := proc.Environ() + envs, err := getEnvs(proc) if err != nil { return "", nil } - return s.serviceDetector.GetServiceName(cmdline, envsToMap(env...)), nil + return s.serviceDetector.GetServiceName(cmdline, envs), nil } // getService gets information for a single service. diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go index 37601526954e1..b1b9a02d53d39 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go @@ -294,6 +294,25 @@ func TestServiceName(t *testing.T) { }, 30*time.Second, 100*time.Millisecond) } +func TestInjectedServiceName(t *testing.T) { + url := setupDiscoveryModule(t) + + createEnvsMemfd(t, []string{ + "OTHER_ENV=test", + "DD_SERVICE=injected-service-name", + "YET_ANOTHER_ENV=test", + }) + + listener, err := net.Listen("tcp", "") + require.NoError(t, err) + t.Cleanup(func() { listener.Close() }) + + pid := os.Getpid() + portMap := getServicesMap(t, url) + require.Contains(t, portMap, pid) + require.Equal(t, "injected-service-name", portMap[pid].Name) +} + // Check that we can get listening processes in other namespaces. func TestNamespaces(t *testing.T) { url := setupDiscoveryModule(t) From 7f068e92f61d86dfd35fcfd7b2b8c0e1a775c460 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Mon, 19 Aug 2024 16:25:21 +0200 Subject: [PATCH 028/245] [gitlab] Make AWS credentials error more explicit (#28535) --- tools/ci/aws_ssm_get_wrapper.ps1 | 2 +- tools/ci/aws_ssm_get_wrapper.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/ci/aws_ssm_get_wrapper.ps1 b/tools/ci/aws_ssm_get_wrapper.ps1 index a4f906c8a73d1..158301f4a08c6 100644 --- a/tools/ci/aws_ssm_get_wrapper.ps1 +++ b/tools/ci/aws_ssm_get_wrapper.ps1 @@ -14,7 +14,7 @@ while ($retryCount -lt $maxRetries) { } if ($error -match "Unable to locate credentials") { # See 5th row in https://docs.google.com/spreadsheets/d/1JvdN0N-RdNEeOJKmW_ByjBsr726E3ZocCKU8QoYchAc - Write-Error "Credentials won't be retrieved, no need to retry" + Write-Error "Permanent error: unable to locate AWS credentials, not retrying" exit 1 } diff --git a/tools/ci/aws_ssm_get_wrapper.sh b/tools/ci/aws_ssm_get_wrapper.sh index c9018efdb3f2e..2fb8298145d9b 100755 --- a/tools/ci/aws_ssm_get_wrapper.sh +++ b/tools/ci/aws_ssm_get_wrapper.sh @@ -18,7 +18,7 @@ while [[ $retry_count -lt $max_retries ]]; do fi if [[ "$error" =~ "Unable to locate credentials" ]]; then # See 5th row in https://docs.google.com/spreadsheets/d/1JvdN0N-RdNEeOJKmW_ByjBsr726E3ZocCKU8QoYchAc - >&2 echo "Credentials won't be retrieved, no need to retry" + >&2 echo "Permanent error: unable to locate AWS credentials, not retrying" exit 1 fi retry_count=$((retry_count+1)) From c36a42130301a9e5b4dce0604db61868751eee5a Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Mon, 19 Aug 2024 10:27:22 -0400 Subject: [PATCH 029/245] Tag truncated logs (#28339) Co-authored-by: Jen Gilbert Co-authored-by: Vickenty Fesunov --- pkg/config/setup/config.go | 3 ++ .../internal/decoder/line_handler_test.go | 7 ++++ .../internal/decoder/multiline_handler.go | 9 ++++-- .../internal/decoder/single_line_handler.go | 2 ++ pkg/logs/message/message.go | 10 ++++-- pkg/logs/tailers/file/tailer.go | 12 ++++++- pkg/logs/tailers/file/tailer_test.go | 32 +++++++++++++++++++ .../tag-truncated-logs-7306a23c68a494f8.yaml | 13 ++++++++ 8 files changed, 82 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/tag-truncated-logs-7306a23c68a494f8.yaml diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 0c6fa60a9afca..fd1ccf3164518 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -1506,6 +1506,9 @@ func logsagent(config pkgconfigmodel.Setup) { // Max size in MB to allow for integrations logs files config.BindEnvAndSetDefault("logs_config.integrations_logs_files_max_size", 100) + + // Add a tag to file logs that are truncated by the agent + config.BindEnvAndSetDefault("logs_config.tag_truncated_logs", false) } func vector(config pkgconfigmodel.Setup) { diff --git a/pkg/logs/internal/decoder/line_handler_test.go b/pkg/logs/internal/decoder/line_handler_test.go index e7de9ba2fb9fb..24584506dfa9d 100644 --- a/pkg/logs/internal/decoder/line_handler_test.go +++ b/pkg/logs/internal/decoder/line_handler_test.go @@ -128,6 +128,7 @@ func TestMultiLineHandler(t *testing.T) { output = <-outputChan assert.Equal(t, "3. stringssssssize20...TRUNCATED...", string(output.GetContent())) + assert.True(t, output.ParsingExtra.IsTruncated) assert.Equal(t, len("3. stringssssssize20"), output.RawDataLen) assertNothingInChannel(t, outputChan) @@ -135,6 +136,7 @@ func TestMultiLineHandler(t *testing.T) { output = <-outputChan assert.Equal(t, "...TRUNCATED...con", string(output.GetContent())) + assert.True(t, output.ParsingExtra.IsTruncated) assert.Equal(t, 4, output.RawDataLen) // second line + TRUNCATED too long @@ -143,10 +145,12 @@ func TestMultiLineHandler(t *testing.T) { output = <-outputChan assert.Equal(t, "4. stringssssssize20...TRUNCATED...", string(output.GetContent())) + assert.True(t, output.ParsingExtra.IsTruncated) assert.Equal(t, len("4. stringssssssize20"), output.RawDataLen) output = <-outputChan assert.Equal(t, "...TRUNCATED...continue...TRUNCATED...", string(output.GetContent())) + assert.True(t, output.ParsingExtra.IsTruncated) assert.Equal(t, 9, output.RawDataLen) // continuous too long lines @@ -159,14 +163,17 @@ func TestMultiLineHandler(t *testing.T) { output = <-outputChan assert.Equal(t, "5. stringssssssize20...TRUNCATED...", string(output.GetContent())) + assert.True(t, output.ParsingExtra.IsTruncated) assert.Equal(t, len("5. stringssssssize20"), output.RawDataLen) output = <-outputChan assert.Equal(t, "...TRUNCATED...continu ...TRUNCATED...", string(output.GetContent())) + assert.True(t, output.ParsingExtra.IsTruncated) assert.Equal(t, len(longLineTracingSpaces), output.RawDataLen) output = <-outputChan assert.Equal(t, "...TRUNCATED...end", string(output.GetContent())) + assert.True(t, output.ParsingExtra.IsTruncated) assert.Equal(t, len("end\n"), output.RawDataLen) assertNothingInChannel(t, outputChan) diff --git a/pkg/logs/internal/decoder/multiline_handler.go b/pkg/logs/internal/decoder/multiline_handler.go index dc36002561215..4d0e5a6402edb 100644 --- a/pkg/logs/internal/decoder/multiline_handler.go +++ b/pkg/logs/internal/decoder/multiline_handler.go @@ -29,6 +29,7 @@ type MultiLineHandler struct { flushTimer *time.Timer lineLimit int shouldTruncate bool + isBufferTruncated bool linesLen int status string timestamp string @@ -111,6 +112,7 @@ func (h *MultiLineHandler) process(msg *message.Message) { // the new line is just a remainder, // adding the truncated flag at the beginning of the content h.buffer.Write(message.TruncatedFlag) + h.isBufferTruncated = true } h.buffer.Write(msg.GetContent()) @@ -119,6 +121,7 @@ func (h *MultiLineHandler) process(msg *message.Message) { // the multiline message is too long, it needs to be cut off and send, // adding the truncated flag the end of the content h.buffer.Write(message.TruncatedFlag) + h.isBufferTruncated = true h.sendBuffer() h.shouldTruncate = true } @@ -141,6 +144,7 @@ func (h *MultiLineHandler) sendBuffer() { h.linesLen = 0 h.linesCombined = 0 h.shouldTruncate = false + h.isBufferTruncated = false }() data := bytes.TrimSpace(h.buffer.Bytes()) @@ -156,7 +160,8 @@ func (h *MultiLineHandler) sendBuffer() { telemetry.GetStatsTelemetryProvider().Count(linesCombinedTelemetryMetricName, float64(linesCombined), []string{}) } } - - h.outputFn(message.NewRawMessage(content, h.status, h.linesLen, h.timestamp)) + msg := message.NewRawMessage(content, h.status, h.linesLen, h.timestamp) + msg.ParsingExtra.IsTruncated = h.isBufferTruncated + h.outputFn(msg) } } diff --git a/pkg/logs/internal/decoder/single_line_handler.go b/pkg/logs/internal/decoder/single_line_handler.go index e3192474fe04e..6661edf9b370d 100644 --- a/pkg/logs/internal/decoder/single_line_handler.go +++ b/pkg/logs/internal/decoder/single_line_handler.go @@ -52,6 +52,7 @@ func (h *SingleLineHandler) process(msg *message.Message) { // the new line is just a remainder, // adding the truncated flag at the beginning of the content content = append(message.TruncatedFlag, content...) + msg.ParsingExtra.IsTruncated = true } // how should we detect logs which are too long before rendering them? @@ -63,6 +64,7 @@ func (h *SingleLineHandler) process(msg *message.Message) { // adding the truncated flag the end of the content content = append(content, message.TruncatedFlag...) msg.SetContent(content) // refresh the content in the message + msg.ParsingExtra.IsTruncated = true h.outputFn(msg) // make sure the following part of the line will be cut off as well h.shouldTruncate = true diff --git a/pkg/logs/message/message.go b/pkg/logs/message/message.go index 9f7728f768601..cc46aaf51eb27 100644 --- a/pkg/logs/message/message.go +++ b/pkg/logs/message/message.go @@ -19,6 +19,9 @@ import ( // or/and at the end of every trucated lines. var TruncatedFlag = []byte("...TRUNCATED...") +// TruncatedTag is added to truncated log messages (if enabled). +const TruncatedTag = "truncated" + // EscapedLineFeed is used to escape new line character // for multiline message. // New line character needs to be escaped because they are used @@ -166,9 +169,10 @@ func (m *MessageContent) SetEncoded(content []byte) { // E.g. Timestamp is used by the docker parsers to transmit a tailing offset. type ParsingExtra struct { // Used by docker parsers to transmit an offset. - Timestamp string - IsPartial bool - Tags []string + Timestamp string + IsPartial bool + IsTruncated bool + Tags []string } // ServerlessExtra ships extra information from logs processing in serverless envs. diff --git a/pkg/logs/tailers/file/tailer.go b/pkg/logs/tailers/file/tailer.go index 6a9c675f41ee5..4718ce65c5cee 100644 --- a/pkg/logs/tailers/file/tailer.go +++ b/pkg/logs/tailers/file/tailer.go @@ -344,7 +344,17 @@ func (t *Tailer) forwardMessages() { origin := message.NewOrigin(t.file.Source.UnderlyingSource()) origin.Identifier = identifier origin.Offset = strconv.FormatInt(offset, 10) - origin.SetTags(append(append(t.tags, t.tagProvider.GetTags()...), output.ParsingExtra.Tags...)) + + tags := make([]string, len(t.tags)) + copy(tags, t.tags) + tags = append(tags, t.tagProvider.GetTags()...) + + if output.ParsingExtra.IsTruncated && coreConfig.Datadog().GetBool("logs_config.tag_truncated_logs") { + tags = append(tags, message.TruncatedTag) + } + + tags = append(tags, output.ParsingExtra.Tags...) + origin.SetTags(tags) // Ignore empty lines once the registry offset is updated if len(output.GetContent()) == 0 { continue diff --git a/pkg/logs/tailers/file/tailer_test.go b/pkg/logs/tailers/file/tailer_test.go index 66900b2307ced..affdeb992ab65 100644 --- a/pkg/logs/tailers/file/tailer_test.go +++ b/pkg/logs/tailers/file/tailer_test.go @@ -351,6 +351,38 @@ func (suite *TailerTestSuite) TestBuildTagsFileDir() { }, tags) } +func (suite *TailerTestSuite) TestTruncatedTag() { + coreConfig.Datadog().SetWithoutSource("logs_config.max_message_size_bytes", 3) + coreConfig.Datadog().SetWithoutSource("logs_config.tag_truncated_logs", true) + defer coreConfig.Datadog().SetWithoutSource("logs_config.max_message_size_bytes", coreConfig.DefaultMaxMessageSizeBytes) + defer coreConfig.Datadog().SetWithoutSource("logs_config.tag_truncated_logs", false) + + source := sources.NewLogSource("", &config.LogsConfig{ + Type: config.FileType, + Path: suite.testPath, + }) + sleepDuration := 10 * time.Millisecond + info := status.NewInfoRegistry() + + tailerOptions := &TailerOptions{ + OutputChan: suite.outputChan, + File: NewFile(suite.testPath, source, true), + SleepDuration: sleepDuration, + Decoder: decoder.NewDecoderFromSource(suite.source, info), + Info: info, + } + + suite.tailer = NewTailer(tailerOptions) + suite.tailer.StartFromBeginning() + + _, err := suite.testFile.WriteString("1234\n") + suite.Nil(err) + + msg := <-suite.outputChan + tags := msg.Tags() + suite.Contains(tags, message.TruncatedTag) +} + func (suite *TailerTestSuite) TestMutliLineAutoDetect() { lines := "Jul 12, 2021 12:55:15 PM test message 1\n" lines += "Jul 12, 2021 12:55:15 PM test message 2\n" diff --git a/releasenotes/notes/tag-truncated-logs-7306a23c68a494f8.yaml b/releasenotes/notes/tag-truncated-logs-7306a23c68a494f8.yaml new file mode 100644 index 0000000000000..e562dd9d20ff1 --- /dev/null +++ b/releasenotes/notes/tag-truncated-logs-7306a23c68a494f8.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Added config option ``logs_config.tag_truncated_logs``. When + enabled, file logs will come with a tag ``truncated:true`` if + they were truncated by the Agent. From 64cb95d637d82664d73fe48b91eb1d3345152113 Mon Sep 17 00:00:00 2001 From: Brian Floersch Date: Mon, 19 Aug 2024 11:07:27 -0400 Subject: [PATCH 030/245] V2 Auto Multiline Detection - Add timestamp detection heuristic (#28335) --- cmd/serverless/dependencies_linux_amd64.txt | 1 + cmd/serverless/dependencies_linux_arm64.txt | 1 + pkg/config/setup/config.go | 3 + .../auto_multiline_detection/labeler.go | 7 +- .../timestamp_detector.go | 123 ++++++++ .../timestamp_detector_test.go | 121 ++++++++ .../auto_multiline_detection/token_graph.go | 114 +++++++ .../token_graph_test.go | 60 ++++ .../auto_multiline_detection/tokenizer.go | 277 +++++++----------- .../tokenizer_test.go | 31 +- .../auto_multiline_detection/tokens/tokens.go | 82 ++++++ .../decoder/auto_multiline_handler.go | 4 +- 12 files changed, 646 insertions(+), 178 deletions(-) create mode 100644 pkg/logs/internal/decoder/auto_multiline_detection/timestamp_detector.go create mode 100644 pkg/logs/internal/decoder/auto_multiline_detection/timestamp_detector_test.go create mode 100644 pkg/logs/internal/decoder/auto_multiline_detection/token_graph.go create mode 100644 pkg/logs/internal/decoder/auto_multiline_detection/token_graph_test.go create mode 100644 pkg/logs/internal/decoder/auto_multiline_detection/tokens/tokens.go diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt index 16be4684f5886..dd71baaed30d2 100644 --- a/cmd/serverless/dependencies_linux_amd64.txt +++ b/cmd/serverless/dependencies_linux_amd64.txt @@ -156,6 +156,7 @@ github.com/DataDog/datadog-agent/pkg/logs/client/tcp github.com/DataDog/datadog-agent/pkg/logs/diagnostic github.com/DataDog/datadog-agent/pkg/logs/internal/decoder github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection +github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection/tokens github.com/DataDog/datadog-agent/pkg/logs/internal/framer github.com/DataDog/datadog-agent/pkg/logs/internal/parsers github.com/DataDog/datadog-agent/pkg/logs/internal/parsers/dockerfile diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/dependencies_linux_arm64.txt index 4493cef9488ae..ad4c33cc32177 100644 --- a/cmd/serverless/dependencies_linux_arm64.txt +++ b/cmd/serverless/dependencies_linux_arm64.txt @@ -156,6 +156,7 @@ github.com/DataDog/datadog-agent/pkg/logs/client/tcp github.com/DataDog/datadog-agent/pkg/logs/diagnostic github.com/DataDog/datadog-agent/pkg/logs/internal/decoder github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection +github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection/tokens github.com/DataDog/datadog-agent/pkg/logs/internal/framer github.com/DataDog/datadog-agent/pkg/logs/internal/parsers github.com/DataDog/datadog-agent/pkg/logs/internal/parsers/dockerfile diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index fd1ccf3164518..9a8188b0313e5 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -1471,6 +1471,9 @@ func logsagent(config pkgconfigmodel.Setup) { config.BindEnvAndSetDefault("logs_config.auto_multi_line_default_match_timeout", 30) // Seconds config.BindEnvAndSetDefault("logs_config.auto_multi_line_default_match_threshold", 0.48) + config.BindEnvAndSetDefault("logs_config.auto_multi_line.timestamp_detector_match_threshold", 0.5) + config.BindEnvAndSetDefault("logs_config.auto_multi_line.tokenizer_max_input_bytes", 60) + // If true, the agent looks for container logs in the location used by podman, rather // than docker. This is a temporary configuration parameter to support podman logs until // a more substantial refactor of autodiscovery is made to determine this automatically. diff --git a/pkg/logs/internal/decoder/auto_multiline_detection/labeler.go b/pkg/logs/internal/decoder/auto_multiline_detection/labeler.go index 177bbf313e0f2..3aec9522da3df 100644 --- a/pkg/logs/internal/decoder/auto_multiline_detection/labeler.go +++ b/pkg/logs/internal/decoder/auto_multiline_detection/labeler.go @@ -6,6 +6,8 @@ // Package automultilinedetection contains auto multiline detection and aggregation logic. package automultilinedetection +import "github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection/tokens" + // Label is a label for a log message. type Label uint32 @@ -19,8 +21,9 @@ type messageContext struct { rawMessage []byte // NOTE: tokens can be nil if the heuristic runs before the tokenizer. // Heuristic implementations must check if tokens is nil before using it. - tokens []Token - label Label + tokens []tokens.Token + tokenIndicies []int + label Label } // Heuristic is an interface representing a strategy to label log messages. diff --git a/pkg/logs/internal/decoder/auto_multiline_detection/timestamp_detector.go b/pkg/logs/internal/decoder/auto_multiline_detection/timestamp_detector.go new file mode 100644 index 0000000000000..85c3db0c17447 --- /dev/null +++ b/pkg/logs/internal/decoder/auto_multiline_detection/timestamp_detector.go @@ -0,0 +1,123 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package automultilinedetection contains auto multiline detection and aggregation logic. +package automultilinedetection + +import ( + "github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection/tokens" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// knownTimestampFormats is a list of known timestamp formats used to build the TokenGraph. +// Adding similar or partial duplicate timestamps does not impact accuracy since related +// tokens are inherently deduped in the graph. Be sure to test the accuracy of the heuristic +// after adding new formtas. +var knownTimestampFormats = []string{ + "2024-03-28T13:45:30.123456Z", + "28/Mar/2024:13:45:30", + "Sun, 28 Mar 2024 13:45:30", + "2024-03-28 13:45:30", + "2024-03-28 13:45:30,123", + "02 Jan 06 15:04 MST", + "2024-03-28T14:33:53.743350Z", + "2024-03-28T15:19:38.578639+00:00", + "2024-03-28 15:44:53", + "2024-08-20'T'13:20:10*633+0000", + "2024 Mar 03 05:12:41.211 PDT", + "Jan 21 18:20:11 +0000 2024", + "19/Apr/2024:06:36:15", + "Dec 2, 2024 2:39:58 AM", + "Jun 09 2024 15:28:14", + "Apr 20 00:00:35 2010", + "Sep 28 19:00:00 +0000", + "Mar 16 08:12:04", + "Jul 1 09:00:55", + "2024-10-14T22:11:20+0000", + "2024-07-01T14:59:55.711'+0000'", + "2024-07-01T14:59:55.711Z", + "2024-08-19 12:17:55-0400", + "2024-06-26 02:31:29,573", + "2024/04/12*19:37:50", + "2024 Apr 13 22:08:13.211*PDT", + "2024 Mar 10 01:44:20.392", + "2024-03-10 14:30:12,655+0000", + "2024-02-27 15:35:20.311", + "2024-07-22'T'16:28:55.444", + "2024-11-22'T'10:10:15.455", + "2024-02-11'T'18:31:44", + "2024-10-30*02:47:33:899", + "2024-07-04*13:23:55", + "24-02-11 16:47:35,985 +0000", + "24-06-26 02:31:29,573", + "24-04-19 12:00:17", + "06/01/24 04:11:05", + "08/10/24*13:33:56", + "11/24/2024*05:13:11", + "05/09/2024*08:22:14*612", + "04/23/24 04:34:22 +0000", + "2024/04/25 14:57:42", + "11:42:35.173", + "11:42:35,173", + "23/Apr 11:42:35,173", + "23/Apr/2024:11:42:35", + "23/Apr/2024 11:42:35", + "23-Apr-2024 11:42:35", + "23-Apr-2024 11:42:35.883", + "23 Apr 2024 11:42:35", + "23 Apr 2024 10:32:35*311", + "8/5/2024 3:31:18 AM:234", + "9/28/2024 2:23:15 PM", + "2023-03.28T14-33:53-7430Z", + "2017-05-16_13:53:08", +} + +// staticTokenGraph is never mutated after construction so this is safe to share between all instances of TimestampDetector. +var staticTokenGraph = makeStaticTokenGraph() + +// minimumTokenLength is the minimum number of tokens needed to evaluate a timestamp probability. +// This is not configurable because it has a large impact of the relative accuracy of the heuristic. +// For example, a string 12:30:2017 is tokenized to 5 tokens DD:DD:DDDD which can easily be confused +// with other non timestamp string. Enforcing more tokens to determine a likely timetamp decreases +// the likelihood of a false positive. 8 was chosen by iterative testing using the tests in timestamp_detector_test.go. +var minimumTokenLength = 8 + +func makeStaticTokenGraph() *TokenGraph { + tokenizer := NewTokenizer(100) // 100 is arbitrary, anything larger than the longest knownTimestampFormat is fine. + inputData := make([][]tokens.Token, len(knownTimestampFormats)) + for i, format := range knownTimestampFormats { + tokens, _ := tokenizer.tokenize([]byte(format)) + inputData[i] = tokens + } + return NewTokenGraph(minimumTokenLength, inputData) +} + +// TimestampDetector is a heuristic to detect timestamps. +type TimestampDetector struct { + tokenGraph *TokenGraph + matchThreshold float64 +} + +// NewTimestampDetector returns a new Timestamp detection heuristic. +func NewTimestampDetector(matchThreshold float64) *TimestampDetector { + return &TimestampDetector{ + tokenGraph: staticTokenGraph, + matchThreshold: matchThreshold, + } +} + +// Process checks if a message is likely to be a timestamp. +func (t *TimestampDetector) Process(context *messageContext) bool { + if context.tokens == nil { + log.Error("Tokens are required to detect timestamps") + return true + } + + if t.tokenGraph.MatchProbability(context.tokens).probability > t.matchThreshold { + context.label = startGroup + } + + return true +} diff --git a/pkg/logs/internal/decoder/auto_multiline_detection/timestamp_detector_test.go b/pkg/logs/internal/decoder/auto_multiline_detection/timestamp_detector_test.go new file mode 100644 index 0000000000000..ab6f1b342f5ac --- /dev/null +++ b/pkg/logs/internal/decoder/auto_multiline_detection/timestamp_detector_test.go @@ -0,0 +1,121 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package automultilinedetection contains auto multiline detection and aggregation logic. +package automultilinedetection + +import ( + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/pkg/config" +) + +type testInput struct { + label Label + input string +} + +// Use this dataset to improve the accuracy of the timestamp detector. +// Ideally logs with the start group label should be very close to a 1.0 match probability +// and logs with the aggregate label should be very close to a 0.0 match probability. +var inputs = []testInput{ + // Likely contain timestamps for aggregation + {startGroup, "2021-03-28 13:45:30 App started successfully"}, + {startGroup, "13:45:30 2021-03-28"}, + {startGroup, "foo bar 13:45:30 2021-03-28"}, + {startGroup, "2023-03-28T14:33:53.743350Z App started successfully"}, + {startGroup, "2023-03-27 12:34:56 INFO App started successfully"}, + {startGroup, "2023-03.28T14-33:53-7430Z App started successfully"}, + {startGroup, "Datadog Agent 2023-03.28T14-33:53-7430Z App started successfully"}, + {startGroup, "[2023-03-27 12:34:56] [INFO] App started successfully"}, + {startGroup, "9/28/2022 2:23:15 PM"}, + {startGroup, "2024-05-15 17:04:12,369 - root - DEBUG -"}, + {startGroup, "[2024-05-15T18:03:23.501Z] Info : All routes applied."}, + {startGroup, "2024-05-15 14:03:13 EDT | CORE | INFO | (pkg/logs/tailers/file/tailer.go:353 in forwardMessages) | "}, + {startGroup, "Jun 14 15:16:01 combo sshd(pam_unix)[19939]: authentication failure; logname= uid=0 euid=0 tty=NODEVssh ruser= rhost=123.456.2.4 "}, + {startGroup, "Jul 1 09:00:55 calvisitor-10-105-160-95 kernel[0]: IOThunderboltSwitch<0>(0x0)::listenerCallback -"}, + {startGroup, "[Sun Dec 04 04:47:44 2005] [notice] workerEnv.init() ok /etc/httpd/conf/workers2.properties"}, + {startGroup, "2024/05/16 14:47:42 Datadog Tracer v1.64"}, + {startGroup, "2024/05/16 19:46:15 Datadog Tracer v1.64.0-rc.1 "}, + {startGroup, "127.0.0.1 - - [16/May/2024:19:49:17 +0000]"}, + {startGroup, "127.0.0.1 - - [17/May/2024:13:51:52 +0000] \"GET /probe?debug=1 HTTP/1.1\" 200 0 "}, + {startGroup, "nova-api.log.1.2017-05-16_13:53:08 2017-05-16 00:00:00.008 25746 INFO nova.osapi"}, + + // A case where the timestamp has a non-matching token in the midddle of it. + {startGroup, "acb def 10:10:10 foo 2024-05-15 hijk lmop"}, + + // Likely do not contain timestamps for aggreagtion + {aggregate, "12:30:2017 - info App started successfully"}, + {aggregate, "12:30:20 - info App started successfully"}, + {aggregate, "20171223-22:15:29:606|Step_LSC|30002312|onStandStepChanged 3579"}, + {aggregate, " . a at some log"}, + {aggregate, "abc this 13:45:30 is a log "}, + {aggregate, "abc this 13 45:30 is a log "}, + {aggregate, " [java] 1234-12-12"}, + {aggregate, " at system.com.blah"}, + {aggregate, "Info - this is an info message App started successfully"}, + {aggregate, "[INFO] App started successfully"}, + {aggregate, "[INFO] test.swift:123 App started successfully"}, + {aggregate, "ERROR in | myFile.go:53:123 App started successfully"}, + {aggregate, "a2de9888a8f1fc289547f77d4834e66 - - -] 10.11.10.1 "}, + {aggregate, "'/conf.d/..data/foobar_lifecycle.yaml' "}, + {aggregate, "commit: a2de9888a8f1fc289547f77d4834e669bf993e7e"}, + {aggregate, " auth.handler: auth handler stopped"}, + {aggregate, "10:10:10 foo :10: bar 10:10"}, + {aggregate, "1234-1234-1234-123-21-1"}, +} + +func TestCorrectLabelIsAssigned(t *testing.T) { + tokenizer := NewTokenizer(config.Datadog().GetInt("logs_config.auto_multi_line.tokenizer_max_input_bytes")) + timestampDetector := NewTimestampDetector(config.Datadog().GetFloat64("logs_config.auto_multi_line.timestamp_detector_match_threshold")) + + for _, testInput := range inputs { + context := &messageContext{ + rawMessage: []byte(testInput.input), + label: aggregate, + } + + assert.True(t, tokenizer.Process(context)) + assert.True(t, timestampDetector.Process(context)) + match := timestampDetector.tokenGraph.MatchProbability(context.tokens) + assert.Equal(t, testInput.label, context.label, fmt.Sprintf("input: %s had the wrong label with probability: %f", testInput.input, match.probability)) + + // To assist with debugging and tuning - this prints the probability and an underline of where the input was matched + printMatchUnderline(context, testInput.input, match) + } +} + +func printMatchUnderline(context *messageContext, input string, match MatchContext) { + maxLen := config.Datadog().GetInt("logs_config.auto_multi_line.tokenizer_max_input_bytes") + fmt.Printf("%.2f\t\t%v\n", match.probability, input) + + if match.start == match.end { + return + } + + evalStr := input + if len(input) > maxLen { + evalStr = input[:maxLen] + } + dbgStr := "" + printChar := " " + last := context.tokenIndicies[0] + for i, idx := range context.tokenIndicies { + dbgStr += strings.Repeat(printChar, idx-last) + if i == match.start { + printChar = "^" + } + if i == match.end+1 { + printChar = " " + } + last = idx + } + dbgStr += strings.Repeat(printChar, len(evalStr)-last) + fmt.Printf("\t\t\t%v\n", dbgStr) +} diff --git a/pkg/logs/internal/decoder/auto_multiline_detection/token_graph.go b/pkg/logs/internal/decoder/auto_multiline_detection/token_graph.go new file mode 100644 index 0000000000000..f9c65fcbe8005 --- /dev/null +++ b/pkg/logs/internal/decoder/auto_multiline_detection/token_graph.go @@ -0,0 +1,114 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package automultilinedetection contains auto multiline detection and aggregation logic. +package automultilinedetection + +import "github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection/tokens" + +// TokenGraph is a directed cyclic graph of tokens that model the relationship between any two tokens. +// It is used to calculate the probability of an unknown sequence of tokens being represented by the graph. +type TokenGraph struct { + adjacencies [][]bool + minimumTokenLength int +} + +// MatchContext is the context of a match. +type MatchContext struct { + probability float64 + // start and end are the indices of the token subsequence that produced the highest probability. + start int + end int +} + +// NewTokenGraph returns a new TokenGraph. +func NewTokenGraph(minimumTokenLength int, inputData [][]tokens.Token) *TokenGraph { + g := &TokenGraph{ + adjacencies: make([][]bool, tokens.End), + minimumTokenLength: minimumTokenLength, + } + for _, tokens := range inputData { + g.add(tokens) + } + return g +} + +// add adds a sequence of tokens to the graph. +func (m *TokenGraph) add(ts []tokens.Token) { + lastToken := ts[0] + for _, token := range ts[1:] { + if m.adjacencies[lastToken] == nil { + m.adjacencies[lastToken] = make([]bool, tokens.End) + } + m.adjacencies[lastToken][token] = true + lastToken = token + } +} + +// MatchProbability returns the probability of a sequence of tokens being represented by the graph. +func (m *TokenGraph) MatchProbability(ts []tokens.Token) MatchContext { + if len(ts) < m.minimumTokenLength { + return MatchContext{} + } + + lastToken := ts[0] + // A function used by maxSubsequence to look up a match in the graph for a pair of tokens. + matchForIndex := func(idx int) int { + match := -1 + if m.adjacencies[lastToken] != nil && m.adjacencies[lastToken][ts[idx+1]] { + match = 1 + } + lastToken = ts[idx+1] + return match + } + + // Look up each token transition and mark it with a 1 (match) or -1 (no match). From this + // we must compute the subsequences that have the highest probability of being a match. + // This code may seem overcomplicated but it's designed this way to avoid allocating an additional buffer to + // store the matches while remaining testable and clear. + avg, start, end := maxSubsequence(len(ts)-1, matchForIndex) + + // Reject sequences of tokens that are less than the minimum token length. + if end-start < m.minimumTokenLength { + return MatchContext{} + } + + return MatchContext{ + probability: avg, + start: start, + end: end, + } +} + +// maxSubsequence is a modified Kadane’s Algorithm that returns the average, start, and end of the largest subsequence. +// It takes a length of the target input, and a function used to look up values for each index. +func maxSubsequence(length int, matchForIndex func(idx int) int) (float64, int, int) { + if length == 0 { + return 0, 0, 0 + } + maxSum := matchForIndex(0) + currentSum := maxSum + start := 0 + end := 0 + tempStart := 0 + + for i := 1; i < length; i++ { + v := matchForIndex(i) + if v > currentSum+v { + currentSum = v + tempStart = i + } else { + currentSum += v + } + + if currentSum > maxSum { + maxSum = currentSum + start = tempStart + end = i + } + } + end++ + return float64(maxSum) / float64(end-start), start, end +} diff --git a/pkg/logs/internal/decoder/auto_multiline_detection/token_graph_test.go b/pkg/logs/internal/decoder/auto_multiline_detection/token_graph_test.go new file mode 100644 index 0000000000000..6d86f574fd4d8 --- /dev/null +++ b/pkg/logs/internal/decoder/auto_multiline_detection/token_graph_test.go @@ -0,0 +1,60 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package automultilinedetection contains auto multiline detection and aggregation logic. +package automultilinedetection + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection/tokens" +) + +func TestMatchEmpty(t *testing.T) { + emptyTokenGraph := NewTokenGraph(2, nil) + assert.Equal(t, float64(0), emptyTokenGraph.MatchProbability([]tokens.Token{}).probability) +} + +func TestExpectedMatch(t *testing.T) { + graph := NewTokenGraph(0, [][]tokens.Token{{1, 2, 3}}) + assert.Equal(t, float64(1), graph.MatchProbability([]tokens.Token{1, 2, 3}).probability, "Input should match exactly") + assert.Equal(t, float64(-1), graph.MatchProbability([]tokens.Token{3, 2, 1}).probability, "Backwards input should not match because the graph is directed") + assert.Equal(t, float64(-1), graph.MatchProbability([]tokens.Token{4, 5, 6}).probability, "Unknown input should not match") + + graph = NewTokenGraph(0, [][]tokens.Token{{1, 2, 3}, {3, 2, 1}}) + assert.Equal(t, float64(1), graph.MatchProbability([]tokens.Token{1, 2, 3}).probability, "Input should match exactly") + assert.Equal(t, float64(1), graph.MatchProbability([]tokens.Token{3, 2, 1}).probability, "Backwards input should match") + assert.Equal(t, float64(-1), graph.MatchProbability([]tokens.Token{4, 5, 6}).probability, "Unknown input should not match") + + graph = NewTokenGraph(0, [][]tokens.Token{{1, 2, 3, 4, 5, 6}}) + assert.Equal(t, float64(1), graph.MatchProbability([]tokens.Token{7, 2, 3, 4, 5, 8}).probability, "Input should match because unmatch tokens are trimmed") +} + +func TestMaxSubsequence(t *testing.T) { + tests := []struct { + input []int + expected []int + }{ + {[]int{}, []int{}}, + {[]int{1, 1, 1, 1, 1}, []int{1, 1, 1, 1, 1}}, + {[]int{-1, -1, 1, -1, -1}, []int{1}}, + {[]int{-1, 1, 1}, []int{1, 1}}, + {[]int{1, 1, -1}, []int{1, 1}}, + {[]int{-1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, 1}, []int{1, 1, 1, 1}}, + {[]int{-1, 1, 1, 1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, 1}, []int{1, 1, 1, -1, -1, -1, 1, 1, 1, 1}}, + {[]int{1, 1, 1, -1, -1, -1, -1, 1, -1, 1, 1, 1}, []int{1, 1, 1}}, + {[]int{1, -1, 1, 1, 1, -1, -1, -1, -1, 1, 1, 1}, []int{1, -1, 1, 1, 1}}, + } + + for _, test := range tests { + _, start, end := maxSubsequence(len(test.input), func(idx int) int { + return test.input[idx] + }) + + assert.Equal(t, test.expected, test.input[start:end]) + } +} diff --git a/pkg/logs/internal/decoder/auto_multiline_detection/tokenizer.go b/pkg/logs/internal/decoder/auto_multiline_detection/tokenizer.go index eb8cadc534385..d636ad704b633 100644 --- a/pkg/logs/internal/decoder/auto_multiline_detection/tokenizer.go +++ b/pkg/logs/internal/decoder/auto_multiline_detection/tokenizer.go @@ -10,82 +10,14 @@ import ( "bytes" "strings" "unicode" -) -// Token is the type that represents a single token. -type Token byte + "github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection/tokens" +) // maxRun is the maximum run of a char or digit before it is capped. // Note: This must not exceed d10 or c10 below. const maxRun = 10 -const ( - space Token = iota - - // Special Characters - colon // : - semicolon // ; - dash // - - underscore // _ - fslash // / - bslash // \ - period // . - comma // , - singlequote // ' - doublequote // " - backtick // ` - tilda // ~ - star // * - plus // + - equal // = - parenopen // ( - parenclose // ) - braceopen // { - braceclose // } - bracketopen // [ - bracketclose // ] - ampersand // & - exclamation // ! - at // @ - pound // # - dollar // $ - percent // % - uparrow // ^ - - // Digit runs - d1 - d2 - d3 - d4 - d5 - d6 - d7 - d8 - d9 - d10 - - // Char runs - c1 - c2 - c3 - c4 - c5 - c6 - c7 - c8 - c9 - c10 - - // Special tokens - month - day - apm // am or pm - zone // Represents a timezone - t // t (often `T`) denotes a time separator in many timestamp formats - - end // Not a valid token. Used to mark the end of the token list or as a terminator. -) - // Tokenizer is a heuristic to compute tokens from a log message. // The tokenizer is used to convert a log message (string of bytes) into a list of tokens that // represents the underlying structure of the log. The string of tokens is a compact slice of bytes @@ -111,18 +43,23 @@ func (t *Tokenizer) Process(context *messageContext) bool { if maxBytes > t.maxEvalBytes { maxBytes = t.maxEvalBytes } - context.tokens = t.tokenize(context.rawMessage[:maxBytes]) + tokens, indicies := t.tokenize(context.rawMessage[:maxBytes]) + context.tokens = tokens + context.tokenIndicies = indicies return true } // tokenize converts a byte slice to a list of tokens. -func (t *Tokenizer) tokenize(input []byte) []Token { - // len(tokens) will always be <= len(input) - tokens := make([]Token, 0, len(input)) +// This function return the slice of tokens, and a slice of indices where each token starts. +func (t *Tokenizer) tokenize(input []byte) ([]tokens.Token, []int) { + // len(ts) will always be <= len(input) + ts := make([]tokens.Token, 0, len(input)) + indicies := make([]int, 0, len(input)) if len(input) == 0 { - return tokens + return ts, indicies } + idx := 0 run := 0 lastToken := getToken(input[0]) t.strBuf.Reset() @@ -135,29 +72,33 @@ func (t *Tokenizer) tokenize(input []byte) []Token { }() // Only test for special tokens if the last token was a charcater (Special tokens are currently only A-Z). - if lastToken == c1 { + if lastToken == tokens.C1 { if t.strBuf.Len() == 1 { - if specialToken := getSpecialShortToken(t.strBuf.Bytes()[0]); specialToken != end { - tokens = append(tokens, specialToken) + if specialToken := getSpecialShortToken(t.strBuf.Bytes()[0]); specialToken != tokens.End { + ts = append(ts, specialToken) + indicies = append(indicies, idx) return } } else if t.strBuf.Len() > 1 { // Only test special long tokens if buffer is > 1 token - if specialToken := getSpecialLongToken(t.strBuf.String()); specialToken != end { - tokens = append(tokens, specialToken) + if specialToken := getSpecialLongToken(t.strBuf.String()); specialToken != tokens.End { + ts = append(ts, specialToken) + indicies = append(indicies, idx-run) return } } } // Check for char or digit runs - if lastToken == c1 || lastToken == d1 { + if lastToken == tokens.C1 || lastToken == tokens.D1 { + indicies = append(indicies, idx-run) // Limit max run size if run >= maxRun { run = maxRun - 1 } - tokens = append(tokens, lastToken+Token(run)) + ts = append(ts, lastToken+tokens.Token(run)) } else { - tokens = append(tokens, lastToken) + ts = append(ts, lastToken) + indicies = append(indicies, idx-run) } } @@ -168,208 +109,208 @@ func (t *Tokenizer) tokenize(input []byte) []Token { } else { run++ } - if currentToken == c1 { + if currentToken == tokens.C1 { // Store upper case A-Z characters for matching special tokens t.strBuf.WriteRune(unicode.ToUpper(rune(char))) } else { t.strBuf.WriteByte(char) } lastToken = currentToken + idx++ } // Flush any remaining buffered tokens insertToken() - return tokens + return ts, indicies } // getToken returns a single token from a single byte. -func getToken(char byte) Token { +func getToken(char byte) tokens.Token { if unicode.IsDigit(rune(char)) { - return d1 + return tokens.D1 } else if unicode.IsSpace(rune(char)) { - return space + return tokens.Space } switch char { case ':': - return colon + return tokens.Colon case ';': - return semicolon + return tokens.Semicolon case '-': - return dash + return tokens.Dash case '_': - return underscore + return tokens.Underscore case '/': - return fslash + return tokens.Fslash case '\\': - return bslash + return tokens.Bslash case '.': - return period + return tokens.Period case ',': - return comma + return tokens.Comma case '\'': - return singlequote + return tokens.Singlequote case '"': - return doublequote + return tokens.Doublequote case '`': - return backtick + return tokens.Backtick case '~': - return tilda + return tokens.Tilda case '*': - return star + return tokens.Star case '+': - return plus + return tokens.Plus case '=': - return equal + return tokens.Equal case '(': - return parenopen + return tokens.Parenopen case ')': - return parenclose + return tokens.Parenclose case '{': - return braceopen + return tokens.Braceopen case '}': - return braceclose + return tokens.Braceclose case '[': - return bracketopen + return tokens.Bracketopen case ']': - return bracketclose + return tokens.Bracketclose case '&': - return ampersand + return tokens.Ampersand case '!': - return exclamation + return tokens.Exclamation case '@': - return at + return tokens.At case '#': - return pound + return tokens.Pound case '$': - return dollar + return tokens.Dollar case '%': - return percent + return tokens.Percent case '^': - return uparrow + return tokens.Uparrow } - return c1 + return tokens.C1 } -func getSpecialShortToken(char byte) Token { +func getSpecialShortToken(char byte) tokens.Token { switch char { case 'T': - return t + return tokens.T case 'Z': - return zone + return tokens.Zone } - return end + return tokens.End } // getSpecialLongToken returns a special token that is > 1 character. // NOTE: This set of tokens is non-exhaustive and can be expanded. -func getSpecialLongToken(input string) Token { +func getSpecialLongToken(input string) tokens.Token { switch input { case "JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT", "NOV", "DEC": - return month + return tokens.Month case "MON", "TUE", "WED", "THU", "FRI", "SAT", "SUN": - return day + return tokens.Day case "AM", "PM": - return apm + return tokens.Apm case "UTC", "GMT", "EST", "EDT", "CST", "CDT", "MST", "MDT", "PST", "PDT", "JST", "KST", "IST", "MSK", "CEST", "CET", "BST", "NZST", "NZDT", "ACST", "ACDT", "AEST", "AEDT", "AWST", "AWDT", "AKST", "AKDT", "HST", "HDT", "CHST", "CHDT", "NST", "NDT": - return zone + return tokens.Zone } - return end + return tokens.End } // tokenToString converts a single token to a debug string. -func tokenToString(token Token) string { - if token >= d1 && token <= d10 { - return strings.Repeat("D", int(token-d1)+1) - } else if token >= c1 && token <= c10 { - return strings.Repeat("C", int(token-c1)+1) +func tokenToString(token tokens.Token) string { + if token >= tokens.D1 && token <= tokens.D10 { + return strings.Repeat("D", int(token-tokens.D1)+1) + } else if token >= tokens.C1 && token <= tokens.C10 { + return strings.Repeat("C", int(token-tokens.C1)+1) } switch token { - case space: + case tokens.Space: return " " - case colon: + case tokens.Colon: return ":" - case semicolon: + case tokens.Semicolon: return ";" - case dash: + case tokens.Dash: return "-" - case underscore: + case tokens.Underscore: return "_" - case fslash: + case tokens.Fslash: return "/" - case bslash: + case tokens.Bslash: return "\\" - case period: + case tokens.Period: return "." - case comma: + case tokens.Comma: return "," - case singlequote: + case tokens.Singlequote: return "'" - case doublequote: + case tokens.Doublequote: return "\"" - case backtick: + case tokens.Backtick: return "`" - case tilda: + case tokens.Tilda: return "~" - case star: + case tokens.Star: return "*" - case plus: + case tokens.Plus: return "+" - case equal: + case tokens.Equal: return "=" - case parenopen: + case tokens.Parenopen: return "(" - case parenclose: + case tokens.Parenclose: return ")" - case braceopen: + case tokens.Braceopen: return "{" - case braceclose: + case tokens.Braceclose: return "}" - case bracketopen: + case tokens.Bracketopen: return "[" - case bracketclose: + case tokens.Bracketclose: return "]" - case ampersand: + case tokens.Ampersand: return "&" - case exclamation: + case tokens.Exclamation: return "!" - case at: + case tokens.At: return "@" - case pound: + case tokens.Pound: return "#" - case dollar: + case tokens.Dollar: return "$" - case percent: + case tokens.Percent: return "%" - case uparrow: + case tokens.Uparrow: return "^" - case month: + case tokens.Month: return "MTH" - case day: + case tokens.Day: return "DAY" - case apm: + case tokens.Apm: return "PM" - case t: + case tokens.T: return "T" - case zone: + case tokens.Zone: return "ZONE" } - return "" } // tokensToString converts a list of tokens to a debug string. -func tokensToString(tokens []Token) string { +func tokensToString(tokens []tokens.Token) string { str := "" for _, t := range tokens { str += tokenToString(t) diff --git a/pkg/logs/internal/decoder/auto_multiline_detection/tokenizer_test.go b/pkg/logs/internal/decoder/auto_multiline_detection/tokenizer_test.go index d48c25fb8f763..6301317107fd3 100644 --- a/pkg/logs/internal/decoder/auto_multiline_detection/tokenizer_test.go +++ b/pkg/logs/internal/decoder/auto_multiline_detection/tokenizer_test.go @@ -10,6 +10,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection/tokens" ) type testCase struct { @@ -47,26 +49,29 @@ func TestTokenizer(t *testing.T) { tokenizer := NewTokenizer(0) for _, tc := range testCases { - actualToken := tokensToString(tokenizer.tokenize([]byte(tc.input))) + tokens, _ := tokenizer.tokenize([]byte(tc.input)) + actualToken := tokensToString(tokens) assert.Equal(t, tc.expectedToken, actualToken) } } func TestTokenizerMaxCharRun(t *testing.T) { - tokens := tokensToString(NewTokenizer(0).tokenize([]byte("ABCDEFGHIJKLMNOP"))) - assert.Equal(t, "CCCCCCCCCC", tokens) + tokens, indicies := NewTokenizer(0).tokenize([]byte("ABCDEFGHIJKLMNOP")) + assert.Equal(t, "CCCCCCCCCC", tokensToString(tokens)) + assert.Equal(t, []int{0}, indicies) } func TestTokenizerMaxDigitRun(t *testing.T) { - tokens := tokensToString(NewTokenizer(0).tokenize([]byte("0123456789012345"))) - assert.Equal(t, "DDDDDDDDDD", tokens) + tokens, indicies := NewTokenizer(0).tokenize([]byte("0123456789012345")) + assert.Equal(t, "DDDDDDDDDD", tokensToString(tokens)) + assert.Equal(t, []int{0}, indicies) } func TestAllSymbolsAreHandled(t *testing.T) { - for i := space; i < d1; i++ { + for i := tokens.Space; i < tokens.D1; i++ { str := tokenToString(i) assert.NotEmpty(t, str, "Token %d is not converted to a debug string", i) - assert.NotEqual(t, getToken(byte(str[0])), c1, "Token %v is not tokenizable", str) + assert.NotEqual(t, getToken(byte(str[0])), tokens.C1, "Token %v is not tokenizable", str) } } @@ -79,8 +84,20 @@ func TestTokenizerHeuristic(t *testing.T) { msg = &messageContext{rawMessage: []byte("12-12-12T12:12:12.12T12:12Z123")} assert.True(t, tokenizer.Process(msg)) assert.Equal(t, "DD-DD-DDTD", tokensToString(msg.tokens), "Tokens should be limited to the first 10 bytes") + assert.Equal(t, []int{0, 2, 3, 5, 6, 8, 9}, msg.tokenIndicies) msg = &messageContext{rawMessage: []byte("abc 123")} assert.True(t, tokenizer.Process(msg)) assert.Equal(t, "CCC DDD", tokensToString(msg.tokens)) + assert.Equal(t, []int{0, 3, 4}, msg.tokenIndicies) + + msg = &messageContext{rawMessage: []byte("Jan 123")} + assert.True(t, tokenizer.Process(msg)) + assert.Equal(t, "MTH DDD", tokensToString(msg.tokens)) + assert.Equal(t, []int{0, 3, 4}, msg.tokenIndicies) + + msg = &messageContext{rawMessage: []byte("123Z")} + assert.True(t, tokenizer.Process(msg)) + assert.Equal(t, "DDDZONE", tokensToString(msg.tokens)) + assert.Equal(t, []int{0, 3}, msg.tokenIndicies) } diff --git a/pkg/logs/internal/decoder/auto_multiline_detection/tokens/tokens.go b/pkg/logs/internal/decoder/auto_multiline_detection/tokens/tokens.go new file mode 100644 index 0000000000000..5d12d8ba05bc3 --- /dev/null +++ b/pkg/logs/internal/decoder/auto_multiline_detection/tokens/tokens.go @@ -0,0 +1,82 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package tokens contains the token definitions for the tokenizer. +package tokens + +// Token is the type that represents a single token. +type Token byte + +// Disable linter since the token list is self explanatory, or documented where needed. +// +//revive:disable +const ( + Space Token = iota + + // Special Characters + Colon // : + Semicolon // ; + Dash // - + Underscore // _ + Fslash // / + Bslash // \ + Period // . + Comma // , + Singlequote // ' + Doublequote // " + Backtick // ` + Tilda // ~ + Star // * + Plus // + + Equal // = + Parenopen // ( + Parenclose // ) + Braceopen // { + Braceclose // } + Bracketopen // [ + Bracketclose // ] + Ampersand // & + Exclamation // ! + At // @ + Pound // # + Dollar // $ + Percent // % + Uparrow // ^ + + // Digit runs + D1 + D2 + D3 + D4 + D5 + D6 + D7 + D8 + D9 + D10 + + // Char runs + C1 + C2 + C3 + C4 + C5 + C6 + C7 + C8 + C9 + C10 + + // Special tokens + Month + Day + Apm // am or pm + Zone // Represents a timezone + T // t (often `T`) denotes a time separator in many timestamp formats + + End // Not a valid token. Used to mark the end of the token list or as a terminator. +) + +//revive:enable diff --git a/pkg/logs/internal/decoder/auto_multiline_handler.go b/pkg/logs/internal/decoder/auto_multiline_handler.go index f3e5a84b2e858..740f79cd244a5 100644 --- a/pkg/logs/internal/decoder/auto_multiline_handler.go +++ b/pkg/logs/internal/decoder/auto_multiline_handler.go @@ -8,6 +8,7 @@ package decoder import ( "time" + "github.com/DataDog/datadog-agent/pkg/config" automultilinedetection "github.com/DataDog/datadog-agent/pkg/logs/internal/decoder/auto_multiline_detection" "github.com/DataDog/datadog-agent/pkg/logs/message" ) @@ -24,7 +25,8 @@ func NewAutoMultilineHandler(outputFn func(m *message.Message), maxContentSize i // Order is important heuristics := []automultilinedetection.Heuristic{ automultilinedetection.NewJSONDetector(), - automultilinedetection.NewTokenizer(40), // TODO: (brian) this will be configruable in a future change. + automultilinedetection.NewTokenizer(config.Datadog().GetInt("logs_config.auto_multi_line.tokenizer_max_input_bytes")), + automultilinedetection.NewTimestampDetector(config.Datadog().GetFloat64("logs_config.auto_multi_line.timestamp_detector_match_threshold")), } return &AutoMultilineHandler{ From cc7a427829105ca996ee2d69f7f956f0ee72dd6c Mon Sep 17 00:00:00 2001 From: Hasan Mahmood <6599778+hmahmood@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:12:16 -0500 Subject: [PATCH 031/245] Add config to enable/disable ebpf conntracker (#28447) --- pkg/config/setup/system_probe.go | 1 + pkg/network/config/config.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/config/setup/system_probe.go b/pkg/config/setup/system_probe.go index e7754f72c842b..c7ae697fee115 100644 --- a/pkg/config/setup/system_probe.go +++ b/pkg/config/setup/system_probe.go @@ -206,6 +206,7 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Config) { cfg.BindEnvAndSetDefault(join(netNS, "ignore_conntrack_init_failure"), false, "DD_SYSTEM_PROBE_NETWORK_IGNORE_CONNTRACK_INIT_FAILURE") cfg.BindEnvAndSetDefault(join(netNS, "conntrack_init_timeout"), 10*time.Second) cfg.BindEnvAndSetDefault(join(netNS, "allow_netlink_conntracker_fallback"), true) + cfg.BindEnvAndSetDefault(join(netNS, "enable_ebpf_conntracker"), true) cfg.BindEnvAndSetDefault(join(spNS, "source_excludes"), map[string][]string{}) cfg.BindEnvAndSetDefault(join(spNS, "dest_excludes"), map[string][]string{}) diff --git a/pkg/network/config/config.go b/pkg/network/config/config.go index 026842a9c289f..ec9db0aae1e9a 100644 --- a/pkg/network/config/config.go +++ b/pkg/network/config/config.go @@ -386,7 +386,7 @@ func New() *Config { EnableConntrackAllNamespaces: cfg.GetBool(join(spNS, "enable_conntrack_all_namespaces")), IgnoreConntrackInitFailure: cfg.GetBool(join(netNS, "ignore_conntrack_init_failure")), ConntrackInitTimeout: cfg.GetDuration(join(netNS, "conntrack_init_timeout")), - EnableEbpfConntracker: true, + EnableEbpfConntracker: cfg.GetBool(join(netNS, "enable_ebpf_conntracker")), EnableGatewayLookup: cfg.GetBool(join(netNS, "enable_gateway_lookup")), From cecc0e0a8fccfd1f067afbcf38865b812529ef88 Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Mon, 19 Aug 2024 17:54:00 +0200 Subject: [PATCH 032/245] (fleet) add scafholding for policies handling at the agent install (#28234) Co-authored-by: BaptisteFoy --- cmd/system-probe/config/config.go | 3 + comp/core/config/setup.go | 3 + pkg/config/setup/config.go | 1 + pkg/config/setup/system_probe.go | 3 + pkg/fleet/env/env.go | 15 +- pkg/fleet/installer/default_packages.go | 2 - pkg/fleet/installer/installer.go | 109 ++++++---- pkg/fleet/installer/installer_test.go | 27 +-- .../installer/repository/repositories.go | 15 +- .../installer/repository/repositories_test.go | 16 ++ .../installer/repository/repository_test.go | 14 -- pkg/fleet/installer/service/datadog_agent.go | 34 ++++ .../service/datadog_agent_windows.go | 21 +- .../embedded/datadog-agent-exp.service | 1 + .../datadog-agent-process-exp.service | 1 + .../embedded/datadog-agent-process.service | 1 + .../datadog-agent-security-exp.service | 1 + .../embedded/datadog-agent-security.service | 1 + .../datadog-agent-sysprobe-exp.service | 1 + .../embedded/datadog-agent-sysprobe.service | 1 + .../embedded/datadog-agent-trace-exp.service | 1 + .../embedded/datadog-agent-trace.service | 1 + .../service/embedded/datadog-agent.service | 1 + pkg/fleet/internal/cdn/cdn.go | 73 +++++++ pkg/fleet/internal/cdn/config.go | 55 +++++ pkg/fleet/internal/cdn/config_test.go | 49 +++++ pkg/fleet/internal/cdn/merge.go | 63 ++++++ pkg/fleet/internal/cdn/merge_test.go | 191 ++++++++++++++++++ pkg/fleet/internal/exec/installer_exec.go | 4 +- pkg/fleet/internal/paths/installer_paths.go | 13 +- .../internal/paths/installer_paths_windows.go | 23 +-- 31 files changed, 636 insertions(+), 108 deletions(-) create mode 100644 pkg/fleet/internal/cdn/cdn.go create mode 100644 pkg/fleet/internal/cdn/config.go create mode 100644 pkg/fleet/internal/cdn/config_test.go create mode 100644 pkg/fleet/internal/cdn/merge.go create mode 100644 pkg/fleet/internal/cdn/merge_test.go diff --git a/cmd/system-probe/config/config.go b/cmd/system-probe/config/config.go index 29678bac66657..ff21f22169103 100644 --- a/cmd/system-probe/config/config.go +++ b/cmd/system-probe/config/config.go @@ -83,6 +83,9 @@ func newSysprobeConfig(configPath string, fleetPoliciesDirPath string) (*types.C } // Load the remote configuration + if fleetPoliciesDirPath == "" { + fleetPoliciesDirPath = aconfig.SystemProbe().GetString("fleet_policies_dir") + } if fleetPoliciesDirPath != "" { err := aconfig.SystemProbe().MergeFleetPolicy(path.Join(fleetPoliciesDirPath, "system-probe.yaml")) if err != nil { diff --git a/comp/core/config/setup.go b/comp/core/config/setup.go index 8c6bf36e8701d..b3dc261f37e3d 100644 --- a/comp/core/config/setup.go +++ b/comp/core/config/setup.go @@ -77,6 +77,9 @@ func setupConfig(config pkgconfigmodel.Config, deps configDependencies) (*pkgcon } // Load the remote configuration + if p.FleetPoliciesDirPath == "" { + p.FleetPoliciesDirPath = config.GetString("fleet_policies_dir") + } if p.FleetPoliciesDirPath != "" { // Main config file err := config.MergeFleetPolicy(path.Join(p.FleetPoliciesDirPath, "datadog.yaml")) diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 9a8188b0313e5..bd74e7c29e1ef 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -955,6 +955,7 @@ func InitConfig(config pkgconfigmodel.Config) { config.BindEnvAndSetDefault("remote_updates", false) config.BindEnvAndSetDefault("installer.registry.url", "") config.BindEnvAndSetDefault("installer.registry.auth", "") + config.BindEnv("fleet_policies_dir") // Data Jobs Monitoring config config.BindEnvAndSetDefault("djm_config.enabled", false) diff --git a/pkg/config/setup/system_probe.go b/pkg/config/setup/system_probe.go index c7ae697fee115..3998a6519a708 100644 --- a/pkg/config/setup/system_probe.go +++ b/pkg/config/setup/system_probe.go @@ -393,6 +393,9 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Config) { // Discovery config cfg.BindEnvAndSetDefault(join(discoveryNS, "enabled"), false) + // Fleet policies + cfg.BindEnv("fleet_policies_dir") + initCWSSystemProbeConfig(cfg) } diff --git a/pkg/fleet/env/env.go b/pkg/fleet/env/env.go index a9b46e91e1f62..f423eafb4fd0e 100644 --- a/pkg/fleet/env/env.go +++ b/pkg/fleet/env/env.go @@ -20,6 +20,7 @@ const ( envAPIKey = "DD_API_KEY" envSite = "DD_SITE" envRemoteUpdates = "DD_REMOTE_UPDATES" + envRemotePolicies = "DD_REMOTE_POLICIES" envRegistryURL = "DD_INSTALLER_REGISTRY_URL" envRegistryAuth = "DD_INSTALLER_REGISTRY_AUTH" envDefaultPackageVersion = "DD_INSTALLER_DEFAULT_PKG_VERSION" @@ -56,9 +57,10 @@ type ApmLibVersion string // Env contains the configuration for the installer. type Env struct { - APIKey string - Site string - RemoteUpdates bool + APIKey string + Site string + RemoteUpdates bool + RemotePolicies bool RegistryOverride string RegistryAuthOverride string @@ -79,9 +81,10 @@ type Env struct { // FromEnv returns an Env struct with values from the environment. func FromEnv() *Env { return &Env{ - APIKey: getEnvOrDefault(envAPIKey, defaultEnv.APIKey), - Site: getEnvOrDefault(envSite, defaultEnv.Site), - RemoteUpdates: os.Getenv(envRemoteUpdates) == "true", + APIKey: getEnvOrDefault(envAPIKey, defaultEnv.APIKey), + Site: getEnvOrDefault(envSite, defaultEnv.Site), + RemoteUpdates: os.Getenv(envRemoteUpdates) == "true", + RemotePolicies: os.Getenv(envRemotePolicies) == "true", RegistryOverride: getEnvOrDefault(envRegistryURL, defaultEnv.RegistryOverride), RegistryAuthOverride: getEnvOrDefault(envRegistryAuth, defaultEnv.RegistryAuthOverride), diff --git a/pkg/fleet/installer/default_packages.go b/pkg/fleet/installer/default_packages.go index c00688dc3d113..38102cd00e833 100644 --- a/pkg/fleet/installer/default_packages.go +++ b/pkg/fleet/installer/default_packages.go @@ -35,8 +35,6 @@ var PackagesList = []Package{ {Name: "datadog-agent", version: agentVersion, released: false, releasedWithRemoteUpdates: true}, } -var packageDependencies = map[string][]string{} - var apmPackageDefaultVersions = map[string]string{ "datadog-apm-library-java": "1", "datadog-apm-library-ruby": "2", diff --git a/pkg/fleet/installer/installer.go b/pkg/fleet/installer/installer.go index 88d571deba895..b51f585ecd2a2 100644 --- a/pkg/fleet/installer/installer.go +++ b/pkg/fleet/installer/installer.go @@ -18,6 +18,7 @@ import ( "sync" "time" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/cdn" "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" "github.com/DataDog/datadog-agent/pkg/fleet/env" @@ -66,42 +67,48 @@ type Installer interface { type installerImpl struct { m sync.Mutex - db *db.PackagesDB - downloader *oci.Downloader - repositories *repository.Repositories - configsDir string - packagesDir string - tmpDirPath string + env *env.Env + cdn *cdn.CDN + db *db.PackagesDB + downloader *oci.Downloader + packages *repository.Repositories + configs *repository.Repositories + + packagesDir string + userConfigsDir string } // NewInstaller returns a new Package Manager. func NewInstaller(env *env.Env) (Installer, error) { - err := ensurePackageDirExists() + err := ensureRepositoriesExist() if err != nil { - return nil, fmt.Errorf("could not ensure packages directory exists: %w", err) + return nil, fmt.Errorf("could not ensure packages and config directory exists: %w", err) } db, err := db.New(filepath.Join(paths.PackagesPath, "packages.db"), db.WithTimeout(10*time.Second)) if err != nil { return nil, fmt.Errorf("could not create packages db: %w", err) } return &installerImpl{ - db: db, - downloader: oci.NewDownloader(env, http.DefaultClient), - repositories: repository.NewRepositories(paths.PackagesPath, paths.LocksPack), - configsDir: paths.DefaultConfigsDir, - tmpDirPath: paths.TmpDirPath, - packagesDir: paths.PackagesPath, + env: env, + cdn: cdn.New(env), + db: db, + downloader: oci.NewDownloader(env, http.DefaultClient), + packages: repository.NewRepositories(paths.PackagesPath, paths.LocksPath), + configs: repository.NewRepositories(paths.ConfigsPath, paths.LocksPath), + + userConfigsDir: paths.DefaultUserConfigsDir, + packagesDir: paths.PackagesPath, }, nil } // State returns the state of a package. func (i *installerImpl) State(pkg string) (repository.State, error) { - return i.repositories.GetPackageState(pkg) + return i.packages.GetPackageState(pkg) } // States returns the states of all packages. func (i *installerImpl) States() (map[string]repository.State, error) { - return i.repositories.GetState() + return i.packages.GetState() } // IsInstalled checks if a package is installed. @@ -138,18 +145,6 @@ func (i *installerImpl) Install(ctx context.Context, url string, args []string) span.SetTag(ext.ResourceName, pkg.Name) span.SetTag("package_version", pkg.Version) } - - for _, dependency := range packageDependencies[pkg.Name] { - installed, err := i.IsInstalled(ctx, dependency) - if err != nil { - return fmt.Errorf("could not check if required package %s is installed: %w", dependency, err) - } - if !installed { - // TODO: we should resolve the dependency version & install it instead - return fmt.Errorf("required package %s is not installed", dependency) - } - } - dbPkg, err := i.db.GetPackage(pkg.Name) if err != nil && !errors.Is(err, db.ErrPackageNotFound) { return fmt.Errorf("could not get package: %w", err) @@ -162,12 +157,12 @@ func (i *installerImpl) Install(ctx context.Context, url string, args []string) if err != nil { return fmt.Errorf("not enough disk space: %w", err) } - tmpDir, err := os.MkdirTemp(i.tmpDirPath, fmt.Sprintf("tmp-install-stable-%s-*", pkg.Name)) // * is replaced by a random string + tmpDir, err := i.packages.MkdirTemp() if err != nil { return fmt.Errorf("could not create temporary directory: %w", err) } defer os.RemoveAll(tmpDir) - configDir := filepath.Join(i.configsDir, pkg.Name) + configDir := filepath.Join(i.userConfigsDir, pkg.Name) err = pkg.ExtractLayers(oci.DatadogPackageLayerMediaType, tmpDir) if err != nil { return fmt.Errorf("could not extract package layers: %w", err) @@ -176,10 +171,14 @@ func (i *installerImpl) Install(ctx context.Context, url string, args []string) if err != nil { return fmt.Errorf("could not extract package config layer: %w", err) } - err = i.repositories.Create(pkg.Name, pkg.Version, tmpDir) + err = i.packages.Create(pkg.Name, pkg.Version, tmpDir) if err != nil { return fmt.Errorf("could not create repository: %w", err) } + err = i.configurePackage(ctx, pkg.Name) + if err != nil { + return fmt.Errorf("could not configure package: %w", err) + } err = i.setupPackage(ctx, pkg.Name, args) if err != nil { return fmt.Errorf("could not setup package: %w", err) @@ -207,12 +206,12 @@ func (i *installerImpl) InstallExperiment(ctx context.Context, url string) error if err != nil { return fmt.Errorf("not enough disk space: %w", err) } - tmpDir, err := os.MkdirTemp(i.tmpDirPath, fmt.Sprintf("tmp-install-experiment-%s-*", pkg.Name)) // * is replaced by a random string + tmpDir, err := i.packages.MkdirTemp() if err != nil { return fmt.Errorf("could not create temporary directory: %w", err) } defer os.RemoveAll(tmpDir) - configDir := filepath.Join(i.configsDir, pkg.Name) + configDir := filepath.Join(i.userConfigsDir, pkg.Name) err = pkg.ExtractLayers(oci.DatadogPackageLayerMediaType, tmpDir) if err != nil { return fmt.Errorf("could not extract package layers: %w", err) @@ -221,7 +220,7 @@ func (i *installerImpl) InstallExperiment(ctx context.Context, url string) error if err != nil { return fmt.Errorf("could not extract package config layer: %w", err) } - repository := i.repositories.Get(pkg.Name) + repository := i.packages.Get(pkg.Name) err = repository.SetExperiment(pkg.Version, tmpDir) if err != nil { return fmt.Errorf("could not set experiment: %w", err) @@ -234,7 +233,7 @@ func (i *installerImpl) RemoveExperiment(ctx context.Context, pkg string) error i.m.Lock() defer i.m.Unlock() - repository := i.repositories.Get(pkg) + repository := i.packages.Get(pkg) if runtime.GOOS != "windows" && pkg == packageDatadogInstaller { // Special case for the Linux installer since `stopExperiment` // will kill the current process, delete the experiment first. @@ -264,7 +263,7 @@ func (i *installerImpl) PromoteExperiment(ctx context.Context, pkg string) error i.m.Lock() defer i.m.Unlock() - repository := i.repositories.Get(pkg) + repository := i.packages.Get(pkg) err := repository.PromoteExperiment() if err != nil { return fmt.Errorf("could not promote experiment: %w", err) @@ -297,12 +296,16 @@ func (i *installerImpl) Purge(ctx context.Context) { log.Warnf("could not remove installer: %v", err) } + err = os.RemoveAll(paths.ConfigsPath) + if err != nil { + log.Warnf("could not delete configs dir: %v", err) + } // remove all from disk span, _ := tracer.StartSpanFromContext(ctx, "remove_all") err = os.RemoveAll(paths.PackagesPath) defer span.Finish(tracer.WithError(err)) if err != nil { - log.Warnf("could not remove path: %v", err) + log.Warnf("could not delete packages dir: %v", err) } } @@ -314,7 +317,7 @@ func (i *installerImpl) Remove(ctx context.Context, pkg string) error { if err != nil { return fmt.Errorf("could not remove package: %w", err) } - err = i.repositories.Delete(ctx, pkg) + err = i.packages.Delete(ctx, pkg) if err != nil { return fmt.Errorf("could not delete repository: %w", err) } @@ -329,8 +332,15 @@ func (i *installerImpl) Remove(ctx context.Context, pkg string) error { func (i *installerImpl) GarbageCollect(_ context.Context) error { i.m.Lock() defer i.m.Unlock() - - return i.repositories.Cleanup() + err := i.packages.Cleanup() + if err != nil { + return fmt.Errorf("could not cleanup packages: %w", err) + } + err = i.configs.Cleanup() + if err != nil { + return fmt.Errorf("could not cleanup configs: %w", err) + } + return nil } // InstrumentAPMInjector instruments the APM injector. @@ -432,6 +442,19 @@ func (i *installerImpl) removePackage(ctx context.Context, pkg string) error { } } +func (i *installerImpl) configurePackage(ctx context.Context, pkg string) error { + if !i.env.RemotePolicies { + return nil + } + + switch pkg { + case packageDatadogAgent: + return service.ConfigureAgent(ctx, i.cdn, i.configs) + default: + return nil + } +} + const ( packageUnknownSize = 2 << 30 // 2GiB installerOverhead = 10 << 20 // 10MiB @@ -465,10 +488,14 @@ func checkAvailableDiskSpace(pkg *oci.DownloadedPackage, path string) error { return nil } -func ensurePackageDirExists() error { +func ensureRepositoriesExist() error { err := os.MkdirAll(paths.PackagesPath, 0755) if err != nil { return fmt.Errorf("error creating packages directory: %w", err) } + err = os.MkdirAll(paths.ConfigsPath, 0755) + if err != nil { + return fmt.Errorf("error creating configs directory: %w", err) + } return nil } diff --git a/pkg/fleet/installer/installer_test.go b/pkg/fleet/installer/installer_test.go index a548fd2c596fe..069c2e04cf9e8 100644 --- a/pkg/fleet/installer/installer_test.go +++ b/pkg/fleet/installer/installer_test.go @@ -7,12 +7,13 @@ package installer import ( "context" - "github.com/stretchr/testify/assert" "io/fs" "os" "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/pkg/fleet/env" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" "github.com/DataDog/datadog-agent/pkg/fleet/internal/db" @@ -27,23 +28,23 @@ type testPackageManager struct { } func newTestPackageManager(t *testing.T, s *fixtures.Server, rootPath string, locksPath string) *testPackageManager { - repositories := repository.NewRepositories(rootPath, locksPath) + packages := repository.NewRepositories(rootPath, locksPath) db, err := db.New(filepath.Join(rootPath, "packages.db")) assert.NoError(t, err) return &testPackageManager{ installerImpl{ - db: db, - downloader: oci.NewDownloader(&env.Env{}, s.Client()), - repositories: repositories, - configsDir: t.TempDir(), - tmpDirPath: rootPath, - packagesDir: rootPath, + env: &env.Env{}, + db: db, + downloader: oci.NewDownloader(&env.Env{}, s.Client()), + packages: packages, + userConfigsDir: t.TempDir(), + packagesDir: rootPath, }, } } func (i *testPackageManager) ConfigFS(f fixtures.Fixture) fs.FS { - return os.DirFS(filepath.Join(i.configsDir, f.Package)) + return os.DirFS(filepath.Join(i.userConfigsDir, f.Package)) } func TestInstallStable(t *testing.T) { @@ -53,7 +54,7 @@ func TestInstallStable(t *testing.T) { err := installer.Install(testCtx, s.PackageURL(fixtures.FixtureSimpleV1), nil) assert.NoError(t, err) - r := installer.repositories.Get(fixtures.FixtureSimpleV1.Package) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) state, err := r.GetState() assert.NoError(t, err) assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) @@ -71,7 +72,7 @@ func TestInstallExperiment(t *testing.T) { assert.NoError(t, err) err = installer.InstallExperiment(testCtx, s.PackageURL(fixtures.FixtureSimpleV2)) assert.NoError(t, err) - r := installer.repositories.Get(fixtures.FixtureSimpleV1.Package) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) state, err := r.GetState() assert.NoError(t, err) assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) @@ -92,7 +93,7 @@ func TestInstallPromoteExperiment(t *testing.T) { assert.NoError(t, err) err = installer.PromoteExperiment(testCtx, fixtures.FixtureSimpleV1.Package) assert.NoError(t, err) - r := installer.repositories.Get(fixtures.FixtureSimpleV1.Package) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) state, err := r.GetState() assert.NoError(t, err) assert.Equal(t, fixtures.FixtureSimpleV2.Version, state.Stable) @@ -112,7 +113,7 @@ func TestUninstallExperiment(t *testing.T) { assert.NoError(t, err) err = installer.RemoveExperiment(testCtx, fixtures.FixtureSimpleV1.Package) assert.NoError(t, err) - r := installer.repositories.Get(fixtures.FixtureSimpleV1.Package) + r := installer.packages.Get(fixtures.FixtureSimpleV1.Package) state, err := r.GetState() assert.NoError(t, err) assert.Equal(t, fixtures.FixtureSimpleV1.Version, state.Stable) diff --git a/pkg/fleet/installer/repository/repositories.go b/pkg/fleet/installer/repository/repositories.go index 66e08d70bd342..4122a83156303 100644 --- a/pkg/fleet/installer/repository/repositories.go +++ b/pkg/fleet/installer/repository/repositories.go @@ -13,6 +13,10 @@ import ( "strings" ) +const ( + tempDirPrefix = "tmp-i-" +) + // Repositories manages multiple repositories. type Repositories struct { rootPath string @@ -44,8 +48,8 @@ func (r *Repositories) loadRepositories() (map[string]*Repository, error) { if !d.IsDir() { continue } - if strings.HasPrefix(d.Name(), "tmp-install") { - // Temporary extraction dir, ignore + if strings.HasPrefix(d.Name(), tempDirPrefix) { + // Temporary dir created by Repositories.MkdirTemp, ignore continue } repo := r.newRepository(d.Name()) @@ -116,3 +120,10 @@ func (r *Repositories) Cleanup() error { } return nil } + +// MkdirTemp creates a temporary directory in the same partition as the root path. +// This ensures that the temporary directory can be moved to the root path without copying. +// The caller is responsible for cleaning up the directory. +func (r *Repositories) MkdirTemp() (string, error) { + return os.MkdirTemp(r.rootPath, tempDirPrefix+"*") +} diff --git a/pkg/fleet/installer/repository/repositories_test.go b/pkg/fleet/installer/repository/repositories_test.go index 28d7bcc888049..bac3114c3fcde 100644 --- a/pkg/fleet/installer/repository/repositories_test.go +++ b/pkg/fleet/installer/repository/repositories_test.go @@ -6,6 +6,8 @@ package repository import ( + "os" + "path" "testing" "github.com/stretchr/testify/assert" @@ -59,3 +61,17 @@ func TestRepositoriesReopen(t *testing.T) { assert.Equal(t, state["repo1"], State{Stable: "v1"}) assert.Equal(t, state["repo2"], State{Stable: "v1"}) } + +func TestLoadRepositories(t *testing.T) { + rootDir := t.TempDir() + runDir := t.TempDir() + + os.Mkdir(path.Join(rootDir, "datadog-agent"), 0755) + os.Mkdir(path.Join(rootDir, tempDirPrefix+"2394812349"), 0755) + + repositories, err := NewRepositories(rootDir, runDir).loadRepositories() + assert.NoError(t, err) + assert.Len(t, repositories, 1) + assert.Contains(t, repositories, "datadog-agent") + assert.NotContains(t, repositories, tempDirPrefix+"2394812349") +} diff --git a/pkg/fleet/installer/repository/repository_test.go b/pkg/fleet/installer/repository/repository_test.go index 5092181e0c570..13bcf7266ed43 100644 --- a/pkg/fleet/installer/repository/repository_test.go +++ b/pkg/fleet/installer/repository/repository_test.go @@ -191,17 +191,3 @@ func TestDeleteExperimentWithLockedPackage(t *testing.T) { assert.NoFileExists(t, path.Join(repository.locksPath, "v2", "-1")) assert.FileExists(t, path.Join(repository.locksPath, "v2", fmt.Sprint(os.Getpid()))) } - -func TestLoadRepositories(t *testing.T) { - rootDir := t.TempDir() - runDir := t.TempDir() - - os.Mkdir(path.Join(rootDir, "datadog-agent"), 0755) - os.Mkdir(path.Join(rootDir, "tmp-install-stable-datadog-agent"), 0755) - - repositories, err := NewRepositories(rootDir, runDir).loadRepositories() - assert.NoError(t, err) - assert.Len(t, repositories, 1) - assert.Contains(t, repositories, "datadog-agent") - assert.NotContains(t, repositories, "tmp-install-stable-datadog-agent") -} diff --git a/pkg/fleet/installer/service/datadog_agent.go b/pkg/fleet/installer/service/datadog_agent.go index 5c92bbb947932..feddc3ddb6347 100644 --- a/pkg/fleet/installer/service/datadog_agent.go +++ b/pkg/fleet/installer/service/datadog_agent.go @@ -16,12 +16,15 @@ import ( "os/exec" "path/filepath" + "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/cdn" "github.com/DataDog/datadog-agent/pkg/util/installinfo" "github.com/DataDog/datadog-agent/pkg/util/log" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) const ( + agentPackage = "datadog-agent" pathOldAgent = "/opt/datadog-agent" agentSymlink = "/usr/bin/datadog-agent" agentUnit = "datadog-agent.service" @@ -34,6 +37,7 @@ const ( processAgentExp = "datadog-agent-process-exp.service" systemProbeExp = "datadog-agent-sysprobe-exp.service" securityAgentExp = "datadog-agent-security-exp.service" + configDatadogYAML = "datadog.yaml" ) var ( @@ -222,3 +226,33 @@ func StopAgentExperiment(ctx context.Context) error { func PromoteAgentExperiment(ctx context.Context) error { return StopAgentExperiment(ctx) } + +// ConfigureAgent configures the stable agent +func ConfigureAgent(ctx context.Context, cdn *cdn.CDN, configs *repository.Repositories) error { + config, err := cdn.Get(ctx) + if err != nil { + return fmt.Errorf("could not get cdn config: %w", err) + } + tmpDir, err := configs.MkdirTemp() + if err != nil { + return fmt.Errorf("could not create temporary directory: %w", err) + } + defer os.RemoveAll(tmpDir) + ddAgentUID, ddAgentGID, err := getAgentIDs() + if err != nil { + return fmt.Errorf("error getting dd-agent user and group IDs: %w", err) + } + err = os.WriteFile(filepath.Join(tmpDir, configDatadogYAML), []byte(config.Datadog), 0640) + if err != nil { + return fmt.Errorf("could not write datadog.yaml: %w", err) + } + err = os.Chown(filepath.Join(tmpDir, configDatadogYAML), ddAgentUID, ddAgentGID) + if err != nil { + return fmt.Errorf("could not chown datadog.yaml: %w", err) + } + err = configs.Create(agentPackage, config.Version, tmpDir) + if err != nil { + return fmt.Errorf("could not create repository: %w", err) + } + return nil +} diff --git a/pkg/fleet/installer/service/datadog_agent_windows.go b/pkg/fleet/installer/service/datadog_agent_windows.go index 6ad1595c6d293..96d7feb77d9b5 100644 --- a/pkg/fleet/installer/service/datadog_agent_windows.go +++ b/pkg/fleet/installer/service/datadog_agent_windows.go @@ -11,12 +11,14 @@ package service import ( "context" "fmt" + "os/exec" + "path/filepath" + + "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/cdn" "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" "github.com/DataDog/datadog-agent/pkg/util/log" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" - "os" - "os/exec" - "path/filepath" ) func msiexec(target, operation string, args []string) (err error) { @@ -31,13 +33,7 @@ func msiexec(target, operation string, args []string) (err error) { return fmt.Errorf("no MSIs in package") } - tmpDir, err := os.MkdirTemp(paths.TmpDirPath, fmt.Sprintf("install-%s-*", filepath.Base(msis[0]))) - if err != nil { - return fmt.Errorf("could not create temporary directory: %w", err) - } - - logPath := filepath.Join(tmpDir, "install.log") - cmd := exec.Command("msiexec", append([]string{operation, msis[0], "/qn", "/l", logPath, "MSIFASTINSTALL=7"}, args...)...) + cmd := exec.Command("msiexec", append([]string{operation, msis[0], "/qn", "MSIFASTINSTALL=7"}, args...)...) return cmd.Run() } @@ -100,3 +96,8 @@ func RemoveAgent(ctx context.Context) (err error) { }() return msiexec("stable", "/x", nil) } + +// ConfigureAgent noop +func ConfigureAgent(_ context.Context, _ *cdn.CDN, _ *repository.Repositories) error { + return nil +} diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-exp.service index ed500b148089b..336d98f781a0a 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-exp.service @@ -12,6 +12,7 @@ Type=oneshot PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/agent.pid User=dd-agent EnvironmentFile=-/etc/datadog-agent/environment +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/experiment/bin/agent/agent run -p /opt/datadog-packages/datadog-agent/experiment/run/agent.pid ExecStart=/bin/false ExecStop=/bin/false diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-process-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-process-exp.service index 426d019f4c62a..87c88adbc8c50 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-process-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-process-exp.service @@ -9,6 +9,7 @@ PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/process-agent.pid User=dd-agent Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/experiment/embedded/bin/process-agent --cfgpath=/etc/datadog-agent/datadog.yaml --sysprobe-config=/etc/datadog-agent/system-probe.yaml --pid=/opt/datadog-packages/datadog-agent/experiment/run/process-agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-process.service b/pkg/fleet/installer/service/embedded/datadog-agent-process.service index e990c47e28748..23e892f1ef5da 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-process.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-process.service @@ -9,6 +9,7 @@ PIDFile=/opt/datadog-packages/datadog-agent/stable/run/process-agent.pid User=dd-agent Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/stable/embedded/bin/process-agent --cfgpath=/etc/datadog-agent/datadog.yaml --sysprobe-config=/etc/datadog-agent/system-probe.yaml --pid=/opt/datadog-packages/datadog-agent/stable/run/process-agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-security-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-security-exp.service index dd2db087469cd..87927afd96cdd 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-security-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-security-exp.service @@ -9,6 +9,7 @@ Type=simple PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/security-agent.pid Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/experiment/embedded/bin/security-agent -c /etc/datadog-agent/datadog.yaml --pidfile /opt/datadog-packages/datadog-agent/experiment/run/security-agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-security.service b/pkg/fleet/installer/service/embedded/datadog-agent-security.service index 935070f68ce33..a71b46142be9b 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-security.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-security.service @@ -9,6 +9,7 @@ Type=simple PIDFile=/opt/datadog-packages/datadog-agent/stable/run/security-agent.pid Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/stable/embedded/bin/security-agent -c /etc/datadog-agent/datadog.yaml --pidfile /opt/datadog-packages/datadog-agent/stable/run/security-agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe-exp.service index 6b0d435fac472..81f3192e4feda 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe-exp.service @@ -9,6 +9,7 @@ ConditionPathExists=/etc/datadog-agent/system-probe.yaml Type=simple PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/system-probe.pid Restart=on-failure +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/experiment/embedded/bin/system-probe run --config=/etc/datadog-agent/system-probe.yaml --pid=/opt/datadog-packages/datadog-agent/experiment/run/system-probe.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe.service b/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe.service index 5bcfeb717e984..f79685a577aa4 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe.service @@ -10,6 +10,7 @@ ConditionPathExists=/etc/datadog-agent/system-probe.yaml Type=simple PIDFile=/opt/datadog-packages/datadog-agent/stable/run/system-probe.pid Restart=on-failure +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/stable/embedded/bin/system-probe run --config=/etc/datadog-agent/system-probe.yaml --pid=/opt/datadog-packages/datadog-agent/stable/run/system-probe.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-trace-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-trace-exp.service index bcb3306b608c6..22b73c91bfe73 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-trace-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-trace-exp.service @@ -8,6 +8,7 @@ PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/trace-agent.pid User=dd-agent Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/experiment/embedded/bin/trace-agent --config /etc/datadog-agent/datadog.yaml --pidfile /opt/datadog-packages/datadog-agent/experiment/run/trace-agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-trace.service b/pkg/fleet/installer/service/embedded/datadog-agent-trace.service index 483b47a1ead24..c3568e7004738 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-trace.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-trace.service @@ -9,6 +9,7 @@ PIDFile=/opt/datadog-packages/datadog-agent/stable/run/trace-agent.pid User=dd-agent Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/stable/embedded/bin/trace-agent --config /etc/datadog-agent/datadog.yaml --pidfile /opt/datadog-packages/datadog-agent/stable/run/trace-agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent.service b/pkg/fleet/installer/service/embedded/datadog-agent.service index cb8d9455d4648..baf05b20796b8 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent.service @@ -11,6 +11,7 @@ PIDFile=/opt/datadog-packages/datadog-agent/stable/run/agent.pid User=dd-agent Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" ExecStart=/opt/datadog-packages/datadog-agent/stable/bin/agent/agent run -p /opt/datadog-packages/datadog-agent/stable/run/agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/internal/cdn/cdn.go b/pkg/fleet/internal/cdn/cdn.go new file mode 100644 index 0000000000000..0d159b9fcebb5 --- /dev/null +++ b/pkg/fleet/internal/cdn/cdn.go @@ -0,0 +1,73 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package cdn provides access to the Remote Config CDN. +package cdn + +import ( + "context" + "crypto/sha256" + "fmt" + + "github.com/DataDog/datadog-agent/pkg/fleet/env" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +// CDN provides access to the Remote Config CDN. +type CDN struct { + env *env.Env +} + +// Config represents the configuration from the CDN. +type Config struct { + Version string + Datadog []byte + SecurityAgent []byte + SystemProbe []byte +} + +// New creates a new CDN. +func New(env *env.Env) *CDN { + return &CDN{ + env: env, + } +} + +// Get gets the configuration from the CDN. +func (c *CDN) Get(ctx context.Context) (_ Config, err error) { + span, ctx := tracer.StartSpanFromContext(ctx, "cdn.Get") + defer func() { span.Finish(tracer.WithError(err)) }() + return getFakeCDNConfig(ctx) +} + +// HACK (arthur): this is a temporary function that returns a fake CDN config to unblock development. +func getFakeCDNConfig(_ context.Context) (Config, error) { + baseLayer := layer{ + ID: "base", + Content: map[interface{}]interface{}{ + "extra_tags": []string{"layer:base"}, + }, + } + overrideLayer := layer{ + ID: "override", + Content: map[interface{}]interface{}{ + "extra_tags": []string{"layer:override"}, + }, + } + config, err := newConfig(baseLayer, overrideLayer) + if err != nil { + return Config{}, err + } + serializedConfig, err := config.Marshal() + if err != nil { + return Config{}, err + } + hash := sha256.New() + hash.Write(serializedConfig) + return Config{ + Version: fmt.Sprintf("%x", hash.Sum(nil)), + Datadog: serializedConfig, + }, nil +} diff --git a/pkg/fleet/internal/cdn/config.go b/pkg/fleet/internal/cdn/config.go new file mode 100644 index 0000000000000..a4feb97ac914a --- /dev/null +++ b/pkg/fleet/internal/cdn/config.go @@ -0,0 +1,55 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package cdn + +import ( + "bytes" + + "gopkg.in/yaml.v2" +) + +const ( + layerKeys = "__fleet_layers" + doNotEditDisclaimer = `# This configuration was generated by Datadog's Fleet Automation. DO NOT EDIT.` +) + +// config is a configuration for the package. +type config map[interface{}]interface{} + +// layer is a config layer that can be merged with other layers into a config. +type layer struct { + ID string + Content map[interface{}]interface{} +} + +// newConfig creates a new config from a list of layers. +func newConfig(layers ...layer) (config, error) { + config := make(map[interface{}]interface{}) + var layerIDs []string + for _, l := range layers { + merged, err := merge(config, l.Content) + if err != nil { + return nil, err + } + config = merged.(map[interface{}]interface{}) + layerIDs = append(layerIDs, l.ID) + } + config[layerKeys] = layerIDs + return config, nil +} + +// Marshal marshals the config as YAML. +func (c *config) Marshal() ([]byte, error) { + var b bytes.Buffer + b.WriteString(doNotEditDisclaimer) + b.WriteString("\n") + rawConfig, err := yaml.Marshal(c) + if err != nil { + return nil, err + } + b.Write(rawConfig) + return b.Bytes(), nil +} diff --git a/pkg/fleet/internal/cdn/config_test.go b/pkg/fleet/internal/cdn/config_test.go new file mode 100644 index 0000000000000..697fc6b0f0481 --- /dev/null +++ b/pkg/fleet/internal/cdn/config_test.go @@ -0,0 +1,49 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package cdn + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfig(t *testing.T) { + baseLayer := layer{ + ID: "base", + Content: map[interface{}]interface{}{ + "api_key": "1234", + "apm": map[interface{}]interface{}{ + "enabled": true, + "sampling_rate": 0.5, + }, + }, + } + overrideLayer := layer{ + ID: "override", + Content: map[interface{}]interface{}{ + "apm": map[interface{}]interface{}{ + "sampling_rate": 0.7, + "env": "prod", + }, + }, + } + config, err := newConfig(baseLayer, overrideLayer) + assert.NoError(t, err) + serializedConfig, err := config.Marshal() + assert.NoError(t, err) + exprectedConfig := doNotEditDisclaimer + ` +__fleet_layers: +- base +- override +api_key: "1234" +apm: + enabled: true + env: prod + sampling_rate: 0.7 +` + assert.Equal(t, exprectedConfig, string(serializedConfig)) +} diff --git a/pkg/fleet/internal/cdn/merge.go b/pkg/fleet/internal/cdn/merge.go new file mode 100644 index 0000000000000..7f38e84fc281e --- /dev/null +++ b/pkg/fleet/internal/cdn/merge.go @@ -0,0 +1,63 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package cdn + +import "fmt" + +func isList(i interface{}) bool { + _, ok := i.([]interface{}) + return ok +} + +func isMap(i interface{}) bool { + _, ok := i.(map[interface{}]interface{}) + return ok +} + +func isScalar(i interface{}) bool { + return !isList(i) && !isMap(i) +} + +// merge merges two layers into a single layer +// +// The override layer takes precedence over the base layer. The values are merged as follows: +// - Scalars: the override value is used +// - Lists: the override list is used +// - Maps: the override map is recursively merged into the base map +func merge(base interface{}, override interface{}) (interface{}, error) { + if base == nil { + return override, nil + } + if override == nil { + // this allows to override a value with nil + return nil, nil + } + if isScalar(base) && isScalar(override) { + return override, nil + } + if isList(base) && isList(override) { + return override, nil + } + if isMap(base) && isMap(override) { + return mergeMap(base.(map[interface{}]interface{}), override.(map[interface{}]interface{})) + } + return nil, fmt.Errorf("could not merge %T with %T", base, override) +} + +func mergeMap(base, override map[interface{}]interface{}) (map[interface{}]interface{}, error) { + merged := make(map[interface{}]interface{}) + for k, v := range base { + merged[k] = v + } + for k := range override { + v, err := merge(base[k], override[k]) + if err != nil { + return nil, fmt.Errorf("could not merge key %v: %w", k, err) + } + merged[k] = v + } + return merged, nil +} diff --git a/pkg/fleet/internal/cdn/merge_test.go b/pkg/fleet/internal/cdn/merge_test.go new file mode 100644 index 0000000000000..e066082dc5953 --- /dev/null +++ b/pkg/fleet/internal/cdn/merge_test.go @@ -0,0 +1,191 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package cdn + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMergeScalar(t *testing.T) { + tests := []struct { + name string + base interface{} + override interface{} + expected interface{} + expectedErr bool + }{ + { + name: "nil base and override", + base: nil, + override: nil, + expected: nil, + }, + { + name: "nil base", + base: nil, + override: "override", + expected: "override", + }, + { + name: "nil override", + base: "base", + override: nil, + expected: nil, + }, + { + name: "override", + base: "base", + override: "override", + expected: "override", + }, + { + name: "scalar and list error", + base: "base", + override: []interface{}{"override"}, + expectedErr: true, + }, + { + name: "scalar and map error", + base: "base", + override: map[interface{}]interface{}{"key": "value"}, + expectedErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + merged, err := merge(tt.base, tt.override) + if tt.expectedErr { + assert.Error(t, err, "expected an error") + } else { + assert.Equal(t, tt.expected, merged) + } + }) + } +} + +func TestMergeList(t *testing.T) { + tests := []struct { + name string + base interface{} + override interface{} + expected interface{} + expectedErr bool + }{ + { + name: "nil override", + base: []interface{}{"base"}, + override: nil, + expected: nil, + }, + { + name: "override", + base: []interface{}{"base"}, + override: []interface{}{"override"}, + expected: []interface{}{"override"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + merged, err := merge(tt.base, tt.override) + if tt.expectedErr { + assert.Error(t, err, "expected an error") + } else { + assert.Equal(t, tt.expected, merged) + } + }) + } +} + +func TestMergeMap(t *testing.T) { + tests := []struct { + name string + base interface{} + override interface{} + expected interface{} + expectedErr bool + }{ + { + name: "nil override", + base: map[interface{}]interface{}{ + "base": "value", + }, + override: nil, + expected: nil, + }, + { + name: "override", + base: map[interface{}]interface{}{ + "base": "value", + }, + override: map[interface{}]interface{}{ + "base": "override", + }, + expected: map[interface{}]interface{}{ + "base": "override", + }, + }, + { + name: "add key", + base: map[interface{}]interface{}{ + "base": "value", + }, + override: map[interface{}]interface{}{ + "override": "value", + }, + expected: map[interface{}]interface{}{ + "base": "value", + "override": "value", + }, + }, + { + name: "nested", + base: map[interface{}]interface{}{ + "base": map[interface{}]interface{}{ + "key": "value", + }, + }, + override: map[interface{}]interface{}{ + "base": map[interface{}]interface{}{ + "key": "override", + }, + }, + expected: map[interface{}]interface{}{ + "base": map[interface{}]interface{}{ + "key": "override", + }, + }, + }, + { + name: "nested scalar and list error", + base: map[interface{}]interface{}{ + "base": map[interface{}]interface{}{ + "key": []interface{}{"value"}, + }, + }, + override: map[interface{}]interface{}{ + "base": map[interface{}]interface{}{ + "key": "override", + }, + }, + expectedErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + merged, err := merge(tt.base, tt.override) + if tt.expectedErr { + assert.Error(t, err, "expected an error") + } else { + assert.Equal(t, tt.expected, merged) + } + }) + } +} diff --git a/pkg/fleet/internal/exec/installer_exec.go b/pkg/fleet/internal/exec/installer_exec.go index b0e1c08739473..a1beaf19764db 100644 --- a/pkg/fleet/internal/exec/installer_exec.go +++ b/pkg/fleet/internal/exec/installer_exec.go @@ -169,13 +169,13 @@ func (i *InstallerExec) DefaultPackages(ctx context.Context) (_ []string, err er // State returns the state of a package. func (i *InstallerExec) State(pkg string) (repository.State, error) { - repositories := repository.NewRepositories(paths.PackagesPath, paths.LocksPack) + repositories := repository.NewRepositories(paths.PackagesPath, paths.LocksPath) return repositories.Get(pkg).GetState() } // States returns the states of all packages. func (i *InstallerExec) States() (map[string]repository.State, error) { - repositories := repository.NewRepositories(paths.PackagesPath, paths.LocksPack) + repositories := repository.NewRepositories(paths.PackagesPath, paths.LocksPath) states, err := repositories.GetState() log.Debugf("repositories states: %v", states) return states, err diff --git a/pkg/fleet/internal/paths/installer_paths.go b/pkg/fleet/internal/paths/installer_paths.go index f20408c23705e..a3d913a727634 100644 --- a/pkg/fleet/internal/paths/installer_paths.go +++ b/pkg/fleet/internal/paths/installer_paths.go @@ -11,10 +11,11 @@ package paths const ( // PackagesPath is the path to the packages directory. PackagesPath = "/opt/datadog-packages" - // TmpDirPath is the path to the temporary directory used for package installation. - TmpDirPath = "/opt/datadog-packages" - // LocksPack is the path to the locks directory. - LocksPack = "/var/run/datadog-installer/locks" - // DefaultConfigsDir is the default Agent configuration directory - DefaultConfigsDir = "/etc" + // ConfigsPath is the path to the Fleet-managed configuration directory. + ConfigsPath = "/etc/datadog-packages" + // LocksPath is the path to the packages locks directory. + LocksPath = "/var/run/datadog-installer/locks" + + // DefaultUserConfigsDir is the default Agent configuration directory. + DefaultUserConfigsDir = "/etc" ) diff --git a/pkg/fleet/internal/paths/installer_paths_windows.go b/pkg/fleet/internal/paths/installer_paths_windows.go index 47d16726e50f4..456cb7e6af7c2 100644 --- a/pkg/fleet/internal/paths/installer_paths_windows.go +++ b/pkg/fleet/internal/paths/installer_paths_windows.go @@ -9,29 +9,28 @@ package paths import ( + "path/filepath" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/winregistry" "golang.org/x/sys/windows" - "path/filepath" ) var ( // PackagesPath is the path to the packages directory. PackagesPath string + // ConfigsPath is the path to the Fleet-managed configuration directory + ConfigsPath string + // LocksPath is the path to the locks directory. + LocksPath string - // TmpDirPath is the path to the temporary directory used for package installation. - TmpDirPath string - - // LocksPack is the path to the locks directory. - LocksPack string - - // DefaultConfigsDir is the default Agent configuration directory - DefaultConfigsDir string + // DefaultUserConfigsDir is the default Agent configuration directory + DefaultUserConfigsDir string ) func init() { datadogInstallerData, _ := winregistry.GetProgramDataDirForProduct("Datadog Installer") - TmpDirPath = filepath.Join(datadogInstallerData, "temp") PackagesPath = filepath.Join(datadogInstallerData, "packages") - LocksPack = filepath.Join(datadogInstallerData, "locks") - DefaultConfigsDir, _ = windows.KnownFolderPath(windows.FOLDERID_ProgramData, 0) + ConfigsPath = filepath.Join(datadogInstallerData, "configs") + LocksPath = filepath.Join(datadogInstallerData, "locks") + DefaultUserConfigsDir, _ = windows.KnownFolderPath(windows.FOLDERID_ProgramData, 0) } From c480864cb17f78fbaa5adce4ba031fbe6f95f8c9 Mon Sep 17 00:00:00 2001 From: Kevin Fairise <132568982+KevinFairise2@users.noreply.github.com> Date: Mon, 19 Aug 2024 18:16:22 +0200 Subject: [PATCH 033/245] Handle base branch different from main (#28531) --- tasks/gotest.py | 4 +++- tasks/libs/common/git.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tasks/gotest.py b/tasks/gotest.py index 833352eb7b157..49bb9df504a39 100644 --- a/tasks/gotest.py +++ b/tasks/gotest.py @@ -33,6 +33,7 @@ from tasks.libs.common.git import get_modified_files from tasks.libs.common.junit_upload_core import enrich_junitxml, produce_junit_tar from tasks.libs.common.utils import clean_nested_paths, get_build_flags, gitlab_section +from tasks.libs.releasing.json import _get_release_json_value from tasks.modules import DEFAULT_MODULES, GoModule from tasks.test_core import ModuleTestResult, process_input_args, process_module_results, test_core from tasks.testwasher import TestWasher @@ -833,7 +834,8 @@ def should_run_all_tests(files, trigger_files): def get_go_modified_files(ctx): - files = get_modified_files(ctx) + base_branch = _get_release_json_value("base_branch") + files = get_modified_files(ctx, base_branch=base_branch) return [ file for file in files diff --git a/tasks/libs/common/git.py b/tasks/libs/common/git.py index fc85b12241629..c16a5c42e0cbe 100644 --- a/tasks/libs/common/git.py +++ b/tasks/libs/common/git.py @@ -45,8 +45,8 @@ def get_staged_files(ctx, commit="HEAD", include_deleted_files=False) -> Iterabl yield file -def get_modified_files(ctx) -> list[str]: - last_main_commit = ctx.run("git merge-base HEAD origin/main", hide=True).stdout +def get_modified_files(ctx, base_branch="main") -> list[str]: + last_main_commit = ctx.run(f"git merge-base HEAD origin/{base_branch}", hide=True).stdout return ctx.run(f"git diff --name-only --no-renames {last_main_commit}", hide=True).stdout.splitlines() From 32c3865764dda45e92ee4c939658e07cd33a2a5c Mon Sep 17 00:00:00 2001 From: Vickenty Fesunov Date: Mon, 19 Aug 2024 18:16:27 +0200 Subject: [PATCH 034/245] AMLII-1983 Remove redundant error message (#28498) --- pkg/collector/scheduler.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/collector/scheduler.go b/pkg/collector/scheduler.go index 0b32dd70150cc..e61b3f7dc7690 100644 --- a/pkg/collector/scheduler.go +++ b/pkg/collector/scheduler.go @@ -214,10 +214,6 @@ func (s *CheckScheduler) getChecks(config integration.Config) ([]check.Check, er } } - if len(checks) == 0 { - return checks, fmt.Errorf("unable to load any check from config '%s'", config.Name) - } - return checks, nil } From 76a301476413a3d58edd3bbb187e244a7cef0181 Mon Sep 17 00:00:00 2001 From: Scott Opell Date: Mon, 19 Aug 2024 12:20:24 -0400 Subject: [PATCH 035/245] Updates regression detector instructions and a lading patch version bump (#28493) --- test/regression/README.md | 21 +++++++++------------ test/regression/config.yaml | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/test/regression/README.md b/test/regression/README.md index 82c59fe4e732b..fca1b33bce783 100644 --- a/test/regression/README.md +++ b/test/regression/README.md @@ -12,24 +12,24 @@ contact #single-machine-performance; we'll be glad to help. In order for SMP's tooling to properly read a experiment directory please adhere to the following structure. Starting at the root: -* `cases/` -- __Required__ The directory that contains each regression - experiment. Each sub-directory is a separate experiment and the name of the +* `config.yaml` -- __Required__ Configuration that applies to all experiments. +* `cases/` -- __Required__ The directory that contains each experiment. + Each sub-directory is a separate experiment and the name of the directory is the name of the experiment, for instance `tcp_syslog_to_blackhole`. We call these sub-directories 'cases'. The structure of each case is as follows: * `lading/lading.yaml` -- __Required__ The [lading] configuration inside its own - directory. Directory will be mount read-only in the container built from - `Dockerfile` above at `/etc/lading`. + directory. * `datadog-agent/` -- __Required__ This is the configuration directory of your program. Will be mounted read-only in the container build from `Dockerfile` above at `/etc/datadog-agent`. -* `experiment.yaml` -- __Optional__ This file can be used to set a - single optimization goal for each experiment. - - In this file, the optimization goal is set via a snippet like +* `experiment.yaml` -- __Required__ Set any experiment-specific configuration. + The "optimization goal" determines what metric the Regression Detector + will analyze at the conclusion of the experiment. + Eg: ```yaml optimization_goal: ingress_throughput ``` @@ -37,9 +37,6 @@ The structure of each case is as follows: Supported values of `optimization_goal` are `ingress_throughput` and `egress_throughput`. - If an experiment omits this file, the optimization goal defaults to - `ingress_throughput`. - [Vector]: https://github.com/vectordotdev/vector/tree/master/regression [lading]: https://github.com/DataDog/lading @@ -54,5 +51,5 @@ See full docs [here](https://github.com/DataDog/single-machine-performance/blob/ An example command may look like this: ``` -smp local-run --experiment-dir ~/dev/datadog-agent/test/regression/ --case uds_to_blackhole --target-image datadog/agent-dev:nightly-main-1bf80594-py3 --lading-path ~/lading/target/release/lading --target-command "/bin/entrypoint.sh" --target datadog-agent +smp local-run --experiment-dir ~/dev/datadog-agent/test/regression/ --case uds_dogstatsd_to_api --target-image datadog/agent-dev:nightly-main-fe13dead-py3 ``` diff --git a/test/regression/config.yaml b/test/regression/config.yaml index 76f70e11a2091..cf3dbf4ef947b 100644 --- a/test/regression/config.yaml +++ b/test/regression/config.yaml @@ -1,5 +1,5 @@ lading: - version: 0.21.0 + version: 0.21.1 target: cpu_allotment: 8 From af65bccde21ec86bf6e0581c26280b0df8add0f5 Mon Sep 17 00:00:00 2001 From: Scott Opell Date: Mon, 19 Aug 2024 12:20:29 -0400 Subject: [PATCH 036/245] Refines SMP experiment focused on generating tags from a python check. (#23890) Co-authored-by: GeorgeHahn --- .../datadog-agent/conf.d/my-check.d/conf.yaml | 10 ----- .../datadog-agent/checks.d/my-check.py | 37 ++++++++++++------- .../datadog-agent/conf.d/my-check.d/conf.yaml | 12 ++++++ .../datadog-agent/datadog.yaml | 0 .../experiment.yaml | 5 +-- .../lading/lading.yaml | 0 6 files changed, 37 insertions(+), 27 deletions(-) delete mode 100644 test/regression/cases/pycheck_1000_100byte_tags/datadog-agent/conf.d/my-check.d/conf.yaml rename test/regression/cases/{pycheck_1000_100byte_tags => pycheck_lots_of_tags}/datadog-agent/checks.d/my-check.py (66%) create mode 100644 test/regression/cases/pycheck_lots_of_tags/datadog-agent/conf.d/my-check.d/conf.yaml rename test/regression/cases/{pycheck_1000_100byte_tags => pycheck_lots_of_tags}/datadog-agent/datadog.yaml (100%) rename test/regression/cases/{pycheck_1000_100byte_tags => pycheck_lots_of_tags}/experiment.yaml (88%) rename test/regression/cases/{pycheck_1000_100byte_tags => pycheck_lots_of_tags}/lading/lading.yaml (100%) diff --git a/test/regression/cases/pycheck_1000_100byte_tags/datadog-agent/conf.d/my-check.d/conf.yaml b/test/regression/cases/pycheck_1000_100byte_tags/datadog-agent/conf.d/my-check.d/conf.yaml deleted file mode 100644 index 1792246671ee4..0000000000000 --- a/test/regression/cases/pycheck_1000_100byte_tags/datadog-agent/conf.d/my-check.d/conf.yaml +++ /dev/null @@ -1,10 +0,0 @@ -instances: - - seed: abcdef - num_tagsets: 10 - tags_per_set: 1000 - tag_length: 100 - - - seed: 12255457845 - num_tagsets: 10 - tags_per_set: 1000 - tag_length: 100 diff --git a/test/regression/cases/pycheck_1000_100byte_tags/datadog-agent/checks.d/my-check.py b/test/regression/cases/pycheck_lots_of_tags/datadog-agent/checks.d/my-check.py similarity index 66% rename from test/regression/cases/pycheck_1000_100byte_tags/datadog-agent/checks.d/my-check.py rename to test/regression/cases/pycheck_lots_of_tags/datadog-agent/checks.d/my-check.py index 8cce2f10e49ea..17d5e2ac9beaa 100644 --- a/test/regression/cases/pycheck_1000_100byte_tags/datadog-agent/checks.d/my-check.py +++ b/test/regression/cases/pycheck_lots_of_tags/datadog-agent/checks.d/my-check.py @@ -14,14 +14,22 @@ def generate_tag_sets(rng, num_sets, tags_per_set, tag_length, unique_tagset_rat - num_sets (int): Number of tag sets to generate. - tags_per_set (int): Number of tags in each set. - tag_length (int): Total length of each tag, including the delimiter. - - unique_tagset_ratio (float): Value between 1 and 0, indicating the ratio of unique tag sets. + - unique_tag_ratio (float): Value between 0 and 1 inclusive. Indicates the ratio of unique tags. + If this value is 1, every tag will be unique. If 0, all will be re-used. - seed (int): Seed value for random number generator to ensure reproducibility. Returns: - List[List[str]]: A list of tag sets. """ + individual_tags = [] + def generate_tag(tag_length): + if rng.random() >= unique_tagset_ratio and len(individual_tags) != 0: + # sample from existing tags + return rng.choice(individual_tags) + # Else, generate a new unique tag + if tag_length % 2 == 0: half_length = tag_length // 2 - 1 else: @@ -36,20 +44,17 @@ def generate_tag(tag_length): left_part = ''.join(rng.choice(string.ascii_letters + string.digits) for _ in range(left_length)) right_part = ''.join(rng.choice(string.ascii_letters + string.digits) for _ in range(right_length)) - return f"{left_part}:{right_part}" + tag = f"{left_part}:{right_part}" + individual_tags.append(tag) + return tag tag_sets = [] for _ in range(num_sets): - if rng.random() <= unique_tagset_ratio or not tag_sets: - # Generate a unique tag set - current_set = set() - while len(current_set) < tags_per_set: - current_set.add(generate_tag(tag_length)) - tag_sets.append(list(current_set)) - else: - # Reuse an entire tag set from the previously generated ones - tag_sets.append(rng.choice(tag_sets).copy()) + current_set = set() + while len(current_set) < tags_per_set: + current_set.add(generate_tag(tag_length)) + tag_sets.append(list(current_set)) return tag_sets @@ -63,8 +68,12 @@ def check(self, instance): num_tagsets = instance.get("num_tagsets", 10) tags_per_set = instance.get("tags_per_set", 10) tag_length = instance.get("tag_length", 100) - unique_tagset_ratio = instance.get("unique_tagset_ratio", 0.5) + unique_tagset_ratio = instance.get("unique_tagset_ratio", 0.11) + num_metrics = instance.get("num_metrics", 100) tag_sets = generate_tag_sets(rng, num_tagsets, tags_per_set, tag_length, unique_tagset_ratio) - for tag_set in tag_sets: - self.gauge('hello.world', rng.random() * 1000, tags=tag_set) + for _ in range(num_metrics): + # For each metric that gets submitted, choose a tagset at random + # This will average out to + # contexts = len(tag_sets) as long as num_metrics is greater than num_tagsets + self.gauge('hello.world', rng.random() * 1000, tags=rng.choice(tag_sets)) diff --git a/test/regression/cases/pycheck_lots_of_tags/datadog-agent/conf.d/my-check.d/conf.yaml b/test/regression/cases/pycheck_lots_of_tags/datadog-agent/conf.d/my-check.d/conf.yaml new file mode 100644 index 0000000000000..fd3d6cde5774d --- /dev/null +++ b/test/regression/cases/pycheck_lots_of_tags/datadog-agent/conf.d/my-check.d/conf.yaml @@ -0,0 +1,12 @@ +instances: + - seed: abcdef + num_tagsets: 100 + tags_per_set: 10 + tag_length: 100 + num_metrics: 150 + + - seed: 12255457845 + num_tagsets: 100 + tags_per_set: 10 + tag_length: 100 + num_metrics: 150 diff --git a/test/regression/cases/pycheck_1000_100byte_tags/datadog-agent/datadog.yaml b/test/regression/cases/pycheck_lots_of_tags/datadog-agent/datadog.yaml similarity index 100% rename from test/regression/cases/pycheck_1000_100byte_tags/datadog-agent/datadog.yaml rename to test/regression/cases/pycheck_lots_of_tags/datadog-agent/datadog.yaml diff --git a/test/regression/cases/pycheck_1000_100byte_tags/experiment.yaml b/test/regression/cases/pycheck_lots_of_tags/experiment.yaml similarity index 88% rename from test/regression/cases/pycheck_1000_100byte_tags/experiment.yaml rename to test/regression/cases/pycheck_lots_of_tags/experiment.yaml index 4b692716ccd87..f07f0e3cfdc16 100644 --- a/test/regression/cases/pycheck_1000_100byte_tags/experiment.yaml +++ b/test/regression/cases/pycheck_lots_of_tags/experiment.yaml @@ -12,8 +12,6 @@ target: DD_DD_URL: http://127.0.0.1:9092 profiling_environment: - DD_INTEGRATION_PROFILING: true - DD_TRACE_AGENT_URL: unix:///var/run/datadog/apm.socket DD_INTERNAL_PROFILING_BLOCK_PROFILE_RATE: 10000 DD_INTERNAL_PROFILING_CPU_DURATION: 1m DD_INTERNAL_PROFILING_DELTA_PROFILES: true @@ -25,5 +23,6 @@ target: DD_PROFILING_EXECUTION_TRACE_ENABLED: true DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m DD_PROFILING_WAIT_PROFILE: true + DD_TRACE_AGENT_URL: unix:///var/run/datadog/apm.socket - DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:pycheck_1000_100byte_tags + DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:pycheck_lots_of_tags diff --git a/test/regression/cases/pycheck_1000_100byte_tags/lading/lading.yaml b/test/regression/cases/pycheck_lots_of_tags/lading/lading.yaml similarity index 100% rename from test/regression/cases/pycheck_1000_100byte_tags/lading/lading.yaml rename to test/regression/cases/pycheck_lots_of_tags/lading/lading.yaml From 1b8236b86e7c83443cb282345a44da039d347518 Mon Sep 17 00:00:00 2001 From: Hasan Mahmood <6599778+hmahmood@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:21:30 -0500 Subject: [PATCH 037/245] [NPM-3378] Adding eBPF-less network tracer POC (#27100) Co-authored-by: Bryce Kahle --- cmd/system-probe/config/adjust_npm.go | 22 + .../modules/network_tracer_linux.go | 5 +- go.mod | 2 + go.sum | 5 +- pkg/config/setup/system_probe.go | 2 + pkg/ebpf/ebpftest/buildmode.go | 19 + pkg/ebpf/ebpftest/buildmode_linux.go | 4 + pkg/network/config/config.go | 4 + pkg/network/dns/driver_windows.go | 5 +- pkg/network/dns/monitor_linux.go | 60 +- pkg/network/dns/packet_source_windows.go | 9 +- pkg/network/dns/snooper.go | 27 +- pkg/network/dns/snooper_test.go | 2 +- pkg/network/event_common.go | 12 + pkg/network/filter/packet_source.go | 33 + pkg/network/filter/packet_source_linux.go | 164 +++- pkg/network/filter/socket_filter.go | 4 +- pkg/network/netlink/conntracker.go | 17 + pkg/network/port.go | 4 +- pkg/network/port_test.go | 10 +- pkg/network/state.go | 3 - pkg/network/tracer/connection/cookie.go | 44 ++ pkg/network/tracer/connection/ebpf_tracer.go | 733 +++++++++++++++++ .../tracer/connection/ebpfless/payload.go | 102 +++ .../tracer/connection/ebpfless/ports.go | 130 +++ .../tracer/connection/ebpfless_tracer.go | 417 ++++++++++ pkg/network/tracer/connection/tracer.go | 745 +----------------- pkg/network/tracer/tracer.go | 52 +- pkg/network/tracer/tracer_linux_test.go | 43 +- pkg/network/tracer/tracer_test.go | 41 +- pkg/network/tracer/utils_linux.go | 12 +- pkg/network/tracer/utils_unsupported.go | 2 +- pkg/network/usm/config/config.go | 27 +- 33 files changed, 1863 insertions(+), 898 deletions(-) create mode 100644 pkg/network/filter/packet_source.go create mode 100644 pkg/network/tracer/connection/cookie.go create mode 100644 pkg/network/tracer/connection/ebpf_tracer.go create mode 100644 pkg/network/tracer/connection/ebpfless/payload.go create mode 100644 pkg/network/tracer/connection/ebpfless/ports.go create mode 100644 pkg/network/tracer/connection/ebpfless_tracer.go diff --git a/cmd/system-probe/config/adjust_npm.go b/cmd/system-probe/config/adjust_npm.go index 58cfe1c3ac8a3..ab0fc468bf553 100644 --- a/cmd/system-probe/config/adjust_npm.go +++ b/cmd/system-probe/config/adjust_npm.go @@ -25,6 +25,8 @@ const ( ) func adjustNetwork(cfg config.Config) { + ebpflessEnabled := cfg.GetBool(netNS("enable_ebpfless")) + limitMaxInt(cfg, spNS("max_conns_per_message"), maxConnsMessageBatchSize) if cfg.GetBool(spNS("disable_tcp")) { @@ -99,4 +101,24 @@ func adjustNetwork(cfg config.Config) { log.Warn("disabling NPM connection rollups since USM connection rollups are not enabled") cfg.Set(netNS("enable_connection_rollup"), false, model.SourceAgentRuntime) } + + // disable features that are not supported on certain + // configs/platforms + var disableConfigs []struct { + key, reason string + } + if ebpflessEnabled { + const notSupportedEbpfless = "not supported when ebpf-less is enabled" + disableConfigs = append(disableConfigs, []struct{ key, reason string }{ + {netNS("enable_protocol_classification"), notSupportedEbpfless}, + {evNS("network_process", "enabled"), notSupportedEbpfless}}..., + ) + } + + for _, c := range disableConfigs { + if cfg.GetBool(c.key) { + log.Warnf("disabling %s: %s", c.key, c.reason) + cfg.Set(c.key, false, model.SourceAgentRuntime) + } + } } diff --git a/cmd/system-probe/modules/network_tracer_linux.go b/cmd/system-probe/modules/network_tracer_linux.go index c3274c39df631..9481d9d9208e5 100644 --- a/cmd/system-probe/modules/network_tracer_linux.go +++ b/cmd/system-probe/modules/network_tracer_linux.go @@ -10,6 +10,7 @@ package modules import ( "github.com/DataDog/datadog-agent/cmd/system-probe/api/module" "github.com/DataDog/datadog-agent/cmd/system-probe/config" + "github.com/DataDog/datadog-agent/pkg/network/tracer" ) // NetworkTracer is a factory for NPM's tracer @@ -17,7 +18,5 @@ var NetworkTracer = module.Factory{ Name: config.NetworkTracerModule, ConfigNamespaces: networkTracerModuleConfigNamespaces, Fn: createNetworkTracerModule, - NeedsEBPF: func() bool { - return true - }, + NeedsEBPF: tracer.NeedsEBPF, } diff --git a/go.mod b/go.mod index fc6c85fe356d4..3135c86490b5a 100644 --- a/go.mod +++ b/go.mod @@ -1075,3 +1075,5 @@ replace ( // Prevent a false-positive detection by the Google and Ikarus security vendors on VirusTotal exclude go.opentelemetry.io/proto/otlp v1.1.0 + +replace github.com/google/gopacket v1.1.19 => github.com/DataDog/gopacket v0.0.0-20240626205202-4ac4cee31f14 diff --git a/go.sum b/go.sum index 019d7521a3dcb..b3bf2e8247cde 100644 --- a/go.sum +++ b/go.sum @@ -718,6 +718,8 @@ github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4ti github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee/go.mod h1:nTot/Iy0kW16bXgXr6blEc8gFeAS7vTqYlhAxh+dbc0= +github.com/DataDog/gopacket v0.0.0-20240626205202-4ac4cee31f14 h1:t34NfJA77KgFZsh8kcNFW57LZLa0kW2YSUs4MvLKRxU= +github.com/DataDog/gopacket v0.0.0-20240626205202-4ac4cee31f14/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA= github.com/DataDog/gopsutil v1.2.2 h1:8lmthwyyCXa1NKiYcHlrtl9AAFdfbNI2gPcioCJcBPU= github.com/DataDog/gopsutil v1.2.2/go.mod h1:glkxNt/qRu9lnpmUEQwOIAXW+COWDTBOTEAHqbgBPts= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= @@ -1507,8 +1509,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= -github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA= github.com/google/licenseclassifier/v2 v2.0.0/go.mod h1:cOjbdH0kyC9R22sdQbYsFkto4NGCAc+ZSwbeThazEtM= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= @@ -2567,6 +2567,7 @@ github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8ok github.com/vibrantbyte/go-antpath v1.1.1 h1:SWDIMx4pSjyo7QoAsgTkpNU7QD0X9O0JAgr5O3TsYKk= github.com/vibrantbyte/go-antpath v1.1.1/go.mod h1:ZqMGIk+no3BL2o6OdEZ3ZDiWfIteuastNSaTFv7kgUY= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vito/go-sse v1.0.0 h1:e6/iTrrvy8BRrOwJwmQmlndlil+TLdxXvHi55ZDzH6M= diff --git a/pkg/config/setup/system_probe.go b/pkg/config/setup/system_probe.go index 3998a6519a708..5fcf419faf449 100644 --- a/pkg/config/setup/system_probe.go +++ b/pkg/config/setup/system_probe.go @@ -299,6 +299,8 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Config) { // connection aggregation with port rollups cfg.BindEnvAndSetDefault(join(netNS, "enable_connection_rollup"), false) + cfg.BindEnvAndSetDefault(join(netNS, "enable_ebpfless"), false) + // windows config cfg.BindEnvAndSetDefault(join(spNS, "windows.enable_monotonic_count"), false) diff --git a/pkg/ebpf/ebpftest/buildmode.go b/pkg/ebpf/ebpftest/buildmode.go index 6fcc977f80af4..76e774f68cbbb 100644 --- a/pkg/ebpf/ebpftest/buildmode.go +++ b/pkg/ebpf/ebpftest/buildmode.go @@ -16,6 +16,7 @@ var ( RuntimeCompiled BuildMode CORE BuildMode Fentry BuildMode + Ebpfless BuildMode ) func init() { @@ -23,6 +24,7 @@ func init() { RuntimeCompiled = runtimeCompiled{} CORE = core{} Fentry = fentry{} + Ebpfless = ebpfless{} } // BuildMode is an eBPF build mode @@ -95,6 +97,23 @@ func (f fentry) Env() map[string]string { } } +type ebpfless struct{} + +func (e ebpfless) String() string { + return "eBPFless" +} + +func (e ebpfless) Env() map[string]string { + return map[string]string{ + "NETWORK_TRACER_FENTRY_TESTS": "false", + "DD_ENABLE_RUNTIME_COMPILER": "false", + "DD_ENABLE_CO_RE": "false", + "DD_ALLOW_RUNTIME_COMPILED_FALLBACK": "false", + "DD_ALLOW_PRECOMPILED_FALLBACK": "false", + "DD_NETWORK_CONFIG_ENABLE_EBPFLESS": "true", + } +} + // GetBuildMode returns which build mode the current environment matches, if any func GetBuildMode() BuildMode { for _, mode := range []BuildMode{Prebuilt, RuntimeCompiled, CORE, Fentry} { diff --git a/pkg/ebpf/ebpftest/buildmode_linux.go b/pkg/ebpf/ebpftest/buildmode_linux.go index a16d7bbc8dc2d..0e88f0f528271 100644 --- a/pkg/ebpf/ebpftest/buildmode_linux.go +++ b/pkg/ebpf/ebpftest/buildmode_linux.go @@ -30,6 +30,10 @@ func SupportedBuildModes() []BuildMode { (runtime.GOARCH == "amd64" && (hostPlatform == "amazon" || hostPlatform == "amzn") && kv.Major() == 5 && kv.Minor() == 10) { modes = append(modes, Fentry) } + if os.Getenv("TEST_EBPFLESS_OVERRIDE") == "true" { + modes = append(modes, Ebpfless) + } + return modes } diff --git a/pkg/network/config/config.go b/pkg/network/config/config.go index ec9db0aae1e9a..5c443e0a70a41 100644 --- a/pkg/network/config/config.go +++ b/pkg/network/config/config.go @@ -302,6 +302,8 @@ type Config struct { // buffers (>=5.8) will result in forcing the use of Perf Maps instead. EnableUSMRingBuffers bool + EnableEbpfless bool + // EnableUSMEventStream enables USM to use the event stream instead // of netlink for receiving process events. EnableUSMEventStream bool @@ -406,6 +408,8 @@ func New() *Config { EnableNPMConnectionRollup: cfg.GetBool(join(netNS, "enable_connection_rollup")), + EnableEbpfless: cfg.GetBool(join(netNS, "enable_ebpfless")), + // Service Monitoring EnableJavaTLSSupport: cfg.GetBool(join(smjtNS, "enabled")), JavaAgentDebug: cfg.GetBool(join(smjtNS, "debug")), diff --git a/pkg/network/dns/driver_windows.go b/pkg/network/dns/driver_windows.go index 82e0cb3e88529..3056d38c25cdd 100644 --- a/pkg/network/dns/driver_windows.go +++ b/pkg/network/dns/driver_windows.go @@ -23,6 +23,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/telemetry" "github.com/DataDog/datadog-agent/pkg/network/driver" + "github.com/DataDog/datadog-agent/pkg/network/filter" ) const ( @@ -99,7 +100,7 @@ func (d *dnsDriver) SetDataFilters(filters []driver.FilterDefinition) error { } // ReadDNSPacket visits a raw DNS packet if one is available. -func (d *dnsDriver) ReadDNSPacket(visit func([]byte, time.Time) error) (didRead bool, err error) { +func (d *dnsDriver) ReadDNSPacket(visit func(data []byte, info filter.PacketInfo, t time.Time) error) (didRead bool, err error) { var bytesRead uint32 var key uintptr // returned by GetQueuedCompletionStatus, then ignored var ol *windows.Overlapped @@ -125,7 +126,7 @@ func (d *dnsDriver) ReadDNSPacket(visit func([]byte, time.Time) error) (didRead start := driver.FilterPacketHeaderSize - if err := visit(buf.data[start:], captureTime); err != nil { + if err := visit(buf.data[start:], nil, captureTime); err != nil { return false, err } diff --git a/pkg/network/dns/monitor_linux.go b/pkg/network/dns/monitor_linux.go index af87db20f2ee7..bd8c606415f0f 100644 --- a/pkg/network/dns/monitor_linux.go +++ b/pkg/network/dns/monitor_linux.go @@ -11,17 +11,14 @@ import ( "fmt" "math" - "golang.org/x/net/bpf" - - "github.com/vishvananda/netns" - manager "github.com/DataDog/ebpf-manager" + "github.com/vishvananda/netns" "github.com/DataDog/datadog-agent/comp/core/telemetry" ddebpf "github.com/DataDog/datadog-agent/pkg/ebpf" "github.com/DataDog/datadog-agent/pkg/network/config" "github.com/DataDog/datadog-agent/pkg/network/ebpf/probes" - filterpkg "github.com/DataDog/datadog-agent/pkg/network/filter" + "github.com/DataDog/datadog-agent/pkg/network/filter" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -33,6 +30,26 @@ type dnsMonitor struct { // NewReverseDNS starts snooping on DNS traffic to allow IP -> domain reverse resolution func NewReverseDNS(cfg *config.Config, _ telemetry.Component) (ReverseDNS, error) { + // Create the RAW_SOCKET inside the root network namespace + var ( + packetSrc *filter.AFPacketSource + srcErr error + ns netns.NsHandle + ) + ns, err := cfg.GetRootNetNs() + if err != nil { + return nil, err + } + defer ns.Close() + + err = kernel.WithNS(ns, func() error { + packetSrc, srcErr = filter.NewAFPacketSource(4 << 20) // 4 MB total + return srcErr + }) + if err != nil { + return nil, err + } + currKernelVersion, err := kernel.HostVersion() if err != nil { // if the platform couldn't be determined, treat it as new kernel case @@ -42,12 +59,11 @@ func NewReverseDNS(cfg *config.Config, _ telemetry.Component) (ReverseDNS, error pre410Kernel := currKernelVersion < kernel.VersionCode(4, 1, 0) var p *ebpfProgram - var filter *manager.Probe - var bpfFilter []bpf.RawInstruction - if pre410Kernel { - bpfFilter, err = generateBPFFilter(cfg) - if err != nil { + if pre410Kernel || cfg.EnableEbpfless { + if bpfFilter, err := generateBPFFilter(cfg); err != nil { return nil, fmt.Errorf("error creating bpf classic filter: %w", err) + } else if err = packetSrc.SetBPF(bpfFilter); err != nil { + return nil, fmt.Errorf("could not set BPF filter on packet source: %w", err) } } else { p, err = newEBPFProgram(cfg) @@ -59,35 +75,21 @@ func NewReverseDNS(cfg *config.Config, _ telemetry.Component) (ReverseDNS, error return nil, fmt.Errorf("error initializing ebpf programs: %w", err) } - filter, _ = p.GetProbe(manager.ProbeIdentificationPair{EBPFFuncName: probes.SocketDNSFilter, UID: probeUID}) + filter, _ := p.GetProbe(manager.ProbeIdentificationPair{EBPFFuncName: probes.SocketDNSFilter, UID: probeUID}) if filter == nil { return nil, fmt.Errorf("error retrieving socket filter") } - } - // Create the RAW_SOCKET inside the root network namespace - var ( - packetSrc *filterpkg.AFPacketSource - srcErr error - ns netns.NsHandle - ) - if ns, err = cfg.GetRootNetNs(); err != nil { - return nil, err - } - defer ns.Close() - - err = kernel.WithNS(ns, func() error { - packetSrc, srcErr = filterpkg.NewPacketSource(filter, bpfFilter) - return srcErr - }) - if err != nil { - return nil, err + if err = packetSrc.SetEbpf(filter); err != nil { + return nil, fmt.Errorf("could not set file descriptor for eBPF program: %w", err) + } } snoop, err := newSocketFilterSnooper(cfg, packetSrc) if err != nil { return nil, err } + return &dnsMonitor{ snoop, p, diff --git a/pkg/network/dns/packet_source_windows.go b/pkg/network/dns/packet_source_windows.go index 5a00280591eda..52ae79ffa4cef 100644 --- a/pkg/network/dns/packet_source_windows.go +++ b/pkg/network/dns/packet_source_windows.go @@ -14,16 +14,17 @@ import ( "github.com/google/gopacket/layers" "github.com/DataDog/datadog-agent/comp/core/telemetry" + "github.com/DataDog/datadog-agent/pkg/network/filter" ) -var _ packetSource = &windowsPacketSource{} +var _ filter.PacketSource = &windowsPacketSource{} type windowsPacketSource struct { di *dnsDriver } // newWindowsPacketSource constructs a new packet source -func newWindowsPacketSource(telemetrycomp telemetry.Component) (packetSource, error) { +func newWindowsPacketSource(telemetrycomp telemetry.Component) (filter.PacketSource, error) { di, err := newDriver(telemetrycomp) if err != nil { return nil, err @@ -31,7 +32,7 @@ func newWindowsPacketSource(telemetrycomp telemetry.Component) (packetSource, er return &windowsPacketSource{di: di}, nil } -func (p *windowsPacketSource) VisitPackets(exit <-chan struct{}, visit func([]byte, time.Time) error) error { +func (p *windowsPacketSource) VisitPackets(exit <-chan struct{}, visit func([]byte, filter.PacketInfo, time.Time) error) error { for { didReadPacket, err := p.di.ReadDNSPacket(visit) if err != nil { @@ -50,7 +51,7 @@ func (p *windowsPacketSource) VisitPackets(exit <-chan struct{}, visit func([]by } } -func (p *windowsPacketSource) PacketType() gopacket.LayerType { +func (p *windowsPacketSource) LayerType() gopacket.LayerType { return layers.LayerTypeIPv4 } diff --git a/pkg/network/dns/snooper.go b/pkg/network/dns/snooper.go index e392a96bca550..c1bc700a33b3c 100644 --- a/pkg/network/dns/snooper.go +++ b/pkg/network/dns/snooper.go @@ -11,9 +11,8 @@ import ( "sync" "time" - "github.com/google/gopacket" - "github.com/DataDog/datadog-agent/pkg/network/config" + "github.com/DataDog/datadog-agent/pkg/network/filter" "github.com/DataDog/datadog-agent/pkg/process/util" "github.com/DataDog/datadog-agent/pkg/telemetry" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -45,7 +44,7 @@ var _ ReverseDNS = &socketFilterSnooper{} // socketFilterSnooper is a DNS traffic snooper built on top of an eBPF SOCKET_FILTER type socketFilterSnooper struct { - source packetSource + source filter.PacketSource parser *dnsParser cache *reverseDNSCache statKeeper *dnsStatKeeper @@ -62,24 +61,8 @@ func (s *socketFilterSnooper) WaitForDomain(domain string) error { return s.statKeeper.WaitForDomain(domain) } -// packetSource reads raw packet data -type packetSource interface { - // VisitPackets reads all new raw packets that are available, invoking the given callback for each packet. - // If no packet is available, VisitPacket returns immediately. - // The format of the packet is dependent on the implementation of packetSource -- i.e. it may be an ethernet frame, or a IP frame. - // The data buffer is reused between invocations of VisitPacket and thus should not be pointed to. - // If the cancel channel is closed, VisitPackets will stop reading. - VisitPackets(cancel <-chan struct{}, visitor func(data []byte, timestamp time.Time) error) error - - // PacketType returns the type of packet this source reads - PacketType() gopacket.LayerType - - // Close closes the packet source - Close() -} - // newSocketFilterSnooper returns a new socketFilterSnooper -func newSocketFilterSnooper(cfg *config.Config, source packetSource) (*socketFilterSnooper, error) { +func newSocketFilterSnooper(cfg *config.Config, source filter.PacketSource) (*socketFilterSnooper, error) { cache := newReverseDNSCache(dnsCacheSize, dnsCacheExpirationPeriod) var statKeeper *dnsStatKeeper if cfg.CollectDNSStats { @@ -93,7 +76,7 @@ func newSocketFilterSnooper(cfg *config.Config, source packetSource) (*socketFil } snooper := &socketFilterSnooper{ source: source, - parser: newDNSParser(source.PacketType(), cfg), + parser: newDNSParser(source.LayerType(), cfg), cache: cache, statKeeper: statKeeper, translation: new(translation), @@ -154,7 +137,7 @@ func (s *socketFilterSnooper) Close() { // The *translation is recycled and re-used in subsequent calls and it should not be accessed concurrently. // The second parameter `ts` is the time when the packet was captured off the wire. This is used for latency calculation // and much more reliable than calling time.Now() at the user layer. -func (s *socketFilterSnooper) processPacket(data []byte, ts time.Time) error { +func (s *socketFilterSnooper) processPacket(data []byte, _ filter.PacketInfo, ts time.Time) error { t := s.getCachedTranslation() pktInfo := dnsPacketInfo{} diff --git a/pkg/network/dns/snooper_test.go b/pkg/network/dns/snooper_test.go index 3c79c25bb7e59..3a0d334c80488 100644 --- a/pkg/network/dns/snooper_test.go +++ b/pkg/network/dns/snooper_test.go @@ -390,7 +390,7 @@ func TestParsingError(t *testing.T) { reverseDNS := rdns.(*dnsMonitor) // Pass a byte array of size 1 which should result in parsing error - err = reverseDNS.processPacket(make([]byte, 1), time.Now()) + err = reverseDNS.processPacket(make([]byte, 1), 0, time.Now()) require.NoError(t, err) assert.True(t, cacheTelemetry.length.Load() == 0) assert.True(t, snooperTelemetry.decodingErrors.Load() == 1) diff --git a/pkg/network/event_common.go b/pkg/network/event_common.go index 7ec7a5390e34a..3c5c4f9769741 100644 --- a/pkg/network/event_common.go +++ b/pkg/network/event_common.go @@ -30,6 +30,9 @@ const ( maxByteCountChange uint64 = 375 << 30 // use typical small MTU size, 1300, to get max packet count maxPacketCountChange uint64 = maxByteCountChange / 1300 + + // ConnectionByteKeyMaxLen represents the maximum size in bytes of a connection byte key + ConnectionByteKeyMaxLen = 41 ) // ConnectionType will be either TCP or UDP @@ -332,6 +335,15 @@ func (c ConnectionStats) ByteKeyNAT(buf []byte) []byte { return generateConnectionKey(c, buf, true) } +// IsValid returns `true` if the connection has a valid source and dest +// ports and IPs +func (c ConnectionStats) IsValid() bool { + return c.Source.IsValid() && + c.Dest.IsValid() && + c.SPort > 0 && + c.DPort > 0 +} + const keyFmt = "p:%d|src:%s:%d|dst:%s:%d|f:%d|t:%d" // BeautifyKey returns a human readable byte key (used for debugging purposes) diff --git a/pkg/network/filter/packet_source.go b/pkg/network/filter/packet_source.go new file mode 100644 index 0000000000000..c75ad1222e9bf --- /dev/null +++ b/pkg/network/filter/packet_source.go @@ -0,0 +1,33 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +// Package filter exposes interfaces and implementations for packet capture +package filter + +import ( + "time" + + "github.com/google/gopacket" +) + +// PacketInfo holds OS dependent packet information +// about a packet +type PacketInfo interface{} + +// PacketSource reads raw packet data +type PacketSource interface { + // VisitPackets reads all new raw packets that are available, invoking the given callback for each packet. + // If no packet is available, VisitPacket returns immediately. + // The format of the packet is dependent on the implementation of PacketSource -- i.e. it may be an ethernet frame, or a IP frame. + // The data buffer is reused between invocations of VisitPacket and thus should not be pointed to. + // If the cancel channel is closed, VisitPackets will stop reading. + VisitPackets(cancel <-chan struct{}, visitor func(data []byte, info PacketInfo, timestamp time.Time) error) error + + // LayerType returns the type of packet this source reads + LayerType() gopacket.LayerType + + // Close closes the packet source + Close() +} diff --git a/pkg/network/filter/packet_source_linux.go b/pkg/network/filter/packet_source_linux.go index 8e21c04e7d5f2..65fbbe9f9270f 100644 --- a/pkg/network/filter/packet_source_linux.go +++ b/pkg/network/filter/packet_source_linux.go @@ -3,13 +3,14 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build linux_bpf +//go:build linux -//nolint:revive // TODO(NET) Fix revive linter +// Package filter exposes interfaces and implementations for packet capture package filter import ( "fmt" + "os" "reflect" "syscall" "time" @@ -19,12 +20,16 @@ import ( "github.com/google/gopacket/afpacket" "github.com/google/gopacket/layers" "golang.org/x/net/bpf" + "golang.org/x/sys/unix" "github.com/DataDog/datadog-agent/pkg/telemetry" "github.com/DataDog/datadog-agent/pkg/util/log" ) -const telemetryModuleName = "network_tracer__dns" +const ( + telemetryModuleName = "network_tracer__filter" + defaultSnapLen = 4096 +) // Telemetry var packetSourceTelemetry = struct { @@ -42,49 +47,91 @@ var packetSourceTelemetry = struct { // AFPacketSource provides a RAW_SOCKET attached to an eBPF SOCKET_FILTER type AFPacketSource struct { *afpacket.TPacket - socketFilter *manager.Probe exit chan struct{} } -// NewPacketSource creates an AFPacketSource using the provided BPF filter -func NewPacketSource(filter *manager.Probe, bpfFilter []bpf.RawInstruction) (*AFPacketSource, error) { +// AFPacketInfo holds information about a packet +type AFPacketInfo struct { + // PktType corresponds to sll_pkttype in the + // sockaddr_ll struct; see packet(7) + // https://man7.org/linux/man-pages/man7/packet.7.html + PktType uint8 +} + +// OptSnapLen specifies the maximum length of the packet to read +// +// Defaults to 4096 bytes +type OptSnapLen int + +// NewAFPacketSource creates an AFPacketSource using the provided BPF filter +func NewAFPacketSource(size int, opts ...interface{}) (*AFPacketSource, error) { + snapLen := defaultSnapLen + for _, opt := range opts { + switch o := opt.(type) { + case OptSnapLen: + snapLen = int(o) + if snapLen <= 0 || snapLen > 65536 { + return nil, fmt.Errorf("snap len should be between 0 and 65536") + } + default: + return nil, fmt.Errorf("unknown option %+v", opt) + } + } + + frameSize, blockSize, numBlocks, err := afpacketComputeSize(size, snapLen, os.Getpagesize()) + if err != nil { + return nil, fmt.Errorf("error computing mmap'ed buffer parameters: %w", err) + } + + log.Debugf("creating tpacket source with frame_size=%d block_size=%d num_blocks=%d", frameSize, blockSize, numBlocks) rawSocket, err := afpacket.NewTPacket( - afpacket.OptPollTimeout(1*time.Second), - // This setup will require ~4Mb that is mmap'd into the process virtual space - // More information here: https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt - afpacket.OptFrameSize(4096), - afpacket.OptBlockSize(4096*128), - afpacket.OptNumBlocks(8), + afpacket.OptPollTimeout(time.Second), + afpacket.OptFrameSize(frameSize), + afpacket.OptBlockSize(blockSize), + afpacket.OptNumBlocks(numBlocks), + afpacket.OptAddPktType(true), ) + if err != nil { return nil, fmt.Errorf("error creating raw socket: %s", err) } - if filter != nil { - // The underlying socket file descriptor is private, hence the use of reflection - // Point socket filter program to the RAW_SOCKET file descriptor - // Note the filter attachment itself is triggered by the ebpf.Manager - filter.SocketFD = int(reflect.ValueOf(rawSocket).Elem().FieldByName("fd").Int()) - } else { - err = rawSocket.SetBPF(bpfFilter) - if err != nil { - return nil, fmt.Errorf("error setting classic bpf filter: %w", err) - } - } - ps := &AFPacketSource{ - TPacket: rawSocket, - socketFilter: filter, - exit: make(chan struct{}), + TPacket: rawSocket, + exit: make(chan struct{}), } go ps.pollStats() return ps, nil } +// SetEbpf attaches an eBPF socket filter to the AFPacketSource +func (p *AFPacketSource) SetEbpf(filter *manager.Probe) error { + // The underlying socket file descriptor is private, hence the use of reflection + // Point socket filter program to the RAW_SOCKET file descriptor + // Note the filter attachment itself is triggered by the ebpf.Manager + f := reflect.ValueOf(p.TPacket).Elem().FieldByName("fd") + if !f.IsValid() { + return fmt.Errorf("could not find fd field in TPacket object") + } + + if !f.CanInt() { + return fmt.Errorf("fd TPacket field is not an int") + } + + filter.SocketFD = int(f.Int()) + return nil +} + +// SetBPF attaches a (classic) BPF socket filter to the AFPacketSource +func (p *AFPacketSource) SetBPF(filter []bpf.RawInstruction) error { + return p.TPacket.SetBPF(filter) +} + // VisitPackets starts reading packets from the source -func (p *AFPacketSource) VisitPackets(exit <-chan struct{}, visit func([]byte, time.Time) error) error { +func (p *AFPacketSource) VisitPackets(exit <-chan struct{}, visit func(data []byte, info PacketInfo, t time.Time) error) error { + pktInfo := &AFPacketInfo{} for { // allow the read loop to be prematurely interrupted select { @@ -108,14 +155,15 @@ func (p *AFPacketSource) VisitPackets(exit <-chan struct{}, visit func([]byte, t return err } - if err := visit(data, stats.Timestamp); err != nil { + pktInfo.PktType = stats.AncillaryData[0].(afpacket.AncillaryPktType).Type + if err := visit(data, pktInfo, stats.Timestamp); err != nil { return err } } } -// PacketType is the gopacket.LayerType for this source -func (p *AFPacketSource) PacketType() gopacket.LayerType { +// LayerType is the gopacket.LayerType for this source +func (p *AFPacketSource) LayerType() gopacket.LayerType { return layers.LayerTypeEthernet } @@ -160,3 +208,57 @@ func (p *AFPacketSource) pollStats() { } } } + +// afpacketComputeSize computes the block_size and the num_blocks in such a way that the +// allocated mmap buffer is close to but smaller than target_size_mb. +// The restriction is that the block_size must be divisible by both the +// frame size and page size. +// +// See https://www.kernel.org/doc/Documentation/networking/packet_mmap.txt +func afpacketComputeSize(targetSize, snaplen, pageSize int) (frameSize, blockSize, numBlocks int, err error) { + frameSize = tpacketAlign(unix.TPACKET_HDRLEN) + tpacketAlign(snaplen) + if frameSize <= pageSize { + frameSize = int(nextPowerOf2(int64(frameSize))) + if frameSize <= pageSize { + blockSize = pageSize + } + } else { + // align frameSize to pageSize + frameSize = (frameSize + pageSize - 1) & ^(pageSize - 1) + blockSize = frameSize + } + + numBlocks = targetSize / blockSize + if numBlocks == 0 { + return 0, 0, 0, fmt.Errorf("buffer size is too small") + } + + blockSizeInc := blockSize + for numBlocks > afpacket.DefaultNumBlocks { + blockSize += blockSizeInc + numBlocks = targetSize / blockSize + } + + return frameSize, blockSize, numBlocks, nil +} + +func tpacketAlign(x int) int { + return (x + unix.TPACKET_ALIGNMENT - 1) & ^(unix.TPACKET_ALIGNMENT - 1) +} + +// nextPowerOf2 rounds up `v` to the next power of 2 +// +// Taken from Hacker's Delight by Henry S. Warren, Jr., +// https://en.wikipedia.org/wiki/Hacker%27s_Delight +func nextPowerOf2(v int64) int64 { + v-- + v |= v >> 1 + v |= v >> 2 + v |= v >> 4 + v |= v >> 8 + v |= v >> 16 + v |= v >> 32 + v++ + + return v +} diff --git a/pkg/network/filter/socket_filter.go b/pkg/network/filter/socket_filter.go index 467cb3931d29f..5c29b2d15cea2 100644 --- a/pkg/network/filter/socket_filter.go +++ b/pkg/network/filter/socket_filter.go @@ -5,15 +5,15 @@ //go:build linux_bpf +// Package filter exposes interfaces and implementations for packet capture package filter import ( "encoding/binary" "runtime" - "golang.org/x/sys/unix" - manager "github.com/DataDog/ebpf-manager" + "golang.org/x/sys/unix" "github.com/DataDog/datadog-agent/pkg/network/config" "github.com/DataDog/datadog-agent/pkg/util/kernel" diff --git a/pkg/network/netlink/conntracker.go b/pkg/network/netlink/conntracker.go index af4314b800cbd..9f066d2f8cb97 100644 --- a/pkg/network/netlink/conntracker.go +++ b/pkg/network/netlink/conntracker.go @@ -10,6 +10,7 @@ package netlink import ( "container/list" "context" + "errors" "fmt" "net" "net/netip" @@ -17,6 +18,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/syndtr/gocapability/capability" "golang.org/x/sys/unix" "github.com/cihub/seelog" @@ -39,6 +41,9 @@ const ( var defaultBuckets = []float64{10, 25, 50, 75, 100, 250, 500, 1000, 10000} +// ErrNotPermitted is the error returned when the current process does not have the required permissions for netlink conntracker +var ErrNotPermitted = errors.New("netlink conntracker requires NET_ADMIN capability") + // Conntracker is a wrapper around go-conntracker that keeps a record of all connections in user space type Conntracker interface { // Describe returns all descriptions of the collector @@ -115,6 +120,18 @@ func NewConntracker(config *config.Config, telemetrycomp telemetryComp.Component conntracker Conntracker ) + // check if we have the right capabilities for the netlink NewConntracker + // NET_ADMIN is required + if caps, err := capability.NewPid2(0); err == nil { + if err = caps.Load(); err != nil { + return nil, fmt.Errorf("could not load process capabilities: %w", err) + } + + if !caps.Get(capability.EFFECTIVE, capability.CAP_NET_ADMIN) { + return nil, ErrNotPermitted + } + } + done := make(chan struct{}) go func() { diff --git a/pkg/network/port.go b/pkg/network/port.go index 6a1ed3f560555..cce191d642382 100644 --- a/pkg/network/port.go +++ b/pkg/network/port.go @@ -30,8 +30,8 @@ var statusMap = map[ConnectionType]int64{ UDP: tcpClose, } -// ReadInitialState reads the /proc filesystem and determines which ports are being listened on -func ReadInitialState(procRoot string, protocol ConnectionType, collectIPv6 bool) (map[PortMapping]uint32, error) { +// ReadListeningPorts reads the /proc filesystem and determines which ports are being listened on +func ReadListeningPorts(procRoot string, protocol ConnectionType, collectIPv6 bool) (map[PortMapping]uint32, error) { start := time.Now() defer func() { log.Debugf("Read initial %s pid->port mapping in %s", protocol.String(), time.Since(start)) diff --git a/pkg/network/port_test.go b/pkg/network/port_test.go index e52c80829cecb..995a4e8ef0a2f 100644 --- a/pkg/network/port_test.go +++ b/pkg/network/port_test.go @@ -89,16 +89,16 @@ func runServerProcess(t *testing.T, proto string, port uint16, ns netns.NsHandle return port, proc } -func TestReadInitialState(t *testing.T) { +func TestReadListeningPorts(t *testing.T) { t.Run("TCP", func(t *testing.T) { - testReadInitialState(t, "tcp") + testReadListeningPorts(t, "tcp") }) t.Run("UDP", func(t *testing.T) { - testReadInitialState(t, "udp") + testReadListeningPorts(t, "udp") }) } -func testReadInitialState(t *testing.T, proto string) { +func testReadListeningPorts(t *testing.T, proto string) { var ns, rootNs netns.NsHandle var err error nsName := netlinktestutil.AddNS(t) @@ -159,7 +159,7 @@ func testReadInitialState(t *testing.T, proto string) { connType, otherConnType = otherConnType, connType } - initialPorts, err := ReadInitialState("/proc", connType, true) + initialPorts, err := ReadListeningPorts("/proc", connType, true) if !assert.NoError(t, err) { return } diff --git a/pkg/network/state.go b/pkg/network/state.go index 708e535adeec5..7bd2684022837 100644 --- a/pkg/network/state.go +++ b/pkg/network/state.go @@ -74,9 +74,6 @@ const ( // constant is not worth the increased memory cost. DNSResponseCodeNoError = 0 - // ConnectionByteKeyMaxLen represents the maximum size in bytes of a connection byte key - ConnectionByteKeyMaxLen = 41 - stateModuleName = "network_tracer__state" shortLivedConnectionThreshold = 2 * time.Minute diff --git a/pkg/network/tracer/connection/cookie.go b/pkg/network/tracer/connection/cookie.go new file mode 100644 index 0000000000000..bc32836d81ff4 --- /dev/null +++ b/pkg/network/tracer/connection/cookie.go @@ -0,0 +1,44 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux && npm + +package connection + +import ( + "encoding/binary" + "hash" + + "github.com/twmb/murmur3" + + "github.com/DataDog/datadog-agent/pkg/network" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +type cookieHasher struct { + hash hash.Hash64 + buf []byte +} + +func newCookieHasher() *cookieHasher { + return &cookieHasher{ + hash: murmur3.New64(), + buf: make([]byte, network.ConnectionByteKeyMaxLen), + } +} + +func (h *cookieHasher) Hash(stats *network.ConnectionStats) { + h.hash.Reset() + if err := binary.Write(h.hash, binary.BigEndian, stats.Cookie); err != nil { + log.Errorf("error writing cookie to hash: %s", err) + return + } + key := stats.ByteKey(h.buf) + if _, err := h.hash.Write(key); err != nil { + log.Errorf("error writing byte key to hash: %s", err) + return + } + stats.Cookie = h.hash.Sum64() +} diff --git a/pkg/network/tracer/connection/ebpf_tracer.go b/pkg/network/tracer/connection/ebpf_tracer.go new file mode 100644 index 0000000000000..558dc2a83218a --- /dev/null +++ b/pkg/network/tracer/connection/ebpf_tracer.go @@ -0,0 +1,733 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux_bpf + +package connection + +import ( + "errors" + "fmt" + "io" + "math" + "sync" + "time" + + manager "github.com/DataDog/ebpf-manager" + "github.com/DataDog/ebpf-manager/tracefs" + "github.com/cihub/seelog" + "github.com/cilium/ebpf" + "github.com/prometheus/client_golang/prometheus" + "go.uber.org/atomic" + "golang.org/x/sys/unix" + + telemetryComponent "github.com/DataDog/datadog-agent/comp/core/telemetry" + ddebpf "github.com/DataDog/datadog-agent/pkg/ebpf" + "github.com/DataDog/datadog-agent/pkg/ebpf/maps" + ebpftelemetry "github.com/DataDog/datadog-agent/pkg/ebpf/telemetry" + "github.com/DataDog/datadog-agent/pkg/network" + "github.com/DataDog/datadog-agent/pkg/network/config" + netebpf "github.com/DataDog/datadog-agent/pkg/network/ebpf" + "github.com/DataDog/datadog-agent/pkg/network/ebpf/probes" + "github.com/DataDog/datadog-agent/pkg/network/protocols" + "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/failure" + "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/fentry" + "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/kprobe" + "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/util" + "github.com/DataDog/datadog-agent/pkg/telemetry" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +const ( + defaultClosedChannelSize = 500 + defaultFailedChannelSize = 500 + connTracerModuleName = "network_tracer__ebpf" +) + +//nolint:revive // TODO(NET) Fix revive linter +var EbpfTracerTelemetry = struct { + connections telemetry.Gauge + tcpFailedConnects *prometheus.Desc + //nolint:revive // TODO(NET) Fix revive linter + TcpSentMiscounts *prometheus.Desc + //nolint:revive // TODO(NET) Fix revive linter + unbatchedTcpClose *prometheus.Desc + //nolint:revive // TODO(NET) Fix revive linter + unbatchedUdpClose *prometheus.Desc + //nolint:revive // TODO(NET) Fix revive linter + UdpSendsProcessed *prometheus.Desc + //nolint:revive // TODO(NET) Fix revive linter + UdpSendsMissed *prometheus.Desc + //nolint:revive // TODO(NET) Fix revive linter + UdpDroppedConns *prometheus.Desc + // doubleFlushAttemptsClose is a counter measuring the number of attempts to flush a closed connection twice from tcp_close + doubleFlushAttemptsClose *prometheus.Desc + // doubleFlushAttemptsDone is a counter measuring the number of attempts to flush a closed connection twice from tcp_done + doubleFlushAttemptsDone *prometheus.Desc + // unsupportedTcpFailures is a counter measuring the number of attempts to flush a TCP failure that is not supported + unsupportedTcpFailures *prometheus.Desc + // tcpDonePidMismatch is a counter measuring the number of TCP connections with a PID mismatch between tcp_connect and tcp_done + tcpDonePidMismatch *prometheus.Desc + PidCollisions *telemetry.StatCounterWrapper + iterationDups telemetry.Counter + iterationAborts telemetry.Counter + + //nolint:revive // TODO(NET) Fix revive linter + lastTcpFailedConnects *atomic.Int64 + //nolint:revive // TODO(NET) Fix revive linter + LastTcpSentMiscounts *atomic.Int64 + //nolint:revive // TODO(NET) Fix revive linter + lastUnbatchedTcpClose *atomic.Int64 + //nolint:revive // TODO(NET) Fix revive linter + lastUnbatchedUdpClose *atomic.Int64 + //nolint:revive // TODO(NET) Fix revive linter + lastUdpSendsProcessed *atomic.Int64 + //nolint:revive // TODO(NET) Fix revive linter + lastUdpSendsMissed *atomic.Int64 + //nolint:revive // TODO(NET) Fix revive linter + lastUdpDroppedConns *atomic.Int64 + // lastDoubleFlushAttemptsClose is a counter measuring the diff between the last two values of doubleFlushAttemptsClose + lastDoubleFlushAttemptsClose *atomic.Int64 + // lastDoubleFlushAttemptsDone is a counter measuring the diff between the last two values of doubleFlushAttemptsDone + lastDoubleFlushAttemptsDone *atomic.Int64 + // lastUnsupportedTcpFailures is a counter measuring the diff between the last two values of unsupportedTcpFailures + lastUnsupportedTcpFailures *atomic.Int64 + // lastTcpDonePidMismatch is a counter measuring the diff between the last two values of tcpDonePidMismatch + lastTcpDonePidMismatch *atomic.Int64 +}{ + telemetry.NewGauge(connTracerModuleName, "connections", []string{"ip_proto", "family"}, "Gauge measuring the number of active connections in the EBPF map"), + prometheus.NewDesc(connTracerModuleName+"__tcp_failed_connects", "Counter measuring the number of failed TCP connections in the EBPF map", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__tcp_sent_miscounts", "Counter measuring the number of miscounted tcp sends in the EBPF map", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__unbatched_tcp_close", "Counter measuring the number of missed TCP close events in the EBPF map", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__unbatched_udp_close", "Counter measuring the number of missed UDP close events in the EBPF map", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__udp_sends_processed", "Counter measuring the number of processed UDP sends in EBPF", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__udp_sends_missed", "Counter measuring failures to process UDP sends in EBPF", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__udp_dropped_conns", "Counter measuring the number of dropped UDP connections in the EBPF map", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__double_flush_attempts_close", "Counter measuring the number of attempts to flush a closed connection twice from tcp_close", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__double_flush_attempts_done", "Counter measuring the number of attempts to flush a closed connection twice from tcp_done", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__unsupported_tcp_failures", "Counter measuring the number of attempts to flush a TCP failure that is not supported", nil, nil), + prometheus.NewDesc(connTracerModuleName+"__tcp_done_pid_mismatch", "Counter measuring the number of TCP connections with a PID mismatch between tcp_connect and tcp_done", nil, nil), + telemetry.NewStatCounterWrapper(connTracerModuleName, "pid_collisions", []string{}, "Counter measuring number of process collisions"), + telemetry.NewCounter(connTracerModuleName, "iteration_dups", []string{}, "Counter measuring the number of connections iterated more than once"), + telemetry.NewCounter(connTracerModuleName, "iteration_aborts", []string{}, "Counter measuring how many times ebpf iteration of connection map was aborted"), + atomic.NewInt64(0), + atomic.NewInt64(0), + atomic.NewInt64(0), + atomic.NewInt64(0), + atomic.NewInt64(0), + atomic.NewInt64(0), + atomic.NewInt64(0), + atomic.NewInt64(0), + atomic.NewInt64(0), + atomic.NewInt64(0), + atomic.NewInt64(0), +} + +type ebpfTracer struct { + m *manager.Manager + + conns *maps.GenericMap[netebpf.ConnTuple, netebpf.ConnStats] + tcpStats *maps.GenericMap[netebpf.ConnTuple, netebpf.TCPStats] + tcpRetransmits *maps.GenericMap[netebpf.ConnTuple, uint32] + config *config.Config + + // tcp_close events + closeConsumer *tcpCloseConsumer + // tcp failure events + failedConnConsumer *failure.TCPFailedConnConsumer + + removeTuple *netebpf.ConnTuple + + closeTracer func() + stopOnce sync.Once + + ebpfTracerType TracerType + + exitTelemetry chan struct{} + + ch *cookieHasher +} + +// NewTracer creates a new tracer +func newEbpfTracer(config *config.Config, _ telemetryComponent.Component) (Tracer, error) { + if _, err := tracefs.Root(); err != nil { + return nil, fmt.Errorf("eBPF based network tracer unsupported: %s", err) + } + + mgrOptions := manager.Options{ + // Extend RLIMIT_MEMLOCK (8) size + // On some systems, the default for RLIMIT_MEMLOCK may be as low as 64 bytes. + // This will result in an EPERM (Operation not permitted) error, when trying to create an eBPF map + // using bpf(2) with BPF_MAP_CREATE. + // + // We are setting the limit to infinity until we have a better handle on the true requirements. + RLimit: &unix.Rlimit{ + Cur: math.MaxUint64, + Max: math.MaxUint64, + }, + MapSpecEditors: map[string]manager.MapSpecEditor{ + probes.ConnMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, + probes.TCPStatsMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, + probes.TCPRetransmitsMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, + probes.PortBindingsMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, + probes.UDPPortBindingsMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, + probes.ConnectionProtocolMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, + probes.ConnectionTupleToSocketSKBConnMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, + }, + ConstantEditors: []manager.ConstantEditor{ + boolConst("tcpv6_enabled", config.CollectTCPv6Conns), + boolConst("udpv6_enabled", config.CollectUDPv6Conns), + }, + DefaultKProbeMaxActive: maxActive, + BypassEnabled: config.BypassEnabled, + } + + begin, end := network.EphemeralRange() + mgrOptions.ConstantEditors = append(mgrOptions.ConstantEditors, + manager.ConstantEditor{Name: "ephemeral_range_begin", Value: uint64(begin)}, + manager.ConstantEditor{Name: "ephemeral_range_end", Value: uint64(end)}) + + closedChannelSize := defaultClosedChannelSize + if config.ClosedChannelSize > 0 { + closedChannelSize = config.ClosedChannelSize + } + var connCloseEventHandler ddebpf.EventHandler + var failedConnsHandler ddebpf.EventHandler + if config.RingBufferSupportedNPM() { + connCloseEventHandler = ddebpf.NewRingBufferHandler(closedChannelSize) + failedConnsHandler = ddebpf.NewRingBufferHandler(defaultFailedChannelSize) + } else { + connCloseEventHandler = ddebpf.NewPerfHandler(closedChannelSize) + failedConnsHandler = ddebpf.NewPerfHandler(defaultFailedChannelSize) + } + + var m *manager.Manager + //nolint:revive // TODO(NET) Fix revive linter + var tracerType TracerType = TracerTypeFentry + var closeTracerFn func() + m, closeTracerFn, err := fentry.LoadTracer(config, mgrOptions, connCloseEventHandler) + if err != nil && !errors.Is(err, fentry.ErrorNotSupported) { + // failed to load fentry tracer + return nil, err + } + + if err != nil { + // load the kprobe tracer + log.Info("fentry tracer not supported, falling back to kprobe tracer") + var kprobeTracerType kprobe.TracerType + m, closeTracerFn, kprobeTracerType, err = kprobe.LoadTracer(config, mgrOptions, connCloseEventHandler, failedConnsHandler) + if err != nil { + return nil, err + } + tracerType = TracerType(kprobeTracerType) + } + m.DumpHandler = dumpMapsHandler + ddebpf.AddNameMappings(m, "npm_tracer") + + numCPUs, err := ebpf.PossibleCPU() + if err != nil { + return nil, fmt.Errorf("could not determine number of CPUs: %w", err) + } + extractor := newBatchExtractor(numCPUs) + batchMgr, err := newConnBatchManager(m, extractor) + if err != nil { + return nil, fmt.Errorf("could not create connection batch manager: %w", err) + } + + closeConsumer := newTCPCloseConsumer(connCloseEventHandler, batchMgr) + + var failedConnConsumer *failure.TCPFailedConnConsumer + // Failed connections are not supported on prebuilt + if tracerType == TracerTypeKProbePrebuilt { + config.TCPFailedConnectionsEnabled = false + } + if config.FailedConnectionsSupported() { + failedConnConsumer = failure.NewFailedConnConsumer(failedConnsHandler, m, config.MaxFailedConnectionsBuffered) + } + + tr := &ebpfTracer{ + m: m, + config: config, + closeConsumer: closeConsumer, + failedConnConsumer: failedConnConsumer, + removeTuple: &netebpf.ConnTuple{}, + closeTracer: closeTracerFn, + ebpfTracerType: tracerType, + exitTelemetry: make(chan struct{}), + ch: newCookieHasher(), + } + + tr.conns, err = maps.GetMap[netebpf.ConnTuple, netebpf.ConnStats](m, probes.ConnMap) + if err != nil { + tr.Stop() + return nil, fmt.Errorf("error retrieving the bpf %s map: %s", probes.ConnMap, err) + } + + tr.tcpStats, err = maps.GetMap[netebpf.ConnTuple, netebpf.TCPStats](m, probes.TCPStatsMap) + if err != nil { + tr.Stop() + return nil, fmt.Errorf("error retrieving the bpf %s map: %s", probes.TCPStatsMap, err) + } + + if tr.tcpRetransmits, err = maps.GetMap[netebpf.ConnTuple, uint32](m, probes.TCPRetransmitsMap); err != nil { + tr.Stop() + return nil, fmt.Errorf("error retrieving the bpf %s map: %s", probes.TCPRetransmitsMap, err) + } + + return tr, nil +} + +func boolConst(name string, value bool) manager.ConstantEditor { + c := manager.ConstantEditor{ + Name: name, + Value: uint64(1), + } + if !value { + c.Value = uint64(0) + } + + return c +} + +func (t *ebpfTracer) Start(callback func(*network.ConnectionStats)) (err error) { + defer func() { + if err != nil { + t.Stop() + } + }() + + err = t.initializePortBindingMaps() + if err != nil { + return fmt.Errorf("error initializing port binding maps: %s", err) + } + + if err := t.m.Start(); err != nil { + return fmt.Errorf("could not start ebpf manager: %s", err) + } + + t.closeConsumer.Start(callback) + t.failedConnConsumer.Start() + return nil +} + +func (t *ebpfTracer) Pause() error { + // add small delay for socket filters to properly detach + time.Sleep(1 * time.Millisecond) + return t.m.Pause() +} + +func (t *ebpfTracer) Resume() error { + err := t.m.Resume() + // add small delay for socket filters to properly attach + time.Sleep(1 * time.Millisecond) + return err +} + +func (t *ebpfTracer) FlushPending() { + t.closeConsumer.FlushPending() +} + +func (t *ebpfTracer) GetFailedConnections() *failure.FailedConns { + if t.failedConnConsumer == nil { + return nil + } + return t.failedConnConsumer.FailedConns +} + +func (t *ebpfTracer) Stop() { + t.stopOnce.Do(func() { + close(t.exitTelemetry) + ddebpf.RemoveNameMappings(t.m) + ebpftelemetry.UnregisterTelemetry(t.m) + _ = t.m.Stop(manager.CleanAll) + t.closeConsumer.Stop() + t.failedConnConsumer.Stop() + if t.closeTracer != nil { + t.closeTracer() + } + }) +} + +func (t *ebpfTracer) GetMap(name string) *ebpf.Map { + switch name { + case probes.ConnectionProtocolMap: + default: + return nil + } + m, _, _ := t.m.GetMap(name) + return m +} + +func (t *ebpfTracer) GetConnections(buffer *network.ConnectionBuffer, filter func(*network.ConnectionStats) bool) error { + // Iterate through all key-value pairs in map + key, stats := &netebpf.ConnTuple{}, &netebpf.ConnStats{} + seen := make(map[netebpf.ConnTuple]struct{}) + // connsByTuple is used to detect whether we are iterating over + // a connection we have previously seen. This can happen when + // ebpf maps are being iterated over and deleted at the same time. + // The iteration can reset when that happens. + // See https://justin.azoff.dev/blog/bpf_map_get_next_key-pitfalls/ + connsByTuple := make(map[netebpf.ConnTuple]uint32) + + // Cached objects + conn := new(network.ConnectionStats) + tcp := new(netebpf.TCPStats) + + var tcp4, tcp6, udp4, udp6 float64 + entries := t.conns.Iterate() + for entries.Next(key, stats) { + if cookie, exists := connsByTuple[*key]; exists && cookie == stats.Cookie { + // already seen the connection in current batch processing, + // due to race between the iterator and bpf_map_delete + EbpfTracerTelemetry.iterationDups.Inc() + continue + } + + populateConnStats(conn, key, stats, t.ch) + connsByTuple[*key] = stats.Cookie + + isTCP := conn.Type == network.TCP + switch conn.Family { + case network.AFINET6: + if isTCP { + tcp6++ + } else { + udp6++ + } + case network.AFINET: + if isTCP { + tcp4++ + } else { + udp4++ + } + } + + if filter != nil && !filter(conn) { + continue + } + + if t.getTCPStats(tcp, key) { + updateTCPStats(conn, tcp, 0) + } + if retrans, ok := t.getTCPRetransmits(key, seen); ok { + updateTCPStats(conn, nil, retrans) + } + + *buffer.Next() = *conn + } + + if err := entries.Err(); err != nil { + if !errors.Is(err, ebpf.ErrIterationAborted) { + return fmt.Errorf("unable to iterate connection map: %w", err) + } + + log.Warn("eBPF conn_stats map iteration aborted. Some connections may not be reported") + EbpfTracerTelemetry.iterationAborts.Inc() + } + + updateTelemetry(tcp4, tcp6, udp4, udp6) + + return nil +} + +func updateTelemetry(tcp4 float64, tcp6 float64, udp4 float64, udp6 float64) { + EbpfTracerTelemetry.connections.Set(tcp4, "tcp", "v4") + EbpfTracerTelemetry.connections.Set(tcp6, "tcp", "v6") + EbpfTracerTelemetry.connections.Set(udp4, "udp", "v4") + EbpfTracerTelemetry.connections.Set(udp6, "udp", "v6") +} + +func removeConnectionFromTelemetry(conn *network.ConnectionStats) { + isTCP := conn.Type == network.TCP + switch conn.Family { + case network.AFINET6: + if isTCP { + EbpfTracerTelemetry.connections.Dec("tcp", "v6") + } else { + EbpfTracerTelemetry.connections.Dec("udp", "v6") + } + case network.AFINET: + if isTCP { + EbpfTracerTelemetry.connections.Dec("tcp", "v4") + } else { + EbpfTracerTelemetry.connections.Dec("udp", "v4") + } + } +} + +func (t *ebpfTracer) Remove(conn *network.ConnectionStats) error { + util.ConnStatsToTuple(conn, t.removeTuple) + + err := t.conns.Delete(t.removeTuple) + if err != nil { + // If this entry no longer exists in the eBPF map it means `tcp_close` has executed + // during this function call. In that case state.StoreClosedConnection() was already called for this connection, + // and we can't delete the corresponding client state, or we'll likely over-report the metric values. + // By skipping to the next iteration and not calling state.RemoveConnections() we'll let + // this connection expire "naturally" when either next connection check runs or the client itself expires. + return err + } + + removeConnectionFromTelemetry(conn) + + if conn.Type == network.TCP { + // We can ignore the error for this map since it will not always contain the entry + _ = t.tcpStats.Delete(t.removeTuple) + // We remove the PID from the tuple as it is not used in the retransmits map + pid := t.removeTuple.Pid + t.removeTuple.Pid = 0 + _ = t.tcpRetransmits.Delete(t.removeTuple) + t.removeTuple.Pid = pid + } + return nil +} + +func (t *ebpfTracer) getEBPFTelemetry() *netebpf.Telemetry { + var zero uint32 + mp, err := maps.GetMap[uint32, netebpf.Telemetry](t.m, probes.TelemetryMap) + if err != nil { + log.Warnf("error retrieving telemetry map: %s", err) + return nil + } + + tm := &netebpf.Telemetry{} + if err := mp.Lookup(&zero, tm); err != nil { + // This can happen if we haven't initialized the telemetry object yet + // so let's just use a trace log + if log.ShouldLog(seelog.TraceLvl) { + log.Tracef("error retrieving the telemetry struct: %s", err) + } + return nil + } + return tm +} + +// Describe returns all descriptions of the collector +func (t *ebpfTracer) Describe(ch chan<- *prometheus.Desc) { + ch <- EbpfTracerTelemetry.tcpFailedConnects + ch <- EbpfTracerTelemetry.TcpSentMiscounts + ch <- EbpfTracerTelemetry.unbatchedTcpClose + ch <- EbpfTracerTelemetry.unbatchedUdpClose + ch <- EbpfTracerTelemetry.UdpSendsProcessed + ch <- EbpfTracerTelemetry.UdpSendsMissed + ch <- EbpfTracerTelemetry.UdpDroppedConns + ch <- EbpfTracerTelemetry.doubleFlushAttemptsClose + ch <- EbpfTracerTelemetry.doubleFlushAttemptsDone + ch <- EbpfTracerTelemetry.unsupportedTcpFailures + ch <- EbpfTracerTelemetry.tcpDonePidMismatch +} + +// Collect returns the current state of all metrics of the collector +func (t *ebpfTracer) Collect(ch chan<- prometheus.Metric) { + ebpfTelemetry := t.getEBPFTelemetry() + if ebpfTelemetry == nil { + return + } + delta := int64(ebpfTelemetry.Tcp_failed_connect) - EbpfTracerTelemetry.lastTcpFailedConnects.Load() + EbpfTracerTelemetry.lastTcpFailedConnects.Store(int64(ebpfTelemetry.Tcp_failed_connect)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.tcpFailedConnects, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Tcp_sent_miscounts) - EbpfTracerTelemetry.LastTcpSentMiscounts.Load() + EbpfTracerTelemetry.LastTcpSentMiscounts.Store(int64(ebpfTelemetry.Tcp_sent_miscounts)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.TcpSentMiscounts, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Unbatched_tcp_close) - EbpfTracerTelemetry.lastUnbatchedTcpClose.Load() + EbpfTracerTelemetry.lastUnbatchedTcpClose.Store(int64(ebpfTelemetry.Unbatched_tcp_close)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.unbatchedTcpClose, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Unbatched_udp_close) - EbpfTracerTelemetry.lastUnbatchedUdpClose.Load() + EbpfTracerTelemetry.lastUnbatchedUdpClose.Store(int64(ebpfTelemetry.Unbatched_udp_close)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.unbatchedUdpClose, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Udp_sends_processed) - EbpfTracerTelemetry.lastUdpSendsProcessed.Load() + EbpfTracerTelemetry.lastUdpSendsProcessed.Store(int64(ebpfTelemetry.Udp_sends_processed)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.UdpSendsProcessed, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Udp_sends_missed) - EbpfTracerTelemetry.lastUdpSendsMissed.Load() + EbpfTracerTelemetry.lastUdpSendsMissed.Store(int64(ebpfTelemetry.Udp_sends_missed)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.UdpSendsMissed, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Udp_dropped_conns) - EbpfTracerTelemetry.lastUdpDroppedConns.Load() + EbpfTracerTelemetry.lastUdpDroppedConns.Store(int64(ebpfTelemetry.Udp_dropped_conns)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.UdpDroppedConns, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Double_flush_attempts_close) - EbpfTracerTelemetry.lastDoubleFlushAttemptsClose.Load() + EbpfTracerTelemetry.lastDoubleFlushAttemptsClose.Store(int64(ebpfTelemetry.Double_flush_attempts_close)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.doubleFlushAttemptsClose, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Double_flush_attempts_done) - EbpfTracerTelemetry.lastDoubleFlushAttemptsDone.Load() + EbpfTracerTelemetry.lastDoubleFlushAttemptsDone.Store(int64(ebpfTelemetry.Double_flush_attempts_done)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.doubleFlushAttemptsDone, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Unsupported_tcp_failures) - EbpfTracerTelemetry.lastUnsupportedTcpFailures.Load() + EbpfTracerTelemetry.lastUnsupportedTcpFailures.Store(int64(ebpfTelemetry.Unsupported_tcp_failures)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.unsupportedTcpFailures, prometheus.CounterValue, float64(delta)) + + delta = int64(ebpfTelemetry.Tcp_done_pid_mismatch) - EbpfTracerTelemetry.lastTcpDonePidMismatch.Load() + EbpfTracerTelemetry.lastTcpDonePidMismatch.Store(int64(ebpfTelemetry.Tcp_done_pid_mismatch)) + ch <- prometheus.MustNewConstMetric(EbpfTracerTelemetry.tcpDonePidMismatch, prometheus.CounterValue, float64(delta)) + +} + +// DumpMaps (for debugging purpose) returns all maps content by default or selected maps from maps parameter. +func (t *ebpfTracer) DumpMaps(w io.Writer, maps ...string) error { + return t.m.DumpMaps(w, maps...) +} + +// Type returns the type of the underlying ebpf tracer that is currently loaded +func (t *ebpfTracer) Type() TracerType { + return t.ebpfTracerType +} + +func (t *ebpfTracer) initializePortBindingMaps() error { + tcpPorts, err := network.ReadListeningPorts(t.config.ProcRoot, network.TCP, t.config.CollectTCPv6Conns) + if err != nil { + return fmt.Errorf("failed to read initial TCP pid->port mapping: %s", err) + } + + tcpPortMap, err := maps.GetMap[netebpf.PortBinding, uint32](t.m, probes.PortBindingsMap) + if err != nil { + return fmt.Errorf("failed to get TCP port binding map: %w", err) + } + for p, count := range tcpPorts { + log.Debugf("adding initial TCP port binding: netns: %d port: %d", p.Ino, p.Port) + pb := netebpf.PortBinding{Netns: p.Ino, Port: p.Port} + err = tcpPortMap.Update(&pb, &count, ebpf.UpdateNoExist) + if err != nil && !errors.Is(err, ebpf.ErrKeyExist) { + return fmt.Errorf("failed to update TCP port binding map: %w", err) + } + } + + udpPorts, err := network.ReadListeningPorts(t.config.ProcRoot, network.UDP, t.config.CollectUDPv6Conns) + if err != nil { + return fmt.Errorf("failed to read initial UDP pid->port mapping: %s", err) + } + + udpPortMap, err := maps.GetMap[netebpf.PortBinding, uint32](t.m, probes.UDPPortBindingsMap) + if err != nil { + return fmt.Errorf("failed to get UDP port binding map: %w", err) + } + for p, count := range udpPorts { + // ignore ephemeral port binds as they are more likely to be from + // clients calling bind with port 0 + if network.IsPortInEphemeralRange(network.AFINET, network.UDP, p.Port) == network.EphemeralTrue { + log.Debugf("ignoring initial ephemeral UDP port bind to %d", p) + continue + } + + log.Debugf("adding initial UDP port binding: netns: %d port: %d", p.Ino, p.Port) + pb := netebpf.PortBinding{Netns: p.Ino, Port: p.Port} + err = udpPortMap.Update(&pb, &count, ebpf.UpdateNoExist) + if err != nil && !errors.Is(err, ebpf.ErrKeyExist) { + return fmt.Errorf("failed to update UDP port binding map: %w", err) + } + } + return nil +} + +func (t *ebpfTracer) getTCPRetransmits(tuple *netebpf.ConnTuple, seen map[netebpf.ConnTuple]struct{}) (uint32, bool) { + if tuple.Type() != netebpf.TCP { + return 0, false + } + + // The PID isn't used as a key in the stats map, we will temporarily set it to 0 here and reset it when we're done + pid := tuple.Pid + tuple.Pid = 0 + + var retransmits uint32 + if err := t.tcpRetransmits.Lookup(tuple, &retransmits); err == nil { + // This is required to avoid (over)reporting retransmits for connections sharing the same socket. + if _, reported := seen[*tuple]; reported { + EbpfTracerTelemetry.PidCollisions.Inc() + retransmits = 0 + } else { + seen[*tuple] = struct{}{} + } + } + + tuple.Pid = pid + return retransmits, true +} + +// getTCPStats reads tcp related stats for the given ConnTuple +func (t *ebpfTracer) getTCPStats(stats *netebpf.TCPStats, tuple *netebpf.ConnTuple) bool { + if tuple.Type() != netebpf.TCP { + return false + } + + return t.tcpStats.Lookup(tuple, stats) == nil +} + +func populateConnStats(stats *network.ConnectionStats, t *netebpf.ConnTuple, s *netebpf.ConnStats, ch *cookieHasher) { + *stats = network.ConnectionStats{ + Pid: t.Pid, + NetNS: t.Netns, + Source: t.SourceAddress(), + Dest: t.DestAddress(), + SPort: t.Sport, + DPort: t.Dport, + Monotonic: network.StatCounters{ + SentBytes: s.Sent_bytes, + RecvBytes: s.Recv_bytes, + SentPackets: uint64(s.Sent_packets), + RecvPackets: uint64(s.Recv_packets), + }, + LastUpdateEpoch: s.Timestamp, + IsAssured: s.IsAssured(), + Cookie: network.StatCookie(s.Cookie), + } + + if s.Duration <= uint64(math.MaxInt64) { + stats.Duration = time.Duration(s.Duration) * time.Nanosecond + } + + stats.ProtocolStack = protocols.Stack{ + API: protocols.API(s.Protocol_stack.Api), + Application: protocols.Application(s.Protocol_stack.Application), + Encryption: protocols.Encryption(s.Protocol_stack.Encryption), + } + + if t.Type() == netebpf.TCP { + stats.Type = network.TCP + } else { + stats.Type = network.UDP + } + + switch t.Family() { + case netebpf.IPv4: + stats.Family = network.AFINET + case netebpf.IPv6: + stats.Family = network.AFINET6 + } + + stats.SPortIsEphemeral = network.IsPortInEphemeralRange(stats.Family, stats.Type, t.Sport) + + switch s.ConnectionDirection() { + case netebpf.Incoming: + stats.Direction = network.INCOMING + case netebpf.Outgoing: + stats.Direction = network.OUTGOING + default: + stats.Direction = network.OUTGOING + } + + if ch != nil { + ch.Hash(stats) + } +} + +func updateTCPStats(conn *network.ConnectionStats, tcpStats *netebpf.TCPStats, retransmits uint32) { + if conn.Type != network.TCP { + return + } + + conn.Monotonic.Retransmits = retransmits + if tcpStats != nil { + conn.Monotonic.TCPEstablished = uint32(tcpStats.State_transitions >> netebpf.Established & 1) + conn.Monotonic.TCPClosed = uint32(tcpStats.State_transitions >> netebpf.Close & 1) + conn.RTT = tcpStats.Rtt + conn.RTTVar = tcpStats.Rtt_var + } +} diff --git a/pkg/network/tracer/connection/ebpfless/payload.go b/pkg/network/tracer/connection/ebpfless/payload.go new file mode 100644 index 0000000000000..50d9e1d190b53 --- /dev/null +++ b/pkg/network/tracer/connection/ebpfless/payload.go @@ -0,0 +1,102 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +// Package ebpfless contains supporting code for the ebpfless tracer +package ebpfless + +import ( + "errors" + "fmt" + + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + + "github.com/DataDog/datadog-agent/pkg/network" +) + +var errZeroLengthUDPPacket = errors.New("UDP packet with length 0") +var errZeroLengthIPPacket = errors.New("IP packet with length 0") + +// UDPPayloadLen returns the UDP payload length from a layers.UDP object +func UDPPayloadLen(udp *layers.UDP) (uint16, error) { + if udp.Length == 0 { + return 0, errZeroLengthUDPPacket + } + + // Length includes the header (8 bytes), + // so we need to exclude that here + return udp.Length - 8, nil +} + +// TCPPayloadLen returns the TCP payload length from a layers.TCP object +func TCPPayloadLen(family network.ConnectionFamily, ip4 *layers.IPv4, ip6 *layers.IPv6, tcp *layers.TCP) (uint16, error) { + var ipl uint16 + var err error + switch family { + case network.AFINET: + ipl, err = ipv4PayloadLen(ip4) + case network.AFINET6: + ipl, err = ipv6PayloadLen(ip6) + default: + return 0, fmt.Errorf("unknown family %s", family) + } + + if err != nil { + return 0, nil + } + + if ipl == 0 { + return 0, errZeroLengthIPPacket + } + + // the data offset field in the TCP header specifies + // the length of the TCP header in 32 bit words, so + // subtracting that here to get the payload size + // + // see https://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_segment_structure + return ipl - uint16(tcp.DataOffset)*4, nil +} + +func ipv4PayloadLen(ip4 *layers.IPv4) (uint16, error) { + // the IHL field specifies the the size of the IP + // header in 32 bit words, so subtracting that here + // to get the payload size + // + // see https://en.wikipedia.org/wiki/IPv4#Header + return ip4.Length - uint16(ip4.IHL)*4, nil +} + +func ipv6PayloadLen(ip6 *layers.IPv6) (uint16, error) { + if ip6.NextHeader == layers.IPProtocolUDP || ip6.NextHeader == layers.IPProtocolTCP { + return ip6.Length, nil + } + + var ipExt layers.IPv6ExtensionSkipper + parser := gopacket.NewDecodingLayerParser(gopacket.LayerTypePayload, &ipExt) + decoded := make([]gopacket.LayerType, 0, 1) + l := ip6.Length + payload := ip6.Payload + for len(payload) > 0 { + err := parser.DecodeLayers(payload, &decoded) + if err != nil { + return 0, fmt.Errorf("error decoding with ipv6 extension skipper: %w", err) + } + + if len(decoded) == 0 { + return l, nil + } + + l -= uint16(len(ipExt.Contents)) + if ipExt.NextHeader == layers.IPProtocolTCP || ipExt.NextHeader == layers.IPProtocolUDP { + break + } + + payload = ipExt.Payload + } + + return l, nil +} diff --git a/pkg/network/tracer/connection/ebpfless/ports.go b/pkg/network/tracer/connection/ebpfless/ports.go new file mode 100644 index 0000000000000..00ebebf273c64 --- /dev/null +++ b/pkg/network/tracer/connection/ebpfless/ports.go @@ -0,0 +1,130 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +// Package ebpfless contains supporting code for the ebpfless tracer +package ebpfless + +import ( + "fmt" + "sync" + "time" + + "github.com/DataDog/datadog-agent/pkg/network" + "github.com/DataDog/datadog-agent/pkg/network/config" + "github.com/DataDog/datadog-agent/pkg/util/kernel" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +type boundPortsKey struct { + proto network.ConnectionType + port uint16 +} + +// BoundPorts is a collection of bound ports on the host +// that is periodically updated from procfs +type BoundPorts struct { + mu sync.RWMutex + + config *config.Config + ports map[boundPortsKey]struct{} + + stop chan struct{} + ino uint32 +} + +// NewBoundPorts returns a new BoundPorts instance +func NewBoundPorts(cfg *config.Config) *BoundPorts { + ino, _ := kernel.GetCurrentIno() + return &BoundPorts{ + config: cfg, + ports: map[boundPortsKey]struct{}{}, + stop: make(chan struct{}), + ino: ino, + } +} + +// Start starts a BoundPorts instance +func (b *BoundPorts) Start() error { + if err := b.update(); err != nil { + return err + } + + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + select { + case <-b.stop: + return + case <-ticker.C: + if err := b.update(); err != nil { + log.Errorf("error updating bound ports, exiting loop: %s", err) + return + } + } + } + }() + + return nil +} + +// Stop stops a BoundPorts instance +func (b *BoundPorts) Stop() { + close(b.stop) +} + +func (b *BoundPorts) update() error { + b.mu.Lock() + defer b.mu.Unlock() + + tcpPorts, err := network.ReadListeningPorts(b.config.ProcRoot, network.TCP, b.config.CollectTCPv6Conns) + if err != nil { + return fmt.Errorf("failed to read initial TCP pid->port mapping: %s", err) + } + + for p := range tcpPorts { + if p.Ino != b.ino { + continue + } + log.Debugf("adding initial TCP port binding: netns: %d port: %d", p.Ino, p.Port) + b.ports[boundPortsKey{network.TCP, p.Port}] = struct{}{} + } + + udpPorts, err := network.ReadListeningPorts(b.config.ProcRoot, network.UDP, b.config.CollectUDPv6Conns) + if err != nil { + return fmt.Errorf("failed to read initial UDP pid->port mapping: %s", err) + } + + for p := range udpPorts { + // ignore ephemeral port binds as they are more likely to be from + // clients calling bind with port 0 + if network.IsPortInEphemeralRange(network.AFINET, network.UDP, p.Port) == network.EphemeralTrue { + log.Debugf("ignoring initial ephemeral UDP port bind to %d", p) + continue + } + + if p.Ino != b.ino { + continue + } + + log.Debugf("adding initial UDP port binding: netns: %d port: %d", p.Ino, p.Port) + b.ports[boundPortsKey{network.UDP, p.Port}] = struct{}{} + } + + return nil + +} + +// Find returns `true` if the given `(proto, port)` exists in +// the BoundPorts collection +func (b *BoundPorts) Find(proto network.ConnectionType, port uint16) bool { + b.mu.RLock() + defer b.mu.RUnlock() + + _, ok := b.ports[boundPortsKey{proto, port}] + return ok +} diff --git a/pkg/network/tracer/connection/ebpfless_tracer.go b/pkg/network/tracer/connection/ebpfless_tracer.go new file mode 100644 index 0000000000000..fd57ec1b3d0ba --- /dev/null +++ b/pkg/network/tracer/connection/ebpfless_tracer.go @@ -0,0 +1,417 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build linux && npm + +package connection + +import ( + "fmt" + "io" + "sync" + "time" + + "github.com/cilium/ebpf" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" + "github.com/prometheus/client_golang/prometheus" + "github.com/vishvananda/netns" + "golang.org/x/sys/unix" + + "github.com/DataDog/datadog-agent/pkg/network" + "github.com/DataDog/datadog-agent/pkg/network/config" + "github.com/DataDog/datadog-agent/pkg/network/filter" + "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/ebpfless" + "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/failure" + "github.com/DataDog/datadog-agent/pkg/process/util" + "github.com/DataDog/datadog-agent/pkg/telemetry" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +const ( + // the segment length to read + // mac header (with vlan) + ip header + tcp header + segmentLen = 18 + 60 + 60 + + ebpfLessTelemetryPrefix = "network_tracer__ebpfless" +) + +var ( + ebpfLessTracerTelemetry = struct { + skippedPackets telemetry.Counter + }{ + telemetry.NewCounter(ebpfLessTelemetryPrefix, "skipped_packets", []string{"reason"}, "Counter measuring skipped packets"), + } +) + +type ebpfLessTracer struct { + m sync.Mutex + + config *config.Config + + packetSrc *filter.AFPacketSource + exit chan struct{} + keyBuf []byte + scratchConn *network.ConnectionStats + + udp *udpProcessor + tcp *tcpProcessor + + // connection maps + conns map[string]*network.ConnectionStats + boundPorts *ebpfless.BoundPorts + cookieHasher *cookieHasher + + ns netns.NsHandle +} + +// newEbpfLessTracer creates a new ebpfLessTracer instance +func newEbpfLessTracer(cfg *config.Config) (*ebpfLessTracer, error) { + packetSrc, err := filter.NewAFPacketSource( + 8<<20, // 8 MB total space + filter.OptSnapLen(segmentLen)) + if err != nil { + return nil, fmt.Errorf("error creating packet source: %w", err) + } + + tr := &ebpfLessTracer{ + config: cfg, + packetSrc: packetSrc, + exit: make(chan struct{}), + keyBuf: make([]byte, network.ConnectionByteKeyMaxLen), + scratchConn: &network.ConnectionStats{}, + udp: &udpProcessor{}, + tcp: newTCPProcessor(), + conns: make(map[string]*network.ConnectionStats, cfg.MaxTrackedConnections), + boundPorts: ebpfless.NewBoundPorts(cfg), + cookieHasher: newCookieHasher(), + } + + tr.ns, err = netns.Get() + if err != nil { + return nil, fmt.Errorf("error getting current net ns: %w", err) + } + + return tr, nil +} + +// Start begins collecting network connection data. +func (t *ebpfLessTracer) Start(func(*network.ConnectionStats)) error { + if err := t.boundPorts.Start(); err != nil { + return fmt.Errorf("could not update bound ports: %w", err) + } + + go func() { + var eth layers.Ethernet + var ip4 layers.IPv4 + var ip6 layers.IPv6 + var tcp layers.TCP + var udp layers.UDP + decoded := make([]gopacket.LayerType, 0, 5) + parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, ð, &ip4, &ip6, &tcp, &udp) + parser.IgnoreUnsupported = true + for { + err := t.packetSrc.VisitPackets(t.exit, func(b []byte, info filter.PacketInfo, ts time.Time) error { + if err := parser.DecodeLayers(b, &decoded); err != nil { + return fmt.Errorf("error decoding packet layers: %w", err) + } + + pktType := info.(*filter.AFPacketInfo).PktType + // only process PACKET_HOST and PACK_OUTGOING packets + if pktType != unix.PACKET_HOST && pktType != unix.PACKET_OUTGOING { + ebpfLessTracerTelemetry.skippedPackets.Inc("unsupported_packet_type") + return nil + } + + if err := t.processConnection(pktType, &ip4, &ip6, &udp, &tcp, decoded); err != nil { + log.Warnf("could not process packet: %s", err) + } + + return nil + }) + + if err != nil { + log.Errorf("exiting packet loop: %s", err) + return + } + } + }() + + return nil +} + +func (t *ebpfLessTracer) processConnection( + pktType uint8, + ip4 *layers.IPv4, + ip6 *layers.IPv6, + udp *layers.UDP, + tcp *layers.TCP, + decoded []gopacket.LayerType, +) error { + t.scratchConn.Source, t.scratchConn.Dest = util.Address{}, util.Address{} + t.scratchConn.SPort, t.scratchConn.DPort = 0, 0 + var udpPresent, tcpPresent bool + for _, layerType := range decoded { + switch layerType { + case layers.LayerTypeIPv4: + t.scratchConn.Source = util.AddressFromNetIP(ip4.SrcIP) + t.scratchConn.Dest = util.AddressFromNetIP(ip4.DstIP) + t.scratchConn.Family = network.AFINET + case layers.LayerTypeIPv6: + t.scratchConn.Source = util.AddressFromNetIP(ip6.SrcIP) + t.scratchConn.Dest = util.AddressFromNetIP(ip6.DstIP) + t.scratchConn.Family = network.AFINET6 + case layers.LayerTypeTCP: + t.scratchConn.SPort = uint16(tcp.SrcPort) + t.scratchConn.DPort = uint16(tcp.DstPort) + t.scratchConn.Type = network.TCP + tcpPresent = true + case layers.LayerTypeUDP: + t.scratchConn.SPort = uint16(udp.SrcPort) + t.scratchConn.DPort = uint16(udp.DstPort) + t.scratchConn.Type = network.UDP + udpPresent = true + } + } + + // check if have all the basic pieces + if !udpPresent && !tcpPresent { + log.Debugf("ignoring packet since its not udp or tcp") + ebpfLessTracerTelemetry.skippedPackets.Inc("not_tcp_udp") + return nil + } + + flipSourceDest(t.scratchConn, pktType) + t.determineConnectionDirection(t.scratchConn, pktType) + + t.m.Lock() + defer t.m.Unlock() + + key := string(t.scratchConn.ByteKey(t.keyBuf)) + conn := t.conns[key] + if conn == nil { + conn = &network.ConnectionStats{} + *conn = *t.scratchConn + t.cookieHasher.Hash(conn) + conn.Duration = time.Duration(time.Now().UnixNano()) + } + + var err error + switch conn.Type { + case network.UDP: + err = t.udp.process(conn, pktType, udp) + case network.TCP: + err = t.tcp.process(conn, pktType, ip4, ip6, tcp) + default: + err = fmt.Errorf("unsupported connection type %d", conn.Type) + } + + if err != nil { + return fmt.Errorf("error processing connection: %w", err) + } + + if conn.Type == network.UDP || conn.Monotonic.TCPEstablished > 0 { + conn.LastUpdateEpoch = uint64(time.Now().UnixNano()) + t.conns[key] = conn + } + + log.TraceFunc(func() string { + return fmt.Sprintf("connection: %s", conn) + }) + return nil +} + +func flipSourceDest(conn *network.ConnectionStats, pktType uint8) { + if pktType == unix.PACKET_HOST { + conn.Dest, conn.Source = conn.Source, conn.Dest + conn.DPort, conn.SPort = conn.SPort, conn.DPort + } +} + +func (t *ebpfLessTracer) determineConnectionDirection(conn *network.ConnectionStats, pktType uint8) { + t.m.Lock() + defer t.m.Unlock() + + ok := t.boundPorts.Find(conn.Type, conn.SPort) + if ok { + // incoming connection + conn.Direction = network.INCOMING + return + } + + if conn.Type == network.TCP { + switch pktType { + case unix.PACKET_HOST: + conn.Direction = network.INCOMING + case unix.PACKET_OUTGOING: + conn.Direction = network.OUTGOING + } + } +} + +// Stop halts all network data collection. +func (t *ebpfLessTracer) Stop() { + if t == nil { + return + } + + close(t.exit) + t.ns.Close() + t.boundPorts.Stop() +} + +// GetConnections returns the list of currently active connections, using the buffer provided. +// The optional filter function is used to prevent unwanted connections from being returned and consuming resources. +func (t *ebpfLessTracer) GetConnections(buffer *network.ConnectionBuffer, filter func(*network.ConnectionStats) bool) error { + t.m.Lock() + defer t.m.Unlock() + + if len(t.conns) == 0 { + return nil + } + + log.Trace(t.conns) + conns := make([]network.ConnectionStats, 0, len(t.conns)) + for _, c := range t.conns { + if filter != nil && !filter(c) { + continue + } + + conns = append(conns, *c) + } + + buffer.Append(conns) + return nil +} + +// FlushPending forces any closed connections waiting for batching to be processed immediately. +func (t *ebpfLessTracer) FlushPending() {} + +// Remove deletes the connection from tracking state. +// It does not prevent the connection from re-appearing later, if additional traffic occurs. +func (t *ebpfLessTracer) Remove(conn *network.ConnectionStats) error { + t.m.Lock() + defer t.m.Unlock() + + delete(t.conns, string(conn.ByteKey(t.keyBuf))) + return nil +} + +// GetMap returns the underlying named map. This is useful if any maps are shared with other eBPF components. +// An individual ebpfLessTracer implementation may choose which maps to expose via this function. +func (t *ebpfLessTracer) GetMap(string) *ebpf.Map { return nil } + +// DumpMaps (for debugging purpose) returns all maps content by default or selected maps from maps parameter. +func (t *ebpfLessTracer) DumpMaps(_ io.Writer, _ ...string) error { + return fmt.Errorf("not implemented") +} + +// Type returns the type of the underlying ebpf ebpfLessTracer that is currently loaded +func (t *ebpfLessTracer) Type() TracerType { + return TracerTypeEbpfless +} + +func (t *ebpfLessTracer) Pause() error { + return fmt.Errorf("not implemented") +} + +func (t *ebpfLessTracer) Resume() error { + return fmt.Errorf("not implemented") +} + +// Describe returns all descriptions of the collector +func (t *ebpfLessTracer) Describe(_ chan<- *prometheus.Desc) {} + +// Collect returns the current state of all metrics of the collector +func (t *ebpfLessTracer) Collect(_ chan<- prometheus.Metric) {} + +// GetFailedConnections returns the underlying map used to store failed connections +func (t *ebpfLessTracer) GetFailedConnections() *failure.FailedConns { return nil } + +var _ Tracer = &ebpfLessTracer{} + +type udpProcessor struct { +} + +func (u *udpProcessor) process(conn *network.ConnectionStats, pktType uint8, udp *layers.UDP) error { + payloadLen, err := ebpfless.UDPPayloadLen(udp) + if err != nil { + return err + } + + switch pktType { + case unix.PACKET_OUTGOING: + conn.Monotonic.SentPackets++ + conn.Monotonic.SentBytes += uint64(payloadLen) + case unix.PACKET_HOST: + conn.Monotonic.RecvPackets++ + conn.Monotonic.RecvBytes += uint64(payloadLen) + } + + return nil +} + +type tcpProcessor struct { + buf []byte + conns map[string]struct { + established bool + closed bool + } +} + +func newTCPProcessor() *tcpProcessor { + return &tcpProcessor{ + buf: make([]byte, network.ConnectionByteKeyMaxLen), + conns: map[string]struct { + established bool + closed bool + }{}, + } +} + +func (t *tcpProcessor) process(conn *network.ConnectionStats, pktType uint8, ip4 *layers.IPv4, ip6 *layers.IPv6, tcp *layers.TCP) error { + payloadLen, err := ebpfless.TCPPayloadLen(conn.Family, ip4, ip6, tcp) + if err != nil { + return err + } + + log.TraceFunc(func() string { + return fmt.Sprintf("tcp processor: pktType=%+v seq=%+v ack=%+v fin=%+v rst=%+v syn=%+v ack=%+v", pktType, tcp.Seq, tcp.Ack, tcp.FIN, tcp.RST, tcp.SYN, tcp.ACK) + }) + key := string(conn.ByteKey(t.buf)) + c := t.conns[key] + log.TraceFunc(func() string { + return fmt.Sprintf("pre ack_seq=%+v", c) + }) + switch pktType { + case unix.PACKET_OUTGOING: + conn.Monotonic.SentPackets++ + conn.Monotonic.SentBytes += uint64(payloadLen) + case unix.PACKET_HOST: + conn.Monotonic.RecvPackets++ + conn.Monotonic.RecvBytes += uint64(payloadLen) + } + + if tcp.FIN || tcp.RST { + if !c.closed { + c.closed = true + conn.Monotonic.TCPClosed++ + conn.Duration = time.Duration(time.Now().UnixNano() - int64(conn.Duration)) + } + delete(t.conns, key) + return nil + } + + if !tcp.SYN && !c.established { + c.established = true + conn.Monotonic.TCPEstablished++ + } + + log.TraceFunc(func() string { + return fmt.Sprintf("ack_seq=%+v", c) + }) + t.conns[key] = c + return nil +} diff --git a/pkg/network/tracer/connection/tracer.go b/pkg/network/tracer/connection/tracer.go index 8c580e43339ad..a8a9a13112b0e 100644 --- a/pkg/network/tracer/connection/tracer.go +++ b/pkg/network/tracer/connection/tracer.go @@ -3,44 +3,20 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build linux_bpf +//go:build linux && npm package connection import ( - "encoding/binary" - "errors" - "fmt" - "hash" "io" - "math" - "sync" - "time" - "github.com/cihub/seelog" "github.com/cilium/ebpf" "github.com/prometheus/client_golang/prometheus" - "github.com/twmb/murmur3" - "go.uber.org/atomic" - "golang.org/x/sys/unix" - manager "github.com/DataDog/ebpf-manager" - - telemetryComponent "github.com/DataDog/datadog-agent/comp/core/telemetry" - ddebpf "github.com/DataDog/datadog-agent/pkg/ebpf" - "github.com/DataDog/datadog-agent/pkg/ebpf/maps" - ebpftelemetry "github.com/DataDog/datadog-agent/pkg/ebpf/telemetry" + "github.com/DataDog/datadog-agent/comp/core/telemetry" "github.com/DataDog/datadog-agent/pkg/network" "github.com/DataDog/datadog-agent/pkg/network/config" - netebpf "github.com/DataDog/datadog-agent/pkg/network/ebpf" - "github.com/DataDog/datadog-agent/pkg/network/ebpf/probes" - "github.com/DataDog/datadog-agent/pkg/network/protocols" "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/failure" - "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/fentry" - "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/kprobe" - "github.com/DataDog/datadog-agent/pkg/network/tracer/connection/util" - "github.com/DataDog/datadog-agent/pkg/telemetry" - "github.com/DataDog/datadog-agent/pkg/util/log" ) // TracerType is the type of the underlying tracer @@ -55,6 +31,8 @@ const ( TracerTypeKProbeCORE //nolint:revive // TODO(NET) Fix revive linter TracerTypeFentry + //nolint:revive // TODO(NET) Fix revive linter + TracerTypeEbpfless ) const ( @@ -96,716 +74,11 @@ type Tracer interface { Collect(metrics chan<- prometheus.Metric) } -const ( - defaultClosedChannelSize = 500 - defaultFailedChannelSize = 500 - connTracerModuleName = "network_tracer__ebpf" -) - -//nolint:revive // TODO(NET) Fix revive linter -var ConnTracerTelemetry = struct { - connections telemetry.Gauge - tcpFailedConnects *prometheus.Desc - //nolint:revive // TODO(NET) Fix revive linter - TcpSentMiscounts *prometheus.Desc - //nolint:revive // TODO(NET) Fix revive linter - unbatchedTcpClose *prometheus.Desc - //nolint:revive // TODO(NET) Fix revive linter - unbatchedUdpClose *prometheus.Desc - //nolint:revive // TODO(NET) Fix revive linter - UdpSendsProcessed *prometheus.Desc - //nolint:revive // TODO(NET) Fix revive linter - UdpSendsMissed *prometheus.Desc - //nolint:revive // TODO(NET) Fix revive linter - UdpDroppedConns *prometheus.Desc - // doubleFlushAttemptsClose is a counter measuring the number of attempts to flush a closed connection twice from tcp_close - doubleFlushAttemptsClose *prometheus.Desc - // doubleFlushAttemptsDone is a counter measuring the number of attempts to flush a closed connection twice from tcp_done - doubleFlushAttemptsDone *prometheus.Desc - // unsupportedTcpFailures is a counter measuring the number of attempts to flush a TCP failure that is not supported - unsupportedTcpFailures *prometheus.Desc - // tcpDonePidMismatch is a counter measuring the number of TCP connections with a PID mismatch between tcp_connect and tcp_done - tcpDonePidMismatch *prometheus.Desc - PidCollisions *telemetry.StatCounterWrapper - iterationDups telemetry.Counter - iterationAborts telemetry.Counter - - //nolint:revive // TODO(NET) Fix revive linter - lastTcpFailedConnects *atomic.Int64 - //nolint:revive // TODO(NET) Fix revive linter - LastTcpSentMiscounts *atomic.Int64 - //nolint:revive // TODO(NET) Fix revive linter - lastUnbatchedTcpClose *atomic.Int64 - //nolint:revive // TODO(NET) Fix revive linter - lastUnbatchedUdpClose *atomic.Int64 - //nolint:revive // TODO(NET) Fix revive linter - lastUdpSendsProcessed *atomic.Int64 - //nolint:revive // TODO(NET) Fix revive linter - lastUdpSendsMissed *atomic.Int64 - //nolint:revive // TODO(NET) Fix revive linter - lastUdpDroppedConns *atomic.Int64 - // lastDoubleFlushAttemptsClose is a counter measuring the diff between the last two values of doubleFlushAttemptsClose - lastDoubleFlushAttemptsClose *atomic.Int64 - // lastDoubleFlushAttemptsDone is a counter measuring the diff between the last two values of doubleFlushAttemptsDone - lastDoubleFlushAttemptsDone *atomic.Int64 - // lastUnsupportedTcpFailures is a counter measuring the diff between the last two values of unsupportedTcpFailures - lastUnsupportedTcpFailures *atomic.Int64 - // lastTcpDonePidMismatch is a counter measuring the diff between the last two values of tcpDonePidMismatch - lastTcpDonePidMismatch *atomic.Int64 -}{ - telemetry.NewGauge(connTracerModuleName, "connections", []string{"ip_proto", "family"}, "Gauge measuring the number of active connections in the EBPF map"), - prometheus.NewDesc(connTracerModuleName+"__tcp_failed_connects", "Counter measuring the number of failed TCP connections in the EBPF map", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__tcp_sent_miscounts", "Counter measuring the number of miscounted tcp sends in the EBPF map", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__unbatched_tcp_close", "Counter measuring the number of missed TCP close events in the EBPF map", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__unbatched_udp_close", "Counter measuring the number of missed UDP close events in the EBPF map", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__udp_sends_processed", "Counter measuring the number of processed UDP sends in EBPF", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__udp_sends_missed", "Counter measuring failures to process UDP sends in EBPF", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__udp_dropped_conns", "Counter measuring the number of dropped UDP connections in the EBPF map", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__double_flush_attempts_close", "Counter measuring the number of attempts to flush a closed connection twice from tcp_close", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__double_flush_attempts_done", "Counter measuring the number of attempts to flush a closed connection twice from tcp_done", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__unsupported_tcp_failures", "Counter measuring the number of attempts to flush a TCP failure that is not supported", nil, nil), - prometheus.NewDesc(connTracerModuleName+"__tcp_done_pid_mismatch", "Counter measuring the number of TCP connections with a PID mismatch between tcp_connect and tcp_done", nil, nil), - telemetry.NewStatCounterWrapper(connTracerModuleName, "pid_collisions", []string{}, "Counter measuring number of process collisions"), - telemetry.NewCounter(connTracerModuleName, "iteration_dups", []string{}, "Counter measuring the number of connections iterated more than once"), - telemetry.NewCounter(connTracerModuleName, "iteration_aborts", []string{}, "Counter measuring how many times ebpf iteration of connection map was aborted"), - atomic.NewInt64(0), - atomic.NewInt64(0), - atomic.NewInt64(0), - atomic.NewInt64(0), - atomic.NewInt64(0), - atomic.NewInt64(0), - atomic.NewInt64(0), - atomic.NewInt64(0), - atomic.NewInt64(0), - atomic.NewInt64(0), - atomic.NewInt64(0), -} - -type tracer struct { - m *manager.Manager - - conns *maps.GenericMap[netebpf.ConnTuple, netebpf.ConnStats] - tcpStats *maps.GenericMap[netebpf.ConnTuple, netebpf.TCPStats] - tcpRetransmits *maps.GenericMap[netebpf.ConnTuple, uint32] - config *config.Config - - // tcp_close events - closeConsumer *tcpCloseConsumer - // tcp failure events - failedConnConsumer *failure.TCPFailedConnConsumer - - removeTuple *netebpf.ConnTuple - - closeTracer func() - stopOnce sync.Once - - ebpfTracerType TracerType - - exitTelemetry chan struct{} - - ch *cookieHasher -} - -// NewTracer creates a new tracer -func NewTracer(config *config.Config, _ telemetryComponent.Component) (Tracer, error) { - mgrOptions := manager.Options{ - // Extend RLIMIT_MEMLOCK (8) size - // On some systems, the default for RLIMIT_MEMLOCK may be as low as 64 bytes. - // This will result in an EPERM (Operation not permitted) error, when trying to create an eBPF map - // using bpf(2) with BPF_MAP_CREATE. - // - // We are setting the limit to infinity until we have a better handle on the true requirements. - RLimit: &unix.Rlimit{ - Cur: math.MaxUint64, - Max: math.MaxUint64, - }, - MapSpecEditors: map[string]manager.MapSpecEditor{ - probes.ConnMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, - probes.TCPStatsMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, - probes.TCPRetransmitsMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, - probes.PortBindingsMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, - probes.UDPPortBindingsMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, - probes.ConnectionProtocolMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, - probes.ConnectionTupleToSocketSKBConnMap: {MaxEntries: config.MaxTrackedConnections, EditorFlag: manager.EditMaxEntries}, - }, - ConstantEditors: []manager.ConstantEditor{ - boolConst("tcpv6_enabled", config.CollectTCPv6Conns), - boolConst("udpv6_enabled", config.CollectUDPv6Conns), - }, - DefaultKProbeMaxActive: maxActive, - BypassEnabled: config.BypassEnabled, +// NewTracer returns a new Tracer +func NewTracer(cfg *config.Config, telemetryComp telemetry.Component) (Tracer, error) { + if cfg.EnableEbpfless { + return newEbpfLessTracer(cfg) } - begin, end := network.EphemeralRange() - mgrOptions.ConstantEditors = append(mgrOptions.ConstantEditors, - manager.ConstantEditor{Name: "ephemeral_range_begin", Value: uint64(begin)}, - manager.ConstantEditor{Name: "ephemeral_range_end", Value: uint64(end)}) - - closedChannelSize := defaultClosedChannelSize - if config.ClosedChannelSize > 0 { - closedChannelSize = config.ClosedChannelSize - } - var connCloseEventHandler ddebpf.EventHandler - var failedConnsHandler ddebpf.EventHandler - if config.RingBufferSupportedNPM() { - connCloseEventHandler = ddebpf.NewRingBufferHandler(closedChannelSize) - failedConnsHandler = ddebpf.NewRingBufferHandler(defaultFailedChannelSize) - } else { - connCloseEventHandler = ddebpf.NewPerfHandler(closedChannelSize) - failedConnsHandler = ddebpf.NewPerfHandler(defaultFailedChannelSize) - } - - var m *manager.Manager - //nolint:revive // TODO(NET) Fix revive linter - var tracerType TracerType = TracerTypeFentry - var closeTracerFn func() - m, closeTracerFn, err := fentry.LoadTracer(config, mgrOptions, connCloseEventHandler) - if err != nil && !errors.Is(err, fentry.ErrorNotSupported) { - // failed to load fentry tracer - return nil, err - } - - if err != nil { - // load the kprobe tracer - log.Info("fentry tracer not supported, falling back to kprobe tracer") - var kprobeTracerType kprobe.TracerType - m, closeTracerFn, kprobeTracerType, err = kprobe.LoadTracer(config, mgrOptions, connCloseEventHandler, failedConnsHandler) - if err != nil { - return nil, err - } - tracerType = TracerType(kprobeTracerType) - } - m.DumpHandler = dumpMapsHandler - ddebpf.AddNameMappings(m, "npm_tracer") - - numCPUs, err := ebpf.PossibleCPU() - if err != nil { - return nil, fmt.Errorf("could not determine number of CPUs: %w", err) - } - extractor := newBatchExtractor(numCPUs) - batchMgr, err := newConnBatchManager(m, extractor) - if err != nil { - return nil, fmt.Errorf("could not create connection batch manager: %w", err) - } - - closeConsumer := newTCPCloseConsumer(connCloseEventHandler, batchMgr) - - var failedConnConsumer *failure.TCPFailedConnConsumer - // Failed connections are not supported on prebuilt - if tracerType == TracerTypeKProbePrebuilt { - config.TCPFailedConnectionsEnabled = false - } - if config.FailedConnectionsSupported() { - failedConnConsumer = failure.NewFailedConnConsumer(failedConnsHandler, m, config.MaxFailedConnectionsBuffered) - } - - tr := &tracer{ - m: m, - config: config, - closeConsumer: closeConsumer, - failedConnConsumer: failedConnConsumer, - removeTuple: &netebpf.ConnTuple{}, - closeTracer: closeTracerFn, - ebpfTracerType: tracerType, - exitTelemetry: make(chan struct{}), - ch: newCookieHasher(), - } - - tr.conns, err = maps.GetMap[netebpf.ConnTuple, netebpf.ConnStats](m, probes.ConnMap) - if err != nil { - tr.Stop() - return nil, fmt.Errorf("error retrieving the bpf %s map: %s", probes.ConnMap, err) - } - - tr.tcpStats, err = maps.GetMap[netebpf.ConnTuple, netebpf.TCPStats](m, probes.TCPStatsMap) - if err != nil { - tr.Stop() - return nil, fmt.Errorf("error retrieving the bpf %s map: %s", probes.TCPStatsMap, err) - } - - if tr.tcpRetransmits, err = maps.GetMap[netebpf.ConnTuple, uint32](m, probes.TCPRetransmitsMap); err != nil { - tr.Stop() - return nil, fmt.Errorf("error retrieving the bpf %s map: %s", probes.TCPRetransmitsMap, err) - } - - return tr, nil -} - -func boolConst(name string, value bool) manager.ConstantEditor { - c := manager.ConstantEditor{ - Name: name, - Value: uint64(1), - } - if !value { - c.Value = uint64(0) - } - - return c -} - -func (t *tracer) Start(callback func(*network.ConnectionStats)) (err error) { - defer func() { - if err != nil { - t.Stop() - } - }() - - err = initializePortBindingMaps(t.config, t.m) - if err != nil { - return fmt.Errorf("error initializing port binding maps: %s", err) - } - - if err := t.m.Start(); err != nil { - return fmt.Errorf("could not start ebpf manager: %s", err) - } - - t.closeConsumer.Start(callback) - t.failedConnConsumer.Start() - return nil -} - -func (t *tracer) Pause() error { - // add small delay for socket filters to properly detach - time.Sleep(1 * time.Millisecond) - return t.m.Pause() -} - -func (t *tracer) Resume() error { - err := t.m.Resume() - // add small delay for socket filters to properly attach - time.Sleep(1 * time.Millisecond) - return err -} - -func (t *tracer) FlushPending() { - t.closeConsumer.FlushPending() -} - -func (t *tracer) GetFailedConnections() *failure.FailedConns { - if t.failedConnConsumer == nil { - return nil - } - return t.failedConnConsumer.FailedConns -} - -func (t *tracer) Stop() { - t.stopOnce.Do(func() { - close(t.exitTelemetry) - ddebpf.RemoveNameMappings(t.m) - ebpftelemetry.UnregisterTelemetry(t.m) - _ = t.m.Stop(manager.CleanAll) - t.closeConsumer.Stop() - t.failedConnConsumer.Stop() - if t.closeTracer != nil { - t.closeTracer() - } - }) -} - -func (t *tracer) GetMap(name string) *ebpf.Map { - switch name { - case probes.ConnectionProtocolMap: - default: - return nil - } - m, _, _ := t.m.GetMap(name) - return m -} - -func (t *tracer) GetConnections(buffer *network.ConnectionBuffer, filter func(*network.ConnectionStats) bool) error { - // Iterate through all key-value pairs in map - key, stats := &netebpf.ConnTuple{}, &netebpf.ConnStats{} - seen := make(map[netebpf.ConnTuple]struct{}) - // connsByTuple is used to detect whether we are iterating over - // a connection we have previously seen. This can happen when - // ebpf maps are being iterated over and deleted at the same time. - // The iteration can reset when that happens. - // See https://justin.azoff.dev/blog/bpf_map_get_next_key-pitfalls/ - connsByTuple := make(map[netebpf.ConnTuple]uint32) - - // Cached objects - conn := new(network.ConnectionStats) - tcp := new(netebpf.TCPStats) - - var tcp4, tcp6, udp4, udp6 float64 - entries := t.conns.Iterate() - for entries.Next(key, stats) { - if cookie, exists := connsByTuple[*key]; exists && cookie == stats.Cookie { - // already seen the connection in current batch processing, - // due to race between the iterator and bpf_map_delete - ConnTracerTelemetry.iterationDups.Inc() - continue - } - - populateConnStats(conn, key, stats, t.ch) - connsByTuple[*key] = stats.Cookie - - isTCP := conn.Type == network.TCP - switch conn.Family { - case network.AFINET6: - if isTCP { - tcp6++ - } else { - udp6++ - } - case network.AFINET: - if isTCP { - tcp4++ - } else { - udp4++ - } - } - - if filter != nil && !filter(conn) { - continue - } - - if t.getTCPStats(tcp, key) { - updateTCPStats(conn, tcp, 0) - } - if retrans, ok := t.getTCPRetransmits(key, seen); ok { - updateTCPStats(conn, nil, retrans) - } - - *buffer.Next() = *conn - } - - if err := entries.Err(); err != nil { - if !errors.Is(err, ebpf.ErrIterationAborted) { - return fmt.Errorf("unable to iterate connection map: %w", err) - } - - log.Warn("eBPF conn_stats map iteration aborted. Some connections may not be reported") - ConnTracerTelemetry.iterationAborts.Inc() - } - - updateTelemetry(tcp4, tcp6, udp4, udp6) - - return nil -} - -func updateTelemetry(tcp4 float64, tcp6 float64, udp4 float64, udp6 float64) { - ConnTracerTelemetry.connections.Set(tcp4, "tcp", "v4") - ConnTracerTelemetry.connections.Set(tcp6, "tcp", "v6") - ConnTracerTelemetry.connections.Set(udp4, "udp", "v4") - ConnTracerTelemetry.connections.Set(udp6, "udp", "v6") -} - -func removeConnectionFromTelemetry(conn *network.ConnectionStats) { - isTCP := conn.Type == network.TCP - switch conn.Family { - case network.AFINET6: - if isTCP { - ConnTracerTelemetry.connections.Dec("tcp", "v6") - } else { - ConnTracerTelemetry.connections.Dec("udp", "v6") - } - case network.AFINET: - if isTCP { - ConnTracerTelemetry.connections.Dec("tcp", "v4") - } else { - ConnTracerTelemetry.connections.Dec("udp", "v4") - } - } -} - -func (t *tracer) Remove(conn *network.ConnectionStats) error { - util.ConnStatsToTuple(conn, t.removeTuple) - - err := t.conns.Delete(t.removeTuple) - if err != nil { - // If this entry no longer exists in the eBPF map it means `tcp_close` has executed - // during this function call. In that case state.StoreClosedConnection() was already called for this connection, - // and we can't delete the corresponding client state, or we'll likely over-report the metric values. - // By skipping to the next iteration and not calling state.RemoveConnections() we'll let - // this connection expire "naturally" when either next connection check runs or the client itself expires. - return err - } - - removeConnectionFromTelemetry(conn) - - if conn.Type == network.TCP { - // We can ignore the error for this map since it will not always contain the entry - _ = t.tcpStats.Delete(t.removeTuple) - // We remove the PID from the tuple as it is not used in the retransmits map - pid := t.removeTuple.Pid - t.removeTuple.Pid = 0 - _ = t.tcpRetransmits.Delete(t.removeTuple) - t.removeTuple.Pid = pid - } - return nil -} - -func (t *tracer) getEBPFTelemetry() *netebpf.Telemetry { - var zero uint32 - mp, err := maps.GetMap[uint32, netebpf.Telemetry](t.m, probes.TelemetryMap) - if err != nil { - log.Warnf("error retrieving telemetry map: %s", err) - return nil - } - - tm := &netebpf.Telemetry{} - if err := mp.Lookup(&zero, tm); err != nil { - // This can happen if we haven't initialized the telemetry object yet - // so let's just use a trace log - if log.ShouldLog(seelog.TraceLvl) { - log.Tracef("error retrieving the telemetry struct: %s", err) - } - return nil - } - return tm -} - -// Describe returns all descriptions of the collector -func (t *tracer) Describe(ch chan<- *prometheus.Desc) { - ch <- ConnTracerTelemetry.tcpFailedConnects - ch <- ConnTracerTelemetry.TcpSentMiscounts - ch <- ConnTracerTelemetry.unbatchedTcpClose - ch <- ConnTracerTelemetry.unbatchedUdpClose - ch <- ConnTracerTelemetry.UdpSendsProcessed - ch <- ConnTracerTelemetry.UdpSendsMissed - ch <- ConnTracerTelemetry.UdpDroppedConns - ch <- ConnTracerTelemetry.doubleFlushAttemptsClose - ch <- ConnTracerTelemetry.doubleFlushAttemptsDone - ch <- ConnTracerTelemetry.unsupportedTcpFailures - ch <- ConnTracerTelemetry.tcpDonePidMismatch -} - -// Collect returns the current state of all metrics of the collector -func (t *tracer) Collect(ch chan<- prometheus.Metric) { - ebpfTelemetry := t.getEBPFTelemetry() - if ebpfTelemetry == nil { - return - } - delta := int64(ebpfTelemetry.Tcp_failed_connect) - ConnTracerTelemetry.lastTcpFailedConnects.Load() - ConnTracerTelemetry.lastTcpFailedConnects.Store(int64(ebpfTelemetry.Tcp_failed_connect)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.tcpFailedConnects, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Tcp_sent_miscounts) - ConnTracerTelemetry.LastTcpSentMiscounts.Load() - ConnTracerTelemetry.LastTcpSentMiscounts.Store(int64(ebpfTelemetry.Tcp_sent_miscounts)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.TcpSentMiscounts, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Unbatched_tcp_close) - ConnTracerTelemetry.lastUnbatchedTcpClose.Load() - ConnTracerTelemetry.lastUnbatchedTcpClose.Store(int64(ebpfTelemetry.Unbatched_tcp_close)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.unbatchedTcpClose, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Unbatched_udp_close) - ConnTracerTelemetry.lastUnbatchedUdpClose.Load() - ConnTracerTelemetry.lastUnbatchedUdpClose.Store(int64(ebpfTelemetry.Unbatched_udp_close)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.unbatchedUdpClose, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Udp_sends_processed) - ConnTracerTelemetry.lastUdpSendsProcessed.Load() - ConnTracerTelemetry.lastUdpSendsProcessed.Store(int64(ebpfTelemetry.Udp_sends_processed)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.UdpSendsProcessed, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Udp_sends_missed) - ConnTracerTelemetry.lastUdpSendsMissed.Load() - ConnTracerTelemetry.lastUdpSendsMissed.Store(int64(ebpfTelemetry.Udp_sends_missed)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.UdpSendsMissed, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Udp_dropped_conns) - ConnTracerTelemetry.lastUdpDroppedConns.Load() - ConnTracerTelemetry.lastUdpDroppedConns.Store(int64(ebpfTelemetry.Udp_dropped_conns)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.UdpDroppedConns, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Double_flush_attempts_close) - ConnTracerTelemetry.lastDoubleFlushAttemptsClose.Load() - ConnTracerTelemetry.lastDoubleFlushAttemptsClose.Store(int64(ebpfTelemetry.Double_flush_attempts_close)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.doubleFlushAttemptsClose, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Double_flush_attempts_done) - ConnTracerTelemetry.lastDoubleFlushAttemptsDone.Load() - ConnTracerTelemetry.lastDoubleFlushAttemptsDone.Store(int64(ebpfTelemetry.Double_flush_attempts_done)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.doubleFlushAttemptsDone, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Unsupported_tcp_failures) - ConnTracerTelemetry.lastUnsupportedTcpFailures.Load() - ConnTracerTelemetry.lastUnsupportedTcpFailures.Store(int64(ebpfTelemetry.Unsupported_tcp_failures)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.unsupportedTcpFailures, prometheus.CounterValue, float64(delta)) - - delta = int64(ebpfTelemetry.Tcp_done_pid_mismatch) - ConnTracerTelemetry.lastTcpDonePidMismatch.Load() - ConnTracerTelemetry.lastTcpDonePidMismatch.Store(int64(ebpfTelemetry.Tcp_done_pid_mismatch)) - ch <- prometheus.MustNewConstMetric(ConnTracerTelemetry.tcpDonePidMismatch, prometheus.CounterValue, float64(delta)) - -} - -// DumpMaps (for debugging purpose) returns all maps content by default or selected maps from maps parameter. -func (t *tracer) DumpMaps(w io.Writer, maps ...string) error { - return t.m.DumpMaps(w, maps...) -} - -// Type returns the type of the underlying ebpf tracer that is currently loaded -func (t *tracer) Type() TracerType { - return t.ebpfTracerType -} - -func initializePortBindingMaps(config *config.Config, m *manager.Manager) error { - tcpPorts, err := network.ReadInitialState(config.ProcRoot, network.TCP, config.CollectTCPv6Conns) - if err != nil { - return fmt.Errorf("failed to read initial TCP pid->port mapping: %s", err) - } - - tcpPortMap, err := maps.GetMap[netebpf.PortBinding, uint32](m, probes.PortBindingsMap) - if err != nil { - return fmt.Errorf("failed to get TCP port binding map: %w", err) - } - for p, count := range tcpPorts { - log.Debugf("adding initial TCP port binding: netns: %d port: %d", p.Ino, p.Port) - pb := netebpf.PortBinding{Netns: p.Ino, Port: p.Port} - err = tcpPortMap.Update(&pb, &count, ebpf.UpdateNoExist) - if err != nil && !errors.Is(err, ebpf.ErrKeyExist) { - return fmt.Errorf("failed to update TCP port binding map: %w", err) - } - } - - udpPorts, err := network.ReadInitialState(config.ProcRoot, network.UDP, config.CollectUDPv6Conns) - if err != nil { - return fmt.Errorf("failed to read initial UDP pid->port mapping: %s", err) - } - - udpPortMap, err := maps.GetMap[netebpf.PortBinding, uint32](m, probes.UDPPortBindingsMap) - if err != nil { - return fmt.Errorf("failed to get UDP port binding map: %w", err) - } - for p, count := range udpPorts { - // ignore ephemeral port binds as they are more likely to be from - // clients calling bind with port 0 - if network.IsPortInEphemeralRange(network.AFINET, network.UDP, p.Port) == network.EphemeralTrue { - log.Debugf("ignoring initial ephemeral UDP port bind to %d", p) - continue - } - - log.Debugf("adding initial UDP port binding: netns: %d port: %d", p.Ino, p.Port) - pb := netebpf.PortBinding{Netns: p.Ino, Port: p.Port} - err = udpPortMap.Update(&pb, &count, ebpf.UpdateNoExist) - if err != nil && !errors.Is(err, ebpf.ErrKeyExist) { - return fmt.Errorf("failed to update UDP port binding map: %w", err) - } - } - return nil -} - -func (t *tracer) getTCPRetransmits(tuple *netebpf.ConnTuple, seen map[netebpf.ConnTuple]struct{}) (uint32, bool) { - if tuple.Type() != netebpf.TCP { - return 0, false - } - - // The PID isn't used as a key in the stats map, we will temporarily set it to 0 here and reset it when we're done - pid := tuple.Pid - tuple.Pid = 0 - - var retransmits uint32 - if err := t.tcpRetransmits.Lookup(tuple, &retransmits); err == nil { - // This is required to avoid (over)reporting retransmits for connections sharing the same socket. - if _, reported := seen[*tuple]; reported { - ConnTracerTelemetry.PidCollisions.Inc() - retransmits = 0 - } else { - seen[*tuple] = struct{}{} - } - } - - tuple.Pid = pid - return retransmits, true -} - -// getTCPStats reads tcp related stats for the given ConnTuple -func (t *tracer) getTCPStats(stats *netebpf.TCPStats, tuple *netebpf.ConnTuple) bool { - if tuple.Type() != netebpf.TCP { - return false - } - - return t.tcpStats.Lookup(tuple, stats) == nil -} - -func populateConnStats(stats *network.ConnectionStats, t *netebpf.ConnTuple, s *netebpf.ConnStats, ch *cookieHasher) { - *stats = network.ConnectionStats{ - Pid: t.Pid, - NetNS: t.Netns, - Source: t.SourceAddress(), - Dest: t.DestAddress(), - SPort: t.Sport, - DPort: t.Dport, - Monotonic: network.StatCounters{ - SentBytes: s.Sent_bytes, - RecvBytes: s.Recv_bytes, - SentPackets: uint64(s.Sent_packets), - RecvPackets: uint64(s.Recv_packets), - }, - LastUpdateEpoch: s.Timestamp, - IsAssured: s.IsAssured(), - Cookie: network.StatCookie(s.Cookie), - } - - if s.Duration <= uint64(math.MaxInt64) { - stats.Duration = time.Duration(s.Duration) * time.Nanosecond - } - - stats.ProtocolStack = protocols.Stack{ - API: protocols.API(s.Protocol_stack.Api), - Application: protocols.Application(s.Protocol_stack.Application), - Encryption: protocols.Encryption(s.Protocol_stack.Encryption), - } - - if t.Type() == netebpf.TCP { - stats.Type = network.TCP - } else { - stats.Type = network.UDP - } - - switch t.Family() { - case netebpf.IPv4: - stats.Family = network.AFINET - case netebpf.IPv6: - stats.Family = network.AFINET6 - } - - stats.SPortIsEphemeral = network.IsPortInEphemeralRange(stats.Family, stats.Type, t.Sport) - - switch s.ConnectionDirection() { - case netebpf.Incoming: - stats.Direction = network.INCOMING - case netebpf.Outgoing: - stats.Direction = network.OUTGOING - default: - stats.Direction = network.OUTGOING - } - - if ch != nil { - ch.Hash(stats) - } -} - -func updateTCPStats(conn *network.ConnectionStats, tcpStats *netebpf.TCPStats, retransmits uint32) { - if conn.Type != network.TCP { - return - } - - conn.Monotonic.Retransmits = retransmits - if tcpStats != nil { - conn.Monotonic.TCPEstablished = uint32(tcpStats.State_transitions >> netebpf.Established & 1) - conn.Monotonic.TCPClosed = uint32(tcpStats.State_transitions >> netebpf.Close & 1) - conn.RTT = tcpStats.Rtt - conn.RTTVar = tcpStats.Rtt_var - } -} - -type cookieHasher struct { - hash hash.Hash64 - buf []byte -} - -func newCookieHasher() *cookieHasher { - return &cookieHasher{ - hash: murmur3.New64(), - buf: make([]byte, network.ConnectionByteKeyMaxLen), - } -} - -func (h *cookieHasher) Hash(stats *network.ConnectionStats) { - h.hash.Reset() - if err := binary.Write(h.hash, binary.BigEndian, stats.Cookie); err != nil { - log.Errorf("error writing cookie to hash: %s", err) - return - } - key := stats.ByteKey(h.buf) - if _, err := h.hash.Write(key); err != nil { - log.Errorf("error writing byte key to hash: %s", err) - return - } - stats.Cookie = h.hash.Sum64() + return newEbpfTracer(cfg, telemetryComp) } diff --git a/pkg/network/tracer/tracer.go b/pkg/network/tracer/tracer.go index db5f0bc95a1a9..55faac7817df1 100644 --- a/pkg/network/tracer/tracer.go +++ b/pkg/network/tracer/tracer.go @@ -15,7 +15,6 @@ import ( "sync" "time" - "github.com/DataDog/ebpf-manager/tracefs" "github.com/cihub/seelog" "github.com/cilium/ebpf" "go.uber.org/atomic" @@ -121,10 +120,6 @@ func NewTracer(config *config.Config, telemetryComponent telemetryComponent.Comp // newTracer is an internal function used by tests primarily // (and NewTracer above) func newTracer(cfg *config.Config, telemetryComponent telemetryComponent.Component) (_ *Tracer, reterr error) { - if _, err := tracefs.Root(); err != nil { - return nil, fmt.Errorf("system-probe unsupported: %s", err) - } - // check if current platform is using old kernel API because it affects what kprobe are we going to enable currKernelVersion, err := kernel.HostVersion() if err != nil { @@ -139,12 +134,16 @@ func newTracer(cfg *config.Config, telemetryComponent telemetryComponent.Compone } if cfg.ServiceMonitoringEnabled { - if !usmconfig.IsUSMSupported() { - errStr := fmt.Sprintf("Universal Service Monitoring (USM) requires a Linux kernel version of %s or higher. We detected %s", usmconfig.MinimumKernelVersion, currKernelVersion) + if err := usmconfig.CheckUSMSupported(cfg); err != nil { + // this is the case where USM is enabled and NPM is not enabled + // in config; we implicitly enable the network tracer module + // in system-probe if USM is enabled if !cfg.NPMEnabled { - return nil, fmt.Errorf(errStr) + return nil, err } - log.Warnf("%s. NPM is explicitly enabled, so system-probe will continue with only NPM features enabled.", errStr) + + log.Warn(err) + log.Warnf("NPM is explicitly enabled, so system-probe will continue with only NPM features enabled") } } @@ -244,30 +243,33 @@ func newConntracker(cfg *config.Config, telemetryComponent telemetryComponent.Co var c netlink.Conntracker var err error - ns, err := cfg.GetRootNetNs() - if err != nil { - log.Warnf("error fetching root net namespace, will not attempt to load nf_conntrack_netlink module: %s", err) - } else { - defer ns.Close() - if err = netlink.LoadNfConntrackKernelModule(ns); err != nil { - log.Warnf("failed to load conntrack kernel module, though it may already be loaded: %s", err) + if !cfg.EnableEbpfless { + ns, err := cfg.GetRootNetNs() + if err != nil { + log.Warnf("error fetching root net namespace, will not attempt to load nf_conntrack_netlink module: %s", err) + } else { + defer ns.Close() + if err = netlink.LoadNfConntrackKernelModule(ns); err != nil { + log.Warnf("failed to load conntrack kernel module, though it may already be loaded: %s", err) + } } - } - if cfg.EnableEbpfConntracker { - if c, err = NewEBPFConntracker(cfg, telemetryComponent); err == nil { - return c, nil + if cfg.EnableEbpfConntracker { + if c, err = NewEBPFConntracker(cfg, telemetryComponent); err == nil { + return c, nil + } + log.Warnf("error initializing ebpf conntracker: %s", err) + } else { + log.Info("ebpf conntracker disabled") } - log.Warnf("error initializing ebpf conntracker: %s", err) - } else { - log.Info("ebpf conntracker disabled") + + log.Info("falling back to netlink conntracker") } - log.Info("falling back to netlink conntracker") if c, err = netlink.NewConntracker(cfg, telemetryComponent); err == nil { return c, nil } - if cfg.IgnoreConntrackInitFailure { + if errors.Is(err, netlink.ErrNotPermitted) || cfg.IgnoreConntrackInitFailure { log.Warnf("could not initialize conntrack, tracer will continue without NAT tracking: %s", err) return netlink.NewNoOpConntracker(), nil } diff --git a/pkg/network/tracer/tracer_linux_test.go b/pkg/network/tracer/tracer_linux_test.go index f86ccbb749ff8..090a105bd3fdf 100644 --- a/pkg/network/tracer/tracer_linux_test.go +++ b/pkg/network/tracer/tracer_linux_test.go @@ -241,7 +241,7 @@ func (s *TracerSuite) TestTCPRetransmitSharedSocket() { // Test if telemetry measuring PID collisions is correct // >= because there can be other connections going on during CI that increase pidCollisions - assert.GreaterOrEqual(t, connection.ConnTracerTelemetry.PidCollisions.Load(), int64(numProcesses-1)) + assert.GreaterOrEqual(t, connection.EbpfTracerTelemetry.PidCollisions.Load(), int64(numProcesses-1)) } func (s *TracerSuite) TestTCPRTT() { @@ -333,7 +333,7 @@ func (s *TracerSuite) TestTCPMiscount() { assert.False(t, uint64(len(x)) == conn.Monotonic.SentBytes) } - assert.NotZero(t, connection.ConnTracerTelemetry.LastTcpSentMiscounts.Load()) + assert.NotZero(t, connection.EbpfTracerTelemetry.LastTcpSentMiscounts.Load()) } func (s *TracerSuite) TestConnectionExpirationRegression() { @@ -387,6 +387,9 @@ func (s *TracerSuite) TestConnectionExpirationRegression() { func (s *TracerSuite) TestConntrackExpiration() { t := s.T() ebpftest.LogLevel(t, "trace") + + cfg := testConfig() + skipOnEbpflessNotSupported(t, cfg) netlinktestutil.SetupDNAT(t) tr := setupTracer(t, testConfig()) @@ -861,6 +864,7 @@ func (s *TracerSuite) TestGatewayLookupCrossNamespace() { }) t.Run("client in other namespace", func(t *testing.T) { + skipOnEbpflessNotSupported(t, cfg) // try connecting to server in test1 namespace test2Ns, err := vnetns.GetFromName(ns2) require.NoError(t, err) @@ -925,6 +929,8 @@ func (s *TracerSuite) TestGatewayLookupCrossNamespace() { func (s *TracerSuite) TestConnectionAssured() { t := s.T() cfg := testConfig() + skipOnEbpflessNotSupported(t, cfg) + tr := setupTracer(t, cfg) server := &UDPServer{ network: "udp4", @@ -1012,9 +1018,11 @@ func (s *TracerSuite) TestUDPConnExpiryTimeout() { func (s *TracerSuite) TestDNATIntraHostIntegration() { t := s.T() + cfg := testConfig() + skipEbpflessTodo(t, cfg) netlinktestutil.SetupDNAT(t) - tr := setupTracer(t, testConfig()) + tr := setupTracer(t, cfg) var serverAddr struct { local, remote net.Addr @@ -1385,12 +1393,13 @@ func testUDPReusePort(t *testing.T, udpnet string, ip string) { func (s *TracerSuite) TestDNSStatsWithNAT() { t := s.T() + cfg := testConfig() + skipEbpflessTodo(t, cfg) testutil.IptablesSave(t) // Setup a NAT rule to translate 2.2.2.2 to 8.8.8.8 and issue a DNS request to 2.2.2.2 cmds := []string{"iptables -t nat -A OUTPUT -d 2.2.2.2 -j DNAT --to-destination 8.8.8.8"} testutil.RunCommands(t, cmds, false) - cfg := testConfig() cfg.CollectDNSStats = true cfg.DNSTimeout = 1 * time.Second tr := setupTracer(t, cfg) @@ -1713,6 +1722,7 @@ func (s *TracerSuite) TestShortWrite() { func (s *TracerSuite) TestKprobeAttachWithKprobeEvents() { t := s.T() cfg := config.New() + skipOnEbpflessNotSupported(t, cfg) cfg.AttachKprobesWithKprobeEventsABI = true tr := setupTracer(t, cfg) @@ -1851,7 +1861,9 @@ func (s *TracerSuite) TestPreexistingConnectionDirection() { m := outgoing.Monotonic assert.Equal(t, clientMessageSize, int(m.SentBytes)) assert.Equal(t, serverMessageSize, int(m.RecvBytes)) - assert.Equal(t, os.Getpid(), int(outgoing.Pid)) + if !tr.config.EnableEbpfless { + assert.Equal(t, os.Getpid(), int(outgoing.Pid)) + } assert.Equal(t, addrPort(server.Address()), int(outgoing.DPort)) assert.Equal(t, c.LocalAddr().(*net.TCPAddr).Port, int(outgoing.SPort)) assert.Equal(t, network.OUTGOING, outgoing.Direction) @@ -1859,7 +1871,9 @@ func (s *TracerSuite) TestPreexistingConnectionDirection() { m = incoming.Monotonic assert.Equal(t, clientMessageSize, int(m.RecvBytes)) assert.Equal(t, serverMessageSize, int(m.SentBytes)) - assert.Equal(t, os.Getpid(), int(incoming.Pid)) + if !tr.config.EnableEbpfless { + assert.Equal(t, os.Getpid(), int(incoming.Pid)) + } assert.Equal(t, addrPort(server.Address()), int(incoming.SPort)) assert.Equal(t, c.LocalAddr().(*net.TCPAddr).Port, int(incoming.DPort)) assert.Equal(t, network.INCOMING, incoming.Direction) @@ -1872,6 +1886,7 @@ func (s *TracerSuite) TestPreexistingEmptyIncomingConnectionDirection() { t.Skip("skipping test as ringbuffers are not supported on this kernel") } c := testConfig() + skipOnEbpflessNotSupported(t, c) c.NPMRingbuffersEnabled = true testPreexistingEmptyIncomingConnectionDirection(t, c) }) @@ -1890,6 +1905,7 @@ func testPreexistingEmptyIncomingConnectionDirection(t *testing.T, config *confi server := tracertestutil.NewTCPServer(func(c net.Conn) { <-ch c.Close() + close(ch) }) require.NoError(t, server.Run()) t.Cleanup(server.Shutdown) @@ -1901,7 +1917,8 @@ func testPreexistingEmptyIncomingConnectionDirection(t *testing.T, config *confi tr := setupTracer(t, config) // close the server connection so the tracer picks it up - close(ch) + ch <- struct{}{} + <-ch var conn *network.ConnectionStats require.Eventually(t, func() bool { @@ -2337,3 +2354,15 @@ func setupDropTrafficRule(tb testing.TB) (ns string) { testutil.RunCommands(tb, cmds, false) return } + +func skipOnEbpflessNotSupported(t *testing.T, cfg *config.Config) { + if cfg.EnableEbpfless { + t.Skip("not supported on ebpf-less") + } +} + +func skipEbpflessTodo(t *testing.T, cfg *config.Config) { + if cfg.EnableEbpfless { + t.Skip("TODO: ebpf-less") + } +} diff --git a/pkg/network/tracer/tracer_test.go b/pkg/network/tracer/tracer_test.go index 7011923c9329d..4bc9d6ce8f0e6 100644 --- a/pkg/network/tracer/tracer_test.go +++ b/pkg/network/tracer/tracer_test.go @@ -156,7 +156,8 @@ func (s *TracerSuite) TestGetStats() { func (s *TracerSuite) TestTCPSendAndReceive() { t := s.T() - tr := setupTracer(t, testConfig()) + cfg := testConfig() + tr := setupTracer(t, cfg) // Create TCP Server which, for every line, sends back a message with size=serverMessageSize server := testutil.NewTCPServer(func(c net.Conn) { @@ -208,10 +209,11 @@ func (s *TracerSuite) TestTCPSendAndReceive() { m := conn.Monotonic assert.Equal(t, 10*clientMessageSize, int(m.SentBytes)) assert.Equal(t, 10*serverMessageSize, int(m.RecvBytes)) - assert.Equal(t, os.Getpid(), int(conn.Pid)) + if !cfg.EnableEbpfless { + assert.Equal(t, os.Getpid(), int(conn.Pid)) + } assert.Equal(t, addrPort(server.Address()), int(conn.DPort)) assert.Equal(t, network.OUTGOING, conn.Direction) - assert.True(t, conn.IntraHost) } func (s *TracerSuite) TestTCPShortLived() { @@ -557,14 +559,16 @@ func (s *TracerSuite) TestLocalDNSCollectionEnabled() { _, err = cn.Write([]byte("test")) assert.NoError(t, err) - found := false - // Iterate through active connections making sure theres at least one connection - for _, c := range getConnections(t, tr).Conns { - found = found || isLocalDNS(c) - } + require.Eventually(t, func() bool { + for _, c := range getConnections(t, tr).Conns { + if isLocalDNS(c) { + return true + } + } - assert.True(t, found) + return false + }, 3*time.Second, 100*time.Millisecond, "could not find connection") } func isLocalDNS(c network.ConnectionStats) bool { @@ -993,8 +997,10 @@ func testDNSStats(t *testing.T, tr *Tracer, domain string, success, failure, tim if !assert.Equal(c, queryMsg.Len(), int(conn.Monotonic.SentBytes)) { return } - if !assert.Equal(c, os.Getpid(), int(conn.Pid)) { - return + if !tr.config.EnableEbpfless { + if !assert.Equal(c, os.Getpid(), int(conn.Pid)) { + return + } } if !assert.Equal(c, dnsServerAddr.Port, int(conn.DPort)) { return @@ -1161,10 +1167,15 @@ func (s *TracerSuite) TestConnectedUDPSendIPv6() { bytesSent, err := conn.Write(message) require.NoError(t, err) - connections := getConnections(t, tr) - outgoing := network.FilterConnections(connections, func(cs network.ConnectionStats) bool { - return cs.DPort == uint16(remotePort) - }) + var outgoing []network.ConnectionStats + require.Eventually(t, func() bool { + connections := getConnections(t, tr) + outgoing = network.FilterConnections(connections, func(cs network.ConnectionStats) bool { + return cs.DPort == uint16(remotePort) + }) + + return len(outgoing) == 1 + }, 3*time.Second, 100*time.Millisecond, "failed to find connection") require.Len(t, outgoing, 1) assert.Equal(t, remoteAddr.IP.String(), outgoing[0].Dest.String()) diff --git a/pkg/network/tracer/utils_linux.go b/pkg/network/tracer/utils_linux.go index 5315b10785a00..d072906eec687 100644 --- a/pkg/network/tracer/utils_linux.go +++ b/pkg/network/tracer/utils_linux.go @@ -3,8 +3,6 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build linux_bpf - package tracer import ( @@ -16,10 +14,16 @@ import ( "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/features" + coreconfig "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" ) +// NeedsEBPF returns `true` if the network-tracer requires eBPF +func NeedsEBPF() bool { + return !coreconfig.SystemProbe().GetBool("network_config.enable_ebpfless") +} + // IsTracerSupportedByOS returns whether the current kernel version supports tracer functionality // along with some context on why it's not supported func IsTracerSupportedByOS(exclusionList []string) (bool, error) { @@ -59,6 +63,10 @@ func verifyOSVersion(kernelCode kernel.Version, platform string, exclusionList [ return false, fmt.Errorf("Known bug for kernel %s on platform %s, see: \n- https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1763454", kernelCode, platform) } + if !NeedsEBPF() { + return true, nil + } + var requiredFuncs = []asm.BuiltinFunc{ asm.FnMapLookupElem, asm.FnMapUpdateElem, diff --git a/pkg/network/tracer/utils_unsupported.go b/pkg/network/tracer/utils_unsupported.go index cf8f70835e1a8..5678e22e579e1 100644 --- a/pkg/network/tracer/utils_unsupported.go +++ b/pkg/network/tracer/utils_unsupported.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !linux_bpf && !windows +//go:build !windows && !linux package tracer diff --git a/pkg/network/usm/config/config.go b/pkg/network/usm/config/config.go index 915e68b15098b..00a4cc9764c5b 100644 --- a/pkg/network/usm/config/config.go +++ b/pkg/network/usm/config/config.go @@ -9,6 +9,8 @@ package config import ( + "errors" + "fmt" "runtime" "strings" @@ -20,6 +22,9 @@ import ( // MinimumKernelVersion indicates the minimum kernel version required for HTTP monitoring var MinimumKernelVersion kernel.Version +// ErrNotSupported is the error returned if USM is not supported on this platform +var ErrNotSupported = errors.New("Universal Service Monitoring (USM) is not supported") + func init() { MinimumKernelVersion = kernel.VersionCode(4, 14, 0) } @@ -44,21 +49,31 @@ func TLSSupported(c *config.Config) bool { return kversion >= MinimumKernelVersion } -// IsUSMSupported We only support http with kernel >= 4.14.0. -func IsUSMSupported() bool { +// CheckUSMSupported returns an error if USM is not supported +// on this platform. Callers can check `errors.Is(err, ErrNotSupported)` +// to verify if USM is supported +func CheckUSMSupported(cfg *config.Config) error { + // TODO: remove this once USM is supported on ebpf-less + if cfg.EnableEbpfless { + return fmt.Errorf("%w: eBPF-less is not supported", ErrNotSupported) + } + kversion, err := kernel.HostVersion() if err != nil { - log.Warn("could not determine the current kernel version. USM disabled.") - return false + return fmt.Errorf("%w: could not determine the current kernel version: %w", ErrNotSupported, err) } - return kversion >= MinimumKernelVersion + if kversion < MinimumKernelVersion { + return fmt.Errorf("%w: a Linux kernel version of %s or higher is required; we detected %s", ErrNotSupported, MinimumKernelVersion, kversion) + } + + return nil } // IsUSMSupportedAndEnabled returns true if USM is supported and enabled func IsUSMSupportedAndEnabled(config *config.Config) bool { // http.Supported is misleading, it should be named usm.Supported. - return config.ServiceMonitoringEnabled && IsUSMSupported() + return config.ServiceMonitoringEnabled && CheckUSMSupported(config) == nil } // NeedProcessMonitor returns true if the process monitor is needed for the given configuration From 88303afa7d89c914eb0f4a0236282e5c6d526f74 Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Mon, 19 Aug 2024 18:59:30 +0200 Subject: [PATCH 038/245] discovery: Report APM instrumentation by injection (#28438) --- .../servicediscovery/apm/detect_nix_test.go | 40 +++++++++++++++++ .../corechecks/servicediscovery/impl_linux.go | 7 +-- .../servicediscovery/impl_linux_test.go | 22 +++++++--- .../servicediscovery/model/model.go | 7 +-- .../servicediscovery/module/impl_linux.go | 44 +++++++++++-------- .../module/impl_linux_test.go | 20 ++++++++- 6 files changed, 109 insertions(+), 31 deletions(-) diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go index 8fa660698140b..9c4978cfe4c6d 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go @@ -11,8 +11,48 @@ import ( "os" "strings" "testing" + + "github.com/stretchr/testify/assert" ) +func TestInjected(t *testing.T) { + data := []struct { + name string + envs map[string]string + result bool + }{ + { + name: "injected", + envs: map[string]string{ + "DD_INJECTION_ENABLED": "tracer", + }, + result: true, + }, + { + name: "one of injected", + envs: map[string]string{ + "DD_INJECTION_ENABLED": "service_name,tracer", + }, + result: true, + }, + { + name: "not injected but with env variable", + envs: map[string]string{ + "DD_INJECTION_ENABLED": "service_name", + }, + }, + { + name: "not injected, no env variable", + }, + } + for _, d := range data { + t.Run(d.name, func(t *testing.T) { + result := isInjected(d.envs) + assert.Equal(t, d.result, result) + }) + } +} + func Test_javaDetector(t *testing.T) { data := []struct { name string diff --git a/pkg/collector/corechecks/servicediscovery/impl_linux.go b/pkg/collector/corechecks/servicediscovery/impl_linux.go index 7060c051493e3..5a039b7489ba5 100644 --- a/pkg/collector/corechecks/servicediscovery/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/impl_linux.go @@ -282,9 +282,10 @@ func (li *linuxImpl) getServiceInfo(p proc, service model.Service) (*serviceInfo serviceType := servicetype.Detect(service.Name, service.Ports) meta := ServiceMetadata{ - Name: service.Name, - Language: string(lang), - Type: string(serviceType), + Name: service.Name, + Language: string(lang), + Type: string(serviceType), + APMInstrumentation: service.APMInstrumentation, } return &serviceInfo{ diff --git a/pkg/collector/corechecks/servicediscovery/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/impl_linux_test.go index 7ae84df2e98a2..6c56599bf1ed1 100644 --- a/pkg/collector/corechecks/servicediscovery/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/impl_linux_test.go @@ -24,6 +24,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/apm" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" ) @@ -119,14 +120,16 @@ var ( Ports: []uint16{22}, } portTCP8080 = model.Service{ - PID: procTestService1.pid, - Name: "test-service-1", - Ports: []uint16{8080}, + PID: procTestService1.pid, + Name: "test-service-1", + Ports: []uint16{8080}, + APMInstrumentation: string(apm.None), } portTCP8080DifferentPID = model.Service{ - PID: procTestService1DifferentPID.pid, - Name: "test-service-1", - Ports: []uint16{8080}, + PID: procTestService1DifferentPID.pid, + Name: "test-service-1", + Ports: []uint16{8080}, + APMInstrumentation: string(apm.Injected), } portTCP8081 = model.Service{ PID: procIgnoreService1.pid, @@ -280,6 +283,7 @@ func Test_linuxImpl(t *testing.T) { Ports: []uint16{8080}, PID: 99, CommandLine: []string{"test-service-1"}, + APMInstrumentation: "none", }, }, { @@ -296,6 +300,7 @@ func Test_linuxImpl(t *testing.T) { Ports: []uint16{8080}, PID: 99, CommandLine: []string{"test-service-1"}, + APMInstrumentation: "none", }, }, { @@ -312,6 +317,7 @@ func Test_linuxImpl(t *testing.T) { Ports: []uint16{8080}, PID: 99, CommandLine: []string{"test-service-1"}, + APMInstrumentation: "none", }, }, { @@ -441,6 +447,7 @@ func Test_linuxImpl(t *testing.T) { Ports: []uint16{8080}, PID: 99, CommandLine: []string{"test-service-1"}, + APMInstrumentation: "none", }, }, { @@ -489,6 +496,7 @@ func Test_linuxImpl(t *testing.T) { Ports: []uint16{8080}, PID: 99, CommandLine: []string{"test-service-1"}, + APMInstrumentation: "none", }, }, }, @@ -562,6 +570,7 @@ func Test_linuxImpl(t *testing.T) { Ports: []uint16{8080}, PID: 99, CommandLine: []string{"test-service-1"}, + APMInstrumentation: "none", }, }, { @@ -578,6 +587,7 @@ func Test_linuxImpl(t *testing.T) { Ports: []uint16{8080}, PID: 102, CommandLine: []string{"test-service-1"}, + APMInstrumentation: "injected", }, }, }, diff --git a/pkg/collector/corechecks/servicediscovery/model/model.go b/pkg/collector/corechecks/servicediscovery/model/model.go index 3f80ea27aded1..a8d776761d9ab 100644 --- a/pkg/collector/corechecks/servicediscovery/model/model.go +++ b/pkg/collector/corechecks/servicediscovery/model/model.go @@ -8,9 +8,10 @@ package model // Service represents a listening process. type Service struct { - PID int `json:"pid"` - Name string `json:"name"` - Ports []uint16 `json:"ports"` + PID int `json:"pid"` + Name string `json:"name"` + Ports []uint16 `json:"ports"` + APMInstrumentation string `json:"apm_instrumentation"` } // ServicesResponse is the response for the system-probe /discovery/services endpoint. diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go index a0b90a834b798..bd0de7db550af 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go @@ -21,6 +21,8 @@ import ( "github.com/DataDog/datadog-agent/cmd/system-probe/utils" "github.com/DataDog/datadog-agent/comp/core/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/apm" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/language" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -34,23 +36,24 @@ const ( // Ensure discovery implements the module.Module interface. var _ module.Module = &discovery{} -// cacheData holds process data that should be cached between calls to the +// serviceInfo holds process data that should be cached between calls to the // endpoint. -type cacheData struct { - serviceName string +type serviceInfo struct { + name string + apmInstrumentation apm.Instrumentation } // discovery is an implementation of the Module interface for the discovery module. type discovery struct { // cache maps pids to data that should be cached between calls to the endpoint. - cache map[int32]cacheData + cache map[int32]*serviceInfo serviceDetector servicediscovery.ServiceDetector } // NewDiscoveryModule creates a new discovery system probe module. func NewDiscoveryModule(*sysconfigtypes.Config, optional.Option[workloadmeta.Component], telemetry.Component) (module.Module, error) { return &discovery{ - cache: make(map[int32]cacheData), + cache: make(map[int32]*serviceInfo), serviceDetector: *servicediscovery.NewServiceDetector(), }, nil } @@ -199,20 +202,24 @@ type parsingContext struct { netNsInfo map[uint32]*namespaceInfo } -// getServiceName gets the service name for a process using the servicedetector -// module. -func (s *discovery) getServiceName(proc *process.Process) (string, error) { +// getServiceInfo gets the service information for a process using the +// servicedetector module. +func (s *discovery) getServiceInfo(proc *process.Process) (*serviceInfo, error) { cmdline, err := proc.CmdlineSlice() if err != nil { - return "", nil + return nil, err } envs, err := getEnvs(proc) if err != nil { - return "", nil + return nil, err } - return s.serviceDetector.GetServiceName(cmdline, envs), nil + name := s.serviceDetector.GetServiceName(cmdline, envs) + // Language passed as unknown for now to only detect injection. + apmInstrumentation := apm.Detect(cmdline, envs, language.Unknown) + + return &serviceInfo{name: name, apmInstrumentation: apmInstrumentation}, nil } // getService gets information for a single service. @@ -267,22 +274,23 @@ func (s *discovery) getService(context parsingContext, pid int32) *model.Service return nil } - var serviceName string + var info *serviceInfo if cached, ok := s.cache[pid]; ok { - serviceName = cached.serviceName + info = cached } else { - serviceName, err = s.getServiceName(proc) + info, err = s.getServiceInfo(proc) if err != nil { return nil } - s.cache[pid] = cacheData{serviceName: serviceName} + s.cache[pid] = info } return &model.Service{ - PID: int(pid), - Name: serviceName, - Ports: ports, + PID: int(pid), + Name: info.name, + Ports: ports, + APMInstrumentation: string(info.apmInstrumentation), } } diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go index b1b9a02d53d39..8885f76f58370 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go @@ -35,6 +35,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/system-probe/config" "github.com/DataDog/datadog-agent/cmd/system-probe/config/types" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/apm" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" protocolUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" @@ -313,6 +314,23 @@ func TestInjectedServiceName(t *testing.T) { require.Equal(t, "injected-service-name", portMap[pid].Name) } +func TestAPMInstrumentationInjected(t *testing.T) { + url := setupDiscoveryModule(t) + + createEnvsMemfd(t, []string{ + "DD_INJECTION_ENABLED=service_name,tracer", + }) + + listener, err := net.Listen("tcp", "") + require.NoError(t, err) + t.Cleanup(func() { listener.Close() }) + + pid := os.Getpid() + portMap := getServicesMap(t, url) + require.Contains(t, portMap, pid) + require.Equal(t, string(apm.Injected), portMap[pid].APMInstrumentation) +} + // Check that we can get listening processes in other namespaces. func TestNamespaces(t *testing.T) { url := setupDiscoveryModule(t) @@ -451,7 +469,7 @@ func TestCache(t *testing.T) { for i, cmd := range cmds { pid := int32(cmd.Process.Pid) - require.Contains(t, discovery.cache[pid].serviceName, serviceNames[i]) + require.Contains(t, discovery.cache[pid].name, serviceNames[i]) } cancel() From a3098b88e0cd3f84a64b6683e3b00b8813fe67c0 Mon Sep 17 00:00:00 2001 From: Amit Slavin <108348428+amitslavin@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:59:34 +0300 Subject: [PATCH 039/245] [USMON-1189] Add Postgres telemetry configuration (#27110) --- pkg/config/setup/system_probe.go | 1 + pkg/network/config/config.go | 32 +++++++++++-------- pkg/network/config/config_test.go | 20 ++++++++++++ .../protocols/postgres/ebpf/types_test.go | 27 ++++++++++++++++ pkg/network/protocols/postgres/protocol.go | 2 +- pkg/network/protocols/postgres/telemetry.go | 27 +++++++++++----- .../protocols/postgres/telemetry_test.go | 28 +++++++++++++++- 7 files changed, 113 insertions(+), 24 deletions(-) create mode 100644 pkg/network/protocols/postgres/ebpf/types_test.go diff --git a/pkg/config/setup/system_probe.go b/pkg/config/setup/system_probe.go index 5fcf419faf449..9567f2df8b0f7 100644 --- a/pkg/config/setup/system_probe.go +++ b/pkg/config/setup/system_probe.go @@ -259,6 +259,7 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Config) { cfg.BindEnv(join(smNS, "max_http_stats_buffered")) cfg.BindEnvAndSetDefault(join(smNS, "max_kafka_stats_buffered"), 100000) cfg.BindEnv(join(smNS, "max_postgres_stats_buffered")) + cfg.BindEnvAndSetDefault(join(smNS, "max_postgres_telemetry_buffer"), 160) cfg.BindEnv(join(smNS, "max_redis_stats_buffered")) cfg.BindEnv(join(smNS, "max_concurrent_requests")) cfg.BindEnv(join(smNS, "enable_quantization")) diff --git a/pkg/network/config/config.go b/pkg/network/config/config.go index 5c443e0a70a41..a5f4c8997c7e3 100644 --- a/pkg/network/config/config.go +++ b/pkg/network/config/config.go @@ -188,6 +188,9 @@ type Config struct { // get flushed on every client request (default 30s check interval) MaxKafkaStatsBuffered int + // MaxPostgresTelemetryBuffer represents the maximum size of the telemetry buffer size for Postgres. + MaxPostgresTelemetryBuffer int + // MaxPostgresStatsBuffered represents the maximum number of Postgres stats we'll buffer in memory. These stats // get flushed on every client request (default 30s check interval) MaxPostgresStatsBuffered int @@ -362,20 +365,21 @@ func New() *Config { NPMRingbuffersEnabled: cfg.GetBool(join(netNS, "enable_ringbuffers")), - EnableHTTPMonitoring: cfg.GetBool(join(smNS, "enable_http_monitoring")), - EnableHTTP2Monitoring: cfg.GetBool(join(smNS, "enable_http2_monitoring")), - EnableKafkaMonitoring: cfg.GetBool(join(smNS, "enable_kafka_monitoring")), - EnablePostgresMonitoring: cfg.GetBool(join(smNS, "enable_postgres_monitoring")), - EnableRedisMonitoring: cfg.GetBool(join(smNS, "enable_redis_monitoring")), - EnableNativeTLSMonitoring: cfg.GetBool(join(smNS, "tls", "native", "enabled")), - EnableIstioMonitoring: cfg.GetBool(join(smNS, "tls", "istio", "enabled")), - EnvoyPath: cfg.GetString(join(smNS, "tls", "istio", "envoy_path")), - EnableNodeJSMonitoring: cfg.GetBool(join(smNS, "tls", "nodejs", "enabled")), - MaxUSMConcurrentRequests: uint32(cfg.GetInt(join(smNS, "max_concurrent_requests"))), - MaxHTTPStatsBuffered: cfg.GetInt(join(smNS, "max_http_stats_buffered")), - MaxKafkaStatsBuffered: cfg.GetInt(join(smNS, "max_kafka_stats_buffered")), - MaxPostgresStatsBuffered: cfg.GetInt(join(smNS, "max_postgres_stats_buffered")), - MaxRedisStatsBuffered: cfg.GetInt(join(smNS, "max_redis_stats_buffered")), + EnableHTTPMonitoring: cfg.GetBool(join(smNS, "enable_http_monitoring")), + EnableHTTP2Monitoring: cfg.GetBool(join(smNS, "enable_http2_monitoring")), + EnableKafkaMonitoring: cfg.GetBool(join(smNS, "enable_kafka_monitoring")), + EnablePostgresMonitoring: cfg.GetBool(join(smNS, "enable_postgres_monitoring")), + EnableRedisMonitoring: cfg.GetBool(join(smNS, "enable_redis_monitoring")), + EnableNativeTLSMonitoring: cfg.GetBool(join(smNS, "tls", "native", "enabled")), + EnableIstioMonitoring: cfg.GetBool(join(smNS, "tls", "istio", "enabled")), + EnvoyPath: cfg.GetString(join(smNS, "tls", "istio", "envoy_path")), + EnableNodeJSMonitoring: cfg.GetBool(join(smNS, "tls", "nodejs", "enabled")), + MaxUSMConcurrentRequests: uint32(cfg.GetInt(join(smNS, "max_concurrent_requests"))), + MaxHTTPStatsBuffered: cfg.GetInt(join(smNS, "max_http_stats_buffered")), + MaxKafkaStatsBuffered: cfg.GetInt(join(smNS, "max_kafka_stats_buffered")), + MaxPostgresStatsBuffered: cfg.GetInt(join(smNS, "max_postgres_stats_buffered")), + MaxPostgresTelemetryBuffer: cfg.GetInt(join(smNS, "max_postgres_telemetry_buffer")), + MaxRedisStatsBuffered: cfg.GetInt(join(smNS, "max_redis_stats_buffered")), MaxTrackedHTTPConnections: cfg.GetInt64(join(smNS, "max_tracked_http_connections")), HTTPNotificationThreshold: cfg.GetInt64(join(smNS, "http_notification_threshold")), diff --git a/pkg/network/config/config_test.go b/pkg/network/config/config_test.go index 30d0598913cde..f8abb6345eaa0 100644 --- a/pkg/network/config/config_test.go +++ b/pkg/network/config/config_test.go @@ -1293,6 +1293,26 @@ service_monitoring_config: }) } +func TestMaxPostgresTelemetryBuffered(t *testing.T) { + t.Run("value set through env var", func(t *testing.T) { + aconfig.ResetSystemProbeConfig(t) + t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_POSTGRES_TELEMETRY_BUFFER", "50000") + + cfg := New() + assert.Equal(t, 50000, cfg.MaxPostgresTelemetryBuffer) + }) + + t.Run("value set through yaml", func(t *testing.T) { + aconfig.ResetSystemProbeConfig(t) + cfg := configurationFromYAML(t, ` +service_monitoring_config: + max_postgres_telemetry_buffer: 30000 +`) + + assert.Equal(t, 30000, cfg.MaxPostgresTelemetryBuffer) + }) +} + func TestMaxPostgresStatsBuffered(t *testing.T) { t.Run("value set through env var", func(t *testing.T) { aconfig.ResetSystemProbeConfig(t) diff --git a/pkg/network/protocols/postgres/ebpf/types_test.go b/pkg/network/protocols/postgres/ebpf/types_test.go new file mode 100644 index 0000000000000..9e64129b2402e --- /dev/null +++ b/pkg/network/protocols/postgres/ebpf/types_test.go @@ -0,0 +1,27 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux_bpf + +package ebpf + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/pkg/network/config" +) + +// TestMaxPostgresTelemetryDefaultConfig validates that the default value matches the one in the Postgres eBPF package. +// This test is located here to avoid importing /pkg/network/protocols/postgres/ebpf/ +// into the /pkg/network/config package. +func TestMaxPostgresTelemetryDefaultConfig(t *testing.T) { + // Validating that the default value matches the one in the Postgres ebpf package. + t.Run("default", func(t *testing.T) { + cfg := config.New() + assert.Equal(t, BufferSize, cfg.MaxPostgresTelemetryBuffer) + }) +} diff --git a/pkg/network/protocols/postgres/protocol.go b/pkg/network/protocols/postgres/protocol.go index cd26e0b9a5a3a..644f74609d2a3 100644 --- a/pkg/network/protocols/postgres/protocol.go +++ b/pkg/network/protocols/postgres/protocol.go @@ -201,7 +201,7 @@ func (p *protocol) processPostgres(events []postgresebpf.EbpfEvent) { tx := &events[i] eventWrapper := NewEventWrapper(tx) p.statskeeper.Process(eventWrapper) - p.telemetry.Count(tx, eventWrapper) + p.telemetry.Count(tx, eventWrapper, CountOptions{TelemetryBufferSize: p.cfg.MaxPostgresTelemetryBuffer}) } } diff --git a/pkg/network/protocols/postgres/telemetry.go b/pkg/network/protocols/postgres/telemetry.go index ed7bd9ddd95c2..1ed62a635f7ee 100644 --- a/pkg/network/protocols/postgres/telemetry.go +++ b/pkg/network/protocols/postgres/telemetry.go @@ -19,13 +19,14 @@ const ( numberOfBuckets = 10 bucketLength = 15 numberOfBucketsSmallerThanMaxBufferSize = 3 - // firstBucketLowerBoundary is the lower boundary of the first bucket. - // We add 1 in order to include BufferSize as the upper boundary of the third bucket. - // Then the first three buckets will include query lengths shorter or equal to BufferSize, - // and the rest will include sizes equal to or above the buffer size. - firstBucketLowerBoundary = ebpf.BufferSize - numberOfBucketsSmallerThanMaxBufferSize*bucketLength + 1 ) +// firstBucketLowerBoundary is the lower boundary of the first bucket. +// We add 1 in order to include BufferSize as the upper boundary of the third bucket. +// Then the first three buckets will include query lengths shorter or equal to BufferSize, +// and the rest will include sizes equal to or above the buffer size. +var firstBucketLowerBoundary = ebpf.BufferSize - numberOfBucketsSmallerThanMaxBufferSize*bucketLength + 1 + // Telemetry is a struct to hold the telemetry for the postgres protocol type Telemetry struct { metricGroup *libtelemetry.MetricGroup @@ -38,6 +39,13 @@ type Telemetry struct { failedOperationExtraction *libtelemetry.Counter } +// CountOptions holds the telemetry buffer size. +// The size is set by default to the ebpf.BufferSize +// but can be overridden by the max_postgres_telemetry_buffer configuration. +type CountOptions struct { + TelemetryBufferSize int +} + // createQueryLengthBuckets initializes the query length buckets // The buckets are defined relative to a `BufferSize` and a `bucketLength` as follows: // Bucket 0: 0 to BufferSize - 2*bucketLength @@ -71,16 +79,19 @@ func NewTelemetry() *Telemetry { } // getBucketIndex returns the index of the bucket for the given query size -func getBucketIndex(querySize int) int { +func getBucketIndex(querySize int, options ...CountOptions) int { + if len(options) > 0 && options[0].TelemetryBufferSize > ebpf.BufferSize { + firstBucketLowerBoundary = options[0].TelemetryBufferSize - numberOfBucketsSmallerThanMaxBufferSize*bucketLength + 1 + } bucketIndex := max(0, querySize-firstBucketLowerBoundary) / bucketLength return min(bucketIndex, numberOfBuckets-1) } // Count increments the telemetry counters based on the event data -func (t *Telemetry) Count(tx *ebpf.EbpfEvent, eventWrapper *EventWrapper) { +func (t *Telemetry) Count(tx *ebpf.EbpfEvent, eventWrapper *EventWrapper, options ...CountOptions) { querySize := int(tx.Tx.Original_query_size) - bucketIndex := getBucketIndex(querySize) + bucketIndex := getBucketIndex(querySize, options...) if bucketIndex >= 0 && bucketIndex < len(t.queryLengthBuckets) { t.queryLengthBuckets[bucketIndex].Add(1) } diff --git a/pkg/network/protocols/postgres/telemetry_test.go b/pkg/network/protocols/postgres/telemetry_test.go index 2680f465e5cfe..5240d2b9a1393 100644 --- a/pkg/network/protocols/postgres/telemetry_test.go +++ b/pkg/network/protocols/postgres/telemetry_test.go @@ -47,10 +47,14 @@ func Test_getBucketIndex(t *testing.T) { } } +// telemetryTestBufferSize serves as example configuration for the telemetry buffer size. +const telemetryTestBufferSize = 2 * ebpf.BufferSize + func TestTelemetry_Count(t *testing.T) { tests := []struct { name string query string + telemetryConfig CountOptions tx []*ebpf.EbpfEvent expectedTelemetry telemetryResults }{ @@ -75,6 +79,28 @@ func TestTelemetry_Count(t *testing.T) { failedTableNameExtraction: 10, }, }, + { + name: "exceeded query length bucket for each bucket ones with telemetry config", + telemetryConfig: CountOptions{TelemetryBufferSize: telemetryTestBufferSize}, + tx: []*ebpf.EbpfEvent{ + createEbpfEvent(telemetryTestBufferSize - 2*bucketLength), + createEbpfEvent(telemetryTestBufferSize - bucketLength), + createEbpfEvent(telemetryTestBufferSize), + createEbpfEvent(telemetryTestBufferSize + 1), + createEbpfEvent(telemetryTestBufferSize + bucketLength + 1), + createEbpfEvent(telemetryTestBufferSize + 2*bucketLength + 1), + createEbpfEvent(telemetryTestBufferSize + 3*bucketLength + 1), + createEbpfEvent(telemetryTestBufferSize + 4*bucketLength + 1), + createEbpfEvent(telemetryTestBufferSize + 5*bucketLength + 1), + createEbpfEvent(telemetryTestBufferSize + 6*bucketLength + 1), + }, + + expectedTelemetry: telemetryResults{ + queryLength: [bucketLength]int64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + failedOperationExtraction: 10, + failedTableNameExtraction: 10, + }, + }, { name: "failed operation extraction", tx: []*ebpf.EbpfEvent{{}}, @@ -114,7 +140,7 @@ func TestTelemetry_Count(t *testing.T) { } for _, tx := range tt.tx { ep := NewEventWrapper(tx) - tel.Count(tx, ep) + tel.Count(tx, ep, tt.telemetryConfig) } verifyTelemetry(t, tel, tt.expectedTelemetry) }) From 818ca548990dcd039e4c3003c23b09f2173b0a77 Mon Sep 17 00:00:00 2001 From: Scott Opell Date: Mon, 19 Aug 2024 13:58:22 -0400 Subject: [PATCH 040/245] Updates agent's SMP cli version to 0.16 (#28540) --- .gitlab/functional_test/regression_detector.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/functional_test/regression_detector.yml b/.gitlab/functional_test/regression_detector.yml index 0f9f06e20c7ec..3819ff4626d4d 100644 --- a/.gitlab/functional_test/regression_detector.yml +++ b/.gitlab/functional_test/regression_detector.yml @@ -17,7 +17,7 @@ single-machine-performance-regression_detector: - outputs/report.html # for debugging, also on S3 when: always variables: - SMP_VERSION: 0.15.1 + SMP_VERSION: 0.16.0 # At present we require two artifacts to exist for the 'baseline' and the # 'comparison'. We are guaranteed by the structure of the pipeline that # 'comparison' exists, not so much with 'baseline' as it has to come from main From 252b243ec5b821d7c2e05d219e40898f95f801fb Mon Sep 17 00:00:00 2001 From: George Hahn Date: Mon, 19 Aug 2024 13:52:56 -0600 Subject: [PATCH 041/245] Add a quality gate for idle experiment memory usage (#28522) --- test/regression/cases/idle/experiment.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/regression/cases/idle/experiment.yaml b/test/regression/cases/idle/experiment.yaml index 514d35ea04bad..8f1662b8b04de 100644 --- a/test/regression/cases/idle/experiment.yaml +++ b/test/regression/cases/idle/experiment.yaml @@ -1,3 +1,6 @@ +# Agent 'out of the box' idle experiment. Represents an agent install with the +# default configuration and no active workload. + optimization_goal: memory erratic: false @@ -24,4 +27,11 @@ target: DD_PROFILING_EXECUTION_TRACE_PERIOD: 1m DD_PROFILING_WAIT_PROFILE: true - DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:idle \ No newline at end of file + DD_INTERNAL_PROFILING_EXTRA_TAGS: experiment:idle + +checks: + - name: memory_usage + description: "Memory usage quality gate. This puts a bound on the total agent memory usage." + bounds: + series: total_rss_bytes + upper_bound: "424.0 MiB" From 87719f5490d29a151d2e3b4d5a3dbbff2d8b3dbe Mon Sep 17 00:00:00 2001 From: Adam Karpowich Date: Mon, 19 Aug 2024 15:58:16 -0400 Subject: [PATCH 042/245] force imdsv2 on subnet lookups (#28437) --- pkg/util/ec2/network.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/ec2/network.go b/pkg/util/ec2/network.go index c7f7277b30714..5fafa6bed62d7 100644 --- a/pkg/util/ec2/network.go +++ b/pkg/util/ec2/network.go @@ -83,14 +83,14 @@ func GetSubnetForHardwareAddr(ctx context.Context, hwAddr net.HardwareAddr) (sub } var resp string - resp, err = getMetadataItem(ctx, fmt.Sprintf("%s/%s/subnet-id", imdsNetworkMacs, hwAddr), false) + resp, err = getMetadataItem(ctx, fmt.Sprintf("%s/%s/subnet-id", imdsNetworkMacs, hwAddr), true) if err != nil { return } subnet.ID = strings.TrimSpace(resp) - resp, err = getMetadataItem(ctx, fmt.Sprintf("%s/%s/subnet-ipv4-cidr-block", imdsNetworkMacs, hwAddr), false) + resp, err = getMetadataItem(ctx, fmt.Sprintf("%s/%s/subnet-ipv4-cidr-block", imdsNetworkMacs, hwAddr), true) if err != nil { return } From 43ae2c524362c53849066783730be9551582cb39 Mon Sep 17 00:00:00 2001 From: Yang Song Date: Mon, 19 Aug 2024 16:07:55 -0400 Subject: [PATCH 043/245] [OTEL-2044] Set span_name_as_resource_name to true by default in dd connector in converged agent (#28521) --- comp/otelcol/converter/impl/autoconfigure.go | 3 + comp/otelcol/converter/impl/converter_test.go | 15 +++++ .../converter/impl/datadogconnector.go | 58 +++++++++++++++++++ .../connectors/already-set/config.yaml | 45 ++++++++++++++ .../connectors/no-dd-connector/config.yaml | 43 ++++++++++++++ .../connectors/set-default/config-result.yaml | 53 +++++++++++++++++ .../connectors/set-default/config.yaml | 49 ++++++++++++++++ .../config-result.yaml | 1 + .../dd-connector/config-result.yaml | 1 + 9 files changed, 268 insertions(+) create mode 100644 comp/otelcol/converter/impl/datadogconnector.go create mode 100644 comp/otelcol/converter/impl/testdata/connectors/already-set/config.yaml create mode 100644 comp/otelcol/converter/impl/testdata/connectors/no-dd-connector/config.yaml create mode 100644 comp/otelcol/converter/impl/testdata/connectors/set-default/config-result.yaml create mode 100644 comp/otelcol/converter/impl/testdata/connectors/set-default/config.yaml diff --git a/comp/otelcol/converter/impl/autoconfigure.go b/comp/otelcol/converter/impl/autoconfigure.go index 40bb2f55ef7e8..4c1023ad2ae05 100644 --- a/comp/otelcol/converter/impl/autoconfigure.go +++ b/comp/otelcol/converter/impl/autoconfigure.go @@ -36,6 +36,9 @@ func enhanceConfig(conf *confmap.Conf) { // prometheus receiver addPrometheusReceiver(conf, prometheusReceiver) + + // datadog connector + changeDefaultConfigsForDatadogConnector(conf) } func componentName(fullName string) string { diff --git a/comp/otelcol/converter/impl/converter_test.go b/comp/otelcol/converter/impl/converter_test.go index 5cd73f87aa483..7b07dd1de1a35 100644 --- a/comp/otelcol/converter/impl/converter_test.go +++ b/comp/otelcol/converter/impl/converter_test.go @@ -51,6 +51,21 @@ func TestConvert(t *testing.T) { provided string expectedResult string }{ + { + name: "connectors/no-dd-connector", + provided: "connectors/no-dd-connector/config.yaml", + expectedResult: "connectors/no-dd-connector/config.yaml", + }, + { + name: "connectors/already-set", + provided: "connectors/already-set/config.yaml", + expectedResult: "connectors/already-set/config.yaml", + }, + { + name: "connectors/set-default", + provided: "connectors/set-default/config.yaml", + expectedResult: "connectors/set-default/config-result.yaml", + }, { name: "extensions/no-extensions", provided: "extensions/no-extensions/config.yaml", diff --git a/comp/otelcol/converter/impl/datadogconnector.go b/comp/otelcol/converter/impl/datadogconnector.go new file mode 100644 index 0000000000000..75c023237bc82 --- /dev/null +++ b/comp/otelcol/converter/impl/datadogconnector.go @@ -0,0 +1,58 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +// Package converterimpl provides the implementation of the otel-agent converter. +package converterimpl + +import ( + "go.opentelemetry.io/collector/confmap" +) + +func changeDefaultConfigsForDatadogConnector(conf *confmap.Conf) { + stringMapConf := conf.ToStringMap() + connectors, ok := stringMapConf["connectors"] + if !ok { + return + } + connectorMap, ok := connectors.(map[string]any) + if !ok { + return + } + changed := false + for name, ccfg := range connectorMap { + if componentName(name) != "datadog" { + continue + } + if ccfg == nil { + connectorMap[name] = map[string]any{ + "traces": map[string]any{"span_name_as_resource_name": true}, + } + changed = true + continue + } + ddconnectorCfg, ok := ccfg.(map[string]any) + if !ok { + continue + } + tcfg, ok := ddconnectorCfg["traces"] + if !ok || tcfg == nil { + ddconnectorCfg["traces"] = map[string]any{"span_name_as_resource_name": true} + changed = true + continue + } + tcfgMap, ok := tcfg.(map[string]any) + if !ok { + continue + } + _, isSet := tcfgMap["span_name_as_resource_name"] + if !isSet { + tcfgMap["span_name_as_resource_name"] = true + changed = true + } + } + if changed { + *conf = *confmap.NewFromStringMap(stringMapConf) + } +} diff --git a/comp/otelcol/converter/impl/testdata/connectors/already-set/config.yaml b/comp/otelcol/converter/impl/testdata/connectors/already-set/config.yaml new file mode 100644 index 0000000000000..2bb36822d711c --- /dev/null +++ b/comp/otelcol/converter/impl/testdata/connectors/already-set/config.yaml @@ -0,0 +1,45 @@ +receivers: + otlp: + prometheus/user-defined: + config: + scrape_configs: + - job_name: 'datadog-agent' + scrape_interval: 10s + static_configs: + - targets: ['0.0.0.0:8888'] + +exporters: + datadog: + api: + key: 12345 + +extensions: + pprof/user-defined: + health_check/user-defined: + zpages/user-defined: + endpoint: "localhost:55679" + datadog/user-defined: + +processors: + infraattributes/user-defined: + +connectors: + datadog/conn: + traces: + span_name_as_resource_name: false + +service: + extensions: [pprof/user-defined, zpages/user-defined, health_check/user-defined, datadog/user-defined] + pipelines: + traces: + receivers: [nop] + processors: [infraattributes/user-defined] + exporters: [datadog, datadog/conn] + metrics: + receivers: [nop, prometheus/user-defined, datadog/conn] + processors: [infraattributes/user-defined] + exporters: [datadog] + logs: + receivers: [nop] + processors: [infraattributes/user-defined] + exporters: [datadog] diff --git a/comp/otelcol/converter/impl/testdata/connectors/no-dd-connector/config.yaml b/comp/otelcol/converter/impl/testdata/connectors/no-dd-connector/config.yaml new file mode 100644 index 0000000000000..381a268dee785 --- /dev/null +++ b/comp/otelcol/converter/impl/testdata/connectors/no-dd-connector/config.yaml @@ -0,0 +1,43 @@ +receivers: + otlp: + prometheus/user-defined: + config: + scrape_configs: + - job_name: 'datadog-agent' + scrape_interval: 10s + static_configs: + - targets: ['0.0.0.0:8888'] + +exporters: + datadog: + api: + key: 12345 + +extensions: + pprof/user-defined: + health_check/user-defined: + zpages/user-defined: + endpoint: "localhost:55679" + datadog/user-defined: + +processors: + infraattributes/user-defined: + +connectors: + nop/conn: + +service: + extensions: [pprof/user-defined, zpages/user-defined, health_check/user-defined, datadog/user-defined] + pipelines: + traces: + receivers: [nop] + processors: [infraattributes/user-defined] + exporters: [datadog, nop/conn] + metrics: + receivers: [nop, prometheus/user-defined] + processors: [infraattributes/user-defined] + exporters: [datadog] + logs: + receivers: [nop] + processors: [infraattributes/user-defined] + exporters: [datadog] diff --git a/comp/otelcol/converter/impl/testdata/connectors/set-default/config-result.yaml b/comp/otelcol/converter/impl/testdata/connectors/set-default/config-result.yaml new file mode 100644 index 0000000000000..ecb812963c7a1 --- /dev/null +++ b/comp/otelcol/converter/impl/testdata/connectors/set-default/config-result.yaml @@ -0,0 +1,53 @@ +receivers: + otlp: + prometheus/user-defined: + config: + scrape_configs: + - job_name: 'datadog-agent' + scrape_interval: 10s + static_configs: + - targets: ['0.0.0.0:8888'] + +exporters: + datadog: + api: + key: 12345 + +extensions: + pprof/user-defined: + health_check/user-defined: + zpages/user-defined: + endpoint: "localhost:55679" + datadog/user-defined: + +processors: + infraattributes/user-defined: + +connectors: + datadog/conn: + traces: + span_name_as_resource_name: true + datadog/conn-2: + traces: + span_name_as_resource_name: true + datadog/conn-3: + traces: + span_name_as_resource_name: true + span_name_remappings: + instrumentation:express.server: express + +service: + extensions: [pprof/user-defined, zpages/user-defined, health_check/user-defined, datadog/user-defined] + pipelines: + traces: + receivers: [nop] + processors: [infraattributes/user-defined] + exporters: [datadog, datadog/conn] + metrics: + receivers: [nop, prometheus/user-defined, datadog/conn] + processors: [infraattributes/user-defined] + exporters: [datadog] + logs: + receivers: [nop] + processors: [infraattributes/user-defined] + exporters: [datadog] diff --git a/comp/otelcol/converter/impl/testdata/connectors/set-default/config.yaml b/comp/otelcol/converter/impl/testdata/connectors/set-default/config.yaml new file mode 100644 index 0000000000000..9487c22344086 --- /dev/null +++ b/comp/otelcol/converter/impl/testdata/connectors/set-default/config.yaml @@ -0,0 +1,49 @@ +receivers: + otlp: + prometheus/user-defined: + config: + scrape_configs: + - job_name: 'datadog-agent' + scrape_interval: 10s + static_configs: + - targets: ['0.0.0.0:8888'] + +exporters: + datadog: + api: + key: 12345 + +extensions: + pprof/user-defined: + health_check/user-defined: + zpages/user-defined: + endpoint: "localhost:55679" + datadog/user-defined: + +processors: + infraattributes/user-defined: + +connectors: + datadog/conn: + datadog/conn-2: + traces: + datadog/conn-3: + traces: + span_name_remappings: + instrumentation:express.server: express + +service: + extensions: [pprof/user-defined, zpages/user-defined, health_check/user-defined, datadog/user-defined] + pipelines: + traces: + receivers: [nop] + processors: [infraattributes/user-defined] + exporters: [datadog, datadog/conn] + metrics: + receivers: [nop, prometheus/user-defined, datadog/conn] + processors: [infraattributes/user-defined] + exporters: [datadog] + logs: + receivers: [nop] + processors: [infraattributes/user-defined] + exporters: [datadog] diff --git a/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config-result.yaml b/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config-result.yaml index eca2c8bf5d14e..26a29a7109aef 100644 --- a/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config-result.yaml +++ b/comp/otelcol/converter/impl/testdata/processors/dd-connector-multi-pipelines/config-result.yaml @@ -25,6 +25,7 @@ processors: connectors: datadog/connector: traces: + span_name_as_resource_name: true compute_top_level_by_span_kind: true peer_tags_aggregation: true compute_stats_by_span_kind: true diff --git a/comp/otelcol/converter/impl/testdata/processors/dd-connector/config-result.yaml b/comp/otelcol/converter/impl/testdata/processors/dd-connector/config-result.yaml index 894aaca27191b..d36497655fb24 100644 --- a/comp/otelcol/converter/impl/testdata/processors/dd-connector/config-result.yaml +++ b/comp/otelcol/converter/impl/testdata/processors/dd-connector/config-result.yaml @@ -25,6 +25,7 @@ processors: connectors: datadog/connector: traces: + span_name_as_resource_name: true compute_top_level_by_span_kind: true peer_tags_aggregation: true compute_stats_by_span_kind: true From 07dbf5b501431c37f00b28e9f6c8e9a2fdfb5026 Mon Sep 17 00:00:00 2001 From: Stephanie Wei <72533858+Stephanie0829@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:10:57 -0400 Subject: [PATCH 044/245] [agent] stripArguments() should account for spaces in the executable path (#28357) Co-authored-by: ccdaniele --- pkg/process/procutil/data_scrubber.go | 10 - ...er_notwin.go => data_scrubber_fallback.go} | 15 +- .../procutil/data_scrubber_fallback_test.go | 61 ++++++ pkg/process/procutil/data_scrubber_windows.go | 66 +++++- .../procutil/data_scrubber_windows_test.go | 192 ++++++++++++++++++ ...trip-arg-with-spaces-88f5ec5ec6b5c380.yaml | 3 + 6 files changed, 335 insertions(+), 12 deletions(-) rename pkg/process/procutil/{data_scrubber_notwin.go => data_scrubber_fallback.go} (56%) create mode 100644 pkg/process/procutil/data_scrubber_fallback_test.go create mode 100644 pkg/process/procutil/data_scrubber_windows_test.go create mode 100644 releasenotes/notes/fix-windows-strip-arg-with-spaces-88f5ec5ec6b5c380.yaml diff --git a/pkg/process/procutil/data_scrubber.go b/pkg/process/procutil/data_scrubber.go index 826cadfb51207..18c3faf62bad1 100644 --- a/pkg/process/procutil/data_scrubber.go +++ b/pkg/process/procutil/data_scrubber.go @@ -186,16 +186,6 @@ func (ds *DataScrubber) ScrubCommand(cmdline []string) ([]string, bool) { return newCmdline, changed } -// Strip away all arguments from the command line -func (ds *DataScrubber) stripArguments(cmdline []string) []string { - // We will sometimes see the entire command line come in via the first element -- splitting guarantees removal - // of arguments in these cases. - if len(cmdline) > 0 { - return []string{strings.Split(cmdline[0], " ")[0]} - } - return cmdline -} - // AddCustomSensitiveWords adds custom sensitive words on the DataScrubber object func (ds *DataScrubber) AddCustomSensitiveWords(words []string) { newPatterns := CompileStringsToRegex(words) diff --git a/pkg/process/procutil/data_scrubber_notwin.go b/pkg/process/procutil/data_scrubber_fallback.go similarity index 56% rename from pkg/process/procutil/data_scrubber_notwin.go rename to pkg/process/procutil/data_scrubber_fallback.go index 015b87d105cc4..1968ea732eb55 100644 --- a/pkg/process/procutil/data_scrubber_notwin.go +++ b/pkg/process/procutil/data_scrubber_fallback.go @@ -5,9 +5,12 @@ //go:build !windows -//nolint:revive // TODO(PROC) Fix revive linter package procutil +import ( + "strings" +) + var ( defaultSensitiveWords = []string{ "*password*", "*passwd*", "*mysql_pwd*", @@ -18,3 +21,13 @@ var ( forbiddenSymbolsRegex = "[^a-zA-Z0-9_*]" ) + +// stripArguments removes all arguments given a command line. +func (ds *DataScrubber) stripArguments(cmdline []string) []string { + // We will sometimes see the entire command line come in via the first element -- splitting guarantees removal + // of arguments in these cases. + if len(cmdline) > 0 { + return []string{strings.SplitN(cmdline[0], " ", 2)[0]} + } + return cmdline +} diff --git a/pkg/process/procutil/data_scrubber_fallback_test.go b/pkg/process/procutil/data_scrubber_fallback_test.go new file mode 100644 index 0000000000000..430b02c2af94d --- /dev/null +++ b/pkg/process/procutil/data_scrubber_fallback_test.go @@ -0,0 +1,61 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !windows + +package procutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStripArguments(t *testing.T) { + for _, tc := range []struct { + name string + cmdline []string + expected string + }{ + { + name: "OS parse", + cmdline: []string{"agent", "-password", "1234"}, + expected: "agent", + }, + { + name: "No OS parse", + cmdline: []string{"python ~/test/run.py -open_password=admin -consul_token 2345 -blocked_from_yamt=1234 &"}, + expected: "python", + }, + { + name: "No OS parse + whitespace", + cmdline: []string{"java -password 1234"}, + expected: "java", + }, + { + name: "Optional dash args", + cmdline: []string{"agent password:1234"}, + expected: "agent", + }, + { + name: "Single dash args", + cmdline: []string{"agent -password:1234"}, + expected: "agent", + }, + { + name: "Double dash args", + cmdline: []string{"agent --password:1234"}, + expected: "agent", + }, + } { + scrubber := setupDataScrubber(t) + scrubber.StripAllArguments = true + + t.Run(tc.name, func(t *testing.T) { + actual := scrubber.stripArguments(tc.cmdline) + assert.Equal(t, actual[0], tc.expected) + }) + } +} diff --git a/pkg/process/procutil/data_scrubber_windows.go b/pkg/process/procutil/data_scrubber_windows.go index b0de24eeb7ec6..6242060a1aadf 100644 --- a/pkg/process/procutil/data_scrubber_windows.go +++ b/pkg/process/procutil/data_scrubber_windows.go @@ -3,9 +3,14 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//nolint:revive // TODO(PROC) Fix revive linter +//go:build windows + package procutil +import ( + "strings" +) + var ( defaultSensitiveWords = []string{ "*password*", "*passwd*", "*mysql_pwd*", @@ -20,3 +25,62 @@ var ( // it's for handling parameters like `/p` and `/rp` in windows forbiddenSymbolsRegex = "[^/a-zA-Z0-9_*]" ) + +var executableExtensions = []string{".com", ".exe", ".bat", ".cmd", ".vbs", ".vbe", ".js", ".jse", ".wsf", ".wsh", ".psc1", ".ps1"} + +// stripArguments identifies windows extension and extracts command. Otherwise, returns the first element of the cmdline before first space. +// If cmdline is empty, stripArguments will return an empty string. +func (ds *DataScrubber) stripArguments(cmdline []string) []string { + if len(cmdline) < 1 { + return cmdline + } + + strCmdline := cmdline[0] + + // Case 1: OS has already completed splitting as there is one token per element, we return first token. + if len(cmdline) > 1 { + return []string{strCmdline} + } + + // Case 2: One string for cmdline, use extensionParser() to find first token. + if !strings.HasPrefix(strCmdline, "\"") { + strippedCmdline := extensionParser(strCmdline, executableExtensions) + + if strippedCmdline != "" { + return []string{strippedCmdline} + } + + // If no extension is found, return first token of cmdline. + return []string{strings.SplitN(strCmdline, " ", 2)[0]} + } + + // Case 2b: One string for cmdline and first token wrapped in quotes, use findEmbeddedQuotes() to find content between quotes. + strippedCmdline := findEmbeddedQuotes(strCmdline) + return []string{strippedCmdline} +} + +// extensionParser returns substring of cmdline up to the first extension (inclusive). +// If no extension is found, returns empty string. +// Example: Input="C:\\Program Files\\Datadog\\agent.vbe check process" Output="C:\\Program Files\\Datadog\\agent.vbe" +func extensionParser(cmdline string, executableExtensions []string) string { + for _, c := range executableExtensions { + // If extension is found before a word break (space or end of line). + if i := strings.Index(cmdline, c); i != -1 && (i+len(c) == len(cmdline) || cmdline[i+len(c)] == ' ') { + processedCmdline := cmdline[:i+len(c)] + return processedCmdline + } + } + return "" +} + +// findEmbeddedQuotes returns the content between the first pair of double quotes in cmdline. +// If there is no pair of double quotes found, function returns original cmdline. +// Example: Input="\"C:\\Program Files\\Datadog\\agent.vbe\" check process" Output="C:\\Program Files\\Datadog\\agent.vbe" +func findEmbeddedQuotes(cmdline string) string { + strippedCmdline := strings.SplitN(cmdline, "\"", 3) + if len(strippedCmdline) < 3 { + return cmdline + } + + return strippedCmdline[1] +} diff --git a/pkg/process/procutil/data_scrubber_windows_test.go b/pkg/process/procutil/data_scrubber_windows_test.go new file mode 100644 index 0000000000000..f7483e23472a3 --- /dev/null +++ b/pkg/process/procutil/data_scrubber_windows_test.go @@ -0,0 +1,192 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build windows + +package procutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStripArguments(t *testing.T) { + for _, tc := range []struct { + name string + cmdline []string + expected string + }{ + { + name: "Empty string", + cmdline: []string{""}, + expected: "", + }, + { + name: "OS parse", + cmdline: []string{"agent", "-password", "1234"}, + expected: "agent", + }, + { + name: "Windows exec + OS parse", + cmdline: []string{"C:\\Program Files\\Datadog\\agent.bat", "check", "process"}, + expected: "C:\\Program Files\\Datadog\\agent.bat", + }, + { + name: "No OS parse", + cmdline: []string{"python ~/test/run.py --password=1234 -password 1234 -open_password=admin -consul_token 2345 -blocked_from_yaml=1234 &"}, + expected: "python", + }, + { + name: "No OS parse + whitespace", + cmdline: []string{"java -password 1234"}, + expected: "java", + }, + { + name: "No OS parse + Optional dash args", + cmdline: []string{"agent password:1234"}, + expected: "agent", + }, + { + name: "Windows exec", + cmdline: []string{"C:\\Program Files\\Datadog\\agent.com"}, + expected: "C:\\Program Files\\Datadog\\agent.com", + }, + { + name: "Windows exec + args", + cmdline: []string{"C:\\Program Files\\Datadog\\agent.exe check process"}, + expected: "C:\\Program Files\\Datadog\\agent.exe", + }, + { + name: "Windows exec + paired quotes", + cmdline: []string{"\"C:\\Program Files\\Datadog\\agent.cmd\" check process"}, + expected: "C:\\Program Files\\Datadog\\agent.cmd", + }, + { + name: "Paired quotes", + cmdline: []string{"\"C:\\Program Files\\agent\" check process"}, + expected: "C:\\Program Files\\agent", + }, + } { + + scrubber := setupDataScrubber(t) + scrubber.StripAllArguments = true + + t.Run(tc.name, func(t *testing.T) { + cmdline := scrubber.stripArguments(tc.cmdline) + assert.Equal(t, cmdline[0], tc.expected) + }) + } +} + +func TestFindEmbeddedQuotes(t *testing.T) { + for _, tc := range []struct { + name string + cmdline string + expected string + }{ + { + name: "Paired quotes", + cmdline: "\"C:\\Program Files\\Datadog\\agent.cmd\" check process ", + expected: "C:\\Program Files\\Datadog\\agent.cmd", + }, + { + name: "One quote", + cmdline: "\"C:\\Program Files\\Datadog\\agent.cmd check process ", + expected: "\"C:\\Program Files\\Datadog\\agent.cmd check process ", + }, + { + name: "Empty string", + cmdline: "", + expected: "", + }, + } { + + t.Run(tc.name, func(t *testing.T) { + actual := findEmbeddedQuotes(tc.cmdline) + assert.Equal(t, actual, tc.expected) + }) + } +} + +func TestExtensionParser(t *testing.T) { + for _, tc := range []struct { + name string + cmdline string + expected string + }{ + { + name: "Extension not found", + cmdline: "python ~/test/run.py --password=1234 -password 1234 -open_password=admin -consul_token 2345 -blocked_from_yaml=1234 & ", + expected: "", + }, + { + name: "Extension at end of line", + cmdline: "C:\\Program Files\\Datadog\\agent.com", + expected: "C:\\Program Files\\Datadog\\agent.com", + }, + { + name: "Extension in first token", + cmdline: "C:\\Program Files\\Datadog\\agent.cmd check process", + expected: "C:\\Program Files\\Datadog\\agent.cmd", + }, + { + name: "Multiple extensions", + cmdline: "C:\\Program Files\\Datadog\\agent.exec.process.cmd check process", + expected: "C:\\Program Files\\Datadog\\agent.exec.process.cmd", + }, + { + name: "Misformed extension", + cmdline: "C:\\Program File\\Datexedog\\agent.exe check process", + expected: "C:\\Program File\\Datexedog\\agent.exe", + }, + { + name: "vbs extension", + cmdline: "C:\\Program Files\\agent.vbs check process", + expected: "C:\\Program Files\\agent.vbs", + }, + { + name: "jse extension", + cmdline: "C:\\Program Files\\Datadog\\agent.jse check process", + expected: "C:\\Program Files\\Datadog\\agent.jse", + }, + { + name: "wsf extension", + cmdline: "C:\\Program Files\\Datadog\\agent.wsf check process", + expected: "C:\\Program Files\\Datadog\\agent.wsf", + }, + { + name: "wsh extension", + cmdline: "C:\\Program Files\\Datadog\\agent.wsh check process", + expected: "C:\\Program Files\\Datadog\\agent.wsh", + }, + { + name: "psc1 extension", + cmdline: "C:\\Program Files\\Datadog\\agent.psc1 check process", + expected: "C:\\Program Files\\Datadog\\agent.psc1", + }, + { + name: "bat extension", + cmdline: "C:\\Program Files\\Datadog\\agent.bat check process", + expected: "C:\\Program Files\\Datadog\\agent.bat", + }, + { + name: "js extension", + cmdline: "C:\\Program Files\\Datadog\\agent.js check process", + expected: "C:\\Program Files\\Datadog\\agent.js", + }, + { + name: "com extension", + cmdline: "C:\\Program Files\\Datadog\\agent.com check process", + expected: "C:\\Program Files\\Datadog\\agent.com", + }, + } { + + t.Run(tc.name, func(t *testing.T) { + actual := extensionParser(tc.cmdline, executableExtensions) + assert.Equal(t, tc.expected, actual) + }) + } +} diff --git a/releasenotes/notes/fix-windows-strip-arg-with-spaces-88f5ec5ec6b5c380.yaml b/releasenotes/notes/fix-windows-strip-arg-with-spaces-88f5ec5ec6b5c380.yaml new file mode 100644 index 0000000000000..2f5ab90e3a5f4 --- /dev/null +++ b/releasenotes/notes/fix-windows-strip-arg-with-spaces-88f5ec5ec6b5c380.yaml @@ -0,0 +1,3 @@ +fixes: + - | + Fix Windows Process Agent argument stripping to account for spaces in the executable path. From 1d07d291730166c919d8a0469442a51d1d341292 Mon Sep 17 00:00:00 2001 From: Romain Marcadier Date: Mon, 19 Aug 2024 22:11:20 +0200 Subject: [PATCH 045/245] fix: get lambda runtime API endpoint dynamically (#28544) Co-authored-by: astuyve --- pkg/serverless/appsec/appsec.go | 8 +++++++- pkg/serverless/trace/trace.go | 11 +++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/serverless/appsec/appsec.go b/pkg/serverless/appsec/appsec.go index 90b2634f105f8..86176b7b140c7 100644 --- a/pkg/serverless/appsec/appsec.go +++ b/pkg/serverless/appsec/appsec.go @@ -11,6 +11,7 @@ import ( "context" "errors" "math/rand" + "os" "time" appsecLog "github.com/DataDog/appsec-internal-go/log" @@ -44,11 +45,16 @@ func NewWithShutdown(demux aggregator.Demultiplexer) (lp *httpsec.ProxyLifecycle return nil, nil, nil // appsec disabled } + lambdaRuntimeAPI := os.Getenv("AWS_LAMBDA_RUNTIME_API") + if lambdaRuntimeAPI == "" { + lambdaRuntimeAPI = "127.0.0.1:9001" + } + // AppSec monitors the invocations by acting as a proxy of the AWS Lambda Runtime API. lp = httpsec.NewProxyLifecycleProcessor(appsecInstance, demux) shutdownProxy := proxy.Start( "127.0.0.1:9000", - "127.0.0.1:9001", + lambdaRuntimeAPI, lp, ) log.Debug("appsec: started successfully using the runtime api proxy monitoring mode") diff --git a/pkg/serverless/trace/trace.go b/pkg/serverless/trace/trace.go index 438b1157287b0..85d24eff9469f 100644 --- a/pkg/serverless/trace/trace.go +++ b/pkg/serverless/trace/trace.go @@ -8,6 +8,7 @@ package trace import ( "context" "fmt" + "os" "strings" "github.com/DataDog/datadog-agent/cmd/serverless-init/cloudservice" @@ -60,8 +61,8 @@ const tcpRemotePortMetaKey = "tcp.remote.port" // dnsAddressMetaKey is the key of the span meta containing the DNS address const dnsAddressMetaKey = "dns.address" -// lambdaRuntimeUrlPrefix is the first part of a URL for a call to the Lambda runtime API -const lambdaRuntimeURLPrefix = "http://127.0.0.1:9001" +// lambdaRuntimeUrlPrefix is the first part of a URL for a call to the Lambda runtime API. The value may be replaced if `AWS_LAMBDA_RUNTIME_API` is set. +var lambdaRuntimeURLPrefix = "http://127.0.0.1:9001" // lambdaExtensionURLPrefix is the first part of a URL for a call from the Datadog Lambda Library to the Lambda Extension const lambdaExtensionURLPrefix = "http://127.0.0.1:8124" @@ -257,3 +258,9 @@ func (t noopTraceAgent) SetTags(map[string]string) {} func (t noopTraceAgent) SetTargetTPS(float64) {} func (t noopTraceAgent) SetSpanModifier(agent.SpanModifier) {} func (t noopTraceAgent) GetSpanModifier() agent.SpanModifier { return nil } + +func init() { + if lambdaRuntime := os.Getenv("AWS_LAMBDA_RUNTIME_API"); lambdaRuntime != "" { + lambdaRuntimeURLPrefix = fmt.Sprintf("http://%s", lambdaRuntime) + } +} From ea509b9cc80385b8ffcadcaa5dca86bed15fa5c5 Mon Sep 17 00:00:00 2001 From: Stan Rozenraukh Date: Mon, 19 Aug 2024 17:13:48 -0400 Subject: [PATCH 046/245] auto-instrumentation: adding /etc/ld.so.preload mount (#28321) --- .../auto_instrumentation_test.go | 35 ++++++--- .../mutate/autoinstrumentation/injector.go | 76 ++++++++++++++----- 2 files changed, 83 insertions(+), 28 deletions(-) diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go index 174cd52642fb6..d4b7e28b39fa3 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/auto_instrumentation_test.go @@ -246,6 +246,9 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { require.Equal(t, volumeName, tt.pod.Spec.Volumes[0].Name, "expected datadog volume to be injected") + require.Equal(t, etcVolume.Name, tt.pod.Spec.Volumes[1].Name, + "expected datadog-etc volume to be injected") + require.Equal(t, len(tt.libInfo.libs)+1, len(tt.pod.Spec.InitContainers), "expected there to be one more than the number of libs to inject for init containers") @@ -253,7 +256,7 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { var injectorMountPath string - if i == 0 { // check init container + if i == 0 { // check inject container require.Equal(t, "datadog-init-apm-inject", c.Name, "expected the first init container to be apm-inject") require.Equal(t, tt.expectedInjectorImage, c.Image, @@ -273,29 +276,39 @@ func TestInjectAutoInstruConfigV2(t *testing.T) { // each of the init containers writes a timestamp to their given volume mount path require.Equal(t, 1, len(c.Args), "expected container args") - _, suffix, found := strings.Cut(c.Args[0], "&&") - require.True(t, found, "expected container args separated by &&") - echoDate, toFile, found := strings.Cut(suffix, ">>") - require.True(t, found, "expected container args with piping redirection") - require.Equal(t, "echo $(date +%s)", strings.TrimSpace(echoDate), "expected container args with date") - require.Equal(t, injectorMountPath+"/c-init-time."+c.Name, strings.TrimSpace(toFile), - "expected to write to file based on the container name") + // the last part of each of the init container's command should be writing + // a timestamp based on the name of the container. + expectedTimestampPath := injectorMountPath + "/c-init-time." + c.Name + cmdTail := "&& echo $(date +%s) >> " + expectedTimestampPath + require.Contains(t, c.Args[0], cmdTail, "expected args to contain %s", cmdTail) + prefix, found := strings.CutSuffix(c.Args[0], cmdTail) + require.True(t, found, "expected args to end with %s ", cmdTail) + + if i == 0 { // inject container + require.Contains(t, prefix, "&& echo /opt/datadog-packages/datadog-apm-inject/stable/inject/launcher.preload.so > /datadog-etc/ld.so.preload") + } } - // two volume mounts + // three volume mounts mounts := tt.pod.Spec.Containers[0].VolumeMounts - require.Equal(t, 2, len(mounts), "expected 2 volume mounts in the application container") + require.Equal(t, 3, len(mounts), "expected 3 volume mounts in the application container") require.Equal(t, corev1.VolumeMount{ Name: volumeName, MountPath: "/opt/datadog-packages/datadog-apm-inject", SubPath: "opt/datadog-packages/datadog-apm-inject", ReadOnly: true, }, mounts[0], "expected first container volume mount to be the injector") + require.Equal(t, corev1.VolumeMount{ + Name: etcVolume.Name, + MountPath: "/etc/ld.so.preload", + SubPath: "ld.so.preload", + ReadOnly: true, + }, mounts[1], "expected first container volume mount to be the injector") require.Equal(t, corev1.VolumeMount{ Name: volumeName, MountPath: "/opt/datadog/apm/library", SubPath: "opt/datadog/apm/library", - }, mounts[1], "expected the second container volume mount to be the language libraries") + }, mounts[2], "expected the second container volume mount to be the language libraries") requireEnv(t, "LD_PRELOAD", true, "/opt/datadog-packages/datadog-apm-inject/stable/inject/launcher.preload.so") requireEnv(t, "DD_INJECT_SENDER_TYPE", true, "k8s") diff --git a/pkg/clusteragent/admission/mutate/autoinstrumentation/injector.go b/pkg/clusteragent/admission/mutate/autoinstrumentation/injector.go index eaa5e1250e9c9..9933c7fdfc536 100644 --- a/pkg/clusteragent/admission/mutate/autoinstrumentation/injector.go +++ b/pkg/clusteragent/admission/mutate/autoinstrumentation/injector.go @@ -15,6 +15,19 @@ import ( corev1 "k8s.io/api/core/v1" ) +const ( + injectPackageDir = "opt/datadog-packages/datadog-apm-inject" + libraryPackagesDir = "opt/datadog/apm/library" +) + +func asAbs(path string) string { + return "/" + path +} + +func injectorFilePath(name string) string { + return injectPackageDir + "/stable/inject/" + name +} + var sourceVolume = volume{ Volume: corev1.Volume{ Name: volumeName, @@ -29,13 +42,32 @@ var v1VolumeMount = sourceVolume.mount(corev1.VolumeMount{ }) var v2VolumeMountInjector = sourceVolume.mount(corev1.VolumeMount{ - MountPath: "/opt/datadog-packages/datadog-apm-inject", - SubPath: "opt/datadog-packages/datadog-apm-inject", + MountPath: asAbs(injectPackageDir), + SubPath: injectPackageDir, }) var v2VolumeMountLibrary = sourceVolume.mount(corev1.VolumeMount{ - MountPath: "/opt/datadog/apm/library", - SubPath: "opt/datadog/apm/library", + MountPath: asAbs(libraryPackagesDir), + SubPath: libraryPackagesDir, +}) + +var etcVolume = volume{ + Volume: corev1.Volume{ + Name: "datadog-auto-instrumentation-etc", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, +} + +var volumeMountETCDPreloadInitContainer = etcVolume.mount(corev1.VolumeMount{ + MountPath: "/datadog-etc", +}) + +var volumeMountETCDPreloadAppContainer = etcVolume.mount(corev1.VolumeMount{ + MountPath: "/etc/ld.so.preload", + SubPath: "ld.so.preload", + ReadOnly: true, }) type injector struct { @@ -62,12 +94,20 @@ func (i *injector) initContainer() initContainer { Image: i.image, Command: []string{"/bin/sh", "-c", "--"}, Args: []string{ + // TODO: We should probably move this into either a script that's in the container _or_ + // something we can do with a go template because this is not great. fmt.Sprintf( - `cp -r /%s/* %s && echo $(date +%%s) >> %s`, - mount.SubPath, mount.MountPath, tsFilePath, + `cp -r /%s/* %s && echo %s > /datadog-etc/ld.so.preload && echo $(date +%%s) >> %s`, + mount.SubPath, + mount.MountPath, + asAbs(injectorFilePath("launcher.preload.so")), + tsFilePath, ), }, - VolumeMounts: []corev1.VolumeMount{mount}, + VolumeMounts: []corev1.VolumeMount{ + mount, + volumeMountETCDPreloadInitContainer.VolumeMount, + }, }, } } @@ -75,20 +115,22 @@ func (i *injector) initContainer() initContainer { func (i *injector) requirements() libRequirement { return libRequirement{ initContainers: []initContainer{i.initContainer()}, - volumes: []volume{sourceVolume}, - volumeMounts: []volumeMount{v2VolumeMountInjector.readOnly().prepended()}, + volumes: []volume{ + sourceVolume, + etcVolume, + }, + volumeMounts: []volumeMount{ + volumeMountETCDPreloadAppContainer.prepended(), + v2VolumeMountInjector.readOnly().prepended(), + }, envVars: []envVar{ { - key: "LD_PRELOAD", - valFunc: identityValFunc( - "/opt/datadog-packages/datadog-apm-inject/stable/inject/launcher.preload.so", - ), + key: "LD_PRELOAD", + valFunc: identityValFunc(asAbs(injectorFilePath("launcher.preload.so"))), }, { - key: "DD_INJECT_SENDER_TYPE", - valFunc: identityValFunc( - "k8s", - ), + key: "DD_INJECT_SENDER_TYPE", + valFunc: identityValFunc("k8s"), }, { key: "DD_INJECT_START_TIME", From f92e03b1014194e2c2226f9f37e928d421548af1 Mon Sep 17 00:00:00 2001 From: "agent-platform-auto-pr[bot]" <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 07:24:06 +0000 Subject: [PATCH 047/245] [Backport main] Update CHANGELOG-DCA.rst for 7.55.3 (#28189) Co-authored-by: FlorentClarret --- CHANGELOG-DCA.rst | 13 +++++++++++++ CHANGELOG.rst | 31 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/CHANGELOG-DCA.rst b/CHANGELOG-DCA.rst index 5db2a8f04d80d..26ca4d803d114 100644 --- a/CHANGELOG-DCA.rst +++ b/CHANGELOG-DCA.rst @@ -2,6 +2,19 @@ Release Notes ============= +.. _Release Notes_7.55.3: + +7.55.3 +================ + +.. _Release Notes_7.55.3_Prelude: + +Prelude +------- + +Released on: 2024-08-01 +Pinned to datadog-agent v7.55.3: `CHANGELOG `_. + .. _Release Notes_7.55.2: 7.55.2 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f1b7112a74497..d93226417d1db 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,37 @@ Release Notes ============= +.. _Release Notes_7.55.3: + +7.55.3 +================ + +.. _Release Notes_7.55.3_Prelude: + +Prelude +------- + +Release on: 2024-08-01 + +- Please refer to the `7.55.3 tag on integrations-core `_ for the list of changes on the Core Checks + + +.. _Release Notes_7.55.3_Enhancement Notes: + +Enhancement Notes +----------------- + +- Agents are now built with Go ``1.21.12``. + + +.. _Release Notes_7.55.3_Security Notes: + +Security Notes +-------------- + +- Fix CVE-2024-41110. + + .. _Release Notes_7.55.2: 7.55.2 From 0318c751827c0513dfb1e5f4786076de35f58d5d Mon Sep 17 00:00:00 2001 From: Adel Haj Hassan <41540817+adel121@users.noreply.github.com> Date: Tue, 20 Aug 2024 10:35:42 +0200 Subject: [PATCH 048/245] [CONTP-277] add kubernetes_resources_annotations_as_tags and kubernetes_resources_labels_as_tags (#27369) --- cmd/serverless/dependencies_linux_amd64.txt | 1 + cmd/serverless/dependencies_linux_arm64.txt | 1 + .../collectors/workloadmeta_extract.go | 97 +++++-- .../collectors/workloadmeta_main.go | 46 ++-- .../collectors/workloadmeta_test.go | 252 +++++++++++++----- .../internal/kubeapiserver/kubeapiserver.go | 42 ++- .../kubeapiserver/kubeapiserver_test.go | 115 ++++++-- .../internal/kubeapiserver/test_helpers.go | 5 +- .../internal/kubeapiserver/utils.go | 22 ++ .../internal/kubemetadata/kubemetadata.go | 7 +- comp/metadata/host/hostimpl/hosttags/tags.go | 2 +- pkg/config/setup/config.go | 18 ++ pkg/config/utils/metadata_as_tags.go | 173 ++++++++++++ pkg/config/utils/metadata_as_tags_test.go | 127 +++++++++ pkg/util/kubernetes/hostinfo/no_tags.go | 16 +- pkg/util/kubernetes/hostinfo/tags.go | 42 ++- pkg/util/kubernetes/hostinfo/tags_test.go | 46 ---- ...k8s_resource_tagging-64b51a2ffaf3e2cc.yaml | 16 ++ ...k8s_resource_tagging-e4806f65fc0a0be1.yaml | 16 ++ 19 files changed, 827 insertions(+), 217 deletions(-) create mode 100644 pkg/config/utils/metadata_as_tags.go create mode 100644 pkg/config/utils/metadata_as_tags_test.go create mode 100644 releasenotes-dca/notes/add_generic_k8s_resource_tagging-64b51a2ffaf3e2cc.yaml create mode 100644 releasenotes/notes/add_generic_k8s_resource_tagging-e4806f65fc0a0be1.yaml diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt index dd71baaed30d2..ba70c83268635 100644 --- a/cmd/serverless/dependencies_linux_amd64.txt +++ b/cmd/serverless/dependencies_linux_amd64.txt @@ -963,6 +963,7 @@ log/internal log/slog log/slog/internal log/slog/internal/buffer +maps math math/big math/bits diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/dependencies_linux_arm64.txt index ad4c33cc32177..dd0b15ffee790 100644 --- a/cmd/serverless/dependencies_linux_arm64.txt +++ b/cmd/serverless/dependencies_linux_arm64.txt @@ -962,6 +962,7 @@ log/internal log/slog log/slog/internal log/slog/internal/buffer +maps math math/big math/bits diff --git a/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go b/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go index d5692a04d53f4..29c95f150bd61 100644 --- a/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go +++ b/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "fmt" - "slices" "strings" k8smetadata "github.com/DataDog/datadog-agent/comp/core/tagger/k8s_metadata" @@ -114,8 +113,6 @@ var ( highCardOrchestratorLabels = map[string]string{ "io.rancher.container.name": tags.RancherContainer, } - - handledKubernetesMetadataResources = []string{"namespaces", "nodes"} ) func (c *WorkloadMetaCollector) processEvents(evBundle workloadmeta.EventBundle) { @@ -156,7 +153,7 @@ func (c *WorkloadMetaCollector) processEvents(evBundle workloadmeta.EventBundle) case workloadmeta.KindProcess: // tagInfos = append(tagInfos, c.handleProcess(ev)...) No tags for now case workloadmeta.KindKubernetesDeployment: - // tagInfos = append(tagInfos, c.handleDeployment(ev)...) No tags for now + tagInfos = append(tagInfos, c.handleKubeDeployment(ev)...) default: log.Errorf("cannot handle event for entity %q with kind %q", entityID.ID, entityID.Kind) } @@ -346,16 +343,24 @@ func (c *WorkloadMetaCollector) handleKubePod(ev workloadmeta.Event) []*types.Ta c.extractTagsFromPodLabels(pod, tagList) + // pod labels as tags + for name, value := range pod.Labels { + k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesLabelsAsTags["pods"], c.globK8sResourcesLabels["pods"], tagList) + } + + // pod annotations as tags for name, value := range pod.Annotations { - k8smetadata.AddMetadataAsTags(name, value, c.annotationsAsTags, c.globAnnotations, tagList) + k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesAnnotationsAsTags["pods"], c.globK8sResourcesAnnotations["pods"], tagList) } + // namespace labels as tags for name, value := range pod.NamespaceLabels { - k8smetadata.AddMetadataAsTags(name, value, c.nsLabelsAsTags, c.globNsLabels, tagList) + k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesLabelsAsTags["namespaces"], c.globK8sResourcesLabels["namespaces"], tagList) } + // namespace annotations as tags for name, value := range pod.NamespaceAnnotations { - k8smetadata.AddMetadataAsTags(name, value, c.nsAnnotationsAsTags, c.globNsAnnotations, tagList) + k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesAnnotationsAsTags["namespaces"], c.globK8sResourcesAnnotations["namespaces"], tagList) } kubeServiceDisabled := false @@ -513,31 +518,79 @@ func (c *WorkloadMetaCollector) handleGardenContainer(container *workloadmeta.Co } } -func (c *WorkloadMetaCollector) handleKubeMetadata(ev workloadmeta.Event) []*types.TagInfo { - kubeMetadata := ev.Entity.(*workloadmeta.KubernetesMetadata) +func (c *WorkloadMetaCollector) handleKubeDeployment(ev workloadmeta.Event) []*types.TagInfo { + deployment := ev.Entity.(*workloadmeta.KubernetesDeployment) - resource := kubeMetadata.GVR.Resource + groupResource := "deployments.apps" - if !slices.Contains(handledKubernetesMetadataResources, resource) { + labelsAsTags := c.k8sResourcesLabelsAsTags[groupResource] + annotationsAsTags := c.k8sResourcesAnnotationsAsTags[groupResource] + + if len(labelsAsTags)+len(annotationsAsTags) == 0 { return nil } + globLabels := c.globK8sResourcesLabels[groupResource] + globAnnotations := c.globK8sResourcesAnnotations[groupResource] + tagList := taglist.NewTagList() - switch resource { - case "nodes": - // No tags for nodes - case "namespaces": - for name, value := range kubeMetadata.Labels { - k8smetadata.AddMetadataAsTags(name, value, c.nsLabelsAsTags, c.globNsLabels, tagList) - } + for name, value := range deployment.Labels { + k8smetadata.AddMetadataAsTags(name, value, labelsAsTags, globLabels, tagList) + } - for name, value := range kubeMetadata.Annotations { - k8smetadata.AddMetadataAsTags(name, value, c.nsAnnotationsAsTags, c.globNsAnnotations, tagList) - } + for name, value := range deployment.Annotations { + k8smetadata.AddMetadataAsTags(name, value, annotationsAsTags, globAnnotations, tagList) + } + + low, orch, high, standard := tagList.Compute() + + if len(low)+len(orch)+len(high)+len(standard) == 0 { + return nil + } + + tagInfos := []*types.TagInfo{ + { + Source: deploymentSource, + Entity: buildTaggerEntityID(deployment.EntityID), + HighCardTags: high, + OrchestratorCardTags: orch, + LowCardTags: low, + StandardTags: standard, + }, + } + + return tagInfos +} + +func (c *WorkloadMetaCollector) handleKubeMetadata(ev workloadmeta.Event) []*types.TagInfo { + kubeMetadata := ev.Entity.(*workloadmeta.KubernetesMetadata) + + tagList := taglist.NewTagList() + + // Generic resource annotations and labels as tags + groupResource := kubeMetadata.GVR.GroupResource().String() + + labelsAsTags := c.k8sResourcesLabelsAsTags[groupResource] + annotationsAsTags := c.k8sResourcesAnnotationsAsTags[groupResource] + + globLabels := c.globK8sResourcesLabels[groupResource] + globAnnotations := c.globK8sResourcesAnnotations[groupResource] + + for name, value := range kubeMetadata.Labels { + k8smetadata.AddMetadataAsTags(name, value, labelsAsTags, globLabels, tagList) + } + + for name, value := range kubeMetadata.Annotations { + k8smetadata.AddMetadataAsTags(name, value, annotationsAsTags, globAnnotations, tagList) } low, orch, high, standard := tagList.Compute() + + if len(low)+len(orch)+len(high)+len(standard) == 0 { + return nil + } + tagInfos := []*types.TagInfo{ { Source: kubeMetadataSource, @@ -575,7 +628,7 @@ func (c *WorkloadMetaCollector) extractTagsFromPodLabels(pod *workloadmeta.Kuber tagList.AddLow(tags.KubeAppManagedBy, value) } - k8smetadata.AddMetadataAsTags(name, value, c.labelsAsTags, c.globLabels, tagList) + k8smetadata.AddMetadataAsTags(name, value, c.k8sResourcesLabelsAsTags["pods"], c.globK8sResourcesLabels["pods"], tagList) } } diff --git a/comp/core/tagger/taggerimpl/collectors/workloadmeta_main.go b/comp/core/tagger/taggerimpl/collectors/workloadmeta_main.go index becd0400a307d..3103c0bb777a6 100644 --- a/comp/core/tagger/taggerimpl/collectors/workloadmeta_main.go +++ b/comp/core/tagger/taggerimpl/collectors/workloadmeta_main.go @@ -16,6 +16,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/types" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/config" + configutils "github.com/DataDog/datadog-agent/pkg/config/utils" "github.com/DataDog/datadog-agent/pkg/status/health" "github.com/DataDog/datadog-agent/pkg/util" "github.com/DataDog/datadog-agent/pkg/util/flavor" @@ -34,6 +35,7 @@ const ( processSource = workloadmetaCollectorName + "-" + string(workloadmeta.KindProcess) hostSource = workloadmetaCollectorName + "-" + string(workloadmeta.KindHost) kubeMetadataSource = workloadmetaCollectorName + "-" + string(workloadmeta.KindKubernetesMetadata) + deploymentSource = workloadmetaCollectorName + "-" + string(workloadmeta.KindKubernetesDeployment) clusterTagNamePrefix = "kube_cluster_name" ) @@ -55,17 +57,13 @@ type WorkloadMetaCollector struct { containerEnvAsTags map[string]string containerLabelsAsTags map[string]string - staticTags map[string]string - labelsAsTags map[string]string - annotationsAsTags map[string]string - nsLabelsAsTags map[string]string - nsAnnotationsAsTags map[string]string - globLabels map[string]glob.Glob - globAnnotations map[string]glob.Glob - globNsLabels map[string]glob.Glob - globNsAnnotations map[string]glob.Glob - globContainerLabels map[string]glob.Glob - globContainerEnvLabels map[string]glob.Glob + staticTags map[string]string + k8sResourcesAnnotationsAsTags map[string]map[string]string + k8sResourcesLabelsAsTags map[string]map[string]string + globContainerLabels map[string]glob.Glob + globContainerEnvLabels map[string]glob.Glob + globK8sResourcesAnnotations map[string]map[string]glob.Glob + globK8sResourcesLabels map[string]map[string]glob.Glob collectEC2ResourceTags bool collectPersistentVolumeClaimsTags bool @@ -76,11 +74,19 @@ func (c *WorkloadMetaCollector) initContainerMetaAsTags(labelsAsTags, envAsTags c.containerEnvAsTags, c.globContainerEnvLabels = k8smetadata.InitMetadataAsTags(envAsTags) } -func (c *WorkloadMetaCollector) initPodMetaAsTags(labelsAsTags, annotationsAsTags, nsLabelsAsTags, nsAnnotationsAsTags map[string]string) { - c.labelsAsTags, c.globLabels = k8smetadata.InitMetadataAsTags(labelsAsTags) - c.annotationsAsTags, c.globAnnotations = k8smetadata.InitMetadataAsTags(annotationsAsTags) - c.nsLabelsAsTags, c.globNsLabels = k8smetadata.InitMetadataAsTags(nsLabelsAsTags) - c.nsAnnotationsAsTags, c.globNsAnnotations = k8smetadata.InitMetadataAsTags(nsAnnotationsAsTags) +func (c *WorkloadMetaCollector) initK8sResourcesMetaAsTags(resourcesLabelsAsTags, resourcesAnnotationsAsTags map[string]map[string]string) { + c.k8sResourcesAnnotationsAsTags = map[string]map[string]string{} + c.k8sResourcesLabelsAsTags = map[string]map[string]string{} + c.globK8sResourcesAnnotations = map[string]map[string]glob.Glob{} + c.globK8sResourcesLabels = map[string]map[string]glob.Glob{} + + for resource, labelsAsTags := range resourcesLabelsAsTags { + c.k8sResourcesLabelsAsTags[resource], c.globK8sResourcesLabels[resource] = k8smetadata.InitMetadataAsTags(labelsAsTags) + } + + for resource, annotationsAsTags := range resourcesAnnotationsAsTags { + c.k8sResourcesAnnotationsAsTags[resource], c.globK8sResourcesAnnotations[resource] = k8smetadata.InitMetadataAsTags(annotationsAsTags) + } } // Run runs the continuous event watching loop and sends new tags to the @@ -178,11 +184,9 @@ func NewWorkloadMetaCollector(_ context.Context, store workloadmeta.Component, p ) c.initContainerMetaAsTags(containerLabelsAsTags, containerEnvAsTags) - labelsAsTags := config.Datadog().GetStringMapString("kubernetes_pod_labels_as_tags") - annotationsAsTags := config.Datadog().GetStringMapString("kubernetes_pod_annotations_as_tags") - nsLabelsAsTags := config.Datadog().GetStringMapString("kubernetes_namespace_labels_as_tags") - nsAnnotationsAsTags := config.Datadog().GetStringMapString("kubernetes_namespace_annotations_as_tags") - c.initPodMetaAsTags(labelsAsTags, annotationsAsTags, nsLabelsAsTags, nsAnnotationsAsTags) + // kubernetes resources metadata as tags + metadataAsTags := configutils.GetMetadataAsTags(config.Datadog()) + c.initK8sResourcesMetaAsTags(metadataAsTags.GetResourcesLabelsAsTags(), metadataAsTags.GetResourcesAnnotationsAsTags()) return c } diff --git a/comp/core/tagger/taggerimpl/collectors/workloadmeta_test.go b/comp/core/tagger/taggerimpl/collectors/workloadmeta_test.go index 303604ad0c003..b77ee191bd389 100644 --- a/comp/core/tagger/taggerimpl/collectors/workloadmeta_test.go +++ b/comp/core/tagger/taggerimpl/collectors/workloadmeta_test.go @@ -21,6 +21,7 @@ import ( logmock "github.com/DataDog/datadog-agent/comp/core/log/mock" "github.com/DataDog/datadog-agent/comp/core/tagger/taglist" "github.com/DataDog/datadog-agent/comp/core/tagger/types" + "github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/util" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" workloadmetafxmock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx-mock" workloadmetamock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/mock" @@ -112,32 +113,38 @@ func TestHandleKubePod(t *testing.T) { }) tests := []struct { - name string - staticTags map[string]string - labelsAsTags map[string]string - annotationsAsTags map[string]string - nsLabelsAsTags map[string]string - nsAnnotationsAsTags map[string]string - pod workloadmeta.KubernetesPod - expected []*types.TagInfo + name string + staticTags map[string]string + k8sResourcesAnnotationsAsTags map[string]map[string]string + k8sResourcesLabelsAsTags map[string]map[string]string + pod workloadmeta.KubernetesPod + expected []*types.TagInfo }{ { name: "fully formed pod (no containers)", - annotationsAsTags: map[string]string{ - "gitcommit": "+gitcommit", - "component": "component", - }, - labelsAsTags: map[string]string{ - "ownerteam": "team", - "tier": "tier", - }, - nsLabelsAsTags: map[string]string{ - "ns_env": "ns_env", - "ns-ownerteam": "ns-team", + k8sResourcesAnnotationsAsTags: map[string]map[string]string{ + "pods": { + "ns_tier": "ns_tier", + "ns_custom": "custom_generic_annotation", + "gitcommit": "+gitcommit", + "component": "component", + }, + "namespaces": { + "ns_tier": "ns_tier", + "namespace_security": "ns_security", + }, }, - nsAnnotationsAsTags: map[string]string{ - "ns_tier": "ns_tier", - "namespace_security": "ns_security", + k8sResourcesLabelsAsTags: map[string]map[string]string{ + "pods": { + "ns_env": "ns_env", + "ns_custom": "custom_generic_label", + "ownerteam": "team", + "tier": "tier", + }, + "namespaces": { + "ns_env": "ns_env", + "ns_ownerteam": "ns_team", + }, }, pod: workloadmeta.KubernetesPod{ EntityID: podEntityID, @@ -149,6 +156,7 @@ func TestHandleKubePod(t *testing.T) { "GitCommit": "foobar", "ignoreme": "ignore", "component": "agent", + "ns_custom": "gee", // Custom tags from map "ad.datadoghq.com/tags": `{"pod_template_version":"1.0.0"}`, @@ -158,6 +166,7 @@ func TestHandleKubePod(t *testing.T) { "OwnerTeam": "container-integrations", "tier": "node", "pod-template-hash": "490794276", + "ns_custom": "zoo", // Standard tags "tags.datadoghq.com/env": env, @@ -177,7 +186,7 @@ func TestHandleKubePod(t *testing.T) { // NS labels as tags NamespaceLabels: map[string]string{ "ns_env": "dev", - "ns-ownerteam": "containers", + "ns_ownerteam": "containers", "foo": "bar", }, @@ -239,7 +248,7 @@ func TestHandleKubePod(t *testing.T) { "kube_service:service2", "kube_qos:guaranteed", "kube_runtime_class:myclass", - "ns-team:containers", + "ns_team:containers", "ns_env:dev", "ns_tier:some_tier", "ns_security:critical", @@ -247,6 +256,8 @@ func TestHandleKubePod(t *testing.T) { "pod_template_version:1.0.0", "team:container-integrations", "tier:node", + "custom_generic_label:zoo", + "custom_generic_annotation:gee", }, standardTags...), StandardTags: standardTags, }, @@ -837,8 +848,7 @@ func TestHandleKubePod(t *testing.T) { t.Run(tt.name, func(t *testing.T) { collector := NewWorkloadMetaCollector(context.Background(), store, nil) collector.staticTags = tt.staticTags - - collector.initPodMetaAsTags(tt.labelsAsTags, tt.annotationsAsTags, tt.nsLabelsAsTags, tt.nsAnnotationsAsTags) + collector.initK8sResourcesMetaAsTags(tt.k8sResourcesLabelsAsTags, tt.k8sResourcesAnnotationsAsTags) actual := collector.handleKubePod(workloadmeta.Event{ Type: workloadmeta.EventTypeSet, @@ -970,8 +980,6 @@ func TestHandleKubePodWithoutPvcAsTags(t *testing.T) { collector := NewWorkloadMetaCollector(context.Background(), store, nil) collector.staticTags = tt.staticTags - collector.initPodMetaAsTags(tt.labelsAsTags, tt.annotationsAsTags, tt.nsLabelsAsTags, tt.nsAnnotationsAsTags) - actual := collector.handleKubePod(workloadmeta.Event{ Type: workloadmeta.EventTypeSet, Entity: &tt.pod, @@ -1118,8 +1126,6 @@ func TestHandleKubePodNoContainerName(t *testing.T) { collector := NewWorkloadMetaCollector(context.Background(), store, nil) collector.staticTags = tt.staticTags - collector.initPodMetaAsTags(tt.labelsAsTags, tt.annotationsAsTags, tt.nsLabelsAsTags, tt.annotationsAsTags) - actual := collector.handleKubePod(workloadmeta.Event{ Type: workloadmeta.EventTypeSet, Entity: &tt.pod, @@ -1138,8 +1144,6 @@ func TestHandleKubeMetadata(t *testing.T) { ID: fmt.Sprintf("namespaces//%s", namespace), } - taggerEntityID := fmt.Sprintf("kubernetes_metadata://%s", kubeMetadataEntityID.ID) - store := fxutil.Test[workloadmetamock.Mock](t, fx.Options( fx.Provide(func() log.Component { return logmock.New(t) }), config.MockModule(), @@ -1159,36 +1163,37 @@ func TestHandleKubeMetadata(t *testing.T) { }) tests := []struct { - name string - labelsAsTags map[string]string - annotationsAsTags map[string]string - nsLabelsAsTags map[string]string - nsAnnotationsAsTags map[string]string - kubeMetadata workloadmeta.KubernetesMetadata - expected []*types.TagInfo + name string + k8sResourcesAnnotationsAsTags map[string]map[string]string + k8sResourcesLabelsAsTags map[string]map[string]string + kubeMetadata workloadmeta.KubernetesMetadata + expected []*types.TagInfo }{ { - name: "namespace", - nsLabelsAsTags: map[string]string{ - "ns_env": "ns_env", - "ns-ownerteam": "ns-team", + name: "namespace with labels and annotations as tags", + k8sResourcesAnnotationsAsTags: map[string]map[string]string{ + "namespaces": { + "ns_tier": "ns_tier", + "ns_custom": "custom_generic_annotation", + "namespace_security": "ns_security", + }, }, - nsAnnotationsAsTags: map[string]string{ - "ns_tier": "ns_tier", - "namespace_security": "ns_security", + k8sResourcesLabelsAsTags: map[string]map[string]string{ + "namespaces": { + "ns_env": "ns_env", + "ns_custom": "custom_generic_label", + "ns_ownerteam": "ns_team", + }, }, kubeMetadata: workloadmeta.KubernetesMetadata{ EntityID: kubeMetadataEntityID, EntityMeta: workloadmeta.EntityMeta{ Name: namespace, Labels: map[string]string{ - "ns_env": "dev", - "ns-ownerteam": "containers", - "foo": "bar", + "a": "dev", }, Annotations: map[string]string{ - "ns_tier": "some_tier", - "namespace_security": "critical", + "b": "some_tier", }, }, GVR: &schema.GroupVersionResource{ @@ -1196,6 +1201,133 @@ func TestHandleKubeMetadata(t *testing.T) { Resource: "namespaces", }, }, + expected: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + collector := NewWorkloadMetaCollector(context.Background(), store, nil) + + collector.initK8sResourcesMetaAsTags(test.k8sResourcesLabelsAsTags, test.k8sResourcesAnnotationsAsTags) + + actual := collector.handleKubeMetadata(workloadmeta.Event{ + Type: workloadmeta.EventTypeSet, + Entity: &test.kubeMetadata, + }) + + assertTagInfoListEqual(tt, test.expected, actual) + }) + } +} + +func TestHandleKubeDeployment(t *testing.T) { + const deploymentName = "fooapp" + + kubeMetadataID := string(util.GenerateKubeMetadataEntityID("apps", "deployments", "default", deploymentName)) + + kubeMetadataEntityID := workloadmeta.EntityID{ + Kind: workloadmeta.KindKubernetesMetadata, + ID: kubeMetadataID, + } + + taggerEntityID := fmt.Sprintf("kubernetes_metadata://%s", kubeMetadataEntityID.ID) + + store := fxutil.Test[workloadmetamock.Mock](t, fx.Options( + fx.Provide(func() log.Component { return logmock.New(t) }), + config.MockModule(), + fx.Supply(workloadmeta.NewParams()), + fx.Supply(context.Background()), + workloadmetafxmock.MockModule(), + )) + + store.Set(&workloadmeta.Container{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindKubernetesMetadata, + ID: kubeMetadataID, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: deploymentName, + Namespace: "default", + }, + }) + + tests := []struct { + name string + k8sResourcesAnnotationsAsTags map[string]map[string]string + k8sResourcesLabelsAsTags map[string]map[string]string + kubeMetadata workloadmeta.KubernetesMetadata + expected []*types.TagInfo + }{ + { + name: "deployment with no matching labels/annotations for annotations/labels as tags. should return nil to avoid empty tagger entity", + k8sResourcesAnnotationsAsTags: map[string]map[string]string{ + "deployments.apps": { + "depl_tier": "depl_tier", + "depl_custom": "custom_generic_annotation", + }, + }, + k8sResourcesLabelsAsTags: map[string]map[string]string{ + "deployments.apps": { + "depl_env": "depl_env", + "depl_custom": "custom_generic_label", + }, + }, + kubeMetadata: workloadmeta.KubernetesMetadata{ + EntityID: kubeMetadataEntityID, + EntityMeta: workloadmeta.EntityMeta{ + Name: deploymentName, + Labels: map[string]string{ + "a": "dev", + }, + Annotations: map[string]string{ + "b": "some_tier", + }, + }, + GVR: &schema.GroupVersionResource{ + Version: "v1", + Group: "apps", + Resource: "deployments", + }, + }, + expected: nil, + }, + { + name: "deployment with generic annotations/labels as tags", + k8sResourcesAnnotationsAsTags: map[string]map[string]string{ + "deployments.apps": { + "depl_tier": "depl_tier", + "depl_custom": "custom_generic_annotation", + }, + }, + k8sResourcesLabelsAsTags: map[string]map[string]string{ + "deployments.apps": { + "depl_env": "depl_env", + "depl_custom": "custom_generic_label", + }, + }, + kubeMetadata: workloadmeta.KubernetesMetadata{ + EntityID: kubeMetadataEntityID, + EntityMeta: workloadmeta.EntityMeta{ + Name: deploymentName, + Labels: map[string]string{ + "depl_env": "dev", + "depl_ownerteam": "containers", + "foo": "bar", + "depl_custom": "zoo", + }, + Annotations: map[string]string{ + "depl_tier": "some_tier", + "depl_security": "critical", + "depl_custom": "gee", + }, + }, + GVR: &schema.GroupVersionResource{ + Version: "v1", + Group: "apps", + Resource: "deployments", + }, + }, expected: []*types.TagInfo{ { Source: kubeMetadataSource, @@ -1203,10 +1335,10 @@ func TestHandleKubeMetadata(t *testing.T) { HighCardTags: []string{}, OrchestratorCardTags: []string{}, LowCardTags: []string{ - "ns_env:dev", - "ns-team:containers", - "ns_tier:some_tier", - "ns_security:critical", + "depl_env:dev", + "depl_tier:some_tier", + "custom_generic_label:zoo", + "custom_generic_annotation:gee", }, StandardTags: []string{}, }, @@ -1214,18 +1346,18 @@ func TestHandleKubeMetadata(t *testing.T) { }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { collector := NewWorkloadMetaCollector(context.Background(), store, nil) - collector.initPodMetaAsTags(tt.labelsAsTags, tt.annotationsAsTags, tt.nsLabelsAsTags, tt.nsAnnotationsAsTags) + collector.initK8sResourcesMetaAsTags(test.k8sResourcesLabelsAsTags, test.k8sResourcesAnnotationsAsTags) actual := collector.handleKubeMetadata(workloadmeta.Event{ Type: workloadmeta.EventTypeSet, - Entity: &tt.kubeMetadata, + Entity: &test.kubeMetadata, }) - assertTagInfoListEqual(t, tt.expected, actual) + assertTagInfoListEqual(tt, test.expected, actual) }) } } diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go index 7aa6449939bd5..0666db1755b75 100644 --- a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver.go @@ -21,6 +21,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + configutils "github.com/DataDog/datadog-agent/pkg/config/utils" "github.com/DataDog/datadog-agent/pkg/status/health" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -42,11 +43,19 @@ type dependencies struct { type storeGenerator func(context.Context, workloadmeta.Component, config.Reader, kubernetes.Interface) (*cache.Reflector, *reflectorStore) func shouldHavePodStore(cfg config.Reader) bool { - return cfg.GetBool("cluster_agent.collect_kubernetes_tags") || cfg.GetBool("autoscaling.workload.enabled") + metadataAsTags := configutils.GetMetadataAsTags(cfg) + hasPodLabelsAsTags := len(metadataAsTags.GetPodLabelsAsTags()) > 0 + hasPodAnnotationsAsTags := len(metadataAsTags.GetPodAnnotationsAsTags()) > 0 + + return cfg.GetBool("cluster_agent.collect_kubernetes_tags") || cfg.GetBool("autoscaling.workload.enabled") || hasPodLabelsAsTags || hasPodAnnotationsAsTags } func shouldHaveDeploymentStore(cfg config.Reader) bool { - return cfg.GetBool("language_detection.enabled") && cfg.GetBool("language_detection.reporting.enabled") + metadataAsTags := configutils.GetMetadataAsTags(cfg) + hasDeploymentsLabelsAsTags := len(metadataAsTags.GetResourcesLabelsAsTags()["deployments.apps"]) > 0 + hasDeploymentsAnnotationsAsTags := len(metadataAsTags.GetResourcesAnnotationsAsTags()["deployments.apps"]) > 0 + + return cfg.GetBool("language_detection.enabled") && cfg.GetBool("language_detection.reporting.enabled") || hasDeploymentsLabelsAsTags || hasDeploymentsAnnotationsAsTags } func storeGenerators(cfg config.Reader) []storeGenerator { @@ -82,10 +91,27 @@ func resourcesWithMetadataCollectionEnabled(cfg config.Reader) []string { func resourcesWithRequiredMetadataCollection(cfg config.Reader) []string { res := []string{"nodes"} // nodes are always needed - namespaceLabelsAsTagsEnabled := len(cfg.GetStringMapString("kubernetes_namespace_labels_as_tags")) > 0 - namespaceAnnotationsAsTagsEnabled := len(cfg.GetStringMapString("kubernetes_namespace_annotations_as_tags")) > 0 - if namespaceLabelsAsTagsEnabled || namespaceAnnotationsAsTagsEnabled { - res = append(res, "namespaces") + metadataAsTags := configutils.GetMetadataAsTags(cfg) + + for groupResource, labelsAsTags := range metadataAsTags.GetResourcesLabelsAsTags() { + + if strings.HasPrefix(groupResource, "pods") || strings.HasPrefix(groupResource, "deployments") || len(labelsAsTags) == 0 { + continue + } + requestedResource := groupResourceToGVRString(groupResource) + if requestedResource != "" { + res = append(res, requestedResource) + } + } + + for groupResource, annotationsAsTags := range metadataAsTags.GetResourcesAnnotationsAsTags() { + if strings.HasPrefix(groupResource, "pods") || strings.HasPrefix(groupResource, "deployments") || len(annotationsAsTags) == 0 { + continue + } + requestedResource := groupResourceToGVRString(groupResource) + if requestedResource != "" { + res = append(res, requestedResource) + } } return res @@ -104,12 +130,12 @@ func resourcesWithExplicitMetadataCollectionEnabled(cfg config.Reader) []string var resources []string requestedResources := cfg.GetStringSlice("cluster_agent.kube_metadata_collection.resources") for _, resource := range requestedResources { - if strings.HasSuffix(resource, "pods") && shouldHavePodStore(cfg) { + if strings.HasSuffix(resource, "pods") { log.Debugf("skipping pods from metadata collection because a separate pod store is initialised in workload metadata store.") continue } - if strings.HasSuffix(resource, "deployments") && shouldHaveDeploymentStore(cfg) { + if strings.HasSuffix(resource, "deployments") { log.Debugf("skipping deployments from metadata collection because a separate deployment store is initialised in workload metadata store.") continue } diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver_test.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver_test.go index c6b34fc9f70ea..3cdddfe24240f 100644 --- a/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver_test.go +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/kubeapiserver_test.go @@ -135,7 +135,27 @@ func Test_metadataCollectionGVRs_WithFunctionalDiscovery(t *testing.T) { }, }, { - name: "only one resource (deployments), only one version, correct resource requested", + name: "only one resource (statefulsets), only one version, correct resource requested", + apiServerResourceList: []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "statefulsets", + Kind: "Statefulset", + Namespaced: true, + }, + }, + }, + }, + expectedGVRs: []schema.GroupVersionResource{{Resource: "statefulsets", Group: "apps", Version: "v1"}}, + cfg: map[string]interface{}{ + "cluster_agent.kube_metadata_collection.enabled": true, + "cluster_agent.kube_metadata_collection.resources": "apps/statefulsets", + }, + }, + { + name: "deployments should be skipped from metadata collection", apiServerResourceList: []*metav1.APIResourceList{ { GroupVersion: "apps/v1", @@ -148,30 +168,50 @@ func Test_metadataCollectionGVRs_WithFunctionalDiscovery(t *testing.T) { }, }, }, - expectedGVRs: []schema.GroupVersionResource{{Resource: "deployments", Group: "apps", Version: "v1"}}, + expectedGVRs: []schema.GroupVersionResource{}, cfg: map[string]interface{}{ "cluster_agent.kube_metadata_collection.enabled": true, "cluster_agent.kube_metadata_collection.resources": "apps/deployments", }, }, { - name: "only one resource (deployments), only one version, correct resource requested, but version is empty (with double slash)", + name: "pods should be skipped from metadata collection", apiServerResourceList: []*metav1.APIResourceList{ { GroupVersion: "apps/v1", APIResources: []metav1.APIResource{ { - Name: "deployments", - Kind: "Deployment", + Name: "pods", + Kind: "Pod", Namespaced: true, }, }, }, }, - expectedGVRs: []schema.GroupVersionResource{{Resource: "deployments", Group: "apps", Version: "v1"}}, + expectedGVRs: []schema.GroupVersionResource{}, cfg: map[string]interface{}{ "cluster_agent.kube_metadata_collection.enabled": true, - "cluster_agent.kube_metadata_collection.resources": "apps//deployments", + "cluster_agent.kube_metadata_collection.resources": "/pods", + }, + }, + { + name: "only one resource (statefulsets), only one version, correct resource requested, but version is empty (with double slash)", + apiServerResourceList: []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "statefulsets", + Kind: "Statefulset", + Namespaced: true, + }, + }, + }, + }, + expectedGVRs: []schema.GroupVersionResource{{Resource: "statefulsets", Group: "apps", Version: "v1"}}, + cfg: map[string]interface{}{ + "cluster_agent.kube_metadata_collection.enabled": true, + "cluster_agent.kube_metadata_collection.resources": "apps//statefulsets", }, }, { @@ -248,7 +288,6 @@ func Test_metadataCollectionGVRs_WithFunctionalDiscovery(t *testing.T) { }, }, expectedGVRs: []schema.GroupVersionResource{ - {Resource: "deployments", Group: "apps", Version: "v1"}, {Resource: "statefulsets", Group: "apps", Version: "v1"}, }, cfg: map[string]interface{}{ @@ -300,7 +339,7 @@ func Test_metadataCollectionGVRs_WithFunctionalDiscovery(t *testing.T) { }, }, }, - expectedGVRs: []schema.GroupVersionResource{{Resource: "deployments", Group: "apps", Version: "v1"}}, + expectedGVRs: []schema.GroupVersionResource{}, cfg: map[string]interface{}{ "cluster_agent.kube_metadata_collection.enabled": true, "cluster_agent.kube_metadata_collection.resources": "apps/deployments", @@ -329,6 +368,26 @@ func Test_metadataCollectionGVRs_WithFunctionalDiscovery(t *testing.T) { }, }, }, + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "daemonsets", + Kind: "Daemonset", + Namespaced: true, + }, + }, + }, + { + GroupVersion: "apps/v1beta1", + APIResources: []metav1.APIResource{ + { + Name: "daemonsets", + Kind: "Daemonset", + Namespaced: true, + }, + }, + }, { GroupVersion: "apps/v1", APIResources: []metav1.APIResource{ @@ -351,11 +410,11 @@ func Test_metadataCollectionGVRs_WithFunctionalDiscovery(t *testing.T) { }, }, expectedGVRs: []schema.GroupVersionResource{ - {Resource: "deployments", Group: "apps", Version: "v1"}, + {Resource: "daemonsets", Group: "apps", Version: "v1"}, }, cfg: map[string]interface{}{ "cluster_agent.kube_metadata_collection.enabled": true, - "cluster_agent.kube_metadata_collection.resources": "apps/deployments apps/statefulsetsy", + "cluster_agent.kube_metadata_collection.resources": "apps/daemonsets apps/statefulsetsy", }, }, } @@ -401,10 +460,22 @@ func TestResourcesWithMetadataCollectionEnabled(t *testing.T) { "cluster_agent.kube_metadata_collection.enabled": true, "cluster_agent.kube_metadata_collection.resources": "apps/deployments apps/statefulsets apps//deployments apps/v1/statefulsets apps/v1/daemonsets", }, - expectedResources: []string{"//nodes", "apps//deployments", "apps/v1/daemonsets"}, + expectedResources: []string{"//nodes", "apps/v1/daemonsets"}, + }, + { + name: "with generic resource tagging based on annotations and/or labels configured", + cfg: map[string]interface{}{ + "language_detection.enabled": false, + "language_detection.reporting.enabled": false, + "cluster_agent.kube_metadata_collection.enabled": false, + "cluster_agent.kube_metadata_collection.resources": "", + "kubernetes_resources_labels_as_tags": `{"deployments.apps": {"x-team": "team"}}`, + "kubernetes_resources_annotations_as_tags": `{"namespaces": {"x-team": "team"}}`, + }, + expectedResources: []string{"//nodes", "//namespaces"}, }, { - name: "deployments needed for language detection should be excluded from metadata collection", + name: "deployments should be excluded from metadata collection", cfg: map[string]interface{}{ "language_detection.enabled": true, "language_detection.reporting.enabled": true, @@ -414,7 +485,7 @@ func TestResourcesWithMetadataCollectionEnabled(t *testing.T) { expectedResources: []string{"apps//daemonsets", "//nodes"}, }, { - name: "pods needed for autoscaling should be excluded from metadata collection", + name: "pods should be excluded from metadata collection", cfg: map[string]interface{}{ "autoscaling.workload.enabled": true, "cluster_agent.kube_metadata_collection.enabled": true, @@ -428,7 +499,7 @@ func TestResourcesWithMetadataCollectionEnabled(t *testing.T) { "cluster_agent.kube_metadata_collection.enabled": true, "cluster_agent.kube_metadata_collection.resources": "apps/deployments apps/statefulsets", }, - expectedResources: []string{"//nodes", "apps//deployments", "apps//statefulsets"}, + expectedResources: []string{"//nodes", "apps//statefulsets"}, }, { name: "namespaces needed for namespace labels as tags", @@ -451,12 +522,8 @@ func TestResourcesWithMetadataCollectionEnabled(t *testing.T) { { name: "namespaces needed for namespace labels and annotations as tags", cfg: map[string]interface{}{ - "kubernetes_namespace_labels_as_tags": map[string]string{ - "label1": "tag1", - }, - "kubernetes_namespace_annotations_as_tags": map[string]string{ - "annotation1": "tag2", - }, + "kubernetes_namespace_labels_as_tags": `{"label1": "tag1"}`, + "kubernetes_namespace_annotations_as_tags": `{"annotation1": "tag2"}`, }, expectedResources: []string{"//nodes", "//namespaces"}, }, @@ -465,11 +532,9 @@ func TestResourcesWithMetadataCollectionEnabled(t *testing.T) { cfg: map[string]interface{}{ "cluster_agent.kube_metadata_collection.enabled": true, "cluster_agent.kube_metadata_collection.resources": "namespaces apps/deployments", - "kubernetes_namespace_labels_as_tags": map[string]string{ - "label1": "tag1", - }, + "kubernetes_namespace_labels_as_tags": `{"label1": "tag1"}`, }, - expectedResources: []string{"//nodes", "//namespaces", "apps//deployments"}, // namespaces are not duplicated + expectedResources: []string{"//nodes", "//namespaces"}, // namespaces are not duplicated }, } diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/test_helpers.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/test_helpers.go index a0c288e4dcb18..7f335852313c0 100644 --- a/comp/core/workloadmeta/collectors/internal/kubeapiserver/test_helpers.go +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/test_helpers.go @@ -9,7 +9,6 @@ package kubeapiserver import ( "context" - "fmt" "testing" "time" @@ -120,10 +119,8 @@ func testCollectMetadataEvent(t *testing.T, createObjects func() []runtime.Objec ctx := context.TODO() // Create a fake metadata client to mock API calls. - - response, err := metadataclient.Resource(gvr).List(ctx, v1.ListOptions{}) + _, err = metadataclient.Resource(gvr).List(ctx, v1.ListOptions{}) assert.NoError(t, err) - fmt.Println("metadata client listing: ", response.String()) store, _ := newMetadataStore(ctx, wlm, wlm.GetConfig(), metadataclient, gvr) stopStore := make(chan struct{}) diff --git a/comp/core/workloadmeta/collectors/internal/kubeapiserver/utils.go b/comp/core/workloadmeta/collectors/internal/kubeapiserver/utils.go index fc737658332c1..6f2d04314ab36 100644 --- a/comp/core/workloadmeta/collectors/internal/kubeapiserver/utils.go +++ b/comp/core/workloadmeta/collectors/internal/kubeapiserver/utils.go @@ -60,6 +60,28 @@ func filterToRegex(filter string) (*regexp.Regexp, error) { return r, nil } +// groupResourceToGVRString is a helper function that converts a group resource string to +// a group-version-resource string +// a group resource string is in the form `{resource}.{group}` or `{resource}` (example: deployments.apps, pods) +// a group version resource string is in the form `{group}/{version}/{resource}` (example: apps/v1/deployments) +// if the groupResource argument is not in the correct format, an empty string is returned +func groupResourceToGVRString(groupResource string) string { + parts := strings.Split(groupResource, ".") + + if len(parts) > 2 { + // incorrect format + log.Errorf("unexpected group resource format %q. correct format should be `{resource}.{group}` or `{resource}`", groupResource) + } else if len(parts) == 1 { + // format is `{resource}` + return parts[0] + } else { + // format is `{resource}/{group}` + return fmt.Sprintf("%s//%s", parts[1], parts[0]) + } + + return "" +} + // cleanDuplicateVersions detects if different versions are requested for the same resource within the same group // it logs an error for each occurrence, and a clean slice that doesn't contain any such duplication func cleanDuplicateVersions(resources []string) []string { diff --git a/comp/core/workloadmeta/collectors/internal/kubemetadata/kubemetadata.go b/comp/core/workloadmeta/collectors/internal/kubemetadata/kubemetadata.go index fd287abed6dc4..19f2d7f782ef4 100644 --- a/comp/core/workloadmeta/collectors/internal/kubemetadata/kubemetadata.go +++ b/comp/core/workloadmeta/collectors/internal/kubemetadata/kubemetadata.go @@ -20,6 +20,7 @@ import ( workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" apiv1 "github.com/DataDog/datadog-agent/pkg/clusteragent/api/v1" "github.com/DataDog/datadog-agent/pkg/config" + configutils "github.com/DataDog/datadog-agent/pkg/config/utils" "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/util/clusteragent" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver" @@ -114,8 +115,10 @@ func (c *collector) Start(_ context.Context, store workloadmeta.Component) error } c.updateFreq = time.Duration(config.Datadog().GetInt("kubernetes_metadata_tag_update_freq")) * time.Second - c.collectNamespaceLabels = len(config.Datadog().GetStringMapString("kubernetes_namespace_labels_as_tags")) > 0 - c.collectNamespaceAnnotations = len(config.Datadog().GetStringMapString("kubernetes_namespace_annotations_as_tags")) > 0 + + metadataAsTags := configutils.GetMetadataAsTags(config.Datadog()) + c.collectNamespaceLabels = len(metadataAsTags.GetNamespaceLabelsAsTags()) > 0 + c.collectNamespaceAnnotations = len(metadataAsTags.GetNamespaceAnnotationsAsTags()) > 0 return err } diff --git a/comp/metadata/host/hostimpl/hosttags/tags.go b/comp/metadata/host/hostimpl/hosttags/tags.go index ef5f783ac571f..3cbe62dc4300f 100644 --- a/comp/metadata/host/hostimpl/hosttags/tags.go +++ b/comp/metadata/host/hostimpl/hosttags/tags.go @@ -57,7 +57,7 @@ func getProvidersDefinitions(conf config.Reader) map[string]*providerDef { } if config.IsFeaturePresent(config.Kubernetes) { - providers["kubernetes"] = &providerDef{10, k8s.GetTags} + providers["kubernetes"] = &providerDef{10, k8s.NewKubeNodeTagsProvider(conf).GetTags} } if config.IsFeaturePresent(config.Docker) { diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index bd74e7c29e1ef..77cfa72ed5d61 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -357,6 +357,24 @@ func InitConfig(config pkgconfigmodel.Config) { config.BindEnvAndSetDefault("kubernetes_node_label_as_cluster_name", "") config.BindEnvAndSetDefault("kubernetes_namespace_labels_as_tags", map[string]string{}) config.BindEnvAndSetDefault("kubernetes_namespace_annotations_as_tags", map[string]string{}) + // kubernetes_resources_annotations_as_tags should be parseable as map[string]map[string]string + // it maps group resources to annotations as tags maps + // a group resource has the format `{resource}.{group}`, or simply `{resource}` if it belongs to the empty group + // examples of group resources: + // - `deployments.apps` + // - `statefulsets.apps` + // - `pods` + // - `nodes` + config.BindEnvAndSetDefault("kubernetes_resources_annotations_as_tags", map[string]map[string]string{}) + // kubernetes_resources_labels_as_tags should be parseable as map[string]map[string]string + // it maps group resources to labels as tags maps + // a group resource has the format `{resource}.{group}`, or simply `{resource}` if it belongs to the empty group + // examples of group resources: + // - `deployments.apps` + // - `statefulsets.apps` + // - `pods` + // - `nodes` + config.BindEnvAndSetDefault("kubernetes_resources_labels_as_tags", map[string]map[string]string{}) config.BindEnvAndSetDefault("kubernetes_persistent_volume_claims_as_tags", true) config.BindEnvAndSetDefault("container_cgroup_prefix", "") diff --git a/pkg/config/utils/metadata_as_tags.go b/pkg/config/utils/metadata_as_tags.go new file mode 100644 index 0000000000000..a3fae541323a3 --- /dev/null +++ b/pkg/config/utils/metadata_as_tags.go @@ -0,0 +1,173 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package utils + +import ( + "encoding/json" + "maps" + "strings" + + "github.com/DataDog/datadog-agent/pkg/util/log" + + pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" +) + +// MetadataAsTags contains the labels as tags and annotations as tags for each kubernetes resource based on the user configurations of the following options ordered in increasing order of priority: +// +// kubernetes_pod_labels_as_tags +// kubernetes_pod_annotations_as_tags +// kubernetes_node_labels_as_tags +// kubernetes_node_annotations_as_tags +// kubernetes_namespace_labels_as_tags +// kubernetes_namespace_annotations_as_tags +// kubernetes_resources_labels_as_tags +// kubernetes_resources_anotations_as_tags +// +// In case of conflict, higher priority configuration value takes precedences +// For example, if kubernetes_pod_labels_as_tags = {`l1`: `v1`, `l2`: `v2`} and kubernetes_resources_labels_as_tags = {`pods`: {`l1`: `x`}}, +// the resulting labels as tags for pods will be {`l1`: `x`, `l2`: `v2`} +type MetadataAsTags struct { + labelsAsTags map[string]map[string]string + annotationsAsTags map[string]map[string]string +} + +// GetPodLabelsAsTags returns pod labels as tags +func (m *MetadataAsTags) GetPodLabelsAsTags() map[string]string { + return m.labelsAsTags["pods"] +} + +// GetPodAnnotationsAsTags returns pod annotations as tags +func (m *MetadataAsTags) GetPodAnnotationsAsTags() map[string]string { + return m.annotationsAsTags["pods"] +} + +// GetNodeLabelsAsTags returns node labels as tags +func (m *MetadataAsTags) GetNodeLabelsAsTags() map[string]string { + return m.labelsAsTags["nodes"] +} + +// GetNodeAnnotationsAsTags returns node annotations as tags +func (m *MetadataAsTags) GetNodeAnnotationsAsTags() map[string]string { + return m.annotationsAsTags["nodes"] +} + +// GetNamespaceLabelsAsTags returns namespace labels as tags +func (m *MetadataAsTags) GetNamespaceLabelsAsTags() map[string]string { + return m.labelsAsTags["namespaces"] +} + +// GetNamespaceAnnotationsAsTags returns namespace annotations as tags +func (m *MetadataAsTags) GetNamespaceAnnotationsAsTags() map[string]string { + return m.annotationsAsTags["namespaces"] +} + +// GetResourcesLabelsAsTags returns a map from group resource to labels as tags +func (m *MetadataAsTags) GetResourcesLabelsAsTags() map[string]map[string]string { + return m.labelsAsTags +} + +// GetResourcesAnnotationsAsTags returns a map from group resource to annotations as tags +func (m *MetadataAsTags) GetResourcesAnnotationsAsTags() map[string]map[string]string { + return m.annotationsAsTags +} + +func (m *MetadataAsTags) mergeGenericResourcesLabelsAsTags(cfg pkgconfigmodel.Reader) { + resourcesToLabelsAsTags := retrieveDoubleMappingFromConfig(cfg, "kubernetes_resources_labels_as_tags") + + for resource, labelsAsTags := range resourcesToLabelsAsTags { + // "pods.", "nodes.", "namespaces." are valid configurations, but they should be replaced here by "pods", "nodes" and "namespaces" respectively to ensure that they override the existing configurations for pods, nodes and namespaces + cleanResource := strings.TrimSuffix(resource, ".") + _, found := m.labelsAsTags[cleanResource] + if !found { + m.labelsAsTags[cleanResource] = map[string]string{} + } + // When a key in `labelsAsTags` exist in `m.labelsAsTags[cleanResource]`, the value in `m.labelsAsTags[cleanResource]` will be overwritten by the value associated with the key in `labelsAsTags` + // source: https://pkg.go.dev/maps#Copy + maps.Copy(m.labelsAsTags[cleanResource], labelsAsTags) + } +} + +func (m *MetadataAsTags) mergeGenericResourcesAnnotationsAsTags(cfg pkgconfigmodel.Reader) { + resourcesToAnnotationsAsTags := retrieveDoubleMappingFromConfig(cfg, "kubernetes_resources_annotations_as_tags") + + for resource, annotationsAsTags := range resourcesToAnnotationsAsTags { + // "pods.", "nodes.", "namespaces." are valid configurations, but they should be replaced here by "pods", "nodes" and "namespaces" respectively to ensure that they override the existing configurations for pods, nodes and namesapces + cleanResource := strings.TrimSuffix(resource, ".") + _, found := m.annotationsAsTags[cleanResource] + if !found { + m.annotationsAsTags[cleanResource] = map[string]string{} + } + // When a key in `annotationsAsTags` exist in `m.annotationsAsTags[cleanResource]`, the value in `m.annotationsAsTags[cleanResource]` will be overwritten by the value associated with the key in `annotationsAsTags` + // source: https://pkg.go.dev/maps#Copy + maps.Copy(m.annotationsAsTags[cleanResource], annotationsAsTags) + } +} + +// GetMetadataAsTags returns a merged configuration of all labels and annotations as tags set by the user +func GetMetadataAsTags(c pkgconfigmodel.Reader) MetadataAsTags { + + metadataAsTags := MetadataAsTags{ + labelsAsTags: map[string]map[string]string{}, + annotationsAsTags: map[string]map[string]string{}, + } + + // node labels/annotations as tags + if nodeLabelsAsTags := c.GetStringMapString("kubernetes_node_labels_as_tags"); nodeLabelsAsTags != nil { + metadataAsTags.labelsAsTags["nodes"] = lowerCaseMapKeys(nodeLabelsAsTags) + } + if nodeAnnotationsAsTags := c.GetStringMapString("kubernetes_node_annotations_as_tags"); nodeAnnotationsAsTags != nil { + metadataAsTags.annotationsAsTags["nodes"] = lowerCaseMapKeys(nodeAnnotationsAsTags) + } + + // namespace labels/annotations as tags + if namespaceLabelsAsTags := c.GetStringMapString("kubernetes_namespace_labels_as_tags"); namespaceLabelsAsTags != nil { + metadataAsTags.labelsAsTags["namespaces"] = lowerCaseMapKeys(namespaceLabelsAsTags) + } + if namespaceAnnotationsAsTags := c.GetStringMapString("kubernetes_namespace_annotations_as_tags"); namespaceAnnotationsAsTags != nil { + metadataAsTags.annotationsAsTags["namespaces"] = lowerCaseMapKeys(namespaceAnnotationsAsTags) + } + + // pod labels/annotations as tags + if podLabelsAsTags := c.GetStringMapString("kubernetes_pod_labels_as_tags"); podLabelsAsTags != nil { + metadataAsTags.labelsAsTags["pods"] = lowerCaseMapKeys(podLabelsAsTags) + } + if podAnnotationsAsTags := c.GetStringMapString("kubernetes_pod_annotations_as_tags"); podAnnotationsAsTags != nil { + metadataAsTags.annotationsAsTags["pods"] = lowerCaseMapKeys(podAnnotationsAsTags) + } + + // generic resources labels/annotations as tags + metadataAsTags.mergeGenericResourcesLabelsAsTags(c) + metadataAsTags.mergeGenericResourcesAnnotationsAsTags(c) + + return metadataAsTags +} + +func retrieveDoubleMappingFromConfig(cfg pkgconfigmodel.Reader, configKey string) map[string]map[string]string { + valueFromConfig := cfg.GetString(configKey) + + var doubleMap map[string]map[string]string + err := json.Unmarshal([]byte(valueFromConfig), &doubleMap) + + if err != nil { + log.Errorf("failed to parse %s with value %s into json: %v", configKey, valueFromConfig, err) + return map[string]map[string]string{} + } + + for resource, tags := range doubleMap { + doubleMap[resource] = lowerCaseMapKeys(tags) + } + + return doubleMap +} + +// lowerCaseMapKeys lowercases all map keys +func lowerCaseMapKeys(m map[string]string) map[string]string { + for label, value := range m { + delete(m, label) + m[strings.ToLower(label)] = value + } + return m +} diff --git a/pkg/config/utils/metadata_as_tags_test.go b/pkg/config/utils/metadata_as_tags_test.go new file mode 100644 index 0000000000000..c1f73c8fa1599 --- /dev/null +++ b/pkg/config/utils/metadata_as_tags_test.go @@ -0,0 +1,127 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package utils + +import ( + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" +) + +func TestGetMetadataAsTagsNoError(t *testing.T) { + + tests := []struct { + name string + podLabelsAsTags map[string]string + podAnnotationsAsTags map[string]string + nodeLabelsAsTags map[string]string + nodeAnnotationsAsTags map[string]string + namespaceLabelsAsTags map[string]string + namespaceAnnotationsAsTags map[string]string + resourcesLabelsAsTags string + resourcesAnnotationsAsTags string + expectedLabelsAsTags map[string]map[string]string + expectedAnnotationsAsTags map[string]map[string]string + }{ + { + name: "no configs", + resourcesLabelsAsTags: "{}", + resourcesAnnotationsAsTags: "{}", + expectedLabelsAsTags: map[string]map[string]string{}, + expectedAnnotationsAsTags: map[string]map[string]string{}, + }, + { + name: "old configurations only", + podLabelsAsTags: map[string]string{"l1": "v1", "l2": "v2"}, + podAnnotationsAsTags: map[string]string{"l3": "v3", "l4": "v4"}, + nodeLabelsAsTags: map[string]string{"L5": "v5", "L6": "v6"}, // keys should be lower-cased automatically + nodeAnnotationsAsTags: map[string]string{"l7": "v7", "l8": "v8"}, + namespaceLabelsAsTags: map[string]string{"l9": "v9", "l10": "v10"}, + namespaceAnnotationsAsTags: map[string]string{"l11": "v11", "l12": "v12"}, + resourcesLabelsAsTags: "{}", + resourcesAnnotationsAsTags: "{}", + expectedLabelsAsTags: map[string]map[string]string{ + "nodes": {"l5": "v5", "l6": "v6"}, + "pods": {"l1": "v1", "l2": "v2"}, + "namespaces": {"l9": "v9", "l10": "v10"}, + }, + expectedAnnotationsAsTags: map[string]map[string]string{ + "nodes": {"l7": "v7", "l8": "v8"}, + "pods": {"l3": "v3", "l4": "v4"}, + "namespaces": {"l11": "v11", "l12": "v12"}, + }, + }, + { + name: "new configurations only", + resourcesLabelsAsTags: `{"pods.": {"l1": "v1", "l2": "v2"}, "deployments.apps": {"l3": "v3", "l4": "v4"}, "namespaces": {"l5": "v5"}}`, + resourcesAnnotationsAsTags: `{"nodes.": {"l6": "v6", "l7": "v7"},"deployments.apps": {"l8": "v8", "l9": "v9"}, "namespaces": {"l10": "v10"}}`, + expectedLabelsAsTags: map[string]map[string]string{ + "pods": {"l1": "v1", "l2": "v2"}, + "deployments.apps": {"l3": "v3", "l4": "v4"}, + "namespaces": {"l5": "v5"}, + }, + expectedAnnotationsAsTags: map[string]map[string]string{ + "nodes": {"l6": "v6", "l7": "v7"}, + "deployments.apps": {"l8": "v8", "l9": "v9"}, + "namespaces": {"l10": "v10"}, + }, + }, + { + name: "old and new configurations | new configuration should take precedence", + podLabelsAsTags: map[string]string{"l1": "v1", "l2": "v2"}, + podAnnotationsAsTags: map[string]string{"l3": "v3", "l4": "v4"}, + nodeLabelsAsTags: map[string]string{"l5": "v5", "l6": "v6"}, + nodeAnnotationsAsTags: map[string]string{"l7": "v7", "l8": "v8"}, + namespaceLabelsAsTags: map[string]string{"l9": "v9", "l10": "v10"}, + namespaceAnnotationsAsTags: map[string]string{"l11": "v11", "l12": "v12"}, + resourcesLabelsAsTags: `{"pods.": {"l1": "x1", "l99": "v99"}, "deployments.apps": {"l3": "v3", "l4": "v4"}}`, + resourcesAnnotationsAsTags: `{"nodes.": {"l6": "v6", "l7": "x7"}}`, + expectedLabelsAsTags: map[string]map[string]string{ + "nodes": {"l5": "v5", "l6": "v6"}, + "pods": {"l1": "x1", "l2": "v2", "l99": "v99"}, + "deployments.apps": {"l3": "v3", "l4": "v4"}, + "namespaces": {"l9": "v9", "l10": "v10"}, + }, + expectedAnnotationsAsTags: map[string]map[string]string{ + "nodes": {"l6": "v6", "l7": "x7", "l8": "v8"}, + "pods": {"l3": "v3", "l4": "v4"}, + "namespaces": {"l11": "v11", "l12": "v12"}, + }, + }, + } + + for _, test := range tests { + + t.Run(test.name, func(tt *testing.T) { + mockConfig := pkgconfigmodel.NewConfig("test", "DD", strings.NewReplacer(".", "_")) + pkgconfigsetup.InitConfig(mockConfig) + + mockConfig.SetWithoutSource("kubernetes_pod_labels_as_tags", test.podLabelsAsTags) + mockConfig.SetWithoutSource("kubernetes_pod_annotations_as_tags", test.podAnnotationsAsTags) + mockConfig.SetWithoutSource("kubernetes_namespace_labels_as_tags", test.namespaceLabelsAsTags) + mockConfig.SetWithoutSource("kubernetes_namespace_annotations_as_tags", test.namespaceAnnotationsAsTags) + mockConfig.SetWithoutSource("kubernetes_node_labels_as_tags", test.nodeLabelsAsTags) + mockConfig.SetWithoutSource("kubernetes_node_annotations_as_tags", test.nodeAnnotationsAsTags) + mockConfig.SetWithoutSource("kubernetes_resources_labels_as_tags", test.resourcesLabelsAsTags) + mockConfig.SetWithoutSource("kubernetes_resources_annotations_as_tags", test.resourcesAnnotationsAsTags) + + metadataAsTags := GetMetadataAsTags(mockConfig) + + assert.NotNil(tt, metadataAsTags) + + labelsAsTags := metadataAsTags.GetResourcesLabelsAsTags() + assert.Truef(tt, reflect.DeepEqual(labelsAsTags, test.expectedLabelsAsTags), "Expected %v, found %v", test.expectedLabelsAsTags, labelsAsTags) + + annotationsAsTags := metadataAsTags.GetResourcesAnnotationsAsTags() + assert.Truef(tt, reflect.DeepEqual(annotationsAsTags, test.expectedAnnotationsAsTags), "Expected %v, found %v", test.expectedAnnotationsAsTags, annotationsAsTags) + }) + } +} diff --git a/pkg/util/kubernetes/hostinfo/no_tags.go b/pkg/util/kubernetes/hostinfo/no_tags.go index d0380bfbff8ec..f81cd46cbd673 100644 --- a/pkg/util/kubernetes/hostinfo/no_tags.go +++ b/pkg/util/kubernetes/hostinfo/no_tags.go @@ -7,11 +7,23 @@ package hostinfo -import "context" +import ( + "context" + + "github.com/DataDog/datadog-agent/pkg/config" +) + +// KubeNodeTagsProvider allows computing node tags based on the user configurations for node labels and annotations as tags +type KubeNodeTagsProvider struct{} + +// NewKubeNodeTagsProvider creates and returns a new kube node tags provider object +func NewKubeNodeTagsProvider(_ config.Reader) KubeNodeTagsProvider { + return KubeNodeTagsProvider{} +} // GetTags gets the tags from the kubernetes apiserver // //nolint:revive // TODO(CINT) Fix revive linter -func GetTags(_ context.Context) ([]string, error) { +func (k KubeNodeTagsProvider) GetTags(_ context.Context) ([]string, error) { return nil, nil } diff --git a/pkg/util/kubernetes/hostinfo/tags.go b/pkg/util/kubernetes/hostinfo/tags.go index 89cb8192915aa..f313c677d5ead 100644 --- a/pkg/util/kubernetes/hostinfo/tags.go +++ b/pkg/util/kubernetes/hostinfo/tags.go @@ -9,23 +9,33 @@ package hostinfo import ( "context" - "strings" k8smetadata "github.com/DataDog/datadog-agent/comp/core/tagger/k8s_metadata" "github.com/DataDog/datadog-agent/comp/core/tagger/taglist" "github.com/DataDog/datadog-agent/pkg/config" + configutils "github.com/DataDog/datadog-agent/pkg/config/utils" "github.com/DataDog/datadog-agent/pkg/util/kubernetes" "github.com/DataDog/datadog-agent/pkg/util/log" ) +// KubeNodeTagsProvider allows computing node tags based on the user configurations for node labels and annotations as tags +type KubeNodeTagsProvider struct { + metadataAsTags configutils.MetadataAsTags +} + +// NewKubeNodeTagsProvider creates and returns a new kube node tags provider object +func NewKubeNodeTagsProvider(conf config.Reader) KubeNodeTagsProvider { + return KubeNodeTagsProvider{configutils.GetMetadataAsTags(conf)} +} + // GetTags gets the tags from the kubernetes apiserver and the kubelet -func GetTags(ctx context.Context) ([]string, error) { - tags, err := getNodeInfoTags(ctx) +func (k KubeNodeTagsProvider) GetTags(ctx context.Context) ([]string, error) { + tags, err := k.getNodeInfoTags(ctx) if err != nil { return nil, err } - annotationsToTags := getAnnotationsToTags() + annotationsToTags := k.metadataAsTags.GetNodeAnnotationsAsTags() if len(annotationsToTags) == 0 { return tags, nil } @@ -40,7 +50,7 @@ func GetTags(ctx context.Context) ([]string, error) { } // getNodeInfoTags gets the tags from the kubelet and the cluster-agent -func getNodeInfoTags(ctx context.Context) ([]string, error) { +func (k KubeNodeTagsProvider) getNodeInfoTags(ctx context.Context) ([]string, error) { nodeInfo, err := NewNodeInfo() if err != nil { log.Debugf("Unable to auto discover node info tags: %s", err) @@ -55,7 +65,7 @@ func getNodeInfoTags(ctx context.Context) ([]string, error) { return nil, err } tags := []string{"kube_node:" + nodeName} - labelsToTags := getLabelsToTags() + labelsToTags := k.metadataAsTags.GetNodeLabelsAsTags() if len(labelsToTags) == 0 { return tags, nil } @@ -78,26 +88,6 @@ func getDefaultLabelsToTags() map[string]string { } } -func getLabelsToTags() map[string]string { - labelsToTags := getDefaultLabelsToTags() - for k, v := range config.Datadog().GetStringMapString("kubernetes_node_labels_as_tags") { - // viper lower-cases map keys from yaml, but not from envvars - labelsToTags[strings.ToLower(k)] = v - } - - return labelsToTags -} - -func getAnnotationsToTags() map[string]string { - annotationsToTags := map[string]string{} - for k, v := range config.Datadog().GetStringMapString("kubernetes_node_annotations_as_tags") { - // viper lower-cases map keys from yaml, but not from envvars - annotationsToTags[strings.ToLower(k)] = v - } - - return annotationsToTags -} - func extractTags(nodeLabels, labelsToTags map[string]string) []string { tagList := taglist.NewTagList() labelsToTags, glob := k8smetadata.InitMetadataAsTags(labelsToTags) diff --git a/pkg/util/kubernetes/hostinfo/tags_test.go b/pkg/util/kubernetes/hostinfo/tags_test.go index 4bd722a0bd864..25e991b882d1b 100644 --- a/pkg/util/kubernetes/hostinfo/tags_test.go +++ b/pkg/util/kubernetes/hostinfo/tags_test.go @@ -11,8 +11,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - - configmock "github.com/DataDog/datadog-agent/pkg/config/mock" ) func TestExtractTags(t *testing.T) { @@ -114,47 +112,3 @@ func TestExtractTags(t *testing.T) { }) } } - -func TestGetLabelsToTags(t *testing.T) { - tests := []struct { - name string - configLabelsAsTags map[string]string - expectLabelsAsTags map[string]string - }{ - { - name: "no labels in config", - expectLabelsAsTags: map[string]string{ - "kubernetes.io/role": "kube_node_role", - }, - }, - { - name: "override node role label", - configLabelsAsTags: map[string]string{ - "kubernetes.io/role": "role", - }, - expectLabelsAsTags: map[string]string{ - "kubernetes.io/role": "role", - }, - }, - { - name: "lower case all labels", - configLabelsAsTags: map[string]string{ - "A": "a", - }, - expectLabelsAsTags: map[string]string{ - "kubernetes.io/role": "kube_node_role", - "a": "a", - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - config := configmock.New(t) - config.SetWithoutSource("kubernetes_node_labels_as_tags", test.configLabelsAsTags) - - actuaLabelsAsTags := getLabelsToTags() - assert.Equal(t, test.expectLabelsAsTags, actuaLabelsAsTags) - }) - } -} diff --git a/releasenotes-dca/notes/add_generic_k8s_resource_tagging-64b51a2ffaf3e2cc.yaml b/releasenotes-dca/notes/add_generic_k8s_resource_tagging-64b51a2ffaf3e2cc.yaml new file mode 100644 index 0000000000000..3189cc8fa7d71 --- /dev/null +++ b/releasenotes-dca/notes/add_generic_k8s_resource_tagging-64b51a2ffaf3e2cc.yaml @@ -0,0 +1,16 @@ +# Each section from every release note are combined when the +# CHANGELOG-DCA.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Added capability to tag any Kubernetes resource based on labels and annotations. + This feature can be configured with `kubernetes_resources_annotations_as_tags` and `kubernetes_resources_labels_as_tags`. + These feature configurations are associate group resources with annotations-to-tags (or labels-to-tags) map + For example, `deployments.apps` can be associated with an annotations-to-tags map to configure annotations as tags for deployments. + Example: + {`deployments.apps`: {`annotationKey1`: `tag1`, `annotationKey2`: `tag2`}} \ No newline at end of file diff --git a/releasenotes/notes/add_generic_k8s_resource_tagging-e4806f65fc0a0be1.yaml b/releasenotes/notes/add_generic_k8s_resource_tagging-e4806f65fc0a0be1.yaml new file mode 100644 index 0000000000000..41ec97735a839 --- /dev/null +++ b/releasenotes/notes/add_generic_k8s_resource_tagging-e4806f65fc0a0be1.yaml @@ -0,0 +1,16 @@ +# Each section from every release note are combined when the +# CHANGELOG-DCA.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Added capability to tag any Kubernetes resource based on labels and annotations. + This feature can be configured with `kubernetes_resources_annotations_as_tags` and `kubernetes_resources_labels_as_tags`. + These feature configurations are associate group resources with annotations-to-tags (or labels-to-tags) map + For example, `pods` can be associated with an annotations-to-tags map to configure annotations as tags for pods. + Example: + {`pods`: {`annotationKey1`: `tag1`, `annotationKey2`: `tag2`}} \ No newline at end of file From dcf139487b39980fb7b3c2bb761b26bfafef8f48 Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Tue, 20 Aug 2024 10:41:58 +0200 Subject: [PATCH 049/245] Stop every created `time.Ticker` (#28453) --- cmd/system-probe/api/module/loader.go | 1 + comp/agent/autoexit/autoexitimpl/manager.go | 3 ++- comp/api/api/utils/stream/stream.go | 1 + comp/core/autodiscovery/listeners/snmp.go | 2 +- comp/netflow/flowaggregator/aggregator.go | 8 ++++++-- comp/snmptraps/forwarder/forwarderimpl/forwarder.go | 5 +++-- .../autoscaling/externalmetrics/autoscaler_watcher.go | 1 + .../autoscaling/externalmetrics/metrics_retriever.go | 1 + pkg/collector/corechecks/containerimage/check.go | 2 +- pkg/collector/corechecks/containerlifecycle/processor.go | 2 +- .../corechecks/snmp/internal/discovery/discovery.go | 2 +- pkg/process/runner/runner.go | 1 + pkg/security/ptracer/proc.go | 2 +- pkg/security/tests/files_generator.go | 2 +- pkg/trace/stats/client_stats_aggregator.go | 1 + pkg/trace/writer/trace.go | 2 ++ pkg/util/clusteragent/clusteragent.go | 3 ++- pkg/util/containers/metrics/provider/registry.go | 1 + .../kubernetes/apiserver/controllers/controller_util.go | 6 ++++-- 19 files changed, 32 insertions(+), 14 deletions(-) diff --git a/cmd/system-probe/api/module/loader.go b/cmd/system-probe/api/module/loader.go index bc67a97fbc818..667341b0503a5 100644 --- a/cmd/system-probe/api/module/loader.go +++ b/cmd/system-probe/api/module/loader.go @@ -200,6 +200,7 @@ func updateStats() { then := time.Now() now := time.Now() ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() for { l.Lock() diff --git a/comp/agent/autoexit/autoexitimpl/manager.go b/comp/agent/autoexit/autoexitimpl/manager.go index 1ec75cfc7fbe5..db71ad7c4e042 100644 --- a/comp/agent/autoexit/autoexitimpl/manager.go +++ b/comp/agent/autoexit/autoexitimpl/manager.go @@ -69,8 +69,9 @@ func startAutoExit(ctx context.Context, sd exitDetector, log log.Component, tick log.Info("Starting auto-exit watcher") lastConditionNotMet := time.Now() - ticker := time.NewTicker(tickerPeriod) go func() { + ticker := time.NewTicker(tickerPeriod) + defer ticker.Stop() for { select { case <-ctx.Done(): diff --git a/comp/api/api/utils/stream/stream.go b/comp/api/api/utils/stream/stream.go index 416890d9ffef4..6ed376af548f3 100644 --- a/comp/api/api/utils/stream/stream.go +++ b/comp/api/api/utils/stream/stream.go @@ -76,6 +76,7 @@ func GetStreamFunc(messageReceiverFunc func() MessageReceiver, streamType, agent defer close(done) logChan := messageReceiver.Filter(&filters, done) flushTimer := time.NewTicker(time.Second) + defer flushTimer.Stop() for { // Handlers for detecting a closed connection (from either the server or client) select { diff --git a/comp/core/autodiscovery/listeners/snmp.go b/comp/core/autodiscovery/listeners/snmp.go index a16778434212a..fa5fd2937bccc 100644 --- a/comp/core/autodiscovery/listeners/snmp.go +++ b/comp/core/autodiscovery/listeners/snmp.go @@ -220,7 +220,7 @@ func (l *SNMPListener) checkDevices() { } discoveryTicker := time.NewTicker(time.Duration(l.config.DiscoveryInterval) * time.Second) - + defer discoveryTicker.Stop() for { var subnet *snmpSubnet for i := range subnets { diff --git a/comp/netflow/flowaggregator/aggregator.go b/comp/netflow/flowaggregator/aggregator.go index 2f41e2ed85716..93afc304748b8 100644 --- a/comp/netflow/flowaggregator/aggregator.go +++ b/comp/netflow/flowaggregator/aggregator.go @@ -212,12 +212,16 @@ func (agg *FlowAggregator) flushLoop() { var flushFlowsToSendTicker <-chan time.Time if agg.FlushFlowsToSendInterval > 0 { - flushFlowsToSendTicker = time.NewTicker(agg.FlushFlowsToSendInterval).C + flushTicker := time.NewTicker(agg.FlushFlowsToSendInterval) + flushFlowsToSendTicker = flushTicker.C + defer flushTicker.Stop() } else { agg.logger.Debug("flushFlowsToSendInterval set to 0: will never flush automatically") } - rollupTrackersRefresh := time.NewTicker(agg.rollupTrackerRefreshInterval).C + rollupTicker := time.NewTicker(agg.rollupTrackerRefreshInterval) + defer rollupTicker.Stop() + rollupTrackersRefresh := rollupTicker.C // TODO: move rollup tracker refresh to a separate loop (separate PR) to avoid rollup tracker and flush flows impacting each other var lastFlushTime time.Time diff --git a/comp/snmptraps/forwarder/forwarderimpl/forwarder.go b/comp/snmptraps/forwarder/forwarderimpl/forwarder.go index c31e9a4f0a592..52f0c5113078a 100644 --- a/comp/snmptraps/forwarder/forwarderimpl/forwarder.go +++ b/comp/snmptraps/forwarder/forwarderimpl/forwarder.go @@ -98,7 +98,8 @@ func (tf *trapForwarder) Stop() { } func (tf *trapForwarder) run() { - flushTicker := time.NewTicker(10 * time.Second).C + flushTicker := time.NewTicker(10 * time.Second) + defer flushTicker.Stop() for { select { case <-tf.stopChan: @@ -106,7 +107,7 @@ func (tf *trapForwarder) run() { return case packet := <-tf.trapsIn: tf.sendTrap(packet) - case <-flushTicker: + case <-flushTicker.C: tf.sender.Commit() // Commit metrics } } diff --git a/pkg/clusteragent/autoscaling/externalmetrics/autoscaler_watcher.go b/pkg/clusteragent/autoscaling/externalmetrics/autoscaler_watcher.go index 3eb7400489933..5fdfc3ab690ef 100644 --- a/pkg/clusteragent/autoscaling/externalmetrics/autoscaler_watcher.go +++ b/pkg/clusteragent/autoscaling/externalmetrics/autoscaler_watcher.go @@ -144,6 +144,7 @@ func (w *AutoscalerWatcher) Run(stopCh <-chan struct{}) { log.Infof("AutoscalerWatcher started (cache sync finished)") tickerRefreshProcess := time.NewTicker(time.Duration(w.refreshPeriod) * time.Second) + defer tickerRefreshProcess.Stop() for { select { case <-tickerRefreshProcess.C: diff --git a/pkg/clusteragent/autoscaling/externalmetrics/metrics_retriever.go b/pkg/clusteragent/autoscaling/externalmetrics/metrics_retriever.go index 39597675561aa..3729985d90f77 100644 --- a/pkg/clusteragent/autoscaling/externalmetrics/metrics_retriever.go +++ b/pkg/clusteragent/autoscaling/externalmetrics/metrics_retriever.go @@ -58,6 +58,7 @@ func NewMetricsRetriever(refreshPeriod, metricsMaxAge int64, processor autoscale func (mr *MetricsRetriever) Run(stopCh <-chan struct{}) { log.Infof("Starting MetricsRetriever") tickerRefreshProcess := time.NewTicker(time.Duration(mr.refreshPeriod) * time.Second) + defer tickerRefreshProcess.Stop() for { select { case <-tickerRefreshProcess.C: diff --git a/pkg/collector/corechecks/containerimage/check.go b/pkg/collector/corechecks/containerimage/check.go index 26da8511de1c7..bf953b83a8d2c 100644 --- a/pkg/collector/corechecks/containerimage/check.go +++ b/pkg/collector/corechecks/containerimage/check.go @@ -145,7 +145,7 @@ func (c *Check) Run() error { ) imgRefreshTicker := time.NewTicker(time.Duration(c.instance.PeriodicRefreshSeconds) * time.Second) - + defer imgRefreshTicker.Stop() defer c.processor.stop() for { select { diff --git a/pkg/collector/corechecks/containerlifecycle/processor.go b/pkg/collector/corechecks/containerlifecycle/processor.go index f3eebfa521ffc..2f514329c091c 100644 --- a/pkg/collector/corechecks/containerlifecycle/processor.go +++ b/pkg/collector/corechecks/containerlifecycle/processor.go @@ -170,7 +170,7 @@ func (p *processor) processTask(task *workloadmeta.ECSTask) error { // processQueues consumes the data available in the queues func (p *processor) processQueues(ctx context.Context, pollInterval time.Duration) { ticker := time.NewTicker(pollInterval) - + defer ticker.Stop() for { select { case <-ticker.C: diff --git a/pkg/collector/corechecks/snmp/internal/discovery/discovery.go b/pkg/collector/corechecks/snmp/internal/discovery/discovery.go index f8c3337308ea8..edf691e7e804f 100644 --- a/pkg/collector/corechecks/snmp/internal/discovery/discovery.go +++ b/pkg/collector/corechecks/snmp/internal/discovery/discovery.go @@ -139,7 +139,7 @@ func (d *Discovery) discoverDevices() { } discoveryTicker := time.NewTicker(time.Duration(d.config.DiscoveryInterval) * time.Second) - + defer discoveryTicker.Stop() for { log.Debugf("subnet %s: Run discovery", d.config.Network) startingIP := make(net.IP, len(subnet.startingIP)) diff --git a/pkg/process/runner/runner.go b/pkg/process/runner/runner.go index ef4e9269088a0..a585764364b1d 100644 --- a/pkg/process/runner/runner.go +++ b/pkg/process/runner/runner.go @@ -378,6 +378,7 @@ func (l *CheckRunner) basicRunner(c checks.Check) func() { } ticker := time.NewTicker(checks.GetInterval(l.config, c.Name())) + defer ticker.Stop() for { select { case <-ticker.C: diff --git a/pkg/security/ptracer/proc.go b/pkg/security/ptracer/proc.go index c6fc46209cdd9..0484449f2ac94 100644 --- a/pkg/security/ptracer/proc.go +++ b/pkg/security/ptracer/proc.go @@ -282,7 +282,7 @@ func (ctx *CWSPtracerCtx) scanProcfs() { every = 500 * time.Millisecond } ticker := time.NewTicker(every) - + defer ticker.Stop() for { select { case <-ticker.C: diff --git a/pkg/security/tests/files_generator.go b/pkg/security/tests/files_generator.go index cb9d617a62d84..5fc0a2e448590 100644 --- a/pkg/security/tests/files_generator.go +++ b/pkg/security/tests/files_generator.go @@ -348,7 +348,7 @@ func (fgc *FileGeneratorContext) start(wg *sync.WaitGroup) { fgc.mountWordDir() defer fgc.unmountWordDir() remountTicker := time.NewTicker(fgc.config.RemountEvery) - + defer remountTicker.Stop() var testEnd *time.Time started := false fgc.resetFirstStates() diff --git a/pkg/trace/stats/client_stats_aggregator.go b/pkg/trace/stats/client_stats_aggregator.go index 3d8d33423b706..0c3e8fef49ca9 100644 --- a/pkg/trace/stats/client_stats_aggregator.go +++ b/pkg/trace/stats/client_stats_aggregator.go @@ -101,6 +101,7 @@ func (a *ClientStatsAggregator) Start() { // Stop stops the aggregator. Calling Stop twice will panic. func (a *ClientStatsAggregator) Stop() { close(a.exit) + a.flushTicker.Stop() <-a.done } diff --git a/pkg/trace/writer/trace.go b/pkg/trace/writer/trace.go index 205df6762c713..2a7b5ab6b9314 100644 --- a/pkg/trace/writer/trace.go +++ b/pkg/trace/writer/trace.go @@ -138,6 +138,7 @@ func NewTraceWriter( func (w *TraceWriter) reporter() { tck := time.NewTicker(w.tick) + defer tck.Stop() defer w.wg.Done() for { select { @@ -172,6 +173,7 @@ func (w *TraceWriter) Stop() { w.wg.Wait() w.flush() stopSenders(w.senders) + w.flushTicker.Stop() } // FlushSync blocks and sends pending payloads when syncMode is true diff --git a/pkg/util/clusteragent/clusteragent.go b/pkg/util/clusteragent/clusteragent.go index aa1f63b9f55e8..46b2ddb98eb81 100644 --- a/pkg/util/clusteragent/clusteragent.go +++ b/pkg/util/clusteragent/clusteragent.go @@ -151,8 +151,9 @@ func (c *DCAClient) startReconnectHandler(reconnectPeriod time.Duration) { return } - t := time.NewTicker(reconnectPeriod) go func() { + t := time.NewTicker(reconnectPeriod) + defer t.Stop() for { <-t.C err := c.initHTTPClient() diff --git a/pkg/util/containers/metrics/provider/registry.go b/pkg/util/containers/metrics/provider/registry.go index 6a7de440e3129..15dc40da027f0 100644 --- a/pkg/util/containers/metrics/provider/registry.go +++ b/pkg/util/containers/metrics/provider/registry.go @@ -91,6 +91,7 @@ func (cr *collectorRegistry) run(c context.Context, cache *Cache, wmeta optional func (cr *collectorRegistry) collectorDiscovery(c context.Context, cache *Cache, wmeta optional.Option[workloadmeta.Component]) { ticker := time.NewTicker(minRetryInterval) + defer ticker.Stop() for { select { case <-c.Done(): diff --git a/pkg/util/kubernetes/apiserver/controllers/controller_util.go b/pkg/util/kubernetes/apiserver/controllers/controller_util.go index 47995379d8664..ab277b7f040fb 100644 --- a/pkg/util/kubernetes/apiserver/controllers/controller_util.go +++ b/pkg/util/kubernetes/apiserver/controllers/controller_util.go @@ -228,9 +228,11 @@ func (h *autoscalersController) updateExternalMetrics() { // processingLoop is a go routine that schedules the garbage collection and the refreshing of external metrics // in the GlobalStore. func (h *autoscalersController) processingLoop(stopCh <-chan struct{}) { - tickerAutoscalerRefreshProcess := time.NewTicker(time.Duration(h.poller.refreshPeriod) * time.Second) - gcPeriodSeconds := time.NewTicker(time.Duration(h.poller.gcPeriodSeconds) * time.Second) go func() { + tickerAutoscalerRefreshProcess := time.NewTicker(time.Duration(h.poller.refreshPeriod) * time.Second) + defer tickerAutoscalerRefreshProcess.Stop() + gcPeriodSeconds := time.NewTicker(time.Duration(h.poller.gcPeriodSeconds) * time.Second) + defer gcPeriodSeconds.Stop() for { select { case <-stopCh: From 038ec0fc4a5b923a25ec8641a5ee9231213214ca Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Tue, 20 Aug 2024 11:29:25 +0200 Subject: [PATCH 050/245] discovery: Report APM instrumentation for Java (#28457) --- .../corechecks/servicediscovery/apm/detect.go | 8 ++++ .../servicediscovery/apm/detect_nix_test.go | 13 ++++++ .../servicediscovery/module/impl_linux.go | 4 +- .../module/impl_linux_test.go | 46 +++++++++++++++++++ .../module/testutil/fake_server/.gitignore | 2 + .../testutil/fake_server/fake_server.go | 31 +++++++++++++ tasks/kmt.py | 1 + tasks/system_probe.py | 1 + 8 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 pkg/collector/corechecks/servicediscovery/module/testutil/fake_server/.gitignore create mode 100644 pkg/collector/corechecks/servicediscovery/module/testutil/fake_server/fake_server.go diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect.go b/pkg/collector/corechecks/servicediscovery/apm/detect.go index e773dba407358..47d1947acbdcd 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect.go @@ -43,6 +43,10 @@ var ( language.Python: pythonDetector, language.Ruby: rubyDetector, } + // For now, only allow a subset of the above detectors to actually run. + allowedLangs = map[language.Language]struct{}{ + language.Java: {}, + } ) // Detect attempts to detect the type of APM instrumentation for the given service. @@ -52,6 +56,10 @@ func Detect(args []string, envs map[string]string, lang language.Language) Instr return Injected } + if _, ok := allowedLangs[lang]; !ok { + return None + } + // different detection for provided instrumentation for each if detect, ok := detectorMap[lang]; ok { return detect(args, envs) diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go index 9c4978cfe4c6d..56562c6d92ec4 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go @@ -70,6 +70,19 @@ func Test_javaDetector(t *testing.T) { args: strings.Split("java -version", " "), result: None, }, + { + name: "cmdline", + args: []string{"java", "-foo", "-javaagent:/path/to/data dog/dd-java-agent.jar", "-Ddd.profiling.enabled=true"}, + result: Provided, + }, + { + name: "CATALINA_OPTS", + args: []string{"java"}, + envs: map[string]string{ + "CATALINA_OPTS": "-javaagent:dd-java-agent.jar", + }, + result: Provided, + }, } for _, d := range data { t.Run(d.name, func(t *testing.T) { diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go index bd0de7db550af..1cdf38f5d81be 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go @@ -216,8 +216,8 @@ func (s *discovery) getServiceInfo(proc *process.Process) (*serviceInfo, error) } name := s.serviceDetector.GetServiceName(cmdline, envs) - // Language passed as unknown for now to only detect injection. - apmInstrumentation := apm.Detect(cmdline, envs, language.Unknown) + language := language.FindInArgs(cmdline) + apmInstrumentation := apm.Detect(cmdline, envs, language) return &serviceInfo{name: name, apmInstrumentation: apmInstrumentation}, nil } diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go index 8885f76f58370..7989e35067823 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go @@ -17,6 +17,7 @@ import ( "net/http" "os" "os/exec" + "path/filepath" "regexp" "runtime" "syscall" @@ -39,6 +40,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" protocolUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/util/optional" ) @@ -331,6 +333,50 @@ func TestAPMInstrumentationInjected(t *testing.T) { require.Equal(t, string(apm.Injected), portMap[pid].APMInstrumentation) } +func buildFakeServer(t *testing.T) string { + curDir, err := testutil.CurDir() + require.NoError(t, err) + serverBin, err := usmtestutil.BuildGoBinaryWrapper(filepath.Join(curDir, "testutil"), "fake_server") + require.NoError(t, err) + + binDir := filepath.Dir(serverBin) + for _, alias := range []string{"java"} { + aliasPath := filepath.Join(binDir, alias) + + target, err := os.Readlink(aliasPath) + if err == nil && target == serverBin { + continue + } + + os.Remove(aliasPath) + err = os.Symlink(serverBin, aliasPath) + require.NoError(t, err) + } + + return binDir +} + +func TestAPMInstrumentationProvided(t *testing.T) { + serverDir := buildFakeServer(t) + url := setupDiscoveryModule(t) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(func() { cancel() }) + + bin := filepath.Join(serverDir, "java") + cmd := exec.CommandContext(ctx, bin, "-javaagent:/path/to/dd-java-agent.jar", "-jar", "foo.jar") + err := cmd.Start() + require.NoError(t, err) + + pid := cmd.Process.Pid + + require.EventuallyWithT(t, func(collect *assert.CollectT) { + portMap := getServicesMap(t, url) + assert.Contains(collect, portMap, pid) + assert.Equal(collect, string(apm.Provided), portMap[pid].APMInstrumentation) + }, 30*time.Second, 100*time.Millisecond) +} + // Check that we can get listening processes in other namespaces. func TestNamespaces(t *testing.T) { url := setupDiscoveryModule(t) diff --git a/pkg/collector/corechecks/servicediscovery/module/testutil/fake_server/.gitignore b/pkg/collector/corechecks/servicediscovery/module/testutil/fake_server/.gitignore new file mode 100644 index 0000000000000..0db56da3f3ccb --- /dev/null +++ b/pkg/collector/corechecks/servicediscovery/module/testutil/fake_server/.gitignore @@ -0,0 +1,2 @@ +fake_server +java diff --git a/pkg/collector/corechecks/servicediscovery/module/testutil/fake_server/fake_server.go b/pkg/collector/corechecks/servicediscovery/module/testutil/fake_server/fake_server.go new file mode 100644 index 0000000000000..40c71c81b783b --- /dev/null +++ b/pkg/collector/corechecks/servicediscovery/module/testutil/fake_server/fake_server.go @@ -0,0 +1,31 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +// Package main is a simple TCP server which accepts any command line arguments, +// in order to test service discovery which uses the command line for detection. +package main + +import ( + "fmt" + "net" + "os" + "os/signal" + "syscall" +) + +func main() { + listener, err := net.Listen("tcp", "") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + defer listener.Close() + + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + fmt.Println("awaiting signal") + <-sigs + fmt.Println("exiting") +} diff --git a/tasks/kmt.py b/tasks/kmt.py index 808efb6827e64..4721aa63db90b 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -961,6 +961,7 @@ def kmt_sysprobe_prepare( "external_unix_proxy_server", "fmapper", "prefetch_file", + "fake_server", ]: src_file_path = os.path.join(pkg, f"{gobin}.go") if os.path.isdir(pkg) and os.path.isfile(src_file_path): diff --git a/tasks/system_probe.py b/tasks/system_probe.py index 7d8525515aaa8..fa89df422bafb 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -947,6 +947,7 @@ def kitchen_prepare(ctx, kernel_release=None, ci=False, packages=""): "gotls_server", "grpc_external_server", "prefetch_file", + "fake_server", ]: src_file_path = os.path.join(pkg, f"{gobin}.go") if not is_windows and os.path.isdir(pkg) and os.path.isfile(src_file_path): From a9ff793320925bf4618a75487b08ae32751ac61d Mon Sep 17 00:00:00 2001 From: maxime mouial Date: Tue, 20 Aug 2024 11:52:57 +0200 Subject: [PATCH 051/245] Removing BindPFlag method from viper (#28358) --- cmd/dogstatsd/subcommands/start/command.go | 24 ++++++++++++++-------- comp/core/config/params.go | 13 ++++++++++++ comp/core/config/params_test.go | 6 ++++++ comp/core/config/setup.go | 4 ++++ comp/metadata/inventoryagent/README.md | 4 ++-- pkg/cli/subcommands/check/command.go | 2 -- pkg/config/model/go.mod | 2 +- pkg/config/model/types.go | 3 --- pkg/config/model/viper.go | 8 -------- 9 files changed, 41 insertions(+), 25 deletions(-) diff --git a/cmd/dogstatsd/subcommands/start/command.go b/cmd/dogstatsd/subcommands/start/command.go index 5110d9c4244b8..a28d8378ca070 100644 --- a/cmd/dogstatsd/subcommands/start/command.go +++ b/cmd/dogstatsd/subcommands/start/command.go @@ -65,7 +65,8 @@ import ( ) type CLIParams struct { - confPath string + confPath string + socketPath string } type DogstatsdComponents struct { @@ -93,10 +94,7 @@ func MakeCommand(defaultLogFile string) *cobra.Command { // local flags startCmd.PersistentFlags().StringVarP(&cliParams.confPath, "cfgpath", "c", "", "path to directory containing datadog.yaml") - - var socketPath string - startCmd.Flags().StringVarP(&socketPath, "socket", "s", "", "listen to this socket instead of UDP") - pkgconfig.Datadog().BindPFlag("dogstatsd_socket", startCmd.Flags().Lookup("socket")) //nolint:errcheck + startCmd.PersistentFlags().StringVarP(&cliParams.socketPath, "socket", "s", "", "listen to this socket instead of UDP") return startCmd } @@ -109,15 +107,23 @@ func RunDogstatsdFct(cliParams *CLIParams, defaultConfPath string, defaultLogFil params := &Params{ DefaultLogFile: defaultLogFile, } + + configOptions := []func(*config.Params){ + config.WithConfFilePath(cliParams.confPath), + config.WithConfigMissingOK(true), + config.WithConfigName("dogstatsd"), + } + if cliParams.socketPath != "" { + configOptions = append(configOptions, config.WithCLIOverride("dogstatsd_socket", cliParams.socketPath)) + } + return fxutil.OneShot(fct, fx.Supply(cliParams), fx.Supply(params), fx.Supply(config.NewParams( defaultConfPath, - config.WithConfFilePath(cliParams.confPath), - config.WithConfigMissingOK(true), - config.WithConfigName("dogstatsd")), - ), + configOptions..., + )), fx.Provide(func(comp secrets.Component) optional.Option[secrets.Component] { return optional.NewOption[secrets.Component](comp) }), diff --git a/comp/core/config/params.go b/comp/core/config/params.go index 52537b23a57b2..188aa742c2bdf 100644 --- a/comp/core/config/params.go +++ b/comp/core/config/params.go @@ -43,12 +43,17 @@ type Params struct { // defaultConfPath determines the default configuration path. // if defaultConfPath is empty, then no default configuration path is used. defaultConfPath string + + // cliOverride is a list of setting overrides from the CLI given to the configuration. The map associate + // settings name like "logs_config.enabled" to its value. + cliOverride map[string]interface{} } // NewParams creates a new instance of Params func NewParams(defaultConfPath string, options ...func(*Params)) Params { params := Params{ defaultConfPath: defaultConfPath, + cliOverride: map[string]interface{}{}, } for _, o := range options { o(¶ms) @@ -141,6 +146,14 @@ func WithFleetPoliciesDirPath(fleetPoliciesDirPath string) func(*Params) { } } +// WithCLIOverride registers a list of settings overrides from the CLI for the configuration. The map associate settings +// name like "logs_config.enabled" to its value. +func WithCLIOverride(setting string, value interface{}) func(*Params) { + return func(b *Params) { + b.cliOverride[setting] = value + } +} + // These functions are used in unit tests. // ConfigMissingOK determines whether it is a fatal error if the config diff --git a/comp/core/config/params_test.go b/comp/core/config/params_test.go index b80ab5ef4df9a..0709912824c91 100644 --- a/comp/core/config/params_test.go +++ b/comp/core/config/params_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/DataDog/datadog-agent/cmd/agent/common/path" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -52,3 +53,8 @@ func TestNewSecurityAgentParams(t *testing.T) { require.Equal(t, false, configComponentParams.configMissingOK, "configMissingOK values not matching") } } + +func TestWithCLIOverride(t *testing.T) { + params := NewParams("test_path", WithCLIOverride("test.setting", true), WithCLIOverride("test.setting2", "test")) + assert.Equal(t, map[string]interface{}{"test.setting": true, "test.setting2": "test"}, params.cliOverride) +} diff --git a/comp/core/config/setup.go b/comp/core/config/setup.go index b3dc261f37e3d..a3d29f4a5bbf0 100644 --- a/comp/core/config/setup.go +++ b/comp/core/config/setup.go @@ -94,5 +94,9 @@ func setupConfig(config pkgconfigmodel.Config, deps configDependencies) (*pkgcon } } + for k, v := range p.cliOverride { + config.Set(k, v, pkgconfigmodel.SourceCLI) + } + return warnings, nil } diff --git a/comp/metadata/inventoryagent/README.md b/comp/metadata/inventoryagent/README.md index 63c17e36bc9cb..74fcf3e26e661 100644 --- a/comp/metadata/inventoryagent/README.md +++ b/comp/metadata/inventoryagent/README.md @@ -162,8 +162,8 @@ Here an example of an inventory payload: "install_method_tool_version": "", "logs_transport": "HTTP", "full_configuration": "", - "provided_configuration": "api_key: \"***************************aaaaa\"\ncheck_runners: 4\ncmd.check.fullsketches: false\ncontainerd_namespace: []\ncontainerd_namespaces: []\npython_version: \"3\"\ntracemalloc_debug: false\nlog_level: \"warn\"", - "file_configuration": "check_runners: 4\ncmd.check.fullsketches: false\ncontainerd_namespace: []\ncontainerd_namespaces: []\npython_version: \"3\"\ntracemalloc_debug: false", + "provided_configuration": "api_key: \"***************************aaaaa\"\ncheck_runners: 4\ncontainerd_namespace: []\ncontainerd_namespaces: []\npython_version: \"3\"\ntracemalloc_debug: false\nlog_level: \"warn\"", + "file_configuration": "check_runners: 4\ncontainerd_namespace: []\ncontainerd_namespaces: []\npython_version: \"3\"\ntracemalloc_debug: false", "agent_runtime_configuration": "runtime_block_profile_rate: 5000", "environment_variable_configuration": "api_key: \"***************************aaaaa\"", "remote_configuration": "log_level: \"debug\"", diff --git a/pkg/cli/subcommands/check/command.go b/pkg/cli/subcommands/check/command.go index 329bd790e01d8..25335ffc61402 100644 --- a/pkg/cli/subcommands/check/command.go +++ b/pkg/cli/subcommands/check/command.go @@ -242,8 +242,6 @@ func MakeCommand(globalParamsGetter func() GlobalParams) *cobra.Command { cmd.Flags().UintVarP(&cliParams.discoveryRetryInterval, "discovery-retry-interval", "", 1, "(unused)") cmd.Flags().UintVarP(&cliParams.discoveryMinInstances, "discovery-min-instances", "", 1, "minimum number of config instances to be discovered before running the check(s)") - pkgconfig.Datadog().BindPFlag("cmd.check.fullsketches", cmd.Flags().Lookup("full-sketches")) //nolint:errcheck - // Power user flags - mark as hidden createHiddenStringFlag(cmd, &cliParams.profileMemoryDir, "m-dir", "", "an existing directory in which to store memory profiling data, ignoring clean-up") createHiddenStringFlag(cmd, &cliParams.profileMemoryFrames, "m-frames", "", "the number of stack frames to consider") diff --git a/pkg/config/model/go.mod b/pkg/config/model/go.mod index 27bc9e41a664b..b70f906370eb3 100644 --- a/pkg/config/model/go.mod +++ b/pkg/config/model/go.mod @@ -13,7 +13,6 @@ require ( github.com/DataDog/viper v1.13.5 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/spf13/afero v1.1.2 - github.com/spf13/pflag v1.0.3 github.com/stretchr/testify v1.9.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 ) @@ -30,6 +29,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect + github.com/spf13/pflag v1.0.3 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/sys v0.23.0 // indirect golang.org/x/text v0.16.0 // indirect diff --git a/pkg/config/model/types.go b/pkg/config/model/types.go index 441db450ddb13..bede1d74c4b6e 100644 --- a/pkg/config/model/types.go +++ b/pkg/config/model/types.go @@ -12,7 +12,6 @@ import ( "github.com/DataDog/viper" "github.com/spf13/afero" - "github.com/spf13/pflag" ) // Proxy represents the configuration for proxies in the agent @@ -144,8 +143,6 @@ type Compound interface { SetConfigName(in string) SetConfigFile(in string) SetConfigType(in string) - - BindPFlag(key string, flag *pflag.Flag) error } // Config represents an object that can load and store configuration parameters diff --git a/pkg/config/model/viper.go b/pkg/config/model/viper.go index 92f4b41cd373b..195d68bbc8bf4 100644 --- a/pkg/config/model/viper.go +++ b/pkg/config/model/viper.go @@ -23,7 +23,6 @@ import ( "github.com/DataDog/viper" "github.com/mohae/deepcopy" "github.com/spf13/afero" - "github.com/spf13/pflag" "golang.org/x/exp/slices" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -777,13 +776,6 @@ func (c *safeConfig) SetTypeByDefaultValue(in bool) { c.Viper.SetTypeByDefaultValue(in) } -// BindPFlag wraps Viper for concurrent access -func (c *safeConfig) BindPFlag(key string, flag *pflag.Flag) error { - c.Lock() - defer c.Unlock() - return c.Viper.BindPFlag(key, flag) -} - // GetEnvVars implements the Config interface func (c *safeConfig) GetEnvVars() []string { c.RLock() From 1cc96da1767d300909184479ae6d2329ec4759d1 Mon Sep 17 00:00:00 2001 From: Alexandre Menasria <47357713+amenasria@users.noreply.github.com> Date: Tue, 20 Aug 2024 13:57:20 +0200 Subject: [PATCH 052/245] Remove `GOPATH`/`GOBIN` env var set dependency for the `install_custom_golanci_lint` function (#28541) --- tasks/install_tasks.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/tasks/install_tasks.py b/tasks/install_tasks.py index b5e2630202786..f557ded6670ea 100644 --- a/tasks/install_tasks.py +++ b/tasks/install_tasks.py @@ -63,23 +63,18 @@ def install_custom_golanci_lint(ctx): if res.ok: gopath = os.getenv('GOPATH') gobin = os.getenv('GOBIN') + default_gopath = os.path.join(Path.home(), "go") golintci_binary = bin_name('golangci-lint') golintci_lint_backup_binary = bin_name('golangci-lint-backup') - if gopath is None and gobin is None: - print("Not able to install custom golangci-lint binary. golangci-lint won't work as expected") - raise Exit(code=1) + go_binaries_folder = gobin or os.path.join(gopath or default_gopath, "bin") - if gobin is not None and gopath is None: - shutil.move(os.path.join(gobin, golintci_binary), os.path.join(gobin, golintci_lint_backup_binary)) - shutil.move(golintci_binary, os.path.join(gobin, golintci_binary)) - - if gopath is not None: - shutil.move( - os.path.join(gopath, "bin", golintci_binary), os.path.join(gopath, "bin", golintci_lint_backup_binary) - ) - shutil.move(golintci_binary, os.path.join(gopath, "bin", golintci_binary)) + shutil.move( + os.path.join(go_binaries_folder, golintci_binary), + os.path.join(go_binaries_folder, golintci_lint_backup_binary), + ) + shutil.move(golintci_binary, os.path.join(go_binaries_folder, golintci_binary)) print("Installed custom golangci-lint binary successfully") From 77b901460efbfebb1c3d5370df8122b2b94bee90 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Tue, 20 Aug 2024 12:09:41 +0000 Subject: [PATCH 053/245] Handle multiple releases in the `create_rc_pr` workflow (#28539) Co-authored-by: pducolin <45568537+pducolin@users.noreply.github.com> --- .github/workflows/create_rc_pr.yml | 35 ++++++++++++++++++++++------ tasks/libs/ciproviders/github_api.py | 23 ++++++++++++++++++ tasks/release.py | 10 ++++++++ 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index 858d0615f9ad3..b6fea6cdab6a3 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -6,14 +6,14 @@ on: - cron: '0 14 * * 1,3,5' # Run on Monday, Wednesday, and Friday at 14:00 UTC - cron: '0 8 * * 1,3,5' # Same as above but at 08:00 UTC, to warn agent-integrations team about releasing - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: - create_rc_pr: + find_release_branches: runs-on: ubuntu-latest - + outputs: + branches: ${{ steps.branches.outputs.value }} steps: - name: Checkout repository uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 @@ -33,27 +33,48 @@ jobs: pip install -r tasks/libs/requirements-github.txt pip install -r tasks/requirements_release_tasks.txt - - name: Determine the release active branch + - name: Determine the release active branches + id: branches run: | - echo "RELEASE_BRANCH=$(inv -e release.get-active-release-branch)" >> $GITHUB_ENV + echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT - name: Set the warning option if: github.event.schedule == '0 8 * * 1,3,5' run: echo "WARNING='-w'" >> $GITHUB_ENV + create_rc_pr: + runs-on: ubuntu-latest + needs: find_release_branches + strategy: + matrix: + value: ${{fromJSON(needs.find_release_branches.outputs.branches)}} + steps: - name: Checkout release branch uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: - ref: ${{ env.RELEASE_BRANCH }} + ref: ${{ matrix.value }} fetch-depth: 0 + - name: Install python + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + with: + python-version: 3.11 + cache: "pip" + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r tasks/libs/requirements-github.txt + pip install -r tasks/requirements_release_tasks.txt + - name: Check for changes since last RC id: check_for_changes env: ATLASSIAN_USERNAME: ${{ secrets.ATLASSIAN_USERNAME }} ATLASSIAN_PASSWORD: ${{ secrets.ATLASSIAN_PASSWORD }} run: | - echo "CHANGES=$(inv -e release.check-for-changes -r ${{ env.RELEASE_BRANCH }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT + echo "CHANGES=$(inv -e release.check-for-changes -r ${{ matrix.value }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT - name: Create RC PR if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}} diff --git a/tasks/libs/ciproviders/github_api.py b/tasks/libs/ciproviders/github_api.py index 73eb66383f47a..7e5a0285228b7 100644 --- a/tasks/libs/ciproviders/github_api.py +++ b/tasks/libs/ciproviders/github_api.py @@ -4,8 +4,10 @@ import json import os import platform +import re import subprocess from collections.abc import Iterable +from distutils.version import StrictVersion from functools import lru_cache import requests @@ -24,6 +26,8 @@ __all__ = ["GithubAPI"] +RELEASE_BRANCH_PATTERN = re.compile(r"\d+\.\d+\.x") + class GithubAPI: """ @@ -198,6 +202,25 @@ def latest_release(self) -> str: release = self._repository.get_latest_release() return release.title + def latest_unreleased_release_branches(self): + """ + Get all the release branches that are newer than the latest release. + """ + release = self._repository.get_latest_release() + released_version = StrictVersion(release.title) + + for branch in self.release_branches(): + if StrictVersion(branch.name.removesuffix(".x")) > released_version: + yield branch + + def release_branches(self): + """ + Yield all the branches that match the release branch pattern (A.B.x). + """ + for branch in self._repository.get_branches(): + if RELEASE_BRANCH_PATTERN.match(branch.name): + yield branch + def get_rate_limit_info(self): """ Gets the current rate limit info. diff --git a/tasks/release.py b/tasks/release.py index c7e47a4fe1eb7..96756a0041871 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -2,6 +2,7 @@ Release helper tasks """ +import json import os import re import sys @@ -881,6 +882,15 @@ def get_active_release_branch(_): print("main") +@task +def get_unreleased_release_branches(_): + """ + Determine what are the current active release branches for the Agent. + """ + gh = GithubAPI() + print(json.dumps([branch.name for branch in gh.latest_unreleased_release_branches()])) + + def get_next_version(gh): latest_release = gh.latest_release() current_version = _create_version_from_match(VERSION_RE.search(latest_release)) From f713f0e0f5d4f5cd0d8e4792f63716329a3aa2e4 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Tue, 20 Aug 2024 13:17:50 +0000 Subject: [PATCH 054/245] Do not log the link to Test Visibility on failed tests locally (#28575) --- tasks/test_core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tasks/test_core.py b/tasks/test_core.py index a05ae0ef0f44b..4cd2f73593a35 100644 --- a/tasks/test_core.py +++ b/tasks/test_core.py @@ -9,6 +9,7 @@ from tasks.flavor import AgentFlavor from tasks.libs.civisibility import get_test_link_to_test_on_main from tasks.libs.common.color import color_message +from tasks.libs.common.utils import running_in_ci from tasks.modules import DEFAULT_MODULES, GoModule @@ -115,7 +116,9 @@ def get_failure(self, flavor): else: for name in sorted(tests): failure_string += f"- {package} {name}\n" - failure_string += f" See this test name on main in Test Visibility at {get_test_link_to_test_on_main(package, name)}\n" + + if running_in_ci(): + failure_string += f" See this test name on main in Test Visibility at {get_test_link_to_test_on_main(package, name)}\n" else: failure_string += "The test command failed, but no test failures detected in the result json." From 37bf9a0172c7ef4c0f5276850b16bb8bbd9ecc2f Mon Sep 17 00:00:00 2001 From: Yang Song Date: Tue, 20 Aug 2024 09:51:35 -0400 Subject: [PATCH 055/245] [OTEL-2047] Fix flaky APM stats bucket test (#28543) --- .../otlp/components/statsprocessor/agent_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/comp/otelcol/otlp/components/statsprocessor/agent_test.go b/comp/otelcol/otlp/components/statsprocessor/agent_test.go index 8810c0513abcc..cf24fba275034 100644 --- a/comp/otelcol/otlp/components/statsprocessor/agent_test.go +++ b/comp/otelcol/otlp/components/statsprocessor/agent_test.go @@ -89,9 +89,13 @@ func TestTraceAgent(t *testing.T) { case stats = <-out: if len(stats.Stats) != 0 { require.Len(t, stats.Stats, 1) - require.Len(t, stats.Stats[0].Stats, 1) - assert.Greater(t, len(stats.Stats[0].Stats[0].Stats), 0) - actual = append(actual, stats.Stats[0].Stats[0].Stats...) + cspayload := stats.Stats[0] + // stats can be in one or multiple buckets + assert.Greater(t, len(cspayload.Stats), 0) + for _, bucket := range cspayload.Stats { + assert.Greater(t, len(bucket.Stats), 0) + actual = append(actual, bucket.Stats...) + } } case <-timeout: t.Fatal("timed out") From a55c8cd037f13757037ad6b62d594dcfb4e8e297 Mon Sep 17 00:00:00 2001 From: Kylian Serrania Date: Tue, 20 Aug 2024 16:48:14 +0200 Subject: [PATCH 056/245] Revert "Handle multiple releases in the `create_rc_pr` workflow" (#28594) Reverts #28539. This broke on macOS runners because they use Python 3.12, which has removed distutils. #incident-29883 --- .github/workflows/create_rc_pr.yml | 35 ++++++---------------------- tasks/libs/ciproviders/github_api.py | 23 ------------------ tasks/release.py | 10 -------- 3 files changed, 7 insertions(+), 61 deletions(-) diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index b6fea6cdab6a3..858d0615f9ad3 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -6,14 +6,14 @@ on: - cron: '0 14 * * 1,3,5' # Run on Monday, Wednesday, and Friday at 14:00 UTC - cron: '0 8 * * 1,3,5' # Same as above but at 08:00 UTC, to warn agent-integrations team about releasing + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: - find_release_branches: + create_rc_pr: runs-on: ubuntu-latest - outputs: - branches: ${{ steps.branches.outputs.value }} + steps: - name: Checkout repository uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 @@ -33,48 +33,27 @@ jobs: pip install -r tasks/libs/requirements-github.txt pip install -r tasks/requirements_release_tasks.txt - - name: Determine the release active branches - id: branches + - name: Determine the release active branch run: | - echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT + echo "RELEASE_BRANCH=$(inv -e release.get-active-release-branch)" >> $GITHUB_ENV - name: Set the warning option if: github.event.schedule == '0 8 * * 1,3,5' run: echo "WARNING='-w'" >> $GITHUB_ENV - create_rc_pr: - runs-on: ubuntu-latest - needs: find_release_branches - strategy: - matrix: - value: ${{fromJSON(needs.find_release_branches.outputs.branches)}} - steps: - name: Checkout release branch uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: - ref: ${{ matrix.value }} + ref: ${{ env.RELEASE_BRANCH }} fetch-depth: 0 - - name: Install python - uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - with: - python-version: 3.11 - cache: "pip" - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r tasks/libs/requirements-github.txt - pip install -r tasks/requirements_release_tasks.txt - - name: Check for changes since last RC id: check_for_changes env: ATLASSIAN_USERNAME: ${{ secrets.ATLASSIAN_USERNAME }} ATLASSIAN_PASSWORD: ${{ secrets.ATLASSIAN_PASSWORD }} run: | - echo "CHANGES=$(inv -e release.check-for-changes -r ${{ matrix.value }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT + echo "CHANGES=$(inv -e release.check-for-changes -r ${{ env.RELEASE_BRANCH }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT - name: Create RC PR if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}} diff --git a/tasks/libs/ciproviders/github_api.py b/tasks/libs/ciproviders/github_api.py index 7e5a0285228b7..73eb66383f47a 100644 --- a/tasks/libs/ciproviders/github_api.py +++ b/tasks/libs/ciproviders/github_api.py @@ -4,10 +4,8 @@ import json import os import platform -import re import subprocess from collections.abc import Iterable -from distutils.version import StrictVersion from functools import lru_cache import requests @@ -26,8 +24,6 @@ __all__ = ["GithubAPI"] -RELEASE_BRANCH_PATTERN = re.compile(r"\d+\.\d+\.x") - class GithubAPI: """ @@ -202,25 +198,6 @@ def latest_release(self) -> str: release = self._repository.get_latest_release() return release.title - def latest_unreleased_release_branches(self): - """ - Get all the release branches that are newer than the latest release. - """ - release = self._repository.get_latest_release() - released_version = StrictVersion(release.title) - - for branch in self.release_branches(): - if StrictVersion(branch.name.removesuffix(".x")) > released_version: - yield branch - - def release_branches(self): - """ - Yield all the branches that match the release branch pattern (A.B.x). - """ - for branch in self._repository.get_branches(): - if RELEASE_BRANCH_PATTERN.match(branch.name): - yield branch - def get_rate_limit_info(self): """ Gets the current rate limit info. diff --git a/tasks/release.py b/tasks/release.py index 96756a0041871..c7e47a4fe1eb7 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -2,7 +2,6 @@ Release helper tasks """ -import json import os import re import sys @@ -882,15 +881,6 @@ def get_active_release_branch(_): print("main") -@task -def get_unreleased_release_branches(_): - """ - Determine what are the current active release branches for the Agent. - """ - gh = GithubAPI() - print(json.dumps([branch.name for branch in gh.latest_unreleased_release_branches()])) - - def get_next_version(gh): latest_release = gh.latest_release() current_version = _create_version_from_match(VERSION_RE.search(latest_release)) From 50ef7b2b5f472a36e2b8a2e2f494247dbf968a10 Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Tue, 20 Aug 2024 17:18:49 +0200 Subject: [PATCH 057/245] Remove explicit path to golangci-lint config file (#28581) --- .vscode/settings.json.template | 4 +--- tasks/devcontainer.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.vscode/settings.json.template b/.vscode/settings.json.template index 795843112176d..297be4f2f0200 100644 --- a/.vscode/settings.json.template +++ b/.vscode/settings.json.template @@ -3,9 +3,7 @@ "go.lintTool": "golangci-lint", "go.lintFlags": [ "--build-tags", - "{build_tags}", - "--config", - "{workspace_folder}/.golangci.yml" + "{build_tags}" ], "[go]": {{ "editor.formatOnSave": true, diff --git a/tasks/devcontainer.py b/tasks/devcontainer.py index 097803f8404dc..eb820228f423e 100644 --- a/tasks/devcontainer.py +++ b/tasks/devcontainer.py @@ -96,8 +96,6 @@ def setup( "go.lintFlags": [ "--build-tags", local_build_tags, - "--config", - f"{AGENT_REPOSITORY_PATH}/.golangci.yml", ], "[go]": { "editor.formatOnSave": True, From 68f732ed8a97d5b8438219da1e9415bb1d39b7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Mathieu?= Date: Tue, 20 Aug 2024 17:45:40 +0200 Subject: [PATCH 058/245] logs/sds: two behaviours of freezing the logs pipeline until we receive an SDS config. (#27438) Co-authored-by: Vickenty Fesunov --- comp/logs/agent/agentimpl/agent.go | 93 ++++++---- comp/logs/agent/agentimpl/agent_test.go | 2 +- comp/logs/agent/agentimpl/serverless.go | 2 +- .../agentimpl/status_templates/logsagent.tmpl | 2 +- pkg/config/setup/config.go | 4 + pkg/logs/launchers/listener/tcp.go | 4 +- pkg/logs/metrics/metrics.go | 3 + pkg/logs/pipeline/mock/mock.go | 10 +- pkg/logs/pipeline/pipeline.go | 4 +- pkg/logs/pipeline/provider.go | 39 +++-- pkg/logs/processor/go.mod | 2 +- pkg/logs/processor/processor.go | 125 ++++++++++++-- pkg/logs/processor/processor_test.go | 159 ++++++++++++++++++ pkg/logs/sds/go.mod | 2 +- pkg/logs/sds/reconfigure.go | 81 ++++++++- pkg/logs/sds/scanner.go | 34 ++-- pkg/logs/sds/scanner_nosds.go | 4 +- pkg/logs/sds/scanner_test.go | 69 ++++++-- pkg/logs/status/builder.go | 29 ++-- pkg/logs/status/status.go | 31 ++-- pkg/logs/status/test_utils.go | 2 +- 21 files changed, 571 insertions(+), 130 deletions(-) diff --git a/comp/logs/agent/agentimpl/agent.go b/comp/logs/agent/agentimpl/agent.go index 582d698e774ef..80f37e8a86d37 100644 --- a/comp/logs/agent/agentimpl/agent.go +++ b/comp/logs/agent/agentimpl/agent.go @@ -10,6 +10,7 @@ import ( "context" "errors" "fmt" + "sync" "time" "github.com/hashicorp/go-multierror" @@ -115,8 +116,11 @@ type logAgent struct { schedulerProviders []schedulers.Scheduler integrationsLogs integrations.Component + // make sure this is done only once, when we're ready + prepareSchedulers sync.Once + // started is true if the logs agent is running - started *atomic.Bool + started *atomic.Uint32 } func newLogsAgent(deps dependencies) provides { @@ -130,7 +134,7 @@ func newLogsAgent(deps dependencies) provides { config: deps.Config, inventoryAgent: deps.InventoryAgent, hostname: deps.Hostname, - started: atomic.NewBool(false), + started: atomic.NewUint32(status.StatusNotStarted), sources: sources.NewLogSources(), services: service.NewServices(), @@ -190,12 +194,6 @@ func (a *logAgent) start(context.Context) error { } a.startPipeline() - a.log.Info("logs-agent started") - - for _, scheduler := range a.schedulerProviders { - a.AddScheduler(scheduler) - } - return nil } @@ -225,6 +223,10 @@ func (a *logAgent) setupAgent() error { status.AddGlobalWarning(invalidProcessingRules, multiLineWarning) } + if err := sds.ValidateConfigField(a.config); err != nil { + a.log.Error(fmt.Errorf("error while reading configuration, will block until the Agents receive an SDS configuration: %v", err)) + } + a.SetupPipeline(processingRules, a.wmeta, a.integrationsLogs) return nil } @@ -232,7 +234,6 @@ func (a *logAgent) setupAgent() error { // Start starts all the elements of the data pipeline // in the right order to prevent data loss func (a *logAgent) startPipeline() { - a.started.Store(true) // setup the status status.Init(a.started, a.endpoints, a.sources, a.tracker, metrics.LogsExpvars) @@ -243,9 +244,28 @@ func (a *logAgent) startPipeline() { a.pipelineProvider, a.diagnosticMessageReceiver, a.launchers, - a.schedulers, ) starter.Start() + + if !sds.ShouldBlockCollectionUntilSDSConfiguration(a.config) { + a.startSchedulers() + } else { + a.log.Info("logs-agent ready, schedulers not started: waiting for an SDS configuration to start the logs collection") + a.started.Store(status.StatusCollectionNotStarted) + } +} + +func (a *logAgent) startSchedulers() { + a.prepareSchedulers.Do(func() { + a.schedulers.Start() + + for _, scheduler := range a.schedulerProviders { + a.AddScheduler(scheduler) + } + + a.log.Info("logs-agent started") + a.started.Store(status.StatusRunning) + }) } func (a *logAgent) stop(context.Context) error { @@ -318,45 +338,40 @@ func (a *logAgent) GetPipelineProvider() pipeline.Provider { } func (a *logAgent) onUpdateSDSRules(updates map[string]state.RawConfig, applyStateCallback func(string, state.ApplyStatus)) { //nolint:revive - var err error - for _, config := range updates { - if rerr := a.pipelineProvider.ReconfigureSDSStandardRules(config.Config); rerr != nil { - err = multierror.Append(err, rerr) - } - } - - if err != nil { - a.log.Errorf("Can't update SDS standard rules: %v", err) - } - - // Apply the new status to all configs - for cfgPath := range updates { - if err == nil { - applyStateCallback(cfgPath, state.ApplyStatus{State: state.ApplyStateAcknowledged}) - } else { - applyStateCallback(cfgPath, state.ApplyStatus{ - State: state.ApplyStateError, - Error: err.Error(), - }) - } - } - + a.onUpdateSDS(sds.StandardRules, updates, applyStateCallback) } func (a *logAgent) onUpdateSDSAgentConfig(updates map[string]state.RawConfig, applyStateCallback func(string, state.ApplyStatus)) { //nolint:revive + a.onUpdateSDS(sds.AgentConfig, updates, applyStateCallback) +} + +func (a *logAgent) onUpdateSDS(reconfigType sds.ReconfigureOrderType, updates map[string]state.RawConfig, applyStateCallback func(string, state.ApplyStatus)) { //nolint:revive var err error + allScannersActiveWithAllUpdatesApplied := true // We received a hit that new updates arrived, but if the list of updates // is empty, it means we don't have any updates applying to this agent anymore - // Send a reconfiguration with an empty payload, indicating that - // the scanners have to be dropped. + // In this case, send the signal to stop the SDS processing. if len(updates) == 0 { - err = a.pipelineProvider.ReconfigureSDSAgentConfig([]byte("{}")) + err = a.pipelineProvider.StopSDSProcessing() } else { for _, config := range updates { - if rerr := a.pipelineProvider.ReconfigureSDSAgentConfig(config.Config); rerr != nil { + var allScannersActive bool + var rerr error + + if reconfigType == sds.AgentConfig { + allScannersActive, rerr = a.pipelineProvider.ReconfigureSDSAgentConfig(config.Config) + } else if reconfigType == sds.StandardRules { + allScannersActive, rerr = a.pipelineProvider.ReconfigureSDSStandardRules(config.Config) + } + + if rerr != nil { err = multierror.Append(err, rerr) } + + if !allScannersActive { + allScannersActiveWithAllUpdatesApplied = false + } } } @@ -364,6 +379,10 @@ func (a *logAgent) onUpdateSDSAgentConfig(updates map[string]state.RawConfig, ap a.log.Errorf("Can't update SDS configurations: %v", err) } + if allScannersActiveWithAllUpdatesApplied && sds.ShouldBlockCollectionUntilSDSConfiguration(a.config) { + a.startSchedulers() + } + // Apply the new status to all configs for cfgPath := range updates { if err == nil { diff --git a/comp/logs/agent/agentimpl/agent_test.go b/comp/logs/agent/agentimpl/agent_test.go index b92a59a794502..68e0f455621c8 100644 --- a/comp/logs/agent/agentimpl/agent_test.go +++ b/comp/logs/agent/agentimpl/agent_test.go @@ -125,7 +125,7 @@ func createAgent(suite *AgentTestSuite, endpoints *config.Endpoints) (*logAgent, log: deps.Log, config: deps.Config, inventoryAgent: deps.InventoryAgent, - started: atomic.NewBool(false), + started: atomic.NewUint32(0), integrationsLogs: deps.IntegrationsLogs, sources: sources, diff --git a/comp/logs/agent/agentimpl/serverless.go b/comp/logs/agent/agentimpl/serverless.go index 82afcbc110d98..66ec9b23ef7ee 100644 --- a/comp/logs/agent/agentimpl/serverless.go +++ b/comp/logs/agent/agentimpl/serverless.go @@ -23,7 +23,7 @@ func NewServerlessLogsAgent() agent.ServerlessLogsAgent { logsAgent := &logAgent{ log: logComponent.NewTemporaryLoggerWithoutInit(), config: pkgConfig.Datadog(), - started: atomic.NewBool(false), + started: atomic.NewUint32(0), sources: sources.NewLogSources(), services: service.NewServices(), diff --git a/comp/logs/agent/agentimpl/status_templates/logsagent.tmpl b/comp/logs/agent/agentimpl/status_templates/logsagent.tmpl index 2da805ecc9d64..71440cac2e718 100644 --- a/comp/logs/agent/agentimpl/status_templates/logsagent.tmpl +++ b/comp/logs/agent/agentimpl/status_templates/logsagent.tmpl @@ -1,7 +1,7 @@ {{ with .logsStats }} {{- if eq .IsRunning false }} Logs Agent is not running -{{- end }} +{{- end }}{{ if eq .WaitingForSDSConfig true }} (waiting for an SDS configuration){{ end }} {{- if .Endpoints }} diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 77cfa72ed5d61..6a6b8eec1b922 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -1526,6 +1526,10 @@ func logsagent(config pkgconfigmodel.Setup) { // more disk I/O at the wildcard log paths config.BindEnvAndSetDefault("logs_config.file_wildcard_selection_mode", "by_name") + // SDS logs blocking mechanism + config.BindEnvAndSetDefault("logs_config.sds.wait_for_configuration", "") + config.BindEnvAndSetDefault("logs_config.sds.buffer_max_size", 0) + // Max size in MB to allow for integrations logs files config.BindEnvAndSetDefault("logs_config.integrations_logs_files_max_size", 100) diff --git a/pkg/logs/launchers/listener/tcp.go b/pkg/logs/launchers/listener/tcp.go index fa07e054b107d..9b7e61dad7c29 100644 --- a/pkg/logs/launchers/listener/tcp.go +++ b/pkg/logs/launchers/listener/tcp.go @@ -72,7 +72,9 @@ func (l *TCPListener) Stop() { l.mu.Lock() defer l.mu.Unlock() l.stop <- struct{}{} - l.listener.Close() + if l.listener != nil { + l.listener.Close() + } stopper := startstop.NewParallelStopper() for _, tailer := range l.tailers { stopper.Add(tailer) diff --git a/pkg/logs/metrics/metrics.go b/pkg/logs/metrics/metrics.go index 063a4e6afe075..2ba51621ece48 100644 --- a/pkg/logs/metrics/metrics.go +++ b/pkg/logs/metrics/metrics.go @@ -70,6 +70,9 @@ var ( DestinationHttpRespByStatusAndUrl = expvar.Map{} //nolint:revive // TODO(AML) Fix revive linter TlmDestinationHttpRespByStatusAndUrl = telemetry.NewCounter("logs", "destination_http_resp", []string{"status_code", "url"}, "Count of http responses by status code and destination url") + + // TlmLogsDiscardedFromSDSBuffer how many messages were dropped when waiting for an SDS configuration because the buffer is full + TlmLogsDiscardedFromSDSBuffer = telemetry.NewCounter("logs", "sds__dropped_from_buffer", nil, "Count of messages dropped from the buffer while waiting for an SDS configuration") ) func init() { diff --git a/pkg/logs/pipeline/mock/mock.go b/pkg/logs/pipeline/mock/mock.go index 8e7c0bde6387d..3d07560754a79 100644 --- a/pkg/logs/pipeline/mock/mock.go +++ b/pkg/logs/pipeline/mock/mock.go @@ -31,11 +31,15 @@ func (p *mockProvider) Start() {} // Stop does nothing func (p *mockProvider) Stop() {} -func (p *mockProvider) ReconfigureSDSStandardRules(_ []byte) error { - return nil +func (p *mockProvider) ReconfigureSDSStandardRules(_ []byte) (bool, error) { + return false, nil +} + +func (p *mockProvider) ReconfigureSDSAgentConfig(_ []byte) (bool, error) { + return false, nil } -func (p *mockProvider) ReconfigureSDSAgentConfig(_ []byte) error { +func (p *mockProvider) StopSDSProcessing() error { return nil } diff --git a/pkg/logs/pipeline/pipeline.go b/pkg/logs/pipeline/pipeline.go index 096430526c091..0a050d38481ad 100644 --- a/pkg/logs/pipeline/pipeline.go +++ b/pkg/logs/pipeline/pipeline.go @@ -77,7 +77,9 @@ func NewPipeline(outputChan chan *message.Payload, logsSender = sender.NewSender(cfg, senderInput, outputChan, mainDestinations, config.DestinationPayloadChanSize, senderDoneChan, flushWg) inputChan := make(chan *message.Message, config.ChanSize) - processor := processor.New(inputChan, strategyInput, processingRules, encoder, diagnosticMessageReceiver, hostname, pipelineID) + + processor := processor.New(cfg, inputChan, strategyInput, processingRules, + encoder, diagnosticMessageReceiver, hostname, pipelineID) return &Pipeline{ InputChan: inputChan, diff --git a/pkg/logs/pipeline/provider.go b/pkg/logs/pipeline/provider.go index 04812a8aed123..54d3b947a1313 100644 --- a/pkg/logs/pipeline/provider.go +++ b/pkg/logs/pipeline/provider.go @@ -28,8 +28,9 @@ import ( type Provider interface { Start() Stop() - ReconfigureSDSStandardRules(standardRules []byte) error - ReconfigureSDSAgentConfig(config []byte) error + ReconfigureSDSStandardRules(standardRules []byte) (bool, error) + ReconfigureSDSAgentConfig(config []byte) (bool, error) + StopSDSProcessing() error NextPipelineChan() chan *message.Message // Flush flushes all pipeline contained in this Provider Flush(ctx context.Context) @@ -111,8 +112,9 @@ func (p *provider) Stop() { p.outputChan = nil } -func (p *provider) reconfigureSDS(config []byte, orderType sds.ReconfigureOrderType) error { - var responses []chan error +// return true if all processor SDS scanners are active. +func (p *provider) reconfigureSDS(config []byte, orderType sds.ReconfigureOrderType) (bool, error) { + var responses []chan sds.ReconfigureResponse // send a reconfiguration order to every running pipeline @@ -120,7 +122,7 @@ func (p *provider) reconfigureSDS(config []byte, orderType sds.ReconfigureOrderT order := sds.ReconfigureOrder{ Type: orderType, Config: config, - ResponseChan: make(chan error), + ResponseChan: make(chan sds.ReconfigureResponse), } responses = append(responses, order.ResponseChan) @@ -131,28 +133,43 @@ func (p *provider) reconfigureSDS(config []byte, orderType sds.ReconfigureOrderT // reports if at least one error occurred var rerr error + allScannersActive := true for _, response := range responses { - err := <-response - if err != nil { - rerr = multierror.Append(rerr, err) + resp := <-response + + if !resp.IsActive { + allScannersActive = false + } + + if resp.Err != nil { + rerr = multierror.Append(rerr, resp.Err) } + close(response) } - return rerr + return allScannersActive, rerr } // ReconfigureSDSStandardRules stores the SDS standard rules for the given provider. -func (p *provider) ReconfigureSDSStandardRules(standardRules []byte) error { +func (p *provider) ReconfigureSDSStandardRules(standardRules []byte) (bool, error) { return p.reconfigureSDS(standardRules, sds.StandardRules) } // ReconfigureSDSAgentConfig reconfigures the pipeline with the given // configuration received through Remote Configuration. -func (p *provider) ReconfigureSDSAgentConfig(config []byte) error { +// Return true if all SDS scanners are active after applying this configuration. +func (p *provider) ReconfigureSDSAgentConfig(config []byte) (bool, error) { return p.reconfigureSDS(config, sds.AgentConfig) } +// StopSDSProcessing reconfigures the pipeline removing the SDS scanning +// from the processing steps. +func (p *provider) StopSDSProcessing() error { + _, err := p.reconfigureSDS(nil, sds.StopProcessing) + return err +} + // NextPipelineChan returns the next pipeline input channel func (p *provider) NextPipelineChan() chan *message.Message { pipelinesLen := len(p.pipelines) diff --git a/pkg/logs/processor/go.mod b/pkg/logs/processor/go.mod index 4308509e56e8f..537d711c603d0 100644 --- a/pkg/logs/processor/go.mod +++ b/pkg/logs/processor/go.mod @@ -46,6 +46,7 @@ require ( github.com/DataDog/agent-payload/v5 v5.0.106 github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface v0.56.0-rc.3 github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 + github.com/DataDog/datadog-agent/pkg/config/model v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/diagnostic v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/message v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/metrics v0.56.0-rc.3 @@ -61,7 +62,6 @@ require ( github.com/DataDog/datadog-agent/comp/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/config/env v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/pkg/config/model v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/config/setup v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/config/utils v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/logs/status/utils v0.56.0-rc.3 // indirect diff --git a/pkg/logs/processor/processor.go b/pkg/logs/processor/processor.go index 707e1874cd063..f66a8c0c48a4d 100644 --- a/pkg/logs/processor/processor.go +++ b/pkg/logs/processor/processor.go @@ -11,6 +11,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/logs/diagnostic" "github.com/DataDog/datadog-agent/pkg/logs/message" "github.com/DataDog/datadog-agent/pkg/logs/metrics" @@ -38,13 +39,29 @@ type Processor struct { mu sync.Mutex hostname hostnameinterface.Component - sds *sds.Scanner // configured through RC + sds sdsProcessor +} + +type sdsProcessor struct { + // buffer stores the messages for the buffering mechanism in case we didn't + // receive any SDS configuration & wait_for_configuration == "buffer". + buffer []*message.Message + bufferedBytes int + maxBufferSize int + + /// buffering indicates if we're buffering while waiting for an SDS configuration + buffering bool + + scanner *sds.Scanner // configured through RC } // New returns an initialized Processor. -func New(inputChan, outputChan chan *message.Message, processingRules []*config.ProcessingRule, encoder Encoder, - diagnosticMessageReceiver diagnostic.MessageReceiver, hostname hostnameinterface.Component, pipelineID int) *Processor { - sdsScanner := sds.CreateScanner(pipelineID) +func New(cfg pkgconfigmodel.Reader, inputChan, outputChan chan *message.Message, processingRules []*config.ProcessingRule, + encoder Encoder, diagnosticMessageReceiver diagnostic.MessageReceiver, hostname hostnameinterface.Component, + pipelineID int) *Processor { + + waitForSDSConfig := sds.ShouldBufferUntilSDSConfiguration(cfg) + maxBufferSize := sds.WaitForConfigurationBufferMaxSize(cfg) return &Processor{ pipelineID: pipelineID, @@ -54,9 +71,15 @@ func New(inputChan, outputChan chan *message.Message, processingRules []*config. processingRules: processingRules, encoder: encoder, done: make(chan struct{}), - sds: sdsScanner, diagnosticMessageReceiver: diagnosticMessageReceiver, hostname: hostname, + + sds: sdsProcessor{ + // will immediately starts buffering if it has been configured as so + buffering: waitForSDSConfig, + maxBufferSize: maxBufferSize, + scanner: sds.CreateScanner(pipelineID), + }, } } @@ -72,13 +95,14 @@ func (p *Processor) Stop() { <-p.done // once the processor mainloop is not running, it's safe // to delete the sds scanner instance. - if p.sds != nil { - p.sds.Delete() - p.sds = nil + if p.sds.scanner != nil { + p.sds.scanner.Delete() + p.sds.scanner = nil } } // Flush processes synchronously the messages that this processor has to process. +// Mainly (only?) used by the Serverless Agent. func (p *Processor) Flush(ctx context.Context) { p.mu.Lock() defer p.mu.Unlock() @@ -104,27 +128,94 @@ func (p *Processor) run() { for { select { + // Processing, usual main loop + // --------------------------- + case msg, ok := <-p.inputChan: if !ok { // channel has been closed return } - p.processMessage(msg) + + // if we have to wait for an SDS configuration to start processing & forwarding + // the logs, that's here that we buffer the message + if p.sds.buffering { + // buffer until we receive a configuration + p.sds.bufferMsg(msg) + } else { + // process the message + p.processMessage(msg) + } + p.mu.Lock() // block here if we're trying to flush synchronously //nolint:staticcheck p.mu.Unlock() + + // SDS reconfiguration + // ------------------- + case order := <-p.ReconfigChan: p.mu.Lock() - if err := p.sds.Reconfigure(order); err != nil { - log.Errorf("Error while reconfiguring the SDS scanner: %v", err) - order.ResponseChan <- err - } else { - order.ResponseChan <- nil - } + p.applySDSReconfiguration(order) p.mu.Unlock() } } } +func (p *Processor) applySDSReconfiguration(order sds.ReconfigureOrder) { + isActive, err := p.sds.scanner.Reconfigure(order) + response := sds.ReconfigureResponse{ + IsActive: isActive, + Err: err, + } + + if err != nil { + log.Errorf("Error while reconfiguring the SDS scanner: %v", err) + } else { + // no error while reconfiguring the SDS scanner and since it looks active now, + // we should drain the buffered messages if any and stop the + // buffering mechanism. + if p.sds.buffering && isActive { + log.Debug("Processor ready with an SDS configuration.") + p.sds.buffering = false + + // drain the buffer of messages if anything's in there + if len(p.sds.buffer) > 0 { + log.Info("SDS: sending", len(p.sds.buffer), "buffered messages") + for _, msg := range p.sds.buffer { + p.processMessage(msg) + } + } + + p.sds.resetBuffer() + } + // no else case, the buffering is only a startup mechanism, after having + // enabled the SDS scanners, if they become inactive it is because the + // configuration has been sent like that. + } + + order.ResponseChan <- response +} + +func (s *sdsProcessor) bufferMsg(msg *message.Message) { + s.buffer = append(s.buffer, msg) + s.bufferedBytes += len(msg.GetContent()) + + for len(s.buffer) > 0 { + if s.bufferedBytes > s.maxBufferSize { + s.bufferedBytes -= len(s.buffer[0].GetContent()) + s.buffer = s.buffer[1:] + metrics.TlmLogsDiscardedFromSDSBuffer.Inc() + } else { + break + } + } +} + +func (s *sdsProcessor) resetBuffer() { + s.buffer = nil + s.bufferedBytes = 0 +} + func (p *Processor) processMessage(msg *message.Message) { metrics.LogsDecoded.Add(1) metrics.TlmLogsDecoded.Inc() @@ -184,8 +275,8 @@ func (p *Processor) applyRedactingRules(msg *message.Message) bool { // -------------------------- // Global SDS scanner, applied on all log sources - if p.sds.IsReady() { - mutated, evtProcessed, err := p.sds.Scan(content, msg) + if p.sds.scanner.IsReady() { + mutated, evtProcessed, err := p.sds.scanner.Scan(content, msg) if err != nil { log.Error("while using SDS to scan the log:", err) } else if mutated { diff --git a/pkg/logs/processor/processor_test.go b/pkg/logs/processor/processor_test.go index 5da8a7b989fe2..bb2ff56b02461 100644 --- a/pkg/logs/processor/processor_test.go +++ b/pkg/logs/processor/processor_test.go @@ -7,13 +7,17 @@ package processor import ( "regexp" + "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/comp/logs/agent/config" + "github.com/DataDog/datadog-agent/pkg/logs/diagnostic" "github.com/DataDog/datadog-agent/pkg/logs/message" + "github.com/DataDog/datadog-agent/pkg/logs/sds" "github.com/DataDog/datadog-agent/pkg/logs/sources" ) @@ -302,6 +306,161 @@ func TestGetHostname(t *testing.T) { assert.Equal(t, "testHostnameFromEnvVar", p.GetHostname(m)) } +func TestBuffering(t *testing.T) { + assert := assert.New(t) + + if !sds.SDSEnabled { // should not run when SDS is not builtin. + return + } + + hostnameComponent, _ := hostnameinterface.NewMock("testHostnameFromEnvVar") + + p := &Processor{ + encoder: JSONEncoder, + inputChan: make(chan *message.Message), + outputChan: make(chan *message.Message), + ReconfigChan: make(chan sds.ReconfigureOrder), + diagnosticMessageReceiver: diagnostic.NewBufferedMessageReceiver(nil, hostnameComponent), + done: make(chan struct{}), + // configured to buffer (max 3 messages) + sds: sdsProcessor{ + maxBufferSize: len("hello1world") + len("hello2world") + len("hello3world") + 1, + buffering: true, + scanner: sds.CreateScanner(42), + }, + } + + var processedMessages atomic.Int32 + + // consumer + go func() { + for { + <-p.outputChan + processedMessages.Add(1) + } + }() + + // test the buffering when the processor is waiting for an SDS config + // -- + + p.Start() + assert.Len(p.sds.buffer, 0) + + // validates it buffers these 3 messages + src := newSource("exclude_at_match", "", "foobar") + p.inputChan <- newMessage([]byte("hello1world"), &src, "") + p.inputChan <- newMessage([]byte("hello2world"), &src, "") + p.inputChan <- newMessage([]byte("hello3world"), &src, "") + // wait for the other routine to process the messages + messagesDequeue(t, func() bool { return processedMessages.Load() == 0 }, "the messages should not be be procesesd") + + // the limit is configured to 3 messages + p.inputChan <- newMessage([]byte("hello4world"), &src, "") + messagesDequeue(t, func() bool { return processedMessages.Load() == 0 }, "the messages should still not be processed") + + // reconfigure the processor + // -- + + // standard rules + order := sds.ReconfigureOrder{ + Type: sds.StandardRules, + Config: []byte(`{"priority":1,"is_enabled":true,"rules":[ + { + "id":"zero-0", + "description":"zero desc", + "name":"zero", + "definitions": [{"version":1, "pattern":"zero"}] + }]}`), + ResponseChan: make(chan sds.ReconfigureResponse), + } + + p.ReconfigChan <- order + resp := <-order.ResponseChan + assert.Nil(resp.Err) + assert.False(resp.IsActive) + assert.True(p.sds.buffering) + close(order.ResponseChan) + + // agent config, but no active rule + order = sds.ReconfigureOrder{ + Type: sds.AgentConfig, + Config: []byte(` {"is_enabled":true,"rules":[ + { + "id": "random000", + "name":"zero", + "definition":{"standard_rule_id":"zero-0"}, + "match_action":{"type":"Redact","placeholder":"[redacted]"}, + "is_enabled":false + }]}`), + ResponseChan: make(chan sds.ReconfigureResponse), + } + + p.ReconfigChan <- order + resp = <-order.ResponseChan + assert.Nil(resp.Err) + assert.False(resp.IsActive) + assert.True(p.sds.buffering) + close(order.ResponseChan) + + // agent config, but the scanner becomes active: + // * the logs agent should stop buffering + // * it should drains its buffer and process the buffered logs + + // first, check that the buffer is still full + messagesDequeue(t, func() bool { return processedMessages.Load() == 0 }, "no messages should be processed just yet") + + order = sds.ReconfigureOrder{ + Type: sds.AgentConfig, + Config: []byte(` {"is_enabled":true,"rules":[ + { + "id": "random000", + "name":"zero", + "definition":{"standard_rule_id":"zero-0"}, + "match_action":{"type":"Redact","placeholder":"[redacted]"}, + "is_enabled":true + }]}`), + ResponseChan: make(chan sds.ReconfigureResponse), + } + + p.ReconfigChan <- order + resp = <-order.ResponseChan + + assert.Nil(resp.Err) + assert.True(resp.IsActive) + assert.False(p.sds.buffering) // not buffering anymore + close(order.ResponseChan) + + // make sure all messages have been drained and processed + messagesDequeue(t, func() bool { return processedMessages.Load() == 3 }, "all messages must be drained") + + // make sure it continues to process normally without buffering now + p.inputChan <- newMessage([]byte("usual work"), &src, "") + messagesDequeue(t, func() bool { return processedMessages.Load() == 4 }, "should continue processing now") +} + +// messagesDequeue let the other routines being scheduled +// to give some time for the processor routine to dequeue its messages +func messagesDequeue(t *testing.T, f func() bool, errorLog string) { + timerTest := time.NewTimer(10 * time.Millisecond) + timerTimeout := time.NewTimer(5 * time.Second) + for { + select { + case <-timerTimeout.C: + timerTest.Stop() + timerTimeout.Stop() + t.Error("timeout while message dequeuing in the processor") + t.Fatal(errorLog) + break + case <-timerTest.C: + if f() { + timerTest.Stop() + timerTimeout.Stop() + return + } + } + } +} + // helpers // - diff --git a/pkg/logs/sds/go.mod b/pkg/logs/sds/go.mod index e43c03164d1e0..93043c9f9e218 100644 --- a/pkg/logs/sds/go.mod +++ b/pkg/logs/sds/go.mod @@ -45,6 +45,7 @@ replace ( ) require ( + github.com/DataDog/datadog-agent/pkg/config/model v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/logs/message v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/telemetry v0.56.0-rc.3 github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 @@ -59,7 +60,6 @@ require ( github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/collector/check/defaults v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/config/env v0.56.0-rc.3 // indirect - github.com/DataDog/datadog-agent/pkg/config/model v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/config/setup v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/config/utils v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/logs/sources v0.56.0-rc.3 // indirect diff --git a/pkg/logs/sds/reconfigure.go b/pkg/logs/sds/reconfigure.go index 5b3d3fe4c64cd..117aad86090b3 100644 --- a/pkg/logs/sds/reconfigure.go +++ b/pkg/logs/sds/reconfigure.go @@ -6,8 +6,21 @@ //nolint:revive package sds +import ( + "fmt" + + pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" +) + type ReconfigureOrderType string +const waitForConfigField = "logs_config.sds.wait_for_configuration" +const waitForConfigBufferMaxSizeField = "logs_config.sds.buffer_max_size" +const waitForConfigDefaultBufferMaxSize = 1024 * 1024 * 500 + +const waitForConfigNoCollection = "no_collection" +const waitForConfigBuffer = "buffer" + const ( // StandardRules triggers the storage of a new set of standard rules // and reconfigure the internal SDS scanner with an existing user @@ -15,6 +28,9 @@ const ( StandardRules ReconfigureOrderType = "standard_rules" // AgentConfig triggers a reconfiguration of the SDS scanner. AgentConfig ReconfigureOrderType = "agent_config" + // StopProcessing triggers a reconfiguration of the SDS scanner by destroying + // it to remove the SDS processing step. + StopProcessing ReconfigureOrderType = "stop_processing" ) // ReconfigureOrder are used to trigger a reconfiguration @@ -22,5 +38,68 @@ const ( type ReconfigureOrder struct { Type ReconfigureOrderType Config []byte - ResponseChan chan error + ResponseChan chan ReconfigureResponse +} + +// ReconfigureResponse is used to transmit the result from reconfiguring +// the processors. +type ReconfigureResponse struct { + Err error + IsActive bool +} + +// ValidateConfigField returns true if the configuration value for +// wait_for_configuration is valid. +// Validates its value only when SDS is enabled. +func ValidateConfigField(cfg pkgconfigmodel.Reader) error { + str := cfg.GetString(waitForConfigField) + + if !SDSEnabled || + str == "" || str == waitForConfigBuffer || str == waitForConfigNoCollection { + return nil + } + + return fmt.Errorf("invalid value for '%s': %s. Valid values: %s, %s", + waitForConfigField, str, + waitForConfigBuffer, waitForConfigNoCollection) +} + +// ShouldBlockCollectionUntilSDSConfiguration returns true if we want to start the +// collection only after having received an SDS configuration. +func ShouldBlockCollectionUntilSDSConfiguration(cfg pkgconfigmodel.Reader) bool { + if cfg == nil { + return false + } + + // in case of an invalid value for the `wait_for_configuration` field, + // as a safeguard, we want to block collection until we received an SDS configuration. + if SDSEnabled && ValidateConfigField(cfg) != nil { + return true + } + + return SDSEnabled && cfg.GetString(waitForConfigField) == waitForConfigNoCollection +} + +// ShouldBufferUntilSDSConfiguration returns true if we have to buffer until we've +// received an SDS configuration. +func ShouldBufferUntilSDSConfiguration(cfg pkgconfigmodel.Reader) bool { + if cfg == nil { + return false + } + + return SDSEnabled && cfg.GetString(waitForConfigField) == waitForConfigBuffer +} + +// WaitForConfigurationBufferMaxSize returns a size for the buffer used while +// waiting for an SDS configuration. +func WaitForConfigurationBufferMaxSize(cfg pkgconfigmodel.Reader) int { + if cfg == nil { + return waitForConfigDefaultBufferMaxSize + } + + v := cfg.GetInt(waitForConfigBufferMaxSizeField) + if v <= 0 { + v = waitForConfigDefaultBufferMaxSize + } + return v } diff --git a/pkg/logs/sds/scanner.go b/pkg/logs/sds/scanner.go index effde9e28c544..8e7a104252ea0 100644 --- a/pkg/logs/sds/scanner.go +++ b/pkg/logs/sds/scanner.go @@ -86,11 +86,12 @@ const ( // to apply the reconfiguration. // When receiving standard rules, user configuration are reloaded and scanners are // recreated to use the newly received standard rules. +// The boolean return parameter indicates if the SDS scanner has been destroyed. // This method is thread safe, a scan can't happen at the same time. -func (s *Scanner) Reconfigure(order ReconfigureOrder) error { +func (s *Scanner) Reconfigure(order ReconfigureOrder) (bool, error) { if s == nil { log.Warn("Trying to reconfigure a nil Scanner") - return nil + return false, nil } s.Lock() @@ -102,24 +103,28 @@ func (s *Scanner) Reconfigure(order ReconfigureOrder) error { case StandardRules: // reconfigure the standard rules err := s.reconfigureStandardRules(order.Config) + var isActive bool // if we already received a configuration and no errors happened while // reconfiguring the standard rules: reapply the user configuration now. if err == nil && s.rawConfig != nil { - if rerr := s.reconfigureRules(s.rawConfig); rerr != nil { + var rerr error + if isActive, rerr = s.reconfigureRules(s.rawConfig); rerr != nil { log.Error("Can't reconfigure SDS after having received standard rules:", rerr) - s.rawConfig = nil // we drop this configuration because it is unusable + s.rawConfig = nil // we drop this configuration because it seems unusable if err == nil { err = rerr } } } - return err + return isActive, err case AgentConfig: return s.reconfigureRules(order.Config) + case StopProcessing: + return s.reconfigureRules([]byte("{}")) } - return fmt.Errorf("Scanner.Reconfigure: Unknown order type: %v", order.Type) + return false, fmt.Errorf("Scanner.Reconfigure: Unknown order type: %v", order.Type) } // reconfigureStandardRules stores in-memory standard rules received through RC. @@ -159,25 +164,26 @@ func (s *Scanner) reconfigureStandardRules(rawConfig []byte) error { // reconfigureRules reconfigures the internal SDS scanner using the in-memory // standard rules. Could possibly delete and recreate the internal SDS scanner if // necessary. +// The boolean return parameter returns if an SDS scanner is active. // This method is NOT thread safe, caller has to ensure the thread safety. -func (s *Scanner) reconfigureRules(rawConfig []byte) error { +func (s *Scanner) reconfigureRules(rawConfig []byte) (bool, error) { if rawConfig == nil { tlmSDSReconfigError.Inc(s.pipelineID, string(AgentConfig), "nil_config") - return fmt.Errorf("Invalid nil raw configuration received for user configuration") + return s.Scanner != nil, fmt.Errorf("Invalid nil raw configuration received for user configuration") } if s.standardRules == nil || len(s.standardRules) == 0 { // store it for the next try s.rawConfig = rawConfig tlmSDSReconfigError.Inc(s.pipelineID, string(AgentConfig), "no_std_rules") - log.Info("Received an user configuration but no SDS standard rules available.") - return nil + log.Debug("Received an user configuration but no SDS standard rules available.") + return s.Scanner != nil, nil } var config RulesConfig if err := json.Unmarshal(rawConfig, &config); err != nil { tlmSDSReconfigError.Inc(s.pipelineID, string(AgentConfig), "cant_unmarshal") - return fmt.Errorf("Can't unmarshal raw configuration: %v", err) + return s.Scanner != nil, fmt.Errorf("Can't unmarshal raw configuration: %v", err) } // ignore disabled rules @@ -197,7 +203,7 @@ func (s *Scanner) reconfigureRules(rawConfig []byte) error { s.configuredRules = nil tlmSDSReconfigSuccess.Inc(s.pipelineID, "shutdown") } - return nil + return false, nil } // prepare the scanner rules @@ -231,7 +237,7 @@ func (s *Scanner) reconfigureRules(rawConfig []byte) error { var err error if scanner, err = sds.CreateScanner(sdsRules); err != nil { tlmSDSReconfigError.Inc(s.pipelineID, string(AgentConfig), "scanner_error") - return fmt.Errorf("while configuring an SDS Scanner: %v", err) + return s.Scanner != nil, fmt.Errorf("while configuring an SDS Scanner: %v", err) } // destroy the old scanner @@ -255,7 +261,7 @@ func (s *Scanner) reconfigureRules(rawConfig []byte) error { tlmSDSRulesState.Set(float64(totalRulesReceived-len(config.Rules)), s.pipelineID, "disabled") tlmSDSReconfigSuccess.Inc(s.pipelineID, string(AgentConfig)) - return nil + return true, nil } // interpretRCRule interprets a rule as received through RC to return diff --git a/pkg/logs/sds/scanner_nosds.go b/pkg/logs/sds/scanner_nosds.go index 7ed96c1062525..0f1d256f6917a 100644 --- a/pkg/logs/sds/scanner_nosds.go +++ b/pkg/logs/sds/scanner_nosds.go @@ -29,8 +29,8 @@ func CreateScanner(_ int) *Scanner { } // Reconfigure mocks the Reconfigure function. -func (s *Scanner) Reconfigure(_ ReconfigureOrder) error { - return nil +func (s *Scanner) Reconfigure(_ ReconfigureOrder) (bool, error) { + return false, nil } // Delete mocks the Delete function. diff --git a/pkg/logs/sds/scanner_test.go b/pkg/logs/sds/scanner_test.go index 34a747f0a55e7..717173ea12423 100644 --- a/pkg/logs/sds/scanner_test.go +++ b/pkg/logs/sds/scanner_test.go @@ -72,15 +72,16 @@ func TestCreateScanner(t *testing.T) { require.NotNil(s, "the scanner should not be nil after a creation") - err := s.Reconfigure(ReconfigureOrder{ + isActive, err := s.Reconfigure(ReconfigureOrder{ Type: StandardRules, Config: standardRules, }) require.NoError(err, "configuring the standard rules should not fail") + require.False(isActive, "with only standard rules, the scanner can't be active") // now that we have some definitions, we can configure the scanner - err = s.Reconfigure(ReconfigureOrder{ + isActive, err = s.Reconfigure(ReconfigureOrder{ Type: AgentConfig, Config: agentConfig, }) @@ -88,6 +89,7 @@ func TestCreateScanner(t *testing.T) { require.NoError(err, "this one shouldn't fail, all rules are disabled but it's OK as long as there are no rules in the scanner") require.NotNil(s, "the scanner should not become a nil object") + require.False(isActive, "all rules are disabled, the scanner should not be active") if s != nil && len(s.configuredRules) > 0 { t.Errorf("No rules should be configured, they're all disabled. Got (%v) rules configured instead.", len(s.configuredRules)) @@ -98,12 +100,13 @@ func TestCreateScanner(t *testing.T) { agentConfig = bytes.Replace(agentConfig, []byte("\"is_enabled\":false"), []byte("\"is_enabled\":true"), 2) - err = s.Reconfigure(ReconfigureOrder{ + isActive, err = s.Reconfigure(ReconfigureOrder{ Type: AgentConfig, Config: agentConfig, }) require.NoError(err, "this one should not fail since two rules are enabled: %v", err) + require.True(isActive, "the scanner should now be active") require.NotNil(s.Scanner, "the Scanner should've been created, it should not be nil") require.NotNil(s.Scanner.Rules, "the Scanner should use rules") @@ -152,13 +155,14 @@ func TestCreateScanner(t *testing.T) { ]} `) - err = s.Reconfigure(ReconfigureOrder{ + isActive, err = s.Reconfigure(ReconfigureOrder{ Type: AgentConfig, Config: agentConfig, }) require.NoError(err, "this one should not fail since one rule is enabled") require.Len(s.configuredRules, 1, "only one rules should be part of this scanner") + require.True(isActive, "the scanner should be active as one rule is enabled") // order matters, it's ok to test rules by [] access require.Equal(s.configuredRules[0].Name, "one", "incorrect rule selected for configuration") @@ -194,13 +198,14 @@ func TestCreateScanner(t *testing.T) { ]} `) - err = s.Reconfigure(ReconfigureOrder{ + isActive, err = s.Reconfigure(ReconfigureOrder{ Type: AgentConfig, Config: agentConfig, }) require.NoError(err, "no error should happen") require.Len(s.configuredRules, 0, "The group is disabled, no rules should be configured.") + require.False(isActive, "the scanner should've been disabled") } // TestEmptyConfiguration validates that the scanner is destroyed when receiving @@ -244,27 +249,29 @@ func TestEmptyConfiguration(t *testing.T) { require.NotNil(s, "the scanner should not be nil after a creation") - err := s.Reconfigure(ReconfigureOrder{ + isActive, err := s.Reconfigure(ReconfigureOrder{ Type: StandardRules, Config: standardRules, }) require.NoError(err, "configuring the standard rules should not fail") + require.False(isActive, "with only standard rules, the scanner can't be active") // configure with one rule - err = s.Reconfigure(ReconfigureOrder{ + isActive, err = s.Reconfigure(ReconfigureOrder{ Type: AgentConfig, Config: agentConfig, }) require.NoError(err, "this one should not fail since one rule is enabled") require.Len(s.configuredRules, 1, "only one rules should be part of this scanner") + require.True(isActive, "one rule is enabled, the scanner should be active") require.NotNil(s.Scanner) // empty reconfiguration - err = s.Reconfigure(ReconfigureOrder{ + isActive, err = s.Reconfigure(ReconfigureOrder{ Type: AgentConfig, Config: []byte("{}"), }) @@ -272,6 +279,31 @@ func TestEmptyConfiguration(t *testing.T) { require.NoError(err) require.Len(s.configuredRules, 0) require.Nil(s.Scanner) + require.False(isActive, "no active rule, the scanner should be disabled") + + // re-enabling with on rule + + isActive, err = s.Reconfigure(ReconfigureOrder{ + Type: AgentConfig, + Config: agentConfig, + }) + + require.NoError(err, "this one should not fail since one rule is enabled") + require.Len(s.configuredRules, 1, "only one rules should be part of this scanner") + require.True(isActive, "one rule is enabled, the scanner should be active") + require.NotNil(s.Scanner) + + // the StopProcessing signal + + isActive, err = s.Reconfigure(ReconfigureOrder{ + Type: StopProcessing, + Config: nil, + }) + + require.NoError(err) + require.Len(s.configuredRules, 0) + require.Nil(s.Scanner) + require.False(isActive, "no active rule, the scanner should be disabled") } func TestIsReady(t *testing.T) { @@ -323,21 +355,23 @@ func TestIsReady(t *testing.T) { require.NotNil(s, "the scanner should not be nil after a creation") require.False(s.IsReady(), "at this stage, the scanner should not be considered ready, no definitions received") - err := s.Reconfigure(ReconfigureOrder{ + isActive, err := s.Reconfigure(ReconfigureOrder{ Type: StandardRules, Config: standardRules, }) require.NoError(err, "configuring the definitions should not fail") require.False(s.IsReady(), "at this stage, the scanner should not be considered ready, no user config received") + require.False(isActive, "only standard rules configured, the scanner should not be active") // now that we have some definitions, we can configure the scanner - err = s.Reconfigure(ReconfigureOrder{ + isActive, err = s.Reconfigure(ReconfigureOrder{ Type: AgentConfig, Config: agentConfig, }) require.True(s.IsReady(), "at this stage, the scanner should be considered ready") + require.True(isActive, "the scanner has some enabled rules, it should be active") } // TestScan validates that everything fits and works. It's not validating @@ -389,16 +423,21 @@ func TestScan(t *testing.T) { s := CreateScanner(0) require.NotNil(s, "the returned scanner should not be nil") - _ = s.Reconfigure(ReconfigureOrder{ + isActive, _ := s.Reconfigure(ReconfigureOrder{ Type: StandardRules, Config: standardRules, }) - _ = s.Reconfigure(ReconfigureOrder{ + + require.False(isActive, "only standard rules, the scanner should be disabled") + + isActive, _ = s.Reconfigure(ReconfigureOrder{ Type: AgentConfig, Config: agentConfig, }) require.True(s.IsReady(), "at this stage, the scanner should be considered ready") + require.True(isActive, "rules are configured, the scanner should be active") + type result struct { matched bool event string @@ -473,16 +512,18 @@ func TestCloseCycleScan(t *testing.T) { s := CreateScanner(0) require.NotNil(s, "the returned scanner should not be nil") - _ = s.Reconfigure(ReconfigureOrder{ + _, _ = s.Reconfigure(ReconfigureOrder{ Type: StandardRules, Config: standardRules, }) - _ = s.Reconfigure(ReconfigureOrder{ + isActive, _ := s.Reconfigure(ReconfigureOrder{ Type: AgentConfig, Config: agentConfig, }) require.True(s.IsReady(), "at this stage, the scanner should be considered ready") + require.True(isActive, "the scanner should be active") + type result struct { matched bool event string diff --git a/pkg/logs/status/builder.go b/pkg/logs/status/builder.go index 9dc7e1f1db6ee..9dfd646e21891 100644 --- a/pkg/logs/status/builder.go +++ b/pkg/logs/status/builder.go @@ -23,7 +23,7 @@ import ( // Builder is used to build the status. type Builder struct { - isRunning *atomic.Bool + isRunning *atomic.Uint32 endpoints *config.Endpoints sources *sourcesPkg.LogSources tailers *tailers.TailerTracker @@ -33,7 +33,7 @@ type Builder struct { } // NewBuilder returns a new builder. -func NewBuilder(isRunning *atomic.Bool, endpoints *config.Endpoints, sources *sourcesPkg.LogSources, tracker *tailers.TailerTracker, warnings *config.Messages, errors *config.Messages, logExpVars *expvar.Map) *Builder { +func NewBuilder(isRunning *atomic.Uint32, endpoints *config.Endpoints, sources *sourcesPkg.LogSources, tracker *tailers.TailerTracker, warnings *config.Messages, errors *config.Messages, logExpVars *expvar.Map) *Builder { return &Builder{ isRunning: isRunning, endpoints: endpoints, @@ -52,15 +52,16 @@ func (b *Builder) BuildStatus(verbose bool) Status { tailers = b.getTailers() } return Status{ - IsRunning: b.getIsRunning(), - Endpoints: b.getEndpoints(), - Integrations: b.getIntegrations(), - Tailers: tailers, - StatusMetrics: b.getMetricsStatus(), - ProcessFileStats: b.getProcessFileStats(), - Warnings: b.getWarnings(), - Errors: b.getErrors(), - UseHTTP: b.getUseHTTP(), + IsRunning: b.getIsRunning(), + WaitingForSDSConfig: b.getWaitingForSDSConfig(), + Endpoints: b.getEndpoints(), + Integrations: b.getIntegrations(), + Tailers: tailers, + StatusMetrics: b.getMetricsStatus(), + ProcessFileStats: b.getProcessFileStats(), + Warnings: b.getWarnings(), + Errors: b.getErrors(), + UseHTTP: b.getUseHTTP(), } } @@ -68,7 +69,11 @@ func (b *Builder) BuildStatus(verbose bool) Status { // this needs to be thread safe as it can be accessed // from different commands (start, stop, status). func (b *Builder) getIsRunning() bool { - return b.isRunning.Load() + return b.isRunning.Load() == StatusRunning +} + +func (b *Builder) getWaitingForSDSConfig() bool { + return b.isRunning.Load() == StatusCollectionNotStarted } func (b *Builder) getUseHTTP() bool { diff --git a/pkg/logs/status/status.go b/pkg/logs/status/status.go index dac8fb4f848fe..7f7a14c286e4f 100644 --- a/pkg/logs/status/status.go +++ b/pkg/logs/status/status.go @@ -26,6 +26,13 @@ const ( TransportHTTP Transport = "HTTP" // TransportTCP indicates logs-agent is using TCP transport TransportTCP Transport = "TCP" + + // StatusNotStarted means that the logs agent is not started + StatusNotStarted = 0 + // StatusRunning means that the logs agent is running and fully operational + StatusRunning = 1 + // StatusCollectionNotStarted means that the logs agent has not started collecting logs + StatusCollectionNotStarted = 2 ) var ( @@ -69,15 +76,16 @@ type Integration struct { // Status provides some information about logs-agent. type Status struct { - IsRunning bool `json:"is_running"` - Endpoints []string `json:"endpoints"` - StatusMetrics map[string]string `json:"metrics"` - ProcessFileStats map[string]uint64 `json:"process_file_stats"` - Integrations []Integration `json:"integrations"` - Tailers []Tailer `json:"tailers"` - Errors []string `json:"errors"` - Warnings []string `json:"warnings"` - UseHTTP bool `json:"use_http"` + IsRunning bool `json:"is_running"` + WaitingForSDSConfig bool `json:"waiting_for_sds_config"` + Endpoints []string `json:"endpoints"` + StatusMetrics map[string]string `json:"metrics"` + ProcessFileStats map[string]uint64 `json:"process_file_stats"` + Integrations []Integration `json:"integrations"` + Tailers []Tailer `json:"tailers"` + Errors []string `json:"errors"` + Warnings []string `json:"warnings"` + UseHTTP bool `json:"use_http"` } // SetCurrentTransport sets the current transport used by the log agent. @@ -97,7 +105,7 @@ func GetCurrentTransport() Transport { } // Init instantiates the builder that builds the status on the fly. -func Init(isRunning *atomic.Bool, endpoints *config.Endpoints, sources *sources.LogSources, tracker *tailers.TailerTracker, logExpVars *expvar.Map) { +func Init(isRunning *atomic.Uint32, endpoints *config.Endpoints, sources *sources.LogSources, tracker *tailers.TailerTracker, logExpVars *expvar.Map) { globalsLock.Lock() defer globalsLock.Unlock() @@ -123,7 +131,8 @@ func Get(verbose bool) Status { if builder == nil { return Status{ - IsRunning: false, + IsRunning: false, + WaitingForSDSConfig: false, } } return builder.BuildStatus(verbose) diff --git a/pkg/logs/status/test_utils.go b/pkg/logs/status/test_utils.go index cb4bbf3949dae..0c3d6ee515605 100644 --- a/pkg/logs/status/test_utils.go +++ b/pkg/logs/status/test_utils.go @@ -19,7 +19,7 @@ import ( // InitStatus initialize a status builder func InitStatus(coreConfig pkgConfig.Reader, sources *sources.LogSources) { - var isRunning = atomic.NewBool(true) + var isRunning = atomic.NewUint32(StatusRunning) tracker := tailers.NewTailerTracker() endpoints, _ := config.BuildEndpoints(coreConfig, config.HTTPConnectivityFailure, "test-track", "test-proto", "test-source") Init(isRunning, endpoints, sources, tracker, metrics.LogsExpvars) From ab27f5092ad56a4221596802c1ed2f151f09c3d3 Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Tue, 20 Aug 2024 17:54:51 +0200 Subject: [PATCH 059/245] Set devx-loops as codeowners of .vscode (#28595) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b0a076a22a41e..19bf75182061b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,6 +21,7 @@ /.golangci.yml @DataDog/agent-devx-loops /.custom-gcl.yml @DataDog/agent-devx-loops /.pre-commit-config.yaml @DataDog/agent-devx-loops +/.vscode/ @DataDog/agent-devx-loops /CHANGELOG.rst @DataDog/agent-delivery /CHANGELOG-DCA.rst @DataDog/container-integrations @DataDog/container-platform From fe75b815c2f135f0d2ea85d7a57a8fc8cbf56bd9 Mon Sep 17 00:00:00 2001 From: Amit Slavin <108348428+amitslavin@users.noreply.github.com> Date: Tue, 20 Aug 2024 19:32:34 +0300 Subject: [PATCH 060/245] [USM] Add UTs for Postgres decoding (#28597) --- .../protocols/postgres/pgx_testclient.go | 19 ++++ pkg/network/usm/postgres_monitor_test.go | 100 ++++++++++++++++-- 2 files changed, 110 insertions(+), 9 deletions(-) diff --git a/pkg/network/protocols/postgres/pgx_testclient.go b/pkg/network/protocols/postgres/pgx_testclient.go index 03dbb4c4d7ff0..c5a8372a096d3 100644 --- a/pkg/network/protocols/postgres/pgx_testclient.go +++ b/pkg/network/protocols/postgres/pgx_testclient.go @@ -12,6 +12,7 @@ import ( "errors" "time" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) @@ -44,6 +45,24 @@ func (c *PGXClient) Ping() error { return c.DB.Ping(ctx) } +// Begin starts a new transaction. +func (c *PGXClient) Begin() (pgx.Tx, error) { + if c.DB == nil { + return nil, errors.New("db handle is nil") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return c.DB.Begin(ctx) +} + +// Commit commits the transaction. +func (c *PGXClient) Commit(tx pgx.Tx) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return tx.Commit(ctx) +} + // Close closes the connection to the database. func (c *PGXClient) Close() { c.DB.Close() diff --git a/pkg/network/usm/postgres_monitor_test.go b/pkg/network/usm/postgres_monitor_test.go index 5e16cd3117472..f5e9e33e14c7a 100644 --- a/pkg/network/usm/postgres_monitor_test.go +++ b/pkg/network/usm/postgres_monitor_test.go @@ -34,15 +34,17 @@ import ( ) const ( - postgresPort = "5432" - repeatCount = ebpf.BufferSize / len("table_") - createTableQuery = "CREATE TABLE dummy (id SERIAL PRIMARY KEY, foo TEXT)" - updateSingleValueQuery = "UPDATE dummy SET foo = 'updated' WHERE id = 1" - selectAllQuery = "SELECT * FROM dummy" - dropTableQuery = "DROP TABLE IF EXISTS dummy" - deleteTableQuery = "DELETE FROM dummy WHERE id = 1" - alterTableQuery = "ALTER TABLE dummy ADD test VARCHAR(255);" - truncateTableQuery = "TRUNCATE TABLE dummy" + postgresPort = "5432" + repeatCount = ebpf.BufferSize / len("table_") + createTableQuery = "CREATE TABLE dummy (id SERIAL PRIMARY KEY, foo TEXT)" + updateSingleValueQuery = "UPDATE dummy SET foo = 'updated' WHERE id = 1" + selectAllQuery = "SELECT * FROM dummy" + selectParameterizedQuery = "SELECT * FROM dummy WHERE id = $1" + dropTableQuery = "DROP TABLE IF EXISTS dummy" + deleteTableQuery = "DELETE FROM dummy WHERE id = 1" + alterTableQuery = "ALTER TABLE dummy ADD test VARCHAR(255);" + truncateTableQuery = "TRUNCATE TABLE dummy" + showQuery = "SHOW search_path" ) var ( @@ -476,6 +478,86 @@ func testDecoding(t *testing.T, isTLS bool) { }, isTLS) }, }, + // This test validates that the SHOW command is currently not supported. + { + name: "show command", + preMonitorSetup: func(t *testing.T, ctx pgTestContext) { + pg, err := postgres.NewPGXClient(postgres.ConnectionOptions{ + ServerAddress: ctx.serverAddress, + EnableTLS: isTLS, + }) + require.NoError(t, err) + require.NoError(t, pg.Ping()) + ctx.extras["pg"] = pg + require.NoError(t, pg.RunQuery(createTableQuery)) + }, + postMonitorSetup: func(t *testing.T, ctx pgTestContext) { + pg := ctx.extras["pg"].(*postgres.PGXClient) + require.NoError(t, pg.RunQuery(showQuery)) + }, + validation: func(t *testing.T, _ pgTestContext, monitor *Monitor) { + validatePostgres(t, monitor, map[string]map[postgres.Operation]int{ + "UNKNOWN": { + postgres.UnknownOP: adjustCount(1), + }, + }, isTLS) + }, + }, + // This test validates that the sql transaction is not supported. + { + name: "transaction", + preMonitorSetup: func(t *testing.T, ctx pgTestContext) { + pg, err := postgres.NewPGXClient(postgres.ConnectionOptions{ + ServerAddress: ctx.serverAddress, + EnableTLS: isTLS, + }) + require.NoError(t, err) + require.NoError(t, pg.Ping()) + ctx.extras["pg"] = pg + + tx, err := pg.Begin() + require.NoError(t, err) + require.NoError(t, pg.RunQuery(createTableQuery)) + require.NoError(t, pg.Commit(tx)) + }, + postMonitorSetup: func(t *testing.T, ctx pgTestContext) { + pg := ctx.extras["pg"].(*postgres.PGXClient) + + tx, err := pg.Begin() + require.NoError(t, err) + require.NoError(t, pg.RunQuery(selectAllQuery)) + require.NoError(t, pg.Commit(tx)) + }, + validation: func(t *testing.T, _ pgTestContext, monitor *Monitor) { + validatePostgres(t, monitor, map[string]map[postgres.Operation]int{ + "UNKNOWN": { + postgres.UnknownOP: adjustCount(2), + }, + }, isTLS) + }, + }, + // This test validates that parameterized queries are currently not supported. + { + name: "parameterized select", + preMonitorSetup: func(t *testing.T, ctx pgTestContext) { + pg, err := postgres.NewPGXClient(postgres.ConnectionOptions{ + ServerAddress: ctx.serverAddress, + EnableTLS: isTLS, + }) + require.NoError(t, err) + require.NoError(t, pg.Ping()) + ctx.extras["pg"] = pg + require.NoError(t, pg.RunQuery(createTableQuery)) + require.NoError(t, pg.RunQuery(createInsertQuery("value-1"))) + }, + postMonitorSetup: func(t *testing.T, ctx pgTestContext) { + pg := ctx.extras["pg"].(*postgres.PGXClient) + require.NoError(t, pg.RunQuery(selectParameterizedQuery, "value-1")) + }, + validation: func(t *testing.T, _ pgTestContext, monitor *Monitor) { + validatePostgres(t, monitor, map[string]map[postgres.Operation]int{}, isTLS) + }, + }, } for _, tt := range tests { From 244b3414972e70d8348107f0b0d9d1685b2904bc Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Wed, 21 Aug 2024 08:57:21 +0200 Subject: [PATCH 061/245] discovery: Add E2E test (#28534) Co-authored-by: rarguelloF --- .github/CODEOWNERS | 1 + .../fixtures/servicediscovery_bytes | Bin 0 -> 409 bytes .../aggregator/servicediscoveryAggregator.go | 79 +++++++++++ .../servicediscoveryAggregator_test.go | 44 ++++++ test/fakeintake/client/client.go | 27 ++++ test/fakeintake/client/client_test.go | 14 ++ .../client/fixtures/api_v2_telemetry_response | 1 + test/new-e2e/tests/discovery/linux_test.go | 131 ++++++++++++++++++ .../testdata/config/agent_config.yaml | 1 + .../testdata/config/check_config.yaml | 1 + .../testdata/config/system_probe_config.yaml | 5 + .../discovery/testdata/provision/provision.sh | 38 +++++ .../testdata/provision/python/server.py | 40 ++++++ 13 files changed, 382 insertions(+) create mode 100644 test/fakeintake/aggregator/fixtures/servicediscovery_bytes create mode 100644 test/fakeintake/aggregator/servicediscoveryAggregator.go create mode 100644 test/fakeintake/aggregator/servicediscoveryAggregator_test.go create mode 100644 test/fakeintake/client/fixtures/api_v2_telemetry_response create mode 100644 test/new-e2e/tests/discovery/linux_test.go create mode 100644 test/new-e2e/tests/discovery/testdata/config/agent_config.yaml create mode 100644 test/new-e2e/tests/discovery/testdata/config/check_config.yaml create mode 100644 test/new-e2e/tests/discovery/testdata/config/system_probe_config.yaml create mode 100755 test/new-e2e/tests/discovery/testdata/provision/provision.sh create mode 100644 test/new-e2e/tests/discovery/testdata/provision/python/server.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 19bf75182061b..33bacdc1d977a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -592,6 +592,7 @@ /test/new-e2e/tests/agent-shared-components @DataDog/agent-shared-components /test/new-e2e/tests/agent-subcommands @DataDog/agent-shared-components /test/new-e2e/tests/containers @DataDog/container-integrations @DataDog/container-platform +/test/new-e2e/tests/discovery @DataDog/apm-onboarding @DataDog/universal-service-monitoring /test/new-e2e/tests/language-detection @DataDog/processes /test/new-e2e/tests/ndm @DataDog/network-device-monitoring /test/new-e2e/tests/npm @DataDog/Networks diff --git a/test/fakeintake/aggregator/fixtures/servicediscovery_bytes b/test/fakeintake/aggregator/fixtures/servicediscovery_bytes new file mode 100644 index 0000000000000000000000000000000000000000..91185f710001f452eb789c3a04c4c25cb621330e GIT binary patch literal 409 zcmV;K0cQRmiwFP!00000|J;&6Z=)~}h5yT(?Se^55~N=DPehUB;3=_$$6k-g3RV5@ zD=68dQC8Z^=Fkg*(R;)5`@DOHgg*!5tn(NV8sKbFmW=7-S_A#E)XA6FIng8I=gAJT364iR_n47&>PO05$Vr(TW_YLdF_aE8az@q`7=5N z!khG(!;sn~K@2G~G`m%`+6#!zLi4;<0@~ocA**{w2+abfF zk1O7ENcq9FZ?>ENRQpC%FXE*v)$}OO1XgNw4?&2zuqGQDIyu)+Ogui{&b6e0CNdGk zJ$)J${nPZH$AXI~w8$*K7r?r#t1E2PKVd_EKx@&?{BC=ZvVH5K$;a0Mb9>`lzMe+o z`&r8)#}p1I+dyxwyXWDNGkQ78F2?BW>tAljmqh2Avi}VL0RR6100960zmbw~T?GID D)KtrJ literal 0 HcmV?d00001 diff --git a/test/fakeintake/aggregator/servicediscoveryAggregator.go b/test/fakeintake/aggregator/servicediscoveryAggregator.go new file mode 100644 index 0000000000000..b834f08fc5156 --- /dev/null +++ b/test/fakeintake/aggregator/servicediscoveryAggregator.go @@ -0,0 +1,79 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package aggregator + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/DataDog/datadog-agent/test/fakeintake/api" +) + +// ServiceDiscoveryPayload is a payload type for the service_discovery check +type ServiceDiscoveryPayload struct { + collectedTime time.Time + + RequestType string `json:"request_type"` + APIVersion string `json:"api_version"` + Payload struct { + NamingSchemaVersion string `json:"naming_schema_version"` + ServiceName string `json:"service_name"` + HostName string `json:"host_name"` + Env string `json:"env"` + ServiceLanguage string `json:"service_language"` + ServiceType string `json:"service_type"` + StartTime int64 `json:"start_time"` + LastSeen int64 `json:"last_seen"` + APMInstrumentation string `json:"apm_instrumentation"` + ServiceNameSource string `json:"service_name_source"` + } `json:"payload"` +} + +func (s *ServiceDiscoveryPayload) name() string { + return s.RequestType +} + +// GetTags is not implemented. +func (s *ServiceDiscoveryPayload) GetTags() []string { + return nil +} + +// GetCollectedTime returns the time that the payload was received by the fake +// intake. +func (s *ServiceDiscoveryPayload) GetCollectedTime() time.Time { + return s.collectedTime +} + +// ParseServiceDiscoveryPayload parses an api.Payload into a list of +// ServiceDiscoveryPayload. +func ParseServiceDiscoveryPayload(payload api.Payload) ([]*ServiceDiscoveryPayload, error) { + enflated, err := enflate(payload.Data, payload.Encoding) + if err != nil { + return nil, fmt.Errorf("could not enflate payload: %w", err) + } + var payloads []*ServiceDiscoveryPayload + err = json.Unmarshal(enflated, &payloads) + if err != nil { + return nil, err + } + for _, p := range payloads { + p.collectedTime = payload.Timestamp + } + return payloads, nil +} + +// ServiceDiscoveryAggregator is an Aggregator for ServiceDiscoveryPayload. +type ServiceDiscoveryAggregator struct { + Aggregator[*ServiceDiscoveryPayload] +} + +// NewServiceDiscoveryAggregator returns a new ServiceDiscoveryAggregator. +func NewServiceDiscoveryAggregator() ServiceDiscoveryAggregator { + return ServiceDiscoveryAggregator{ + Aggregator: newAggregator(ParseServiceDiscoveryPayload), + } +} diff --git a/test/fakeintake/aggregator/servicediscoveryAggregator_test.go b/test/fakeintake/aggregator/servicediscoveryAggregator_test.go new file mode 100644 index 0000000000000..5dd6e4bcf2d7e --- /dev/null +++ b/test/fakeintake/aggregator/servicediscoveryAggregator_test.go @@ -0,0 +1,44 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package aggregator + +import ( + _ "embed" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/fakeintake/api" +) + +//go:embed fixtures/servicediscovery_bytes +var serviceDiscoveryData []byte + +func TestServiceDiscoveryAggregator(t *testing.T) { + t.Run("empty payload", func(t *testing.T) { + payloads, err := ParseServiceDiscoveryPayload(api.Payload{Data: []byte("")}) + assert.Error(t, err) + assert.Nil(t, payloads) + }) + + t.Run("malformed payload", func(t *testing.T) { + payloads, err := ParseServiceDiscoveryPayload(api.Payload{Data: []byte("not a payload")}) + assert.Error(t, err) + assert.Nil(t, payloads) + }) + + t.Run("valid payload", func(t *testing.T) { + payloads, err := ParseServiceDiscoveryPayload(api.Payload{Data: serviceDiscoveryData, Encoding: encodingGzip}) + require.NoError(t, err) + require.Len(t, payloads, 3) + + assert.Equal(t, "start-service", payloads[0].RequestType) + assert.Equal(t, "chronyd", payloads[0].Payload.ServiceName) + assert.Equal(t, "web_service", payloads[1].Payload.ServiceType) + assert.Equal(t, "ip-10-1-60-129", payloads[2].Payload.HostName) + }) +} diff --git a/test/fakeintake/client/client.go b/test/fakeintake/client/client.go index b114d7d24c84e..54df7e45a9df8 100644 --- a/test/fakeintake/client/client.go +++ b/test/fakeintake/client/client.go @@ -80,6 +80,7 @@ const ( orchestratorManifestEndpoint = "/api/v2/orchmanif" metadataEndpoint = "/api/v1/metadata" ndmflowEndpoint = "/api/v2/ndmflow" + apmTelemetryEndpoint = "/api/v2/apmtelemetry" ) // ErrNoFlareAvailable is returned when no flare is available @@ -118,6 +119,7 @@ type Client struct { orchestratorManifestAggregator aggregator.OrchestratorManifestAggregator metadataAggregator aggregator.MetadataAggregator ndmflowAggregator aggregator.NDMFlowAggregator + serviceDiscoveryAggregator aggregator.ServiceDiscoveryAggregator } // NewClient creates a new fake intake client @@ -143,6 +145,7 @@ func NewClient(fakeIntakeURL string, opts ...Option) *Client { orchestratorManifestAggregator: aggregator.NewOrchestratorManifestAggregator(), metadataAggregator: aggregator.NewMetadataAggregator(), ndmflowAggregator: aggregator.NewNDMFlowAggregator(), + serviceDiscoveryAggregator: aggregator.NewServiceDiscoveryAggregator(), } for _, opt := range opts { opt(client) @@ -319,6 +322,14 @@ func (c *Client) FilterContainerImages(name string, options ...MatchOpt[*aggrega return filterPayload(images, options...) } +func (c *Client) getServiceDiscoveries() error { + payloads, err := c.getFakePayloads(apmTelemetryEndpoint) + if err != nil { + return err + } + return c.serviceDiscoveryAggregator.UnmarshallPayloads(payloads) +} + // GetLatestFlare queries the Fake Intake to fetch flares that were sent by a Datadog Agent and returns the latest flare as a Flare struct // TODO: handle multiple flares / flush when returning latest flare func (c *Client) GetLatestFlare() (flare.Flare, error) { @@ -886,3 +897,19 @@ func filterPayload[T aggregator.PayloadItem](payloads []T, options ...MatchOpt[T } return filteredPayloads, nil } + +// GetServiceDiscoveries fetches fakeintake on `api/v2/apmtelemetry` endpoint and returns +// all received service discovery payloads +func (c *Client) GetServiceDiscoveries() ([]*aggregator.ServiceDiscoveryPayload, error) { + err := c.getServiceDiscoveries() + if err != nil { + return nil, err + } + + names := c.serviceDiscoveryAggregator.GetNames() + payloads := make([]*aggregator.ServiceDiscoveryPayload, 0, len(names)) + for _, name := range names { + payloads = append(payloads, c.serviceDiscoveryAggregator.GetPayloadsByName(name)...) + } + return payloads, nil +} diff --git a/test/fakeintake/client/client_test.go b/test/fakeintake/client/client_test.go index a5c4e17af95ec..bf023eeb3687f 100644 --- a/test/fakeintake/client/client_test.go +++ b/test/fakeintake/client/client_test.go @@ -56,6 +56,9 @@ var apiV1Metadata []byte //go:embed fixtures/api_v2_ndmflow_response var apiV2NDMFlow []byte +//go:embed fixtures/api_v2_telemetry_response +var apiV2Teleemtry []byte + func NewServer(handler http.Handler) *httptest.Server { handlerWitHeader := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Fakeintake-ID", "20000000-0000-0000-0000-000000000000") @@ -566,6 +569,17 @@ func TestClient(t *testing.T) { assert.Empty(t, ndmflows[0].AdditionalFields) }) + t.Run("getServiceDiscoveries", func(t *testing.T) { + ts := NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write(apiV2Teleemtry) + })) + defer ts.Close() + + client := NewClient(ts.URL) + err := client.getServiceDiscoveries() + require.NoError(t, err) + }) + t.Run("test strict fakeintakeid check mode", func(t *testing.T) { defer func() { if r := recover(); r != nil { diff --git a/test/fakeintake/client/fixtures/api_v2_telemetry_response b/test/fakeintake/client/fixtures/api_v2_telemetry_response new file mode 100644 index 0000000000000..ea66c9662a93a --- /dev/null +++ b/test/fakeintake/client/fixtures/api_v2_telemetry_response @@ -0,0 +1 @@ +{"payloads":[{"timestamp":"2024-08-20T13:52:51.271230017Z","data":"H4sIAAAAAAAA/9ySQW+jMBCF/8uc7YJJSxKkXvdPRJHl4CmxhMdej8kKVf3vKyjZpFFW2svm0AuC0XvD8/u8e4eEPwfkrPMYERrgbFKWjOnkWgQBJjp9wsQuEDRwqkBANGMfjIXmHch4R53m9ojeXOkUCFh2aDJ+WtweU6DRgoBj4HyeuihVKZWsS6mqLQhAOkEDV/beUDeYDr9Ol7S/8KAvWefsOrtps1pXz6pW67IS0BvOmhHpz3hbb6ajee2Icxo8Ujb5MzoFwpv0msOQ2iVBDCkzNLtVtdoLiM5C87xWAtrgvSGre0cIzQ6KgVPBB0fF5eTyBwhQsP/4EI8r3nRI+cG1b7ab/1T7RlUv4qUs1fQozwRWanUPQYi5sCYbGzo511BMPD7fzrWkgSYyEcQ9fRoW/dP0n8eCiym0yCy/F8C6rKsrbNU/YUN/QGvRzvxue5GyfeuiycfXAnN741y+nkbj+1nLI8cUDijbQG+uu+fhkTN6Ocsuxujs619uyJdE55uy/w0AAP//AQAA//+/kZJwXQUAAA==","encoding":"gzip","content_type":"application/json"}]} \ No newline at end of file diff --git a/test/new-e2e/tests/discovery/linux_test.go b/test/new-e2e/tests/discovery/linux_test.go new file mode 100644 index 0000000000000..7ccd32b4a33da --- /dev/null +++ b/test/new-e2e/tests/discovery/linux_test.go @@ -0,0 +1,131 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package discovery + +import ( + _ "embed" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" +) + +//go:embed testdata/config/agent_config.yaml +var agentConfigStr string + +//go:embed testdata/config/system_probe_config.yaml +var systemProbeConfigStr string + +//go:embed testdata/config/check_config.yaml +var checkConfigStr string + +type linuxTestSuite struct { + e2e.BaseSuite[environments.Host] +} + +func TestLinuxTestSuite(t *testing.T) { + agentParams := []func(*agentparams.Params) error{ + agentparams.WithAgentConfig(agentConfigStr), + agentparams.WithSystemProbeConfig(systemProbeConfigStr), + agentparams.WithFile("/etc/datadog-agent/conf.d/service_discovery.d/conf.yaml", checkConfigStr, true), + } + options := []e2e.SuiteOption{ + e2e.WithProvisioner(awshost.Provisioner(awshost.WithAgentOptions(agentParams...))), + } + e2e.Run(t, &linuxTestSuite{}, options...) +} + +func (s *linuxTestSuite) SetupSuite() { + s.BaseSuite.SetupSuite() + + s.provisionServer() +} + +func (s *linuxTestSuite) TestServiceDiscoveryCheck() { + t := s.T() + s.startServices() + defer s.stopServices() + + client := s.Env().FakeIntake.Client() + err := client.FlushServerAndResetAggregators() + require.NoError(t, err) + + assert.EventuallyWithT(t, func(t *assert.CollectT) { + assertRunningCheck(t, s.Env().RemoteHost, "service_discovery") + }, 2*time.Minute, 10*time.Second) + + // This is very useful for debugging, but we probably don't want to decode + // and assert based on this in this E2E test since this is an internal + // interface between the agent and system-probe. + services := s.Env().RemoteHost.MustExecute("sudo curl -s --unix /opt/datadog-agent/run/sysprobe.sock http://unix/discovery/services") + t.Log("system-probe services", services) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + payloads, err := client.GetServiceDiscoveries() + require.NoError(t, err) + + found := false + for _, p := range payloads { + name := p.Payload.ServiceName + t.Log("RequestType", p.RequestType, "ServiceName", name) + + if p.RequestType == "start-service" && name == "python" { + found = true + break + } + } + + assert.True(c, found, "service not found") + }, 3*time.Minute, 10*time.Second) +} + +type checkStatus struct { + CheckID string `json:"CheckID"` + CheckName string `json:"CheckName"` + CheckConfigSource string `json:"CheckConfigSource"` + ExecutionTimes []int `json:"ExecutionTimes"` +} + +type runnerStats struct { + Checks map[string]checkStatus `json:"Checks"` +} + +type collectorStatus struct { + RunnerStats runnerStats `json:"runnerStats"` +} + +// assertRunningCheck asserts that the given process agent check is running +func assertRunningCheck(t *assert.CollectT, remoteHost *components.RemoteHost, check string) { + statusOutput := remoteHost.MustExecute("sudo datadog-agent status collector --json") + + var status collectorStatus + err := json.Unmarshal([]byte(statusOutput), &status) + require.NoError(t, err, "failed to unmarshal agent status") + + assert.Contains(t, status.RunnerStats.Checks, check) +} + +func (s *linuxTestSuite) provisionServer() { + err := s.Env().RemoteHost.CopyFolder("testdata/provision", "/home/ubuntu/e2e-test") + require.NoError(s.T(), err) + s.Env().RemoteHost.MustExecute("sudo bash /home/ubuntu/e2e-test/provision.sh") +} + +func (s *linuxTestSuite) startServices() { + s.Env().RemoteHost.MustExecute("sudo systemctl start python-svc") +} + +func (s *linuxTestSuite) stopServices() { + s.Env().RemoteHost.MustExecute("sudo systemctl stop python-svc") +} diff --git a/test/new-e2e/tests/discovery/testdata/config/agent_config.yaml b/test/new-e2e/tests/discovery/testdata/config/agent_config.yaml new file mode 100644 index 0000000000000..5493a6ba85f72 --- /dev/null +++ b/test/new-e2e/tests/discovery/testdata/config/agent_config.yaml @@ -0,0 +1 @@ +log_level: DEBUG diff --git a/test/new-e2e/tests/discovery/testdata/config/check_config.yaml b/test/new-e2e/tests/discovery/testdata/config/check_config.yaml new file mode 100644 index 0000000000000..acab3a6421cab --- /dev/null +++ b/test/new-e2e/tests/discovery/testdata/config/check_config.yaml @@ -0,0 +1 @@ +instances: [{}] diff --git a/test/new-e2e/tests/discovery/testdata/config/system_probe_config.yaml b/test/new-e2e/tests/discovery/testdata/config/system_probe_config.yaml new file mode 100644 index 0000000000000..ee8dd5392b0dd --- /dev/null +++ b/test/new-e2e/tests/discovery/testdata/config/system_probe_config.yaml @@ -0,0 +1,5 @@ +log_level: 'debug' +system_probe_config: + enabled: true +discovery: + enabled: true diff --git a/test/new-e2e/tests/discovery/testdata/provision/provision.sh b/test/new-e2e/tests/discovery/testdata/provision/provision.sh new file mode 100755 index 0000000000000..43e7138ea4f4c --- /dev/null +++ b/test/new-e2e/tests/discovery/testdata/provision/provision.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -e + +apt-get update +apt-get install -y ca-certificates curl gnupg python3 + +# Install our own services +install_systemd_unit () { + name=$1 + command=$2 + port=$3 + + cat > "/etc/systemd/system/${name}.service" <<- EOM +[Unit] +Description=${name} +After=network.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=1 +User=root +ExecStart=${command} +Environment="PORT=${port}" + +[Install] +WantedBy=multi-user.target +EOM +} + +install_systemd_unit "python-svc" "/usr/bin/python3 /home/ubuntu/e2e-test/python/server.py" "8082" + +systemctl daemon-reload + +# leave them stopped +systemctl stop python-svc diff --git a/test/new-e2e/tests/discovery/testdata/provision/python/server.py b/test/new-e2e/tests/discovery/testdata/provision/python/server.py new file mode 100644 index 0000000000000..b739ff1e5ba1e --- /dev/null +++ b/test/new-e2e/tests/discovery/testdata/provision/python/server.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import os +from http.server import BaseHTTPRequestHandler, HTTPServer + + +class Handler(BaseHTTPRequestHandler): + def _set_response(self): + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + + def do_GET(self): + self._set_response() + self.wfile.write(f"GET request for {self.path}".encode()) + + def do_POST(self): + self._set_response() + self.wfile.write(f"POST request for {self.path}".encode()) + + +def run(): + host = '0.0.0.0' + port = 8080 + if 'PORT' in os.environ: + port = int(os.environ['PORT']) + + addr = (host, port) + server = HTTPServer(addr, Handler) + + print(f"Server is running on http://{host}:{port}") + try: + server.serve_forever() + except KeyboardInterrupt: + pass + server.server_close() + + +if __name__ == '__main__': + run() From 5e445a67f6fb49129ee45d4e3d6b74c6d7025c88 Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Wed, 21 Aug 2024 09:04:42 +0200 Subject: [PATCH 062/245] upgrade(fleet): Plug rudimentary RC client for policies (#28482) Co-authored-by: arbll --- .../rctelemetryreporter.go | 8 +- pkg/fleet/installer/service/datadog_agent.go | 58 ++++--- pkg/fleet/internal/cdn/cdn.go | 157 +++++++++++++++--- pkg/fleet/internal/cdn/config.go | 89 ++++++++-- pkg/fleet/internal/cdn/config_test.go | 20 +-- pkg/fleet/internal/cdn/merge.go | 12 +- pkg/fleet/internal/cdn/merge_test.go | 36 ++-- 7 files changed, 282 insertions(+), 98 deletions(-) diff --git a/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl/rctelemetryreporter.go b/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl/rctelemetryreporter.go index 476af2e258eae..34c2658f07c3d 100644 --- a/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl/rctelemetryreporter.go +++ b/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl/rctelemetryreporter.go @@ -35,12 +35,16 @@ type DdRcTelemetryReporter struct { // IncRateLimit increments the DdRcTelemetryReporter BypassRateLimitCounter counter. func (r *DdRcTelemetryReporter) IncRateLimit() { - r.BypassRateLimitCounter.Inc() + if r.BypassRateLimitCounter != nil { + r.BypassRateLimitCounter.Inc() + } } // IncTimeout increments the DdRcTelemetryReporter BypassTimeoutCounter counter. func (r *DdRcTelemetryReporter) IncTimeout() { - r.BypassTimeoutCounter.Inc() + if r.BypassTimeoutCounter != nil { + r.BypassTimeoutCounter.Inc() + } } // newDdRcTelemetryReporter creates a new Remote Config telemetry reporter for sending RC metrics to Datadog diff --git a/pkg/fleet/installer/service/datadog_agent.go b/pkg/fleet/installer/service/datadog_agent.go index feddc3ddb6347..cb7ff98380d9b 100644 --- a/pkg/fleet/installer/service/datadog_agent.go +++ b/pkg/fleet/installer/service/datadog_agent.go @@ -24,20 +24,22 @@ import ( ) const ( - agentPackage = "datadog-agent" - pathOldAgent = "/opt/datadog-agent" - agentSymlink = "/usr/bin/datadog-agent" - agentUnit = "datadog-agent.service" - traceAgentUnit = "datadog-agent-trace.service" - processAgentUnit = "datadog-agent-process.service" - systemProbeUnit = "datadog-agent-sysprobe.service" - securityAgentUnit = "datadog-agent-security.service" - agentExp = "datadog-agent-exp.service" - traceAgentExp = "datadog-agent-trace-exp.service" - processAgentExp = "datadog-agent-process-exp.service" - systemProbeExp = "datadog-agent-sysprobe-exp.service" - securityAgentExp = "datadog-agent-security-exp.service" - configDatadogYAML = "datadog.yaml" + agentPackage = "datadog-agent" + pathOldAgent = "/opt/datadog-agent" + agentSymlink = "/usr/bin/datadog-agent" + agentUnit = "datadog-agent.service" + traceAgentUnit = "datadog-agent-trace.service" + processAgentUnit = "datadog-agent-process.service" + systemProbeUnit = "datadog-agent-sysprobe.service" + securityAgentUnit = "datadog-agent-security.service" + agentExp = "datadog-agent-exp.service" + traceAgentExp = "datadog-agent-trace-exp.service" + processAgentExp = "datadog-agent-process-exp.service" + systemProbeExp = "datadog-agent-sysprobe-exp.service" + securityAgentExp = "datadog-agent-security-exp.service" + configDatadogYAML = "datadog.yaml" + configSecurityAgentYAML = "security-agent.yaml" + configSystemProbeYAML = "system-probe.yaml" ) var ( @@ -242,14 +244,30 @@ func ConfigureAgent(ctx context.Context, cdn *cdn.CDN, configs *repository.Repos if err != nil { return fmt.Errorf("error getting dd-agent user and group IDs: %w", err) } - err = os.WriteFile(filepath.Join(tmpDir, configDatadogYAML), []byte(config.Datadog), 0640) - if err != nil { - return fmt.Errorf("could not write datadog.yaml: %w", err) + + if config.Datadog != nil { + err = os.WriteFile(filepath.Join(tmpDir, configDatadogYAML), []byte(config.Datadog), 0640) + if err != nil { + return fmt.Errorf("could not write datadog.yaml: %w", err) + } + err = os.Chown(filepath.Join(tmpDir, configDatadogYAML), ddAgentUID, ddAgentGID) + if err != nil { + return fmt.Errorf("could not chown datadog.yaml: %w", err) + } } - err = os.Chown(filepath.Join(tmpDir, configDatadogYAML), ddAgentUID, ddAgentGID) - if err != nil { - return fmt.Errorf("could not chown datadog.yaml: %w", err) + if config.SecurityAgent != nil { + err = os.WriteFile(filepath.Join(tmpDir, configSecurityAgentYAML), []byte(config.SecurityAgent), 0600) + if err != nil { + return fmt.Errorf("could not write datadog.yaml: %w", err) + } + } + if config.SystemProbe != nil { + err = os.WriteFile(filepath.Join(tmpDir, configSystemProbeYAML), []byte(config.SystemProbe), 0600) + if err != nil { + return fmt.Errorf("could not write datadog.yaml: %w", err) + } } + err = configs.Create(agentPackage, config.Version, tmpDir) if err != nil { return fmt.Errorf("could not create repository: %w", err) diff --git a/pkg/fleet/internal/cdn/cdn.go b/pkg/fleet/internal/cdn/cdn.go index 0d159b9fcebb5..53d93568777b4 100644 --- a/pkg/fleet/internal/cdn/cdn.go +++ b/pkg/fleet/internal/cdn/cdn.go @@ -8,13 +8,30 @@ package cdn import ( "context" - "crypto/sha256" + "encoding/json" "fmt" + "regexp" + "time" + "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/hosttags" + "github.com/DataDog/datadog-agent/comp/remote-config/rctelemetryreporter/rctelemetryreporterimpl" + detectenv "github.com/DataDog/datadog-agent/pkg/config/env" + "github.com/DataDog/datadog-agent/pkg/config/model" + remoteconfig "github.com/DataDog/datadog-agent/pkg/config/remote/service" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/fleet/env" + pbgo "github.com/DataDog/datadog-agent/pkg/proto/pbgo/core" + pkghostname "github.com/DataDog/datadog-agent/pkg/util/hostname" + "github.com/DataDog/datadog-agent/pkg/version" + "github.com/google/uuid" + "go.uber.org/multierr" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) +var datadogConfigIDRegexp = regexp.MustCompile(`^datadog/\d+/AGENT_CONFIG/([^/]+)/[^/]+$`) + +const configOrderID = "configuration_order" + // CDN provides access to the Remote Config CDN. type CDN struct { env *env.Env @@ -28,6 +45,10 @@ type Config struct { SystemProbe []byte } +type orderConfig struct { + Order []string `json:"order"` +} + // New creates a new CDN. func New(env *env.Env) *CDN { return &CDN{ @@ -36,38 +57,124 @@ func New(env *env.Env) *CDN { } // Get gets the configuration from the CDN. -func (c *CDN) Get(ctx context.Context) (_ Config, err error) { +func (c *CDN) Get(ctx context.Context) (_ *Config, err error) { span, ctx := tracer.StartSpanFromContext(ctx, "cdn.Get") defer func() { span.Finish(tracer.WithError(err)) }() - return getFakeCDNConfig(ctx) + configLayers, err := c.getOrderedLayers(ctx) + if err != nil { + return nil, err + } + return newConfig(configLayers...) } -// HACK (arthur): this is a temporary function that returns a fake CDN config to unblock development. -func getFakeCDNConfig(_ context.Context) (Config, error) { - baseLayer := layer{ - ID: "base", - Content: map[interface{}]interface{}{ - "extra_tags": []string{"layer:base"}, - }, +// getOrderedLayers calls the Remote Config service to get the ordered layers. +// Today it doesn't use the CDN, but it should in the future +func (c *CDN) getOrderedLayers(ctx context.Context) ([]*layer, error) { + // HACK(baptiste): Create a dedicated one-shot RC service just for the configuration + // We should use the CDN instead + config := pkgconfigsetup.Datadog() + config.Set("run_path", "/opt/datadog-packages/datadog-installer/stable/run", model.SourceAgentRuntime) + + detectenv.DetectFeatures(config) + hostname, err := pkghostname.Get(ctx) + if err != nil { + return nil, err } - overrideLayer := layer{ - ID: "override", - Content: map[interface{}]interface{}{ - "extra_tags": []string{"layer:override"}, - }, + options := []remoteconfig.Option{ + remoteconfig.WithAPIKey(c.env.APIKey), + remoteconfig.WithConfigRootOverride(c.env.Site, ""), + remoteconfig.WithDirectorRootOverride(c.env.Site, ""), } - config, err := newConfig(baseLayer, overrideLayer) + service, err := remoteconfig.NewService( + config, + "Datadog Installer", + fmt.Sprintf("https://config.%s", c.env.Site), + hostname, + getHostTags(ctx, config), + &rctelemetryreporterimpl.DdRcTelemetryReporter{}, // No telemetry for this client + version.AgentVersion, + options..., + ) if err != nil { - return Config{}, err + return nil, err } - serializedConfig, err := config.Marshal() + service.Start() + defer func() { _ = service.Stop() }() + // Force a cache bypass + cfgs, err := service.ClientGetConfigs(ctx, &pbgo.ClientGetConfigsRequest{ + Client: &pbgo.Client{ + Id: uuid.New().String(), + Products: []string{"AGENT_CONFIG"}, + IsUpdater: true, + ClientUpdater: &pbgo.ClientUpdater{}, + State: &pbgo.ClientState{ + RootVersion: 1, + TargetsVersion: 1, + }, + }, + }) if err != nil { - return Config{}, err + return nil, err + } + + // Unmarshal RC results + configLayers := map[string]*layer{} + var configOrder *orderConfig + var layersErr error + for _, file := range cfgs.TargetFiles { + matched := datadogConfigIDRegexp.FindStringSubmatch(file.GetPath()) + if len(matched) != 2 { + layersErr = multierr.Append(layersErr, fmt.Errorf("invalid config path: %s", file.GetPath())) + continue + } + configName := matched[1] + + if configName != configOrderID { + configLayer := &layer{} + err = json.Unmarshal(file.GetRaw(), configLayer) + if err != nil { + // If a layer is wrong, fail later to parse the rest and check them all + layersErr = multierr.Append(layersErr, err) + continue + } + configLayers[configName] = configLayer + } else { + configOrder = &orderConfig{} + err = json.Unmarshal(file.GetRaw(), configOrder) + if err != nil { + // Return first - we can't continue without the order + return nil, err + } + } + } + if layersErr != nil { + return nil, layersErr + } + + // Order configs + if configOrder == nil { + return nil, fmt.Errorf("no configuration_order found") + } + orderedLayers := []*layer{} + for _, configName := range configOrder.Order { + if configLayer, ok := configLayers[configName]; ok { + orderedLayers = append(orderedLayers, configLayer) + } + } + + return orderedLayers, nil +} + +func getHostTags(ctx context.Context, config model.Config) func() []string { + return func() []string { + // Host tags are cached on host, but we add a timeout to avoid blocking the RC request + // if the host tags are not available yet and need to be fetched. They will be fetched + // by the first agent metadata V5 payload. + ctx, cc := context.WithTimeout(ctx, time.Second) + defer cc() + hostTags := hosttags.Get(ctx, true, config) + tags := append(hostTags.System, hostTags.GoogleCloudPlatform...) + tags = append(tags, "installer:true") + return tags } - hash := sha256.New() - hash.Write(serializedConfig) - return Config{ - Version: fmt.Sprintf("%x", hash.Sum(nil)), - Datadog: serializedConfig, - }, nil } diff --git a/pkg/fleet/internal/cdn/config.go b/pkg/fleet/internal/cdn/config.go index a4feb97ac914a..deb83163fcf9b 100644 --- a/pkg/fleet/internal/cdn/config.go +++ b/pkg/fleet/internal/cdn/config.go @@ -7,6 +7,9 @@ package cdn import ( "bytes" + "crypto/sha256" + "encoding/json" + "fmt" "gopkg.in/yaml.v2" ) @@ -16,33 +19,85 @@ const ( doNotEditDisclaimer = `# This configuration was generated by Datadog's Fleet Automation. DO NOT EDIT.` ) -// config is a configuration for the package. -type config map[interface{}]interface{} - // layer is a config layer that can be merged with other layers into a config. type layer struct { - ID string - Content map[interface{}]interface{} + ID string `json:"name"` + AgentConfig map[string]interface{} `json:"config"` + SecurityAgentConfig map[string]interface{} `json:"security_agent"` + SystemProbeConfig map[string]interface{} `json:"system_probe"` } // newConfig creates a new config from a list of layers. -func newConfig(layers ...layer) (config, error) { - config := make(map[interface{}]interface{}) - var layerIDs []string +func newConfig(layers ...*layer) (_ *Config, err error) { + layerIDs := []string{} + mergedLayer := &layer{ + AgentConfig: map[string]interface{}{}, + SecurityAgentConfig: map[string]interface{}{}, + SystemProbeConfig: map[string]interface{}{}, + } + + // Merge all layers in order for _, l := range layers { - merged, err := merge(config, l.Content) - if err != nil { - return nil, err - } - config = merged.(map[interface{}]interface{}) layerIDs = append(layerIDs, l.ID) + if l.AgentConfig != nil { + agentConfig, err := merge(mergedLayer.AgentConfig, l.AgentConfig) + if err != nil { + return nil, err + } + mergedLayer.AgentConfig = agentConfig.(map[string]interface{}) + } + + if l.SecurityAgentConfig != nil { + securityAgentConfig, err := merge(mergedLayer.SecurityAgentConfig, l.SecurityAgentConfig) + if err != nil { + return nil, err + } + mergedLayer.SecurityAgentConfig = securityAgentConfig.(map[string]interface{}) + } + + if l.SystemProbeConfig != nil { + systemProbeAgentConfig, err := merge(mergedLayer.SystemProbeConfig, l.SystemProbeConfig) + if err != nil { + return nil, err + } + mergedLayer.SystemProbeConfig = systemProbeAgentConfig.(map[string]interface{}) + } + } + mergedLayer.AgentConfig[layerKeys] = layerIDs // Add a field with the applied layers that will be reported through inventories + + serializedAgentConfig, err := marshalConfig(mergedLayer.AgentConfig) + if err != nil { + return nil, err } - config[layerKeys] = layerIDs - return config, nil + serializedSecurityAgentConfig, err := marshalConfig(mergedLayer.SecurityAgentConfig) + if err != nil { + return nil, err + } + serializedSystemProbeConfig, err := marshalConfig(mergedLayer.SystemProbeConfig) + if err != nil { + return nil, err + } + + hash := sha256.New() + serializedConfig, err := json.Marshal(mergedLayer) + if err != nil { + return nil, err + } + hash.Write(serializedConfig) + + return &Config{ + Version: fmt.Sprintf("%x", hash.Sum(nil)), + Datadog: serializedAgentConfig, + SecurityAgent: serializedSecurityAgentConfig, + SystemProbe: serializedSystemProbeConfig, + }, nil } -// Marshal marshals the config as YAML. -func (c *config) Marshal() ([]byte, error) { +// marshalConfig marshals the config as YAML. +func marshalConfig(c map[string]interface{}) ([]byte, error) { + if len(c) == 0 { + return nil, nil + } var b bytes.Buffer b.WriteString(doNotEditDisclaimer) b.WriteString("\n") diff --git a/pkg/fleet/internal/cdn/config_test.go b/pkg/fleet/internal/cdn/config_test.go index 697fc6b0f0481..cbe1140d9e935 100644 --- a/pkg/fleet/internal/cdn/config_test.go +++ b/pkg/fleet/internal/cdn/config_test.go @@ -12,30 +12,28 @@ import ( ) func TestConfig(t *testing.T) { - baseLayer := layer{ + baseLayer := &layer{ ID: "base", - Content: map[interface{}]interface{}{ + AgentConfig: map[string]interface{}{ "api_key": "1234", - "apm": map[interface{}]interface{}{ + "apm": map[string]interface{}{ "enabled": true, "sampling_rate": 0.5, }, }, } - overrideLayer := layer{ + overrideLayer := &layer{ ID: "override", - Content: map[interface{}]interface{}{ - "apm": map[interface{}]interface{}{ + AgentConfig: map[string]interface{}{ + "apm": map[string]interface{}{ "sampling_rate": 0.7, "env": "prod", }, }, } config, err := newConfig(baseLayer, overrideLayer) - assert.NoError(t, err) - serializedConfig, err := config.Marshal() - assert.NoError(t, err) - exprectedConfig := doNotEditDisclaimer + ` + assert.Nil(t, err) + expectedConfig := doNotEditDisclaimer + ` __fleet_layers: - base - override @@ -45,5 +43,5 @@ apm: env: prod sampling_rate: 0.7 ` - assert.Equal(t, exprectedConfig, string(serializedConfig)) + assert.Equal(t, expectedConfig, string(config.Datadog)) } diff --git a/pkg/fleet/internal/cdn/merge.go b/pkg/fleet/internal/cdn/merge.go index 7f38e84fc281e..05a4b1a08b01b 100644 --- a/pkg/fleet/internal/cdn/merge.go +++ b/pkg/fleet/internal/cdn/merge.go @@ -5,7 +5,9 @@ package cdn -import "fmt" +import ( + "fmt" +) func isList(i interface{}) bool { _, ok := i.([]interface{}) @@ -13,7 +15,7 @@ func isList(i interface{}) bool { } func isMap(i interface{}) bool { - _, ok := i.(map[interface{}]interface{}) + _, ok := i.(map[string]interface{}) return ok } @@ -42,13 +44,13 @@ func merge(base interface{}, override interface{}) (interface{}, error) { return override, nil } if isMap(base) && isMap(override) { - return mergeMap(base.(map[interface{}]interface{}), override.(map[interface{}]interface{})) + return mergeMap(base.(map[string]interface{}), override.(map[string]interface{})) } return nil, fmt.Errorf("could not merge %T with %T", base, override) } -func mergeMap(base, override map[interface{}]interface{}) (map[interface{}]interface{}, error) { - merged := make(map[interface{}]interface{}) +func mergeMap(base, override map[string]interface{}) (map[string]interface{}, error) { + merged := make(map[string]interface{}) for k, v := range base { merged[k] = v } diff --git a/pkg/fleet/internal/cdn/merge_test.go b/pkg/fleet/internal/cdn/merge_test.go index e066082dc5953..8af4b7f81d42b 100644 --- a/pkg/fleet/internal/cdn/merge_test.go +++ b/pkg/fleet/internal/cdn/merge_test.go @@ -52,7 +52,7 @@ func TestMergeScalar(t *testing.T) { { name: "scalar and map error", base: "base", - override: map[interface{}]interface{}{"key": "value"}, + override: map[string]interface{}{"key": "value"}, expectedErr: true, }, } @@ -113,7 +113,7 @@ func TestMergeMap(t *testing.T) { }{ { name: "nil override", - base: map[interface{}]interface{}{ + base: map[string]interface{}{ "base": "value", }, override: nil, @@ -121,56 +121,56 @@ func TestMergeMap(t *testing.T) { }, { name: "override", - base: map[interface{}]interface{}{ + base: map[string]interface{}{ "base": "value", }, - override: map[interface{}]interface{}{ + override: map[string]interface{}{ "base": "override", }, - expected: map[interface{}]interface{}{ + expected: map[string]interface{}{ "base": "override", }, }, { name: "add key", - base: map[interface{}]interface{}{ + base: map[string]interface{}{ "base": "value", }, - override: map[interface{}]interface{}{ + override: map[string]interface{}{ "override": "value", }, - expected: map[interface{}]interface{}{ + expected: map[string]interface{}{ "base": "value", "override": "value", }, }, { name: "nested", - base: map[interface{}]interface{}{ - "base": map[interface{}]interface{}{ + base: map[string]interface{}{ + "base": map[string]interface{}{ "key": "value", }, }, - override: map[interface{}]interface{}{ - "base": map[interface{}]interface{}{ + override: map[string]interface{}{ + "base": map[string]interface{}{ "key": "override", }, }, - expected: map[interface{}]interface{}{ - "base": map[interface{}]interface{}{ + expected: map[string]interface{}{ + "base": map[string]interface{}{ "key": "override", }, }, }, { name: "nested scalar and list error", - base: map[interface{}]interface{}{ - "base": map[interface{}]interface{}{ + base: map[string]interface{}{ + "base": map[string]interface{}{ "key": []interface{}{"value"}, }, }, - override: map[interface{}]interface{}{ - "base": map[interface{}]interface{}{ + override: map[string]interface{}{ + "base": map[string]interface{}{ "key": "override", }, }, From f4221181b4f1b3ac29309b4034ce552803973ba1 Mon Sep 17 00:00:00 2001 From: "agent-platform-auto-pr[bot]" <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 08:17:31 +0000 Subject: [PATCH 063/245] [test-infra-definitions][automated] Bump test-infra-definitions to 926ba0f10de75fab94fcc4653a03a0cd96d3c21f (#28607) Co-authored-by: agent-platform-auto-pr[bot] <153269286+agent-platform-auto-pr[bot]@users.noreply.github.com> --- .gitlab/common/test_infra_version.yml | 2 +- test/new-e2e/go.mod | 20 +++++++------- test/new-e2e/go.sum | 40 +++++++++++++-------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index fa31a5ecb408a..93b5c726ec1f8 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -4,4 +4,4 @@ variables: # and check the job creating the image to make sure you have the right SHA prefix TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 94712f9a273b + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 926ba0f10de7 diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index e0d21c495adbf..7d95821f79f00 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -32,8 +32,8 @@ require ( // `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version // Example: github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB // => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB - github.com/DataDog/test-infra-definitions v0.0.0-20240808172947-94712f9a273b - github.com/aws/aws-sdk-go-v2 v1.30.3 + github.com/DataDog/test-infra-definitions v0.0.0-20240820173820-926ba0f10de7 + github.com/aws/aws-sdk-go-v2 v1.30.4 github.com/aws/aws-sdk-go-v2/config v1.27.19 github.com/aws/aws-sdk-go-v2/service/ec2 v1.164.2 github.com/aws/aws-sdk-go-v2/service/eks v1.44.1 @@ -49,9 +49,9 @@ require ( github.com/pulumi/pulumi-aws/sdk/v6 v6.44.0 github.com/pulumi/pulumi-awsx/sdk/v2 v2.13.0 github.com/pulumi/pulumi-eks/sdk/v2 v2.7.6 - github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1 - github.com/pulumi/pulumi/sdk/v3 v3.126.0 - github.com/samber/lo v1.39.0 + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.17.1 + github.com/pulumi/pulumi/sdk/v3 v3.128.0 + github.com/samber/lo v1.47.0 github.com/sethvargo/go-retry v0.2.4 github.com/stretchr/testify v1.9.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -88,11 +88,11 @@ require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.19 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ecr v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ecr v1.32.1 // indirect github.com/aws/aws-sdk-go-v2/service/ecs v1.42.1 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.14 // indirect @@ -102,7 +102,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.20.12 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.28.13 // indirect - github.com/aws/smithy-go v1.20.3 // indirect + github.com/aws/smithy-go v1.20.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -199,7 +199,7 @@ require ( github.com/pulumi/esc v0.9.1 // indirect github.com/pulumi/pulumi-command/sdk v0.9.2 // indirect github.com/pulumi/pulumi-docker/sdk/v4 v4.5.1 // indirect - github.com/pulumi/pulumi-libvirt/sdk v0.4.5 // indirect + github.com/pulumi/pulumi-libvirt/sdk v0.4.7 // indirect github.com/pulumi/pulumi-random/sdk/v4 v4.16.3 // indirect github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 // indirect github.com/pulumiverse/pulumi-time/sdk v0.0.17 // indirect diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index f70a1780d564a..8e783e3a67b2d 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -14,8 +14,8 @@ github.com/DataDog/datadog-api-client-go/v2 v2.27.0 h1:AGZj41frjnjMufQHQbJH2fzmi github.com/DataDog/datadog-api-client-go/v2 v2.27.0/go.mod h1:QKOu6vscsh87fMY1lHfLEmNSunyXImj8BUaUWJXOehc= github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a h1:m9REhmyaWD5YJ0P53ygRHxKKo+KM+nw+zz0hEdKztMo= github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= -github.com/DataDog/test-infra-definitions v0.0.0-20240808172947-94712f9a273b h1:AZfxqr+qekhQ3MbBMWbBTOi7RpYfOyIMRYE/veuDqgQ= -github.com/DataDog/test-infra-definitions v0.0.0-20240808172947-94712f9a273b/go.mod h1:xg6j46MpSUIUd1LS1uGm7tH77JGaCboAo/dRzbNEtZg= +github.com/DataDog/test-infra-definitions v0.0.0-20240820173820-926ba0f10de7 h1:3fF4LVBMWUwNa2tzc2MV8fsHJEqmi5UvyJHG5F6c6uU= +github.com/DataDog/test-infra-definitions v0.0.0-20240820173820-926ba0f10de7/go.mod h1:nMwFqB7McymqzJqc/O6eKLN5YMi9loxUOKRZ6bwFbFY= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= @@ -45,8 +45,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY= -github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= +github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= github.com/aws/aws-sdk-go-v2/config v1.27.19 h1:+DBS8gJP6VsxYkZ6UEV0/VsRM2rYpbQCYsosW9RRmeQ= @@ -55,18 +55,18 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.19 h1:R18G7nBBGLby51CFEqUBFF2IVl7 github.com/aws/aws-sdk-go-v2/credentials v1.17.19/go.mod h1:xr9kUMnaLTB866HItT6pg58JgiBP77fSQLBwIa//zk8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.6 h1:vVOuhRyslJ6T/HteG71ZWCTas1q2w6f0NKsNbkXHs/A= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.6/go.mod h1:jimWaqLiT0sJGLh51dKCLLtExRYPtMU7MpxuCgtbkxg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12 h1:DXFWyt7ymx/l1ygdyTTS0X923e+Q2wXIxConJzrgwc0= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.12/go.mod h1:mVOr/LbvaNySK1/BTy4cBOCjhCNY2raWBwK4v+WR5J4= github.com/aws/aws-sdk-go-v2/service/ec2 v1.164.2 h1:Rts0EZgdi3tneJMXp+uKrZHbMxQIu0y5O/2MG6a2+hY= github.com/aws/aws-sdk-go-v2/service/ec2 v1.164.2/go.mod h1:j0V2ahvdX3mGIyXQSe9vjdIQvSxz3uaMM0bR7Y+0WCE= -github.com/aws/aws-sdk-go-v2/service/ecr v1.30.1 h1:zV3FlyuyPzfyFOXKu6mJW9JBGzdtOgpdlj3va+naOD8= -github.com/aws/aws-sdk-go-v2/service/ecr v1.30.1/go.mod h1:l0zC7cSb2vAH1fr8+BRlolWT9cwlKpbRC8PjW6tyyIU= +github.com/aws/aws-sdk-go-v2/service/ecr v1.32.1 h1:PxM8EHsv1sd9eWGamMQCvqBEjxytK5kAwjrxlfG3tac= +github.com/aws/aws-sdk-go-v2/service/ecr v1.32.1/go.mod h1:kdk+WJbHcGVbIlRQfSrKyuKkbWDdD8I9NScyS5vZ8eQ= github.com/aws/aws-sdk-go-v2/service/ecs v1.42.1 h1:KZetBzrQZlpu1U1kIheGfzxua2wh9KuhgQgFO52bWK4= github.com/aws/aws-sdk-go-v2/service/ecs v1.42.1/go.mod h1:u3ULg7pG2JIqIq42FZwNBBEFUX7a1En5LMNjl47xI6Q= github.com/aws/aws-sdk-go-v2/service/eks v1.44.1 h1:onUAzZXDsyXzyrmOGw/9p8Csl1NZkTDEs4URZ8covUY= @@ -89,8 +89,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.6 h1:lEE+xEcq3lh9bk362tgErP1+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.6/go.mod h1:2tR0x1DCL5IgnVZ1NQNFDNg5/XL/kiQgWI5l7I/N5Js= github.com/aws/aws-sdk-go-v2/service/sts v1.28.13 h1:TSzmuUeruVJ4XWYp3bYzKCXue70ECpJWmbP3UfEvhYY= github.com/aws/aws-sdk-go-v2/service/sts v1.28.13/go.mod h1:FppRtFjBA9mSWTj2cIAWCP66+bbBPMuPpBfWRXC5Yi0= -github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= -github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= @@ -407,16 +407,16 @@ github.com/pulumi/pulumi-docker/sdk/v4 v4.5.1 h1:gyuuECcHaPPop7baKfjapJJYnra6s/K github.com/pulumi/pulumi-docker/sdk/v4 v4.5.1/go.mod h1:BL+XtKTgkbtt03wA9SOQWyGjl4cIA7BjSHFjvFY+f9U= github.com/pulumi/pulumi-eks/sdk/v2 v2.7.6 h1:hO4d6QZniXeU0Qw8xZrtdp/rujHj1aNAcCKJW3fAW+c= github.com/pulumi/pulumi-eks/sdk/v2 v2.7.6/go.mod h1:ARGNnIZENIpDUVSX21JEQJKrESj/0u0r0iT61rpb86I= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1 h1:Fp7siNqQBjwIoY/7Jaml/v1frOyGO+kYeeMrO4d2k7k= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.13.1/go.mod h1:MZ+ci9Iq8f0K1aOTXgD3X+ENo2+dFbgQQ7Ahh0YZ8/g= -github.com/pulumi/pulumi-libvirt/sdk v0.4.5 h1:QJlrXbEgmK72k9ubC/jkFSPrud1vA+ZvQFxVC0k2SZI= -github.com/pulumi/pulumi-libvirt/sdk v0.4.5/go.mod h1:b7qbE4yPwM2byv95MDAa8KrTVDBBHy4K15Rdj3pUGTw= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.17.1 h1:VDX+hu+qK3fbf2FodgG5kfh2h1bHK0FKirW1YqKWkRc= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.17.1/go.mod h1:e69ohZtUePLLYNLXYgiOWp0FvRGg6ya/3fsq3o00nN0= +github.com/pulumi/pulumi-libvirt/sdk v0.4.7 h1:/BBnqqx/Gbg2vINvJxXIVb58THXzw2lSqFqxlRSXH9M= +github.com/pulumi/pulumi-libvirt/sdk v0.4.7/go.mod h1:VKvjhAm1sGtzKZruYwIhgascabEx7+oVVRCoxp/cPi4= github.com/pulumi/pulumi-random/sdk/v4 v4.16.3 h1:nlN42MRSIuDh5Pc5nLq4b0lwZaX2ZUAW67Nw+OlNOig= github.com/pulumi/pulumi-random/sdk/v4 v4.16.3/go.mod h1:yRfWJSLEAVZvkwgXajr3S9OmFkAZTxfO44Ef2HfixXQ= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 h1:tXemWrzeVTqG8zq6hBdv1TdPFXjgZ+dob63a/6GlF1o= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1/go.mod h1:hODo3iEmmXDFOXqPK+V+vwI0a3Ww7BLjs5Tgamp86Ng= -github.com/pulumi/pulumi/sdk/v3 v3.126.0 h1:6GQVhwG2jgnG7wjRiWgrq0/sU39onctAiBcvTlqb20s= -github.com/pulumi/pulumi/sdk/v3 v3.126.0/go.mod h1:p1U24en3zt51agx+WlNboSOV8eLlPWYAkxMzVEXKbnY= +github.com/pulumi/pulumi/sdk/v3 v3.128.0 h1:5VPFfygxt6rva0bEYVQZXxsGAo2/D1wsb9erGOtXxzk= +github.com/pulumi/pulumi/sdk/v3 v3.128.0/go.mod h1:p1U24en3zt51agx+WlNboSOV8eLlPWYAkxMzVEXKbnY= github.com/pulumiverse/pulumi-time/sdk v0.0.17 h1:JNYVLglXeMAjyD3upIwKZ9o7MnNo7kc3FVsgxs7bc+A= github.com/pulumiverse/pulumi-time/sdk v0.0.17/go.mod h1:NUa1zA74DF002WrM6iF111A6UjX9knPpXufVRvBwNyg= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -432,8 +432,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= -github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= -github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= From 3106e742a50d042b7c4ea339ec85d90e91ee2e14 Mon Sep 17 00:00:00 2001 From: Florent Clarret Date: Wed, 21 Aug 2024 08:27:12 +0000 Subject: [PATCH 064/245] Handle multiple releases in the `create_rc_pr` workflow (#28600) --- .github/workflows/create_rc_pr.yml | 35 ++++++++++++++++++++++------ tasks/libs/ciproviders/github_api.py | 23 ++++++++++++++++++ tasks/release.py | 10 ++++++++ 3 files changed, 61 insertions(+), 7 deletions(-) diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index 858d0615f9ad3..b6fea6cdab6a3 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -6,14 +6,14 @@ on: - cron: '0 14 * * 1,3,5' # Run on Monday, Wednesday, and Friday at 14:00 UTC - cron: '0 8 * * 1,3,5' # Same as above but at 08:00 UTC, to warn agent-integrations team about releasing - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: - create_rc_pr: + find_release_branches: runs-on: ubuntu-latest - + outputs: + branches: ${{ steps.branches.outputs.value }} steps: - name: Checkout repository uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 @@ -33,27 +33,48 @@ jobs: pip install -r tasks/libs/requirements-github.txt pip install -r tasks/requirements_release_tasks.txt - - name: Determine the release active branch + - name: Determine the release active branches + id: branches run: | - echo "RELEASE_BRANCH=$(inv -e release.get-active-release-branch)" >> $GITHUB_ENV + echo "value=$(inv release.get-unreleased-release-branches)" >> $GITHUB_OUTPUT - name: Set the warning option if: github.event.schedule == '0 8 * * 1,3,5' run: echo "WARNING='-w'" >> $GITHUB_ENV + create_rc_pr: + runs-on: ubuntu-latest + needs: find_release_branches + strategy: + matrix: + value: ${{fromJSON(needs.find_release_branches.outputs.branches)}} + steps: - name: Checkout release branch uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: - ref: ${{ env.RELEASE_BRANCH }} + ref: ${{ matrix.value }} fetch-depth: 0 + - name: Install python + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + with: + python-version: 3.11 + cache: "pip" + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r tasks/libs/requirements-github.txt + pip install -r tasks/requirements_release_tasks.txt + - name: Check for changes since last RC id: check_for_changes env: ATLASSIAN_USERNAME: ${{ secrets.ATLASSIAN_USERNAME }} ATLASSIAN_PASSWORD: ${{ secrets.ATLASSIAN_PASSWORD }} run: | - echo "CHANGES=$(inv -e release.check-for-changes -r ${{ env.RELEASE_BRANCH }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT + echo "CHANGES=$(inv -e release.check-for-changes -r ${{ matrix.value }} ${{ env.WARNING }})" >> $GITHUB_OUTPUT - name: Create RC PR if: ${{ steps.check_for_changes.outputs.CHANGES == 'true'}} diff --git a/tasks/libs/ciproviders/github_api.py b/tasks/libs/ciproviders/github_api.py index 73eb66383f47a..73a54248cfd73 100644 --- a/tasks/libs/ciproviders/github_api.py +++ b/tasks/libs/ciproviders/github_api.py @@ -4,6 +4,7 @@ import json import os import platform +import re import subprocess from collections.abc import Iterable from functools import lru_cache @@ -14,6 +15,7 @@ from tasks.libs.common.constants import GITHUB_REPO_NAME try: + import semver from github import Auth, Github, GithubException, GithubIntegration, GithubObject, PullRequest from github.NamedUser import NamedUser except ImportError: @@ -24,6 +26,8 @@ __all__ = ["GithubAPI"] +RELEASE_BRANCH_PATTERN = re.compile(r"\d+\.\d+\.x") + class GithubAPI: """ @@ -198,6 +202,25 @@ def latest_release(self) -> str: release = self._repository.get_latest_release() return release.title + def latest_unreleased_release_branches(self): + """ + Get all the release branches that are newer than the latest release. + """ + release = self._repository.get_latest_release() + released_version = semver.VersionInfo.parse(release.title) + + for branch in self.release_branches(): + if semver.VersionInfo.parse(branch.name.replace("x", "0")) > released_version: + yield branch + + def release_branches(self): + """ + Yield all the branches that match the release branch pattern (A.B.x). + """ + for branch in self._repository.get_branches(): + if RELEASE_BRANCH_PATTERN.match(branch.name): + yield branch + def get_rate_limit_info(self): """ Gets the current rate limit info. diff --git a/tasks/release.py b/tasks/release.py index c7e47a4fe1eb7..96756a0041871 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -2,6 +2,7 @@ Release helper tasks """ +import json import os import re import sys @@ -881,6 +882,15 @@ def get_active_release_branch(_): print("main") +@task +def get_unreleased_release_branches(_): + """ + Determine what are the current active release branches for the Agent. + """ + gh = GithubAPI() + print(json.dumps([branch.name for branch in gh.latest_unreleased_release_branches()])) + + def get_next_version(gh): latest_release = gh.latest_release() current_version = _create_version_from_match(VERSION_RE.search(latest_release)) From b16a644edb8418ab7c6269596f54925f54162d57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Wed, 21 Aug 2024 10:49:25 +0200 Subject: [PATCH 065/245] [EBPF] Fix crash when missing register state in annotate_complexity (#28605) --- pkg/ebpf/verifier/types.go | 1 + pkg/ebpf/verifier/verifier_log_parser.go | 18 +++++-- pkg/ebpf/verifier/verifier_log_parser_test.go | 47 +++++++++++++++++++ tasks/ebpf.py | 7 ++- 4 files changed, 66 insertions(+), 7 deletions(-) diff --git a/pkg/ebpf/verifier/types.go b/pkg/ebpf/verifier/types.go index bed3c5af06667..401c86d6d1b66 100644 --- a/pkg/ebpf/verifier/types.go +++ b/pkg/ebpf/verifier/types.go @@ -51,6 +51,7 @@ type InstructionInfo struct { Code string `json:"code"` RegisterState map[int]*RegisterState `json:"register_state"` // Register state after execution of the instruction RegisterStateRaw string `json:"register_state_raw"` + IsDoubleWidth bool `json:"is_double_width"` } // SourceLineStats holds the aggregate verifier statistics for a given C source line diff --git a/pkg/ebpf/verifier/verifier_log_parser.go b/pkg/ebpf/verifier/verifier_log_parser.go index 65a2252cf2b14..aabef65c81578 100644 --- a/pkg/ebpf/verifier/verifier_log_parser.go +++ b/pkg/ebpf/verifier/verifier_log_parser.go @@ -6,6 +6,7 @@ package verifier import ( + "errors" "fmt" "math" "regexp" @@ -18,6 +19,7 @@ var ( regStateRegex = regexp.MustCompile(`^([0-9]+): (R[0-9]+.*)`) singleRegStateRegex = regexp.MustCompile(`R([0-9]+)(_[^=]+)?=([^ ]+)`) regInfoRegex = regexp.MustCompile(`^(P)?([a-z_]+)?(P)?(-?[0-9]+|\((.*)\))`) + doubleWidthInsRegex = regexp.MustCompile(`r[0-9] = 0x[0-9a-f]{16}$`) ) // verifierLogParser is a struct that maintains the state necessary to parse the verifier log @@ -72,6 +74,8 @@ func (vlp *verifierLogParser) parseVerifierLog(log string) (*ComplexityInfo, err return &vlp.complexity, nil } +var errNotFound = errors.New("instruction not found") + func (vlp *verifierLogParser) tryAttachRegisterState(regData string, insIdx int) error { insinfo, ok := vlp.complexity.InsnMap[insIdx] if !ok { @@ -79,7 +83,7 @@ func (vlp *verifierLogParser) tryAttachRegisterState(regData string, insIdx int) // the verifier outputs register state. For example, in some versions it will generate // the state *before* each instruction, but we want the state *after* the instruction, and // as such the first state we find (state before instruction 1) will have no match, for example. - return nil + return errNotFound } // For ease of parsing, replace certain patterns that introduce spaces and make parsing harder @@ -111,7 +115,14 @@ func (vlp *verifierLogParser) parseLine(line string) error { // Try attach the register state to the previous instruction err = vlp.tryAttachRegisterState(match[2], regInsnIdx-1) - if err != nil { + if errors.Is(err, errNotFound) { + // The previous instruction was not found. Check if the previous one is double width, and try to attach to the one before that + // We need to do this because the instruction index jumps one when we have a double width instruction + if insinfo, ok := vlp.complexity.InsnMap[regInsnIdx-2]; ok && insinfo.IsDoubleWidth { + err = vlp.tryAttachRegisterState(match[2], regInsnIdx-2) + } + } + if err != nil && !errors.Is(err, errNotFound) { return fmt.Errorf("failed to attach register state (line is '%s'): %w", line, err) } return nil @@ -134,12 +145,13 @@ func (vlp *verifierLogParser) parseLine(line string) error { if vlp.progSourceMap != nil { insinfo.Source = vlp.progSourceMap[insIdx] } + insinfo.IsDoubleWidth = doubleWidthInsRegex.MatchString(insinfo.Code) // In some kernel versions (6.5 at least), the register state for the next instruction might be printed after this instruction if len(match) >= 4 && match[3] != "" { regData := match[3][2:] // Remove the leading "; " err = vlp.tryAttachRegisterState(regData, insIdx) - if err != nil { + if err != nil && !errors.Is(err, errNotFound) { return fmt.Errorf("Cannot attach register state to instruction %d: %w", insIdx, err) } } diff --git a/pkg/ebpf/verifier/verifier_log_parser_test.go b/pkg/ebpf/verifier/verifier_log_parser_test.go index 89b7c5dc3d700..e7aa334bafbf3 100644 --- a/pkg/ebpf/verifier/verifier_log_parser_test.go +++ b/pkg/ebpf/verifier/verifier_log_parser_test.go @@ -235,6 +235,53 @@ func TestLogParsingWith(t *testing.T) { }, }, }, + { + name: "DoubleWidthInstruction", + input: "1803: R0=0\n1803: (18) r1 = 0xffff0001f3b06000\n1805: R1=1\n", + progSourceMap: map[int]*SourceLine{ + 1803: { + LineInfo: "file.c:5", + Line: "int a = 2", + }, + 1805: { + LineInfo: "file.c:5", + Line: "int a = 2", + }, + }, + expected: ComplexityInfo{ + InsnMap: map[int]*InstructionInfo{ + 1803: { + Index: 1803, + TimesProcessed: 1, + Source: &SourceLine{ + LineInfo: "file.c:5", + Line: "int a = 2", + }, + Code: "r1 = 0xffff0001f3b06000", + RegisterState: map[int]*RegisterState{ + 1: { + Register: 1, + Live: "", + Type: "scalar", + Value: "1", + Precise: false, + }, + }, + RegisterStateRaw: "R1=1", + IsDoubleWidth: true, + }, + }, + SourceMap: map[string]*SourceLineStats{ + "file.c:5": { + NumInstructions: 1, + MaxPasses: 1, + MinPasses: 1, + TotalInstructionsProcessed: 1, + AssemblyInsns: []int{1803}, + }, + }, + }, + }, } for _, tt := range tests { diff --git a/tasks/ebpf.py b/tasks/ebpf.py index bd44109c32fc1..b90d4fa20f880 100644 --- a/tasks/ebpf.py +++ b/tasks/ebpf.py @@ -479,13 +479,12 @@ def annotate_complexity( ) ) - if show_register_state: + # This is the register state after the statement is executed + reg_state = asm_line.get("register_state") + if show_register_state and reg_state is not None: # Get all the registers that were used in this instruction registers = re.findall(r"r\d+", asm_code) - # This is the register state after the statement is executed - reg_state = asm_line["register_state"] - if show_raw_register_state: total_indent = 4 + 3 + get_total_complexity_stats_len(compinfo_widths) raw_state = asm_line['register_state_raw'].split(':', 1)[1].strip() From 146c198f9d30ca43621c627f99319c7dd68d9e5e Mon Sep 17 00:00:00 2001 From: Gustavo Caso Date: Wed, 21 Aug 2024 10:49:30 +0200 Subject: [PATCH 066/245] [ASCII-2210] Ensure we set the workloadmeta params correctly for the agent JMX subcommands (#28578) --- cmd/agent/subcommands/jmx/command.go | 8 +++----- .../fix-agent-jmx-subcommand-b7338c53cc835660.yaml | 11 +++++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 releasenotes/notes/fix-agent-jmx-subcommand-b7338c53cc835660.yaml diff --git a/cmd/agent/subcommands/jmx/command.go b/cmd/agent/subcommands/jmx/command.go index 9c8f4425f4df2..2cfb7832a3cfc 100644 --- a/cmd/agent/subcommands/jmx/command.go +++ b/cmd/agent/subcommands/jmx/command.go @@ -48,6 +48,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" wmcatalog "github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/catalog" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/comp/core/workloadmeta/defaults" workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx" "github.com/DataDog/datadog-agent/comp/dogstatsd/pidmap" replay "github.com/DataDog/datadog-agent/comp/dogstatsd/replay/def" @@ -57,7 +58,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/aggregator/sender" "github.com/DataDog/datadog-agent/pkg/cli/standalone" pkgcollector "github.com/DataDog/datadog-agent/pkg/collector" - pkgconfig "github.com/DataDog/datadog-agent/pkg/config" "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" @@ -134,9 +134,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { }), // workloadmeta setup wmcatalog.GetCatalog(), - fx.Supply(workloadmeta.Params{ - InitHelper: common.GetWorkloadmetaInit(), - }), + fx.Provide(defaults.DefaultParams), workloadmetafx.Module(), apiimpl.Module(), authtokenimpl.Module(), @@ -304,7 +302,7 @@ func runJmxCommandConsole(config config.Component, // This prevents log-spam from "comp/core/workloadmeta/collectors/internal/remote/process_collector/process_collector.go" // It appears that this collector creates some contention in AD. // Disabling it is both more efficient and gets rid of this log spam - pkgconfig.Datadog().Set("language_detection.enabled", "false", model.SourceAgentRuntime) + config.Set("language_detection.enabled", "false", model.SourceAgentRuntime) senderManager, err := diagnoseSendermanager.LazyGetSenderManager() if err != nil { diff --git a/releasenotes/notes/fix-agent-jmx-subcommand-b7338c53cc835660.yaml b/releasenotes/notes/fix-agent-jmx-subcommand-b7338c53cc835660.yaml new file mode 100644 index 0000000000000..33e6ebf655071 --- /dev/null +++ b/releasenotes/notes/fix-agent-jmx-subcommand-b7338c53cc835660.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +fixes: + - | + Fix ``agent jmx [command]`` subcommands for container environments with annotations-based configs. From 8a3dba707cb20d88fda5b1bd39a7264eb2678c6d Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Wed, 21 Aug 2024 10:53:29 +0200 Subject: [PATCH 067/245] (fleet) always create the experiment symlink pointing to stable by default (#28467) --- pkg/fleet/installer/repository/repository.go | 58 +++++++++++++------ .../installer/repository/repository_test.go | 46 +++++++++++++++ .../embedded/datadog-agent-exp.service | 2 +- .../datadog-agent-process-exp.service | 2 +- .../datadog-agent-security-exp.service | 2 +- .../datadog-agent-sysprobe-exp.service | 2 +- .../embedded/datadog-agent-trace-exp.service | 2 +- test/new-e2e/tests/installer/host/fixtures.go | 1 + .../suites/agent-package/upgrade_test.go | 8 +-- 9 files changed, 97 insertions(+), 26 deletions(-) diff --git a/pkg/fleet/installer/repository/repository.go b/pkg/fleet/installer/repository/repository.go index 202f60c16780b..559bf88854641 100644 --- a/pkg/fleet/installer/repository/repository.go +++ b/pkg/fleet/installer/repository/repository.go @@ -93,9 +93,14 @@ func (r *Repository) GetState() (State, error) { if err != nil { return State{}, err } + stable := repository.stable.Target() + experiment := repository.experiment.Target() + if experiment == stable { + experiment = "" + } return State{ - Stable: repository.stable.Target(), - Experiment: repository.experiment.Target(), + Stable: stable, + Experiment: experiment, }, nil } @@ -157,6 +162,10 @@ func (r *Repository) Create(name string, stableSourcePath string) error { if err != nil { return fmt.Errorf("could not set first stable: %w", err) } + err = repository.setExperimentToStable() + if err != nil { + return fmt.Errorf("could not set first experiment: %w", err) + } return nil } @@ -175,7 +184,10 @@ func (r *Repository) SetExperiment(name string, sourcePath string) error { return fmt.Errorf("could not cleanup repository: %w", err) } if !repository.stable.Exists() { - return fmt.Errorf("stable package does not exist, invalid state") + return fmt.Errorf("stable link does not exist, invalid state") + } + if !repository.experiment.Exists() { + return fmt.Errorf("experiment link does not exist, invalid state") } err = repository.setExperiment(name, sourcePath) if err != nil { @@ -187,9 +199,8 @@ func (r *Repository) SetExperiment(name string, sourcePath string) error { // PromoteExperiment promotes the experiment to stable. // // 1. Cleanup the repository. -// 2. Set the stable link to the experiment package. -// 3. Delete the experiment link. -// 4. Cleanup the repository to remove the previous stable package. +// 2. Set the stable link to the experiment package. The experiment link stays in place. +// 3. Cleanup the repository to remove the previous stable package. func (r *Repository) PromoteExperiment() error { repository, err := readRepository(r.rootPath, r.locksPath) if err != nil { @@ -200,19 +211,18 @@ func (r *Repository) PromoteExperiment() error { return fmt.Errorf("could not cleanup repository: %w", err) } if !repository.stable.Exists() { - return fmt.Errorf("stable package does not exist, invalid state") + return fmt.Errorf("stable link does not exist, invalid state") } if !repository.experiment.Exists() { - return fmt.Errorf("experiment package does not exist, invalid state") + return fmt.Errorf("experiment link does not exist, invalid state") + } + if repository.stable.Target() == repository.experiment.Target() { + return fmt.Errorf("no experiment to promote") } err = repository.stable.Set(*repository.experiment.packagePath) if err != nil { return fmt.Errorf("could not set stable: %w", err) } - err = repository.experiment.Delete() - if err != nil { - return fmt.Errorf("could not delete experiment link: %w", err) - } // Read repository again to re-load the list of locked packages repository, err = readRepository(r.rootPath, r.locksPath) @@ -229,7 +239,7 @@ func (r *Repository) PromoteExperiment() error { // DeleteExperiment deletes the experiment. // // 1. Cleanup the repository. -// 2. Delete the experiment link. +// 2. Sets the experiment link to the stable link. // 3. Cleanup the repository to remove the previous experiment package. func (r *Repository) DeleteExperiment() error { repository, err := readRepository(r.rootPath, r.locksPath) @@ -241,14 +251,14 @@ func (r *Repository) DeleteExperiment() error { return fmt.Errorf("could not cleanup repository: %w", err) } if !repository.stable.Exists() { - return fmt.Errorf("stable package does not exist, invalid state") + return fmt.Errorf("stable link does not exist, invalid state") } if !repository.experiment.Exists() { - return nil + return fmt.Errorf("experiment link does not exist, invalid state") } - err = repository.experiment.Delete() + err = repository.setExperimentToStable() if err != nil { - return fmt.Errorf("could not delete experiment link: %w", err) + return fmt.Errorf("could not set experiment to stable: %w", err) } // Read repository again to re-load the list of locked packages @@ -328,6 +338,11 @@ func (r *repositoryFiles) setExperiment(name string, sourcePath string) error { return r.experiment.Set(path) } +// setExperimentToStable moves the experiment to stable. +func (r *repositoryFiles) setExperimentToStable() error { + return r.experiment.Set(r.stable.linkPath) +} + func (r *repositoryFiles) setStable(name string, sourcePath string) error { path, err := movePackageFromSource(name, r.rootPath, r.lockedPackages, sourcePath) if err != nil { @@ -367,6 +382,15 @@ func movePackageFromSource(packageName string, rootPath string, lockedPackages m } func (r *repositoryFiles) cleanup() error { + // migrate old repositories that are missing the experiment link + if r.stable.Exists() && !r.experiment.Exists() { + err := r.setExperimentToStable() + if err != nil { + return fmt.Errorf("could not migrate old repository without experiment link: %w", err) + } + } + + // remove left-over packages files, err := os.ReadDir(r.rootPath) if err != nil { return fmt.Errorf("could not read root directory: %w", err) diff --git a/pkg/fleet/installer/repository/repository_test.go b/pkg/fleet/installer/repository/repository_test.go index 13bcf7266ed43..9e47e4e48dd86 100644 --- a/pkg/fleet/installer/repository/repository_test.go +++ b/pkg/fleet/installer/repository/repository_test.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -36,12 +37,27 @@ func createTestDownloadedPackage(t *testing.T, dir string, packageName string) s return downloadPath } +func assertLinkTarget(t *testing.T, repository *Repository, link string, target string) { + linkPath := path.Join(repository.rootPath, link) + assert.FileExists(t, linkPath) + linkTarget, err := linkRead(linkPath) + assert.NoError(t, err) + assert.Equal(t, target, filepath.Base(linkTarget)) +} + func TestCreateFresh(t *testing.T) { dir := t.TempDir() repository := createTestRepository(t, dir, "v1") + state, err := repository.GetState() assert.DirExists(t, repository.rootPath) assert.DirExists(t, path.Join(repository.rootPath, "v1")) + assert.NoError(t, err) + assert.True(t, state.HasStable()) + assert.Equal(t, "v1", state.Stable) + assert.False(t, state.HasExperiment()) + assertLinkTarget(t, repository, stableVersionLink, "v1") + assertLinkTarget(t, repository, experimentVersionLink, "v1") } func TestCreateOverwrite(t *testing.T) { @@ -87,8 +103,17 @@ func TestSetExperiment(t *testing.T) { experimentDownloadPackagePath := createTestDownloadedPackage(t, dir, "v2") err := repository.SetExperiment("v2", experimentDownloadPackagePath) + assert.NoError(t, err) + state, err := repository.GetState() + assert.NoError(t, err) assert.DirExists(t, path.Join(repository.rootPath, "v2")) + assert.True(t, state.HasStable()) + assert.Equal(t, "v1", state.Stable) + assert.True(t, state.HasExperiment()) + assert.Equal(t, "v2", state.Experiment) + assertLinkTarget(t, repository, stableVersionLink, "v1") + assertLinkTarget(t, repository, experimentVersionLink, "v2") } func TestSetExperimentTwice(t *testing.T) { @@ -124,8 +149,16 @@ func TestPromoteExperiment(t *testing.T) { assert.NoError(t, err) err = repository.PromoteExperiment() assert.NoError(t, err) + state, err := repository.GetState() + assert.NoError(t, err) + assert.NoDirExists(t, path.Join(repository.rootPath, "v1")) assert.DirExists(t, path.Join(repository.rootPath, "v2")) + assert.True(t, state.HasStable()) + assert.Equal(t, "v2", state.Stable) + assert.False(t, state.HasExperiment()) + assertLinkTarget(t, repository, stableVersionLink, "v2") + assertLinkTarget(t, repository, experimentVersionLink, "v2") } func TestPromoteExperimentWithoutExperiment(t *testing.T) { @@ -191,3 +224,16 @@ func TestDeleteExperimentWithLockedPackage(t *testing.T) { assert.NoFileExists(t, path.Join(repository.locksPath, "v2", "-1")) assert.FileExists(t, path.Join(repository.locksPath, "v2", fmt.Sprint(os.Getpid()))) } + +func TestMigrateRepositoryWithoutExperiment(t *testing.T) { + dir := t.TempDir() + repository := createTestRepository(t, dir, "v1") + + err := os.Remove(path.Join(repository.rootPath, experimentVersionLink)) + assert.NoError(t, err) + assert.NoFileExists(t, path.Join(repository.rootPath, experimentVersionLink)) + err = repository.Cleanup() + assert.NoError(t, err) + assertLinkTarget(t, repository, stableVersionLink, "v1") + assertLinkTarget(t, repository, experimentVersionLink, "v1") +} diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-exp.service index 336d98f781a0a..f094a2854b61a 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-exp.service @@ -12,7 +12,7 @@ Type=oneshot PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/agent.pid User=dd-agent EnvironmentFile=-/etc/datadog-agent/environment -Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/experiment" ExecStart=/opt/datadog-packages/datadog-agent/experiment/bin/agent/agent run -p /opt/datadog-packages/datadog-agent/experiment/run/agent.pid ExecStart=/bin/false ExecStop=/bin/false diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-process-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-process-exp.service index 87c88adbc8c50..9583865d902df 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-process-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-process-exp.service @@ -9,7 +9,7 @@ PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/process-agent.pid User=dd-agent Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment -Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/experiment" ExecStart=/opt/datadog-packages/datadog-agent/experiment/embedded/bin/process-agent --cfgpath=/etc/datadog-agent/datadog.yaml --sysprobe-config=/etc/datadog-agent/system-probe.yaml --pid=/opt/datadog-packages/datadog-agent/experiment/run/process-agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-security-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-security-exp.service index 87927afd96cdd..c86acfd7ee24b 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-security-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-security-exp.service @@ -9,7 +9,7 @@ Type=simple PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/security-agent.pid Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment -Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/experiment" ExecStart=/opt/datadog-packages/datadog-agent/experiment/embedded/bin/security-agent -c /etc/datadog-agent/datadog.yaml --pidfile /opt/datadog-packages/datadog-agent/experiment/run/security-agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe-exp.service index 81f3192e4feda..e07fee41bad04 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-sysprobe-exp.service @@ -9,7 +9,7 @@ ConditionPathExists=/etc/datadog-agent/system-probe.yaml Type=simple PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/system-probe.pid Restart=on-failure -Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/experiment" ExecStart=/opt/datadog-packages/datadog-agent/experiment/embedded/bin/system-probe run --config=/etc/datadog-agent/system-probe.yaml --pid=/opt/datadog-packages/datadog-agent/experiment/run/system-probe.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-trace-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-trace-exp.service index 22b73c91bfe73..17bc119e85290 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-trace-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-trace-exp.service @@ -8,7 +8,7 @@ PIDFile=/opt/datadog-packages/datadog-agent/experiment/run/trace-agent.pid User=dd-agent Restart=on-failure EnvironmentFile=-/etc/datadog-agent/environment -Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/stable" +Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/experiment" ExecStart=/opt/datadog-packages/datadog-agent/experiment/embedded/bin/trace-agent --config /etc/datadog-agent/datadog.yaml --pidfile /opt/datadog-packages/datadog-agent/experiment/run/trace-agent.pid # Since systemd 229, should be in [Unit] but in order to support systemd <229, # it is also supported to have it here. diff --git a/test/new-e2e/tests/installer/host/fixtures.go b/test/new-e2e/tests/installer/host/fixtures.go index edb5e6612412c..ce15fe7308b7d 100644 --- a/test/new-e2e/tests/installer/host/fixtures.go +++ b/test/new-e2e/tests/installer/host/fixtures.go @@ -118,6 +118,7 @@ func (h *Host) SetupFakeAgentExp() FakeAgent { h.remote.MustExecute(fmt.Sprintf("sudo mkdir -p %s/embedded/bin", vBroken)) h.remote.MustExecute(fmt.Sprintf("sudo mkdir -p %s/bin/agent", vBroken)) + h.remote.MustExecute("sudo rm -f /opt/datadog-packages/datadog-agent/experiment") // remove symlink if it exists, next command doesn't overwrite it h.remote.MustExecute(fmt.Sprintf("sudo ln -sf %s /opt/datadog-packages/datadog-agent/experiment", vBroken)) f := FakeAgent{ h: h, diff --git a/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go b/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go index 07e1c6bd5a5c0..08d4cc4fa82ff 100644 --- a/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go +++ b/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go @@ -6,11 +6,12 @@ package agenttests import ( + "testing" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" - "testing" ) type testAgentUpgradeSuite struct { @@ -88,6 +89,5 @@ func (s *testAgentUpgradeSuite) stopExperiment() { WithVersionMatchPredicate(func(version string) { s.Require().Contains(version, s.StableAgentVersion().Version()) }). - DirExists(installerwindows.GetStableDirFor(installerwindows.AgentPackage)). - NoDirExists(installerwindows.GetExperimentDirFor(installerwindows.AgentPackage)) + DirExists(installerwindows.GetStableDirFor(installerwindows.AgentPackage)) } From e9d3bb90488e6ae05d9c891e366eca017ec1c480 Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Wed, 21 Aug 2024 11:45:24 +0200 Subject: [PATCH 068/245] fix(fleet): Set RuntimeDirectory on installer agent units (#28532) --- .../installer/service/datadog_installer.go | 37 ------------------- .../embedded/datadog-agent-exp.service | 1 + .../service/embedded/datadog-agent.service | 1 + .../tests/installer/package_installer_test.go | 3 -- 4 files changed, 2 insertions(+), 40 deletions(-) diff --git a/pkg/fleet/installer/service/datadog_installer.go b/pkg/fleet/installer/service/datadog_installer.go index f04e5a735aa70..5f886a0b29c2c 100644 --- a/pkg/fleet/installer/service/datadog_installer.go +++ b/pkg/fleet/installer/service/datadog_installer.go @@ -14,11 +14,9 @@ import ( "os" "os/exec" "os/user" - "path/filepath" "strconv" "github.com/DataDog/datadog-agent/pkg/util/log" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" ) const ( @@ -98,12 +96,6 @@ func SetupInstaller(ctx context.Context) (err error) { if err != nil { return fmt.Errorf("error changing owner of /var/run/datadog-installer: %w", err) } - if err = os.MkdirAll("/var/run/datadog", 0755); err != nil { - return fmt.Errorf("failed to create /var/run/datadog: %v", err) - } - if err = os.Chown("/var/run/datadog", ddAgentUID, ddAgentGID); err != nil { - return fmt.Errorf("failed to chown /var/run/datadog: %v", err) - } // Enforce that the directory exists. It should be created by the bootstrapper but // older versions don't do it err = os.MkdirAll("/opt/datadog-installer/tmp", 0755) @@ -138,10 +130,6 @@ func SetupInstaller(ctx context.Context) (err error) { return fmt.Errorf("error creating %s: %w", systemdPath, err) } - if err = addSystemDRuntimeConfigOverride(ctx); err != nil { - return err - } - // FIXME(Arthur): enable the daemon unit by default and use the same strategy as the system probe if os.Getenv("DD_REMOTE_UPDATES") != "true" { if err = systemdReload(ctx); err != nil { @@ -240,28 +228,3 @@ func StartInstallerExperiment(ctx context.Context) error { func StopInstallerExperiment(ctx context.Context) error { return startUnit(ctx, installerUnit) } - -// addSystemDRuntimeConfigOverride removes RuntimeConfig from the agent unit -// to avoid folder deletion on agent stop -func addSystemDRuntimeConfigOverride(ctx context.Context) (err error) { - span, _ := tracer.StartSpanFromContext(ctx, "add_systemd_runtime_config_override") - defer func() { span.Finish(tracer.WithError(err)) }() - - content := []byte("[Service]\nRuntimeConfig=\n") - - // We don't need a file mutator here as we're fully hard coding the content. - // We don't really need to remove the file either as it'll just be ignored once the - // unit is removed. - path := filepath.Join(systemdPath, "datadog-agent.service.d", "datadog_runtime_config.conf") - err = os.Mkdir(filepath.Dir(path), 0755) - if err != nil && !os.IsExist(err) { - err = fmt.Errorf("error creating systemd environment override directory: %w", err) - return err - } - err = os.WriteFile(path, content, 0644) - if err != nil { - err = fmt.Errorf("error writing systemd runtime config override: %w", err) - return err - } - return nil -} diff --git a/pkg/fleet/installer/service/embedded/datadog-agent-exp.service b/pkg/fleet/installer/service/embedded/datadog-agent-exp.service index f094a2854b61a..40adc828787db 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent-exp.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent-exp.service @@ -16,6 +16,7 @@ Environment="DD_FLEET_POLICIES_DIR=/etc/datadog-packages/datadog-agent/experimen ExecStart=/opt/datadog-packages/datadog-agent/experiment/bin/agent/agent run -p /opt/datadog-packages/datadog-agent/experiment/run/agent.pid ExecStart=/bin/false ExecStop=/bin/false +RuntimeDirectory=datadog [Install] WantedBy=multi-user.target diff --git a/pkg/fleet/installer/service/embedded/datadog-agent.service b/pkg/fleet/installer/service/embedded/datadog-agent.service index baf05b20796b8..0f527777fc2e1 100644 --- a/pkg/fleet/installer/service/embedded/datadog-agent.service +++ b/pkg/fleet/installer/service/embedded/datadog-agent.service @@ -17,6 +17,7 @@ ExecStart=/opt/datadog-packages/datadog-agent/stable/bin/agent/agent run -p /opt # it is also supported to have it here. StartLimitInterval=10 StartLimitBurst=5 +RuntimeDirectory=datadog [Install] WantedBy=multi-user.target diff --git a/test/new-e2e/tests/installer/package_installer_test.go b/test/new-e2e/tests/installer/package_installer_test.go index d20648b8bae71..a658f68507261 100644 --- a/test/new-e2e/tests/installer/package_installer_test.go +++ b/test/new-e2e/tests/installer/package_installer_test.go @@ -38,7 +38,6 @@ func (s *packageInstallerSuite) TestInstall() { state.AssertDirExists("/var/log/datadog", 0755, "dd-agent", "dd-agent") state.AssertDirExists("/var/run/datadog-installer", 0755, "dd-agent", "dd-agent") state.AssertDirExists("/var/run/datadog-installer/locks", 0777, "root", "root") - state.AssertDirExists("/var/run/datadog", 0755, "dd-agent", "dd-agent") state.AssertDirExists("/opt/datadog-installer", 0755, "root", "root") state.AssertDirExists("/opt/datadog-installer/tmp", 0755, "dd-agent", "dd-agent") @@ -49,8 +48,6 @@ func (s *packageInstallerSuite) TestInstall() { state.AssertSymlinkExists("/usr/bin/datadog-installer", "/opt/datadog-packages/datadog-installer/stable/bin/installer/installer", "root", "root") state.AssertUnitsNotLoaded("datadog-installer.service", "datadog-installer-exp.service") - - state.AssertFileExists("/etc/systemd/system/datadog-agent.service.d/datadog_runtime_config.conf", 0644, "root", "root") } func (s *packageInstallerSuite) TestInstallWithRemoteUpdates() { From a6ef156c9f19fa52be6007a3c44c89155c3ee43b Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Wed, 21 Aug 2024 11:46:21 +0200 Subject: [PATCH 069/245] discovery: Remove unused ServiceDetector type (#28612) --- .../corechecks/servicediscovery/impl_linux.go | 4 +- .../servicediscovery/module/impl_linux.go | 11 ++-- .../servicediscovery/service_detector.go | 41 +------------- .../servicediscovery/service_detector_test.go | 53 ------------------- 4 files changed, 6 insertions(+), 103 deletions(-) delete mode 100644 pkg/collector/corechecks/servicediscovery/service_detector_test.go diff --git a/pkg/collector/corechecks/servicediscovery/impl_linux.go b/pkg/collector/corechecks/servicediscovery/impl_linux.go index 5a039b7489ba5..441ddc90401be 100644 --- a/pkg/collector/corechecks/servicediscovery/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/impl_linux.go @@ -49,8 +49,7 @@ type linuxImpl struct { time timer bootTime uint64 - serviceDetector *ServiceDetector - ignoreCfg map[string]bool + ignoreCfg map[string]bool ignoreProcs map[int]bool aliveServices map[int]*serviceInfo @@ -76,7 +75,6 @@ func newLinuxImpl(ignoreCfg map[string]bool) (osImpl, error) { bootTime: stat.BootTime, getSysProbeClient: getSysProbeClient, time: realTime{}, - serviceDetector: NewServiceDetector(), ignoreCfg: ignoreCfg, ignoreProcs: make(map[int]bool), aliveServices: make(map[int]*serviceInfo), diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go index 1cdf38f5d81be..d739c4e938b83 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go @@ -14,13 +14,12 @@ import ( "github.com/prometheus/procfs" "github.com/shirou/gopsutil/v3/process" - "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery" - "github.com/DataDog/datadog-agent/cmd/system-probe/api/module" sysconfigtypes "github.com/DataDog/datadog-agent/cmd/system-probe/config/types" "github.com/DataDog/datadog-agent/cmd/system-probe/utils" "github.com/DataDog/datadog-agent/comp/core/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/apm" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/language" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" @@ -46,15 +45,13 @@ type serviceInfo struct { // discovery is an implementation of the Module interface for the discovery module. type discovery struct { // cache maps pids to data that should be cached between calls to the endpoint. - cache map[int32]*serviceInfo - serviceDetector servicediscovery.ServiceDetector + cache map[int32]*serviceInfo } // NewDiscoveryModule creates a new discovery system probe module. func NewDiscoveryModule(*sysconfigtypes.Config, optional.Option[workloadmeta.Component], telemetry.Component) (module.Module, error) { return &discovery{ - cache: make(map[int32]*serviceInfo), - serviceDetector: *servicediscovery.NewServiceDetector(), + cache: make(map[int32]*serviceInfo), }, nil } @@ -215,7 +212,7 @@ func (s *discovery) getServiceInfo(proc *process.Process) (*serviceInfo, error) return nil, err } - name := s.serviceDetector.GetServiceName(cmdline, envs) + name := servicediscovery.GetServiceName(cmdline, envs) language := language.FindInArgs(cmdline) apmInstrumentation := apm.Detect(cmdline, envs, language) diff --git a/pkg/collector/corechecks/servicediscovery/service_detector.go b/pkg/collector/corechecks/servicediscovery/service_detector.go index 7988f4b35485d..11d3d75d42fcd 100644 --- a/pkg/collector/corechecks/servicediscovery/service_detector.go +++ b/pkg/collector/corechecks/servicediscovery/service_detector.go @@ -9,25 +9,9 @@ import ( "slices" "strings" - "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/apm" - "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/language" - "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/servicetype" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/usm" - "github.com/DataDog/datadog-agent/pkg/util/log" ) -// ServiceDetector defines the service detector to get metadata about services. -type ServiceDetector struct { - langFinder language.Finder -} - -// NewServiceDetector creates a new ServiceDetector object. -func NewServiceDetector() *ServiceDetector { - return &ServiceDetector{ - langFinder: language.New(), - } -} - // ServiceMetadata stores metadata about a service. type ServiceMetadata struct { Name string @@ -59,30 +43,7 @@ func makeFinalName(meta usm.ServiceMetadata) string { // GetServiceName gets the service name based on the command line arguments and // the list of environment variables. -func (sd *ServiceDetector) GetServiceName(cmdline []string, env map[string]string) string { +func GetServiceName(cmdline []string, env map[string]string) string { meta, _ := usm.ExtractServiceMetadata(cmdline, env) return makeFinalName(meta) } - -// Detect gets metadata for a service. -func (sd *ServiceDetector) Detect(p processInfo) ServiceMetadata { - meta, _ := usm.ExtractServiceMetadata(p.CmdLine, p.Env) - lang, _ := sd.langFinder.Detect(p.CmdLine, p.Env) - svcType := servicetype.Detect(meta.Name, p.Ports) - apmInstr := apm.Detect(p.CmdLine, p.Env, lang) - - log.Debugf("name info - name: %q; additional names: %v", meta.Name, meta.AdditionalNames) - - nameSource := "generated" - if meta.FromDDService { - nameSource = "provided" - } - - return ServiceMetadata{ - Name: makeFinalName(meta), - Language: string(lang), - Type: string(svcType), - APMInstrumentation: string(apmInstr), - NameSource: nameSource, - } -} diff --git a/pkg/collector/corechecks/servicediscovery/service_detector_test.go b/pkg/collector/corechecks/servicediscovery/service_detector_test.go deleted file mode 100644 index 90186b1baabf8..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/service_detector_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package servicediscovery - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_serviceDetector(t *testing.T) { - sd := NewServiceDetector() - - // no need to test many cases here, just ensuring the process data is properly passed down is enough. - pInfo := processInfo{ - PID: 100, - CmdLine: []string{"my-service.py"}, - Env: map[string]string{"PATH": "testdata/test-bin", "DD_INJECTION_ENABLED": "tracer"}, - Stat: procStat{}, - Ports: []uint16{5432}, - } - - want := ServiceMetadata{ - Name: "my-service", - Language: "python", - Type: "db", - APMInstrumentation: "injected", - NameSource: "generated", - } - got := sd.Detect(pInfo) - assert.Equal(t, want, got) - - // pass in nil slices and see if anything blows up - pInfoEmpty := processInfo{ - PID: 0, - CmdLine: nil, - Env: nil, - Stat: procStat{}, - Ports: nil, - } - wantEmpty := ServiceMetadata{ - Name: "", - Language: "UNKNOWN", - Type: "web_service", - APMInstrumentation: "none", - NameSource: "generated", - } - gotEmpty := sd.Detect(pInfoEmpty) - assert.Equal(t, wantEmpty, gotEmpty) -} From 0dffd30f85f3a3692554f3bc63350fc36c3e8cb3 Mon Sep 17 00:00:00 2001 From: Kangyi LI Date: Wed, 21 Aug 2024 11:56:34 +0200 Subject: [PATCH 070/245] CAP-1832 update config template (#28582) Co-authored-by: Rosa Trieu <107086888+rtrieu@users.noreply.github.com> --- pkg/config/config_template.yaml | 7 +++++++ pkg/util/ecs/metadata/testutil/dummy_ecs.go | 9 +++------ pkg/util/ecs/metadata/v3or4/client_test.go | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/config/config_template.yaml b/pkg/config/config_template.yaml index a53836464ca73..e01a2e747831b 100644 --- a/pkg/config/config_template.yaml +++ b/pkg/config/config_template.yaml @@ -3196,6 +3196,13 @@ api_key: # # ecs_metadata_timeout: 500 +## @param ecs_task_collection_enabled - boolean - optional - default: false +## @env DD_ECS_TASK_COLLECTION_ENABLED - boolean - optional - default: false +## The Agent can collect detailed task information from the metadata API exposed by the ECS Agent, +## which is used for the orchestrator ECS check. +# +# ecs_task_collection_enabled: false + {{ end -}} {{- if .CRI }} diff --git a/pkg/util/ecs/metadata/testutil/dummy_ecs.go b/pkg/util/ecs/metadata/testutil/dummy_ecs.go index a1a874b566723..368a8a353d11b 100644 --- a/pkg/util/ecs/metadata/testutil/dummy_ecs.go +++ b/pkg/util/ecs/metadata/testutil/dummy_ecs.go @@ -11,20 +11,19 @@ import ( "net/http" "net/http/httptest" "os" - "sync" + "sync/atomic" "time" ) // DummyECS allows tests to mock ECS metadata server responses type DummyECS struct { - sync.Mutex mux *http.ServeMux fileHandlers map[string]string fileHandlersDelay map[string]time.Duration rawHandlers map[string]string rawHandlersDelay map[string]time.Duration Requests chan *http.Request - RequestCount int + RequestCount atomic.Uint64 } // Option represents an option used to create a new mock of the ECS metadata @@ -100,9 +99,7 @@ func NewDummyECS(ops ...Option) (*DummyECS, error) { // ServeHTTP is used to handle HTTP requests. func (d *DummyECS) ServeHTTP(w http.ResponseWriter, r *http.Request) { fmt.Printf("dummyECS received %s on %s\n", r.Method, r.URL.Path) - d.Lock() - d.RequestCount++ - d.Unlock() + d.RequestCount.Add(1) d.Requests <- r d.mux.ServeHTTP(w, r) } diff --git a/pkg/util/ecs/metadata/v3or4/client_test.go b/pkg/util/ecs/metadata/v3or4/client_test.go index c45aa2546b0a1..f8d55bf8135d7 100644 --- a/pkg/util/ecs/metadata/v3or4/client_test.go +++ b/pkg/util/ecs/metadata/v3or4/client_test.go @@ -53,7 +53,7 @@ func TestGetV4TaskWithTagsWithoutRetryWithDelay(t *testing.T) { // default timeout is 500ms while the delay is 1.5s require.True(t, os.IsTimeout(err)) require.Nil(t, task) - require.Equal(t, 1, dummyECS.RequestCount) + require.Equal(t, uint64(1), dummyECS.RequestCount.Load()) } func TestGetV4TaskWithTagsWithRetryWithDelay(t *testing.T) { @@ -81,7 +81,7 @@ func TestGetV4TaskWithTagsWithRetryWithDelay(t *testing.T) { // 1st request failed: request timeout is 500ms // 2nd request failed: request timeout is 1s // 3rd request succeed: request timeout is 2s - require.Equal(t, 3, dummyECS.RequestCount) + require.Equal(t, uint64(3), dummyECS.RequestCount.Load()) } // expected is an expected Task from ./testdata/task.json From b59e88c0f928f8f82737e0d89af9225c68345c12 Mon Sep 17 00:00:00 2001 From: Alexandre Menasria <47357713+amenasria@users.noreply.github.com> Date: Wed, 21 Aug 2024 12:22:17 +0200 Subject: [PATCH 071/245] [CI] Improve go cache using gimme on macOS Gitlab runners (#28573) --- .gitlab/source_test/macos.yml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml index 84fea058c18b2..0e05f7fb40946 100644 --- a/.gitlab/source_test/macos.yml +++ b/.gitlab/source_test/macos.yml @@ -55,20 +55,11 @@ lint_macos: PYTHON_RUNTIMES: "3" # The Gitlab macOS runners are currently long runners, so we need to clean them beforehand. before_script: - # Remove the Go cache and env if the Go version changed + # Selecting the current Go version - | - GO_REPO_VERSION=$(cat .go-version) - GO_VERSION=$(go version | awk '{print $3}' | sed 's/go//') - if [ "$GO_REPO_VERSION" != "$GO_VERSION" ]; then - echo "Go version $GO_VERSION is different from $GO_REPO_VERSION in .go-version. Cleaning the environment." - go clean -cache -modcache -testcache - rm -rf $HOME/go/bin - echo "Installing Go $GO_REPO_VERSION..." - echo "$(gimme $(cat .go-version))" - eval $(gimme $(cat .go-version)) - else - echo "Go current version $GO_VERSION is the same as .go-version. Keeping the cache." - fi + eval $(gimme $(cat .go-version)) + echo "Don't forget to regularly delete unused versions. Here are the installed versions and their memory usage on the runner:" + du -sh $HOME/.gimme/versions/* # Remove the Python cache and env if the Python version changed - | PYTHON_REPO_VERSION=$(cat .python-version) From 7748aa62fdd12a35cf2f461220f4ca5d483a1ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Wed, 21 Aug 2024 12:22:59 +0200 Subject: [PATCH 072/245] [EBPF] Create kmt.flare command (#28606) --- tasks/kernel_matrix_testing/kmt_os.py | 49 ++++++++++++++++++++++++++- tasks/kmt.py | 11 ++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/tasks/kernel_matrix_testing/kmt_os.py b/tasks/kernel_matrix_testing/kmt_os.py index 40d99545a98da..7bd03ece4126f 100644 --- a/tasks/kernel_matrix_testing/kmt_os.py +++ b/tasks/kernel_matrix_testing/kmt_os.py @@ -3,11 +3,12 @@ import platform import plistlib import sys +from datetime import datetime from pathlib import Path from invoke.context import Context -from tasks.kernel_matrix_testing.tool import Exit +from tasks.kernel_matrix_testing.tool import Exit, info from tasks.system_probe import is_root @@ -104,6 +105,12 @@ def install_requirements(ctx: Context): ctx.run("sudo systemctl start nfs-kernel-server.service") + @staticmethod + def flare(ctx: Context, flare_folder: Path): + ctx.run(f"apt-cache policy{' '.join(Linux.packages)} > {flare_folder / 'packages.txt'}", warn=True) + ctx.run(f"ip r > {flare_folder / 'ip_r.txt'}", warn=True) + ctx.run(f"ip a > {flare_folder / 'ip_a.txt'}", warn=True) + class MacOS: kmt_dir = get_home_macos() @@ -212,3 +219,43 @@ def init_local(ctx: Context): @staticmethod def install_requirements(ctx: Context): ctx.run("brew install " + " ".join(MacOS.packages)) + + @staticmethod + def flare(ctx: Context, flare_folder: Path): + ctx.run(f"brew list {' '.join(MacOS.packages)} > {flare_folder / 'brew_libvirt.txt'}", warn=True) + ctx.run(f"netstat -an > {flare_folder / 'netstat.txt'}", warn=True) + ctx.run(f"ifconfig -a > {flare_folder / 'ifconfig.txt'}", warn=True) + + +def flare(ctx: Context, tmp_flare_folder: Path, dest_folder: Path, keep_uncompressed_files: bool = False): + kmt_os = get_kmt_os() + kmt_os.flare(ctx, tmp_flare_folder) + + ctx.run(f"git rev-parse HEAD > {tmp_flare_folder}/git_version.txt", warn=True) + ctx.run(f"git rev-parse --abbrev-ref HEAD >> {tmp_flare_folder}/git_version.txt", warn=True) + ctx.run(f"mkdir -p {tmp_flare_folder}/stacks") + ctx.run(f"cp -r {kmt_os.stacks_dir} {tmp_flare_folder}/stacks", warn=True) + ctx.run(f"sudo virsh list --all > {tmp_flare_folder}/virsh_list.txt", warn=True) + + virsh_config_folder = tmp_flare_folder / 'vm-configs' + ctx.run(f"mkdir -p {virsh_config_folder}", warn=True) + ctx.run( + f"sudo virsh list --all | grep ddvm | awk '{{ print $2 }}' | xargs -I % -S 1024 /bin/bash -c \"sudo virsh dumpxml % > {virsh_config_folder}/%.xml\"", + warn=True, + ) + ctx.run(f"ps aux | grep -i appgate | grep -v grep > {tmp_flare_folder}/ps_appgate.txt", warn=True) + ctx.run(f"sudo chmod a+rw {tmp_flare_folder}/*", warn=True) + + ctx.run(f"mkdir -p {dest_folder}") + now_ts = datetime.now().strftime("%Y%m%d-%H%M%S") + flare_fname = f"kmt_flare_{now_ts}.tar.gz" + flare_path = dest_folder / flare_fname + ctx.run(f"tar -C {tmp_flare_folder} -czf {flare_path} .") + + info(f"[+] Flare saved to {flare_path}") + + if keep_uncompressed_files: + flare_dest_folder = dest_folder / f"kmt_flare_{now_ts}" + ctx.run(f"mkdir -p {flare_dest_folder}") + ctx.run(f"cp -r {tmp_flare_folder}/* {flare_dest_folder}") + info(f"[+] Flare uncompressed contents saved to {flare_dest_folder}") diff --git a/tasks/kmt.py b/tasks/kmt.py index 4721aa63db90b..4ee901ad3cb25 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -37,6 +37,7 @@ try_get_ssh_key, ) from tasks.kernel_matrix_testing.init_kmt import init_kernel_matrix_testing_system +from tasks.kernel_matrix_testing.kmt_os import flare as flare_kmt_os from tasks.kernel_matrix_testing.kmt_os import get_kmt_os from tasks.kernel_matrix_testing.platforms import get_platforms, platforms_file from tasks.kernel_matrix_testing.stacks import check_and_get_stack, ec2_instance_ids @@ -2153,3 +2154,13 @@ def download_complexity_data(ctx: Context, commit: str, dest_path: str | Path): job_folder.mkdir(parents=True, exist_ok=True) tar.extractall(dest_path / job_folder) print(f"Extracted complexity data for {job.name} successfully, filename {complexity_data_fname}") + + +@task +def flare(ctx: Context, dest_folder: Path | str | None = None, keep_uncompressed_files: bool = False): + if dest_folder is None: + dest_folder = "." + dest_folder = Path(dest_folder) + + with tempfile.TemporaryDirectory() as tmpdir: + flare_kmt_os(ctx, Path(tmpdir), dest_folder, keep_uncompressed_files) From dcc8e4699cad7af161e7efb889fbb81e029d6562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Wed, 21 Aug 2024 12:23:23 +0200 Subject: [PATCH 073/245] [EBPF] Improve complexity changes comment (#28506) --- tasks/ebpf.py | 57 ++++++++++++++++++++++++++++++++++++++++++--------- tasks/kmt.py | 14 +++++++++++-- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/tasks/ebpf.py b/tasks/ebpf.py index b90d4fa20f880..76eb2382b0af0 100644 --- a/tasks/ebpf.py +++ b/tasks/ebpf.py @@ -5,6 +5,7 @@ from tasks.github_tasks import pr_commenter from tasks.kmt import download_complexity_data +from tasks.libs.ciproviders.github_api import GithubAPI from tasks.libs.common.git import get_commit_sha, get_current_branch from tasks.libs.types.arch import Arch @@ -674,14 +675,37 @@ def generate_html_report(ctx: Context, dest_folder: str | Path): shutil.copy(file, dest_folder) -@task -def generate_complexity_summary_for_pr(ctx: Context, skip_github_comment=False): +@task( + help={ + "skip_github_comment": "Do not comment on the PR with the complexity summary", + "branch_name": "Branch name to use for the complexity data. By default, the current branch is used", + "base_branch": "Base branch to compare against. If not provided, we will try to find a PR for the current branch, and use the base branch from there. If that fails, the main branch will be used", + } +) +def generate_complexity_summary_for_pr( + ctx: Context, skip_github_comment=False, branch_name: str | None = None, base_branch: str | None = None +): """Task meant to run in CI. Generates a summary of the complexity data for the current PR""" if tabulate is None: raise Exit("tabulate is required to print the complexity summary") pr_comment_head = 'eBPF complexity changes' - branch_name = get_current_branch(ctx) + + if branch_name is None: + branch_name = get_current_branch(ctx) + + if base_branch is None: + github = GithubAPI() + prs = list(github.get_pr_for_branch(branch_name)) + if len(prs) == 0: + print(f"Warning: No PR found for branch {branch_name}, using main branch as base") + base_branch = "main" + elif len(prs) > 1: + print(f"Warning: Multiple PRs found for branch {branch_name}, using main branch as base") + base_branch = "main" + else: + base_branch = prs[0].base.ref + print(f"Found PR {prs[0].number} for this branch, using base branch {base_branch}") def _exit_or_delete_github_comment(msg: str): if skip_github_comment: @@ -704,9 +728,11 @@ def _exit_or_delete_github_comment(msg: str): return # We have files, now get files for the main branch - common_ancestor = cast(Result, ctx.run("git merge-base HEAD origin/main", hide=True)).stdout.strip() + common_ancestor = cast( + Result, ctx.run(f"git merge-base {branch_name} origin/{base_branch}", hide=True) + ).stdout.strip() main_branch_complexity_path = Path("/tmp/verifier-complexity-main") - print(f"Downloading complexity data for main branch (commit {common_ancestor})...") + print(f"Downloading complexity data for {base_branch} branch (commit {common_ancestor})...") download_complexity_data(ctx, common_ancestor, main_branch_complexity_path) main_complexity_files = list(main_branch_complexity_path.glob("verifier-complexity-*")) @@ -853,16 +879,27 @@ def _exit_or_delete_github_comment(msg: str): if not any(row[-1] for row in rows): continue - # Format rows to make it more compact, remove the changes marker and remove the object name - rows = [[row[0].split("/")[1]] + row[1:-1] for row in rows] + rows = list(rows) # Convert the iterator to a list, so we can iterate over it multiple times + + def _build_table(orig_rows): + # Format rows to make it more compact, remove the changes marker and remove the object name + changed_rows = [[row[0].split("/")[1]] + row[1:-1] for row in orig_rows] + assert tabulate is not None # For typing + return tabulate(changed_rows, headers=headers, tablefmt="github") + + with_changes = [row for row in rows if row[-1]] + without_changes = [row for row in rows if not row[-1]] msg += f"\n
{group} details\n\n" - msg += f"## {group}\n\n" - msg += tabulate(rows, headers=headers, tablefmt="github") + msg += f"## {group} [programs with changes]\n\n" + msg += _build_table(with_changes) + msg += f"\n\n## {group} [programs without changes]\n\n" + msg += _build_table(without_changes) msg += "\n\n
\n" has_any_changes = True - msg += f"\n\nThis report was generated based on the complexity data for the current branch {branch_name} (pipeline [https://gitlab.ddbuild.io/DataDog/datadog-agent/-/pipelines/{pipeline_id}]{pipeline_id}) and the main branch (commit {common_ancestor}). Objects without changes are not reported. Contact [#ebpf-platform](https://dd.enterprise.slack.com/archives/C0424HA1SJK) if you have any questions/feedback." + curr_commit = get_commit_sha(ctx, short=False) + msg += f"\n\nThis report was generated based on the complexity data for the current branch {branch_name} (pipeline [{pipeline_id}](https://gitlab.ddbuild.io/DataDog/datadog-agent/-/pipelines/{pipeline_id}), commit {curr_commit}) and the base branch {base_branch} (commit {common_ancestor}). Objects without changes are not reported. Contact [#ebpf-platform](https://dd.enterprise.slack.com/archives/C0424HA1SJK) if you have any questions/feedback." msg += "\n\nTable complexity legend: 🔵 - new; ⚪ - unchanged; 🟢 - reduced; 🔴 - increased" print(msg) diff --git a/tasks/kmt.py b/tasks/kmt.py index 4ee901ad3cb25..6d38c64af6a91 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -2124,8 +2124,14 @@ def install_ddagent( build_layout(ctx, domains, layout, verbose) -@task -def download_complexity_data(ctx: Context, commit: str, dest_path: str | Path): +@task( + help={ + "commit": "The commit to download the complexity data for", + "dest_path": "The path to save the complexity data to", + "keep_compressed_archives": "Keep the compressed archives after extracting the data. Useful for testing, as it replicates the exact state of the artifacts in CI", + } +) +def download_complexity_data(ctx: Context, commit: str, dest_path: str | Path, keep_compressed_archives: bool = False): gitlab = get_gitlab_repo() dest_path = Path(dest_path) @@ -2149,6 +2155,10 @@ def download_complexity_data(ctx: Context, commit: str, dest_path: str | Path): print(f"Complexity data not found for {job.name} - filename {complexity_data_fname} not found") continue + if keep_compressed_archives: + with open(dest_path / f"{complexity_name}.tar.gz", "wb") as f: + f.write(data) + tar = tarfile.open(fileobj=io.BytesIO(data)) job_folder = dest_path / complexity_name job_folder.mkdir(parents=True, exist_ok=True) From ab0aa993d43e001f5c2335ea4e269aec9f9541e6 Mon Sep 17 00:00:00 2001 From: maxime mouial Date: Wed, 21 Aug 2024 12:43:44 +0200 Subject: [PATCH 074/245] Remove 'ResetSystemProbeConfig' function in favor of the config mocks (#28538) --- cmd/system-probe/config/adjust_npm_test.go | 5 +- cmd/system-probe/config/config_linux_test.go | 4 +- cmd/system-probe/config/config_test.go | 38 +- pkg/config/test_helpers.go | 16 - pkg/network/config/config_linux_test.go | 5 +- pkg/network/config/config_test.go | 1061 +++++++----------- 6 files changed, 445 insertions(+), 684 deletions(-) diff --git a/cmd/system-probe/config/adjust_npm_test.go b/cmd/system-probe/config/adjust_npm_test.go index fac7b1e5e06bf..9aab2b4453eaa 100644 --- a/cmd/system-probe/config/adjust_npm_test.go +++ b/cmd/system-probe/config/adjust_npm_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/DataDog/datadog-agent/pkg/config/model" ) @@ -28,8 +28,7 @@ func TestAdjustConnectionRollup(t *testing.T) { for _, te := range tests { t.Run(fmt.Sprintf("npm_enabled_%t_usm_enabled_%t", te.npmEnabled, te.usmEnabled), func(t *testing.T) { - config.ResetSystemProbeConfig(t) - cfg := config.SystemProbe() + cfg := mock.NewSystemProbe(t) cfg.Set(netNS("enable_connection_rollup"), te.npmEnabled, model.SourceUnknown) cfg.Set(smNS("enable_connection_rollup"), te.usmEnabled, model.SourceUnknown) Adjust(cfg) diff --git a/cmd/system-probe/config/config_linux_test.go b/cmd/system-probe/config/config_linux_test.go index a9a71ca6b2928..b2278f41d7f8f 100644 --- a/cmd/system-probe/config/config_linux_test.go +++ b/cmd/system-probe/config/config_linux_test.go @@ -15,6 +15,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/config/mock" ) func TestNetworkProcessEventMonitoring(t *testing.T) { @@ -60,9 +61,8 @@ func TestDynamicInstrumentation(t *testing.T) { } func TestEventStreamEnabledForSupportedKernelsLinux(t *testing.T) { - config.ResetSystemProbeConfig(t) t.Setenv("DD_SYSTEM_PROBE_EVENT_MONITORING_NETWORK_PROCESS_ENABLED", strconv.FormatBool(true)) - cfg := config.SystemProbe() + cfg := mock.NewSystemProbe(t) Adjust(cfg) if ProcessEventDataStreamSupported() { diff --git a/cmd/system-probe/config/config_test.go b/cmd/system-probe/config/config_test.go index db57327068788..e54d6d5b8127d 100644 --- a/cmd/system-probe/config/config_test.go +++ b/cmd/system-probe/config/config_test.go @@ -9,7 +9,6 @@ package config import ( "fmt" - "os" "runtime" "strconv" "testing" @@ -18,6 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/config/mock" ) func TestEventMonitor(t *testing.T) { @@ -65,9 +65,8 @@ func TestEventStreamEnabledForSupportedKernelsWindowsUnsupported(t *testing.T) { if runtime.GOOS != "windows" { t.Skip("This is only for windows") } - config.ResetSystemProbeConfig(t) t.Setenv("DD_SYSTEM_PROBE_EVENT_MONITORING_NETWORK_PROCESS_ENABLED", strconv.FormatBool(true)) - cfg := config.SystemProbe() + cfg := mock.NewSystemProbe(t) Adjust(cfg) require.False(t, cfg.GetBool("event_monitoring_config.network_process.enabled")) @@ -76,9 +75,8 @@ func TestEventStreamEnabledForSupportedKernelsWindowsUnsupported(t *testing.T) { if runtime.GOOS == "windows" || runtime.GOOS == "linux" { t.Skip("This is only for unsupported") } - config.ResetSystemProbeConfig(t) t.Setenv("DD_SYSTEM_PROBE_EVENT_MONITORING_NETWORK_PROCESS_ENABLED", strconv.FormatBool(true)) - cfg := config.SystemProbe() + cfg := mock.NewSystemProbe(t) Adjust(cfg) require.False(t, cfg.GetBool("event_monitoring_config.network_process.enabled")) @@ -87,37 +85,19 @@ func TestEventStreamEnabledForSupportedKernelsWindowsUnsupported(t *testing.T) { func TestEnableDiscovery(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - config.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -discovery: - enabled: true -`) + cfg := mock.NewSystemProbe(t) + cfg.SetWithoutSource("discovery.enabled", true) assert.True(t, cfg.GetBool(discoveryNS("enabled"))) }) t.Run("via ENV variable", func(t *testing.T) { - config.ResetSystemProbeConfig(t) t.Setenv("DD_DISCOVERY_ENABLED", "true") - assert.True(t, config.SystemProbe().GetBool(discoveryNS("enabled"))) + cfg := mock.NewSystemProbe(t) + assert.True(t, cfg.GetBool(discoveryNS("enabled"))) }) t.Run("default", func(t *testing.T) { - config.ResetSystemProbeConfig(t) - assert.False(t, config.SystemProbe().GetBool(discoveryNS("enabled"))) + cfg := mock.NewSystemProbe(t) + assert.False(t, cfg.GetBool(discoveryNS("enabled"))) }) } - -func configurationFromYAML(t *testing.T, yaml string) config.Config { - f, err := os.CreateTemp(t.TempDir(), "system-probe.*.yaml") - require.NoError(t, err) - defer f.Close() - - b := []byte(yaml) - n, err := f.Write(b) - require.NoError(t, err) - require.Equal(t, len(b), n) - f.Sync() - - _, _ = New(f.Name(), "") - return config.SystemProbe() -} diff --git a/pkg/config/test_helpers.go b/pkg/config/test_helpers.go index cde9d54eb28bb..b463b7980fbf0 100644 --- a/pkg/config/test_helpers.go +++ b/pkg/config/test_helpers.go @@ -8,9 +8,6 @@ package config import ( - "strings" - "testing" - "github.com/DataDog/datadog-agent/pkg/config/env" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" ) @@ -21,19 +18,6 @@ var ( // SetFeaturesNoCleanup is alias from env SetFeaturesNoCleanup = env.SetFeaturesNoCleanup - // SetupConf generates and returns a new configuration - SetupConf = pkgconfigsetup.Conf - // SetupConfFromYAML generates a configuration from the given yaml config SetupConfFromYAML = pkgconfigsetup.ConfFromYAML ) - -// ResetSystemProbeConfig resets the configuration. -func ResetSystemProbeConfig(t *testing.T) { - originalConfig := pkgconfigsetup.SystemProbe() - t.Cleanup(func() { - pkgconfigsetup.SetSystemProbe(originalConfig) - }) - pkgconfigsetup.SetSystemProbe(NewConfig("system-probe", "DD", strings.NewReplacer(".", "_"))) - pkgconfigsetup.InitSystemProbeConfig(pkgconfigsetup.SystemProbe()) -} diff --git a/pkg/network/config/config_linux_test.go b/pkg/network/config/config_linux_test.go index 74244d3d06916..3ac7685449ec4 100644 --- a/pkg/network/config/config_linux_test.go +++ b/pkg/network/config/config_linux_test.go @@ -9,15 +9,14 @@ import ( "os" "testing" + "github.com/DataDog/datadog-agent/pkg/config/mock" "github.com/stretchr/testify/require" "github.com/vishvananda/netns" - - aconfig "github.com/DataDog/datadog-agent/pkg/config" ) func TestDisableRootNetNamespace(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) t.Setenv("DD_NETWORK_CONFIG_ENABLE_ROOT_NETNS", "false") + mock.NewSystemProbe(t) cfg := New() require.False(t, cfg.EnableConntrackAllNamespaces) diff --git a/pkg/network/config/config_test.go b/pkg/network/config/config_test.go index f8abb6345eaa0..d732160b00b09 100644 --- a/pkg/network/config/config_test.go +++ b/pkg/network/config/config_test.go @@ -20,8 +20,7 @@ import ( "github.com/stretchr/testify/require" sysconfig "github.com/DataDog/datadog-agent/cmd/system-probe/config" - aconfig "github.com/DataDog/datadog-agent/pkg/config" - "github.com/DataDog/datadog-agent/pkg/config/model" + "github.com/DataDog/datadog-agent/pkg/config/mock" ) // variables for testing config options @@ -33,27 +32,23 @@ const ( invalidHTTPRequestFragment = 600 ) -func makeYamlConfigString(section, entry string, val int) string { - return fmt.Sprintf("\n%s:\n %s: %d", section, entry, val) -} func TestDisablingDNSInspection(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -system_probe_config: - enabled: true - disable_dns_inspection: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("system_probe_config.enabled", true) + mockSystemProbe.SetWithoutSource("system_probe_config.disable_dns_inspection", true) + cfg := New() assert.False(t, cfg.DNSInspection) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_DISABLE_DNS_INSPECTION", "true") + cfg := New() + _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.False(t, cfg.DNSInspection) }) @@ -61,21 +56,19 @@ system_probe_config: func TestDisablingProtocolClassification(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -network_config: - enable_protocol_classification: false -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.enable_protocol_classification", false) + cfg := New() assert.False(t, cfg.ProtocolClassificationEnabled) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_ENABLE_PROTOCOL_CLASSIFICATION", "false") + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.False(t, cfg.ProtocolClassificationEnabled) }) @@ -83,21 +76,19 @@ network_config: func TestEnableHTTPStatsByStatusCode(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - enable_http_stats_by_status_code: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enable_http_stats_by_status_code", true) + cfg := New() assert.True(t, cfg.EnableHTTPStatsByStatusCode) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_HTTP_STATS_BY_STATUS_CODE", "true") + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnableHTTPStatsByStatusCode) }) @@ -105,107 +96,98 @@ service_monitoring_config: func TestEnableHTTPMonitoring(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -network_config: - enable_http_monitoring: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.enable_http_monitoring", true) + cfg := New() assert.True(t, cfg.EnableHTTPMonitoring) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLE_HTTP_MONITORING", "true") + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnableHTTPMonitoring) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - enable_http_monitoring: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enable_http_monitoring", true) + cfg := New() assert.True(t, cfg.EnableHTTPMonitoring) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_HTTP_MONITORING", "true") + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnableHTTPMonitoring) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLE_HTTP_MONITORING", "true") t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_HTTP_MONITORING", "false") + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.False(t, cfg.EnableHTTPMonitoring) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLE_HTTP_MONITORING", "false") t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_HTTP_MONITORING", "true") + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnableHTTPMonitoring) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLE_HTTP_MONITORING", "true") t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_HTTP_MONITORING", "true") + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnableHTTPMonitoring) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() - assert.False(t, cfg.EnableHTTPMonitoring) }) } func TestEnableJavaTLSSupport(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - tls: - java: - enabled: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.tls.java.enabled", true) + cfg := New() + require.True(t, cfg.EnableJavaTLSSupport) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_JAVA_ENABLED", "true") - cfg := New() require.True(t, cfg.EnableJavaTLSSupport) @@ -215,21 +197,20 @@ service_monitoring_config: func TestEnableHTTP2Monitoring(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - enable_http2_monitoring: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enable_http2_monitoring", true) + cfg := New() assert.True(t, cfg.EnableHTTP2Monitoring) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_HTTP2_MONITORING", "true") + cfg := New() + _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnableHTTP2Monitoring) }) @@ -237,21 +218,20 @@ service_monitoring_config: func TestEnableKafkaMonitoring(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - enable_kafka_monitoring: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enable_kafka_monitoring", true) + cfg := New() assert.True(t, cfg.EnableKafkaMonitoring) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_KAFKA_MONITORING", "true") + cfg := New() + _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnableKafkaMonitoring) }) @@ -259,27 +239,26 @@ service_monitoring_config: func TestEnablePostgresMonitoring(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - enable_postgres_monitoring: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enable_postgres_monitoring", true) + cfg := New() assert.True(t, cfg.EnablePostgresMonitoring) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_POSTGRES_MONITORING", "true") + cfg := New() + _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnablePostgresMonitoring) }) t.Run("default", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() assert.False(t, cfg.EnablePostgresMonitoring) @@ -288,27 +267,26 @@ service_monitoring_config: func TestEnableRedisMonitoring(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - enable_redis_monitoring: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enable_redis_monitoring", true) + cfg := New() assert.True(t, cfg.EnableRedisMonitoring) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_REDIS_MONITORING", "true") + cfg := New() + _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnableRedisMonitoring) }) t.Run("default", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() assert.False(t, cfg.EnableRedisMonitoring) @@ -316,50 +294,47 @@ service_monitoring_config: } func TestDefaultDisabledJavaTLSSupport(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.False(t, cfg.EnableJavaTLSSupport) } func TestDefaultDisabledHTTP2Support(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.False(t, cfg.EnableHTTP2Monitoring) } func TestDisableGatewayLookup(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mockSystemProbe := mock.NewSystemProbe(t) + cfg := New() // default config _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.True(t, cfg.EnableGatewayLookup) - aconfig.ResetSystemProbeConfig(t) - cfg = configurationFromYAML(t, ` -network_config: - enable_gateway_lookup: false -`) + mockSystemProbe.SetWithoutSource("network_config.enable_gateway_lookup", false) + cfg = New() assert.False(t, cfg.EnableGatewayLookup) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLE_GATEWAY_LOOKUP", "false") + cfg := New() _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.False(t, cfg.EnableGatewayLookup) }) @@ -367,21 +342,20 @@ network_config: func TestIgnoreConntrackInitFailure(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -network_config: - ignore_conntrack_init_failure: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.ignore_conntrack_init_failure", true) + cfg := New() assert.True(t, cfg.IgnoreConntrackInitFailure) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_IGNORE_CONNTRACK_INIT_FAILURE", "true") + cfg := New() + _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.Nil(t, err) assert.True(t, cfg.IgnoreConntrackInitFailure) @@ -390,56 +364,53 @@ network_config: func TestEnablingDNSStatsCollection(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -system_probe_config: - collect_dns_stats: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("system_probe_config.collect_dns_stats", true) + cfg := New() assert.True(t, cfg.CollectDNSStats) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_COLLECT_DNS_STATS", "false") + cfg := New() + _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.False(t, cfg.CollectDNSStats) - aconfig.ResetSystemProbeConfig(t) t.Setenv("DD_COLLECT_DNS_STATS", "true") _, err = sysconfig.New("", "") require.NoError(t, err) cfg = New() - assert.True(t, cfg.CollectDNSStats) }) } func TestDisablingDNSDomainCollection(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -system_probe_config: - collect_dns_domains: false - max_dns_stats: 100 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("system_probe_config.collect_dns_domains", false) + cfg := New() + + mockSystemProbe.SetWithoutSource("system_probe_config.max_dns_stats", 100) assert.False(t, cfg.CollectDNSDomains) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_COLLECT_DNS_DOMAINS", "false") + cfg := New() + _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.False(t, cfg.CollectDNSDomains) - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_COLLECT_DNS_DOMAINS", "true") _, err = sysconfig.New("", "") require.NoError(t, err) @@ -451,26 +422,23 @@ system_probe_config: func TestSettingMaxDNSStats(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -system_probe_config: - collect_dns_domains: false - max_dns_stats: 100 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("system_probe_config.collect_dns_domains", false) + mockSystemProbe.SetWithoutSource("system_probe_config.max_dns_stats", 100) + cfg := New() assert.Equal(t, 100, cfg.MaxDNSStats) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) + cfg := New() os.Unsetenv("DD_SYSTEM_PROBE_CONFIG_MAX_DNS_STATS") _, err := sysconfig.New("", "") require.NoError(t, err) - cfg := New() assert.Equal(t, 20000, cfg.MaxDNSStats) // default value - aconfig.ResetSystemProbeConfig(t) t.Setenv("DD_SYSTEM_PROBE_CONFIG_MAX_DNS_STATS", "10000") _, err = sysconfig.New("", "") require.NoError(t, err) @@ -512,19 +480,16 @@ func TestHTTPReplaceRules(t *testing.T) { "pattern": "payment_id" } ] - ` + ` t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -network_config: - http_replace_rules: - - pattern: "/users/(.*)" - repl: "/users/?" - - pattern: "foo" - repl: "bar" - - pattern: "payment_id" -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.http_replace_rules", []map[string]string{ + {"pattern": "/users/(.*)", "repl": "/users/?"}, + {"pattern": "foo", "repl": "bar"}, + {"pattern": "payment_id"}, + }) + cfg := New() require.Len(t, cfg.HTTPReplaceRules, 3) for i, r := range expected { @@ -533,9 +498,8 @@ network_config: }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_HTTP_REPLACE_RULES", envContent) - cfg := New() require.Len(t, cfg.HTTPReplaceRules, 3) @@ -545,16 +509,14 @@ network_config: }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - http_replace_rules: - - pattern: "/users/(.*)" - repl: "/users/?" - - pattern: "foo" - repl: "bar" - - pattern: "payment_id" -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.http_replace_rules", []map[string]string{ + {"pattern": "/users/(.*)", "repl": "/users/?"}, + {"pattern": "foo", "repl": "bar"}, + {"pattern": "payment_id"}, + }) + cfg := New() + require.Len(t, cfg.HTTPReplaceRules, 3) for i, r := range expected { assert.Equal(t, r, cfg.HTTPReplaceRules[i]) @@ -562,9 +524,8 @@ service_monitoring_config: }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_REPLACE_RULES", envContent) - cfg := New() require.Len(t, cfg.HTTPReplaceRules, 3) @@ -574,9 +535,8 @@ service_monitoring_config: }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_HTTP_REPLACE_RULES", envContent) - cfg := New() require.Len(t, cfg.HTTPReplaceRules, 3) @@ -586,9 +546,8 @@ service_monitoring_config: }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_REPLACE_RULES", envContent) - cfg := New() require.Len(t, cfg.HTTPReplaceRules, 3) @@ -598,8 +557,10 @@ service_monitoring_config: }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_REPLACE_RULES", envContent) + cfg := New() + // Setting a different value for the old value, as we should override. t.Setenv("DD_SYSTEM_PROBE_NETWORK_HTTP_REPLACE_RULES", ` [ @@ -609,8 +570,6 @@ service_monitoring_config: ] `) - cfg := New() - require.Len(t, cfg.HTTPReplaceRules, 3) for i, r := range expected { assert.Equal(t, r, cfg.HTTPReplaceRules[i]) @@ -618,7 +577,7 @@ service_monitoring_config: }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() assert.Empty(t, cfg.HTTPReplaceRules) @@ -627,75 +586,66 @@ service_monitoring_config: func TestMaxTrackedHTTPConnections(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -network_config: - max_tracked_http_connections: 1025 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.max_tracked_http_connections", 1025) + cfg := New() require.Equal(t, cfg.MaxTrackedHTTPConnections, int64(1025)) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_MAX_TRACKED_HTTP_CONNECTIONS", "1025") - cfg := New() require.Equal(t, cfg.MaxTrackedHTTPConnections, int64(1025)) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - max_tracked_http_connections: 1025 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.max_tracked_http_connections", 1025) + cfg := New() require.Equal(t, cfg.MaxTrackedHTTPConnections, int64(1025)) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_TRACKED_HTTP_CONNECTIONS", "1025") - cfg := New() require.Equal(t, cfg.MaxTrackedHTTPConnections, int64(1025)) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_MAX_TRACKED_HTTP_CONNECTIONS", "1025") - cfg := New() require.Equal(t, cfg.MaxTrackedHTTPConnections, int64(1025)) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_TRACKED_HTTP_CONNECTIONS", "1025") - cfg := New() require.Equal(t, cfg.MaxTrackedHTTPConnections, int64(1025)) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_MAX_TRACKED_HTTP_CONNECTIONS", "1026") t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_TRACKED_HTTP_CONNECTIONS", "1025") - cfg := New() require.Equal(t, cfg.MaxTrackedHTTPConnections, int64(1025)) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.Equal(t, cfg.MaxTrackedHTTPConnections, int64(1024)) }) @@ -703,27 +653,25 @@ service_monitoring_config: func TestHTTP2DynamicTableMapCleanerInterval(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - http2_dynamic_table_map_cleaner_interval_seconds: 1025 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.http2_dynamic_table_map_cleaner_interval_seconds", 1025) + cfg := New() require.Equal(t, cfg.HTTP2DynamicTableMapCleanerInterval, 1025*time.Second) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP2_DYNAMIC_TABLE_MAP_CLEANER_INTERVAL_SECONDS", "1025") - cfg := New() require.Equal(t, cfg.HTTP2DynamicTableMapCleanerInterval, 1025*time.Second) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.Equal(t, cfg.HTTP2DynamicTableMapCleanerInterval, 30*time.Second) }) @@ -731,75 +679,66 @@ service_monitoring_config: func TestHTTPMapCleanerInterval(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -system_probe_config: - http_map_cleaner_interval_in_s: 1025 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("system_probe_config.http_map_cleaner_interval_in_s", 1025) + cfg := New() require.Equal(t, cfg.HTTPMapCleanerInterval, 1025*time.Second) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_CONFIG_HTTP_MAP_CLEANER_INTERVAL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPMapCleanerInterval, 1025*time.Second) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - http_map_cleaner_interval_in_s: 1025 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.http_map_cleaner_interval_in_s", 1025) + cfg := New() require.Equal(t, cfg.HTTPMapCleanerInterval, 1025*time.Second) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_MAP_CLEANER_INTERVAL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPMapCleanerInterval, 1025*time.Second) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_CONFIG_HTTP_MAP_CLEANER_INTERVAL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPMapCleanerInterval, 1025*time.Second) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_MAP_CLEANER_INTERVAL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPMapCleanerInterval, 1025*time.Second) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_CONFIG_HTTP_MAP_CLEANER_INTERVAL_IN_S", "1026") t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_MAP_CLEANER_INTERVAL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPMapCleanerInterval, 1025*time.Second) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.Equal(t, cfg.HTTPMapCleanerInterval, 300*time.Second) }) @@ -807,74 +746,66 @@ service_monitoring_config: func TestHTTPIdleConnectionTTL(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -system_probe_config: - http_idle_connection_ttl_in_s: 1025 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("system_probe_config.http_idle_connection_ttl_in_s", 1025) + cfg := New() require.Equal(t, cfg.HTTPIdleConnectionTTL, 1025*time.Second) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_CONFIG_HTTP_IDLE_CONNECTION_TTL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPIdleConnectionTTL, 1025*time.Second) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - http_idle_connection_ttl_in_s: 1025 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.http_idle_connection_ttl_in_s", 1025) + cfg := New() + require.Equal(t, cfg.HTTPIdleConnectionTTL, 1025*time.Second) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_IDLE_CONNECTION_TTL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPIdleConnectionTTL, 1025*time.Second) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_CONFIG_HTTP_IDLE_CONNECTION_TTL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPIdleConnectionTTL, 1025*time.Second) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_IDLE_CONNECTION_TTL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPIdleConnectionTTL, 1025*time.Second) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_CONFIG_HTTP_IDLE_CONNECTION_TTL_IN_S", "1026") t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_IDLE_CONNECTION_TTL_IN_S", "1025") - cfg := New() require.Equal(t, cfg.HTTPIdleConnectionTTL, 1025*time.Second) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.Equal(t, cfg.HTTPIdleConnectionTTL, 30*time.Second) }) @@ -882,67 +813,64 @@ service_monitoring_config: func TestHTTPNotificationThreshold(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, makeYamlConfigString("network_config", "http_notification_threshold", validNotificationThreshold)) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.http_notification_threshold", validNotificationThreshold) + cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(validNotificationThreshold)) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(validNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(validNotificationThreshold)) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, makeYamlConfigString("service_monitoring_config", "http_notification_threshold", validNotificationThreshold)) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.http_notification_threshold", validNotificationThreshold) + cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(validNotificationThreshold)) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(validNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(validNotificationThreshold)) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(validNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(validNotificationThreshold)) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(validNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(validNotificationThreshold)) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value. + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(validNotificationThreshold+1)) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(validNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(validNotificationThreshold)) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.Equal(t, cfg.HTTPNotificationThreshold, int64(driverDefaultNotificationThreshold)) }) @@ -951,69 +879,66 @@ func TestHTTPNotificationThreshold(t *testing.T) { // Testing we're not exceeding the limit for http_notification_threshold. func TestHTTPNotificationThresholdOverLimit(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, makeYamlConfigString("network_config", "http_notification_threshold", invalidNotificationThreshold)) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.http_notification_threshold", invalidNotificationThreshold) + cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(driverDefaultNotificationThreshold)) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(invalidNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(driverDefaultNotificationThreshold)) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, makeYamlConfigString("service_monitoring_config", "http_notification_threshold", invalidNotificationThreshold)) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.http_notification_threshold", invalidNotificationThreshold) + cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(driverDefaultNotificationThreshold)) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(invalidNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(driverDefaultNotificationThreshold)) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(invalidNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(driverDefaultNotificationThreshold)) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(invalidNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(driverDefaultNotificationThreshold)) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(invalidNotificationThreshold+1)) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_NOTIFICATION_THRESHOLD", strconv.Itoa(invalidNotificationThreshold)) - cfg := New() require.Equal(t, cfg.HTTPNotificationThreshold, int64(driverDefaultNotificationThreshold)) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.Equal(t, cfg.HTTPNotificationThreshold, int64(driverDefaultNotificationThreshold)) }) @@ -1021,73 +946,66 @@ func TestHTTPNotificationThresholdOverLimit(t *testing.T) { func TestHTTPMaxRequestFragment(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -network_config: - http_max_request_fragment: 155 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.http_max_request_fragment", 155) + cfg := New() + require.Equal(t, cfg.HTTPMaxRequestFragment, int64(155)) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", "155") - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(155)) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - http_max_request_fragment: 155 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.http_max_request_fragment", 155) + cfg := New() + require.Equal(t, cfg.HTTPMaxRequestFragment, int64(155)) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", "155") - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(155)) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", "155") - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(155)) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", "155") - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(155)) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", "151") t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", "155") - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(155)) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.Equal(t, cfg.HTTPMaxRequestFragment, int64(driverMaxFragmentLimit)) }) @@ -1096,69 +1014,66 @@ service_monitoring_config: // Testing we're not exceeding the hard coded limit of http_max_request_fragment. func TestHTTPMaxRequestFragmentLimit(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, makeYamlConfigString("network_config", "http_max_request_fragment", invalidHTTPRequestFragment)) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.http_max_request_fragment", invalidHTTPRequestFragment) + cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(driverMaxFragmentLimit)) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", strconv.Itoa(invalidHTTPRequestFragment)) - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(driverMaxFragmentLimit)) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, makeYamlConfigString("service_monitoring_config", "http_max_request_fragment", invalidHTTPRequestFragment)) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.http_max_request_fragment", invalidHTTPRequestFragment) + cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(driverMaxFragmentLimit)) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", strconv.Itoa(invalidHTTPRequestFragment)) - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(driverMaxFragmentLimit)) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", strconv.Itoa(invalidHTTPRequestFragment)) - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(512)) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", strconv.Itoa(invalidHTTPRequestFragment)) - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(driverMaxFragmentLimit)) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", strconv.Itoa(invalidHTTPRequestFragment)) t.Setenv("DD_SERVICE_MONITORING_CONFIG_HTTP_MAX_REQUEST_FRAGMENT", strconv.Itoa(invalidHTTPRequestFragment+1)) - cfg := New() require.Equal(t, cfg.HTTPMaxRequestFragment, int64(driverMaxFragmentLimit)) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.Equal(t, cfg.HTTPMaxRequestFragment, int64(driverMaxFragmentLimit)) }) @@ -1168,15 +1083,17 @@ func TestMaxClosedConnectionsBuffered(t *testing.T) { maxTrackedConnections := New().MaxTrackedConnections t.Run("value set", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_CONFIG_MAX_CLOSED_CONNECTIONS_BUFFERED", fmt.Sprintf("%d", maxTrackedConnections-1)) cfg := New() + require.Equal(t, maxTrackedConnections-1, cfg.MaxClosedConnectionsBuffered) }) t.Run("value not set", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + require.Equal(t, cfg.MaxTrackedConnections, cfg.MaxClosedConnectionsBuffered) }) } @@ -1185,89 +1102,83 @@ func TestMaxFailedConnectionsBuffered(t *testing.T) { maxTrackedConnections := New().MaxTrackedConnections t.Run("value set", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_NETWORK_CONFIG_MAX_FAILED_CONNECTIONS_BUFFERED", fmt.Sprintf("%d", maxTrackedConnections-1)) cfg := New() + require.Equal(t, maxTrackedConnections-1, cfg.MaxFailedConnectionsBuffered) }) t.Run("value not set", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + require.Equal(t, cfg.MaxTrackedConnections, cfg.MaxFailedConnectionsBuffered) }) } func TestMaxHTTPStatsBuffered(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -network_config: - max_http_stats_buffered: 513 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.max_http_stats_buffered", 513) + cfg := New() require.Equal(t, cfg.MaxHTTPStatsBuffered, 513) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_MAX_HTTP_STATS_BUFFERED", "513") - cfg := New() require.Equal(t, cfg.MaxHTTPStatsBuffered, 513) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - max_http_stats_buffered: 513 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.max_http_stats_buffered", 513) + cfg := New() require.Equal(t, cfg.MaxHTTPStatsBuffered, 513) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_HTTP_STATS_BUFFERED", "513") - cfg := New() require.Equal(t, cfg.MaxHTTPStatsBuffered, 513) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_MAX_HTTP_STATS_BUFFERED", "513") - cfg := New() require.Equal(t, cfg.MaxHTTPStatsBuffered, 513) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_HTTP_STATS_BUFFERED", "513") - cfg := New() require.Equal(t, cfg.MaxHTTPStatsBuffered, 513) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_MAX_HTTP_STATS_BUFFERED", "514") t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_HTTP_STATS_BUFFERED", "513") - cfg := New() require.Equal(t, cfg.MaxHTTPStatsBuffered, 513) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.Equal(t, cfg.MaxHTTPStatsBuffered, 100000) }) @@ -1275,19 +1186,17 @@ service_monitoring_config: func TestMaxKafkaStatsBuffered(t *testing.T) { t.Run("value set through env var", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_KAFKA_STATS_BUFFERED", "50000") - cfg := New() + assert.Equal(t, 50000, cfg.MaxKafkaStatsBuffered) }) t.Run("value set through yaml", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - max_kafka_stats_buffered: 30000 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.max_kafka_stats_buffered", 30000) + cfg := New() assert.Equal(t, 30000, cfg.MaxKafkaStatsBuffered) }) @@ -1295,7 +1204,7 @@ service_monitoring_config: func TestMaxPostgresTelemetryBuffered(t *testing.T) { t.Run("value set through env var", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_POSTGRES_TELEMETRY_BUFFER", "50000") cfg := New() @@ -1303,66 +1212,60 @@ func TestMaxPostgresTelemetryBuffered(t *testing.T) { }) t.Run("value set through yaml", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - max_postgres_telemetry_buffer: 30000 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.max_postgres_telemetry_buffer", 30000) + cfg := New() assert.Equal(t, 30000, cfg.MaxPostgresTelemetryBuffer) }) } func TestMaxPostgresStatsBuffered(t *testing.T) { t.Run("value set through env var", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_POSTGRES_STATS_BUFFERED", "50000") - cfg := New() + assert.Equal(t, 50000, cfg.MaxPostgresStatsBuffered) }) t.Run("value set through yaml", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - max_postgres_stats_buffered: 30000 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.max_postgres_stats_buffered", 30000) + cfg := New() assert.Equal(t, 30000, cfg.MaxPostgresStatsBuffered) }) t.Run("default", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - + mock.NewSystemProbe(t) cfg := New() + assert.Equal(t, 100000, cfg.MaxPostgresStatsBuffered) }) } func TestMaxRedisStatsBuffered(t *testing.T) { t.Run("value set through env var", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_REDIS_STATS_BUFFERED", "50000") - cfg := New() + assert.Equal(t, 50000, cfg.MaxRedisStatsBuffered) }) t.Run("value set through yaml", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - max_redis_stats_buffered: 30000 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.max_redis_stats_buffered", 30000) + cfg := New() assert.Equal(t, 30000, cfg.MaxRedisStatsBuffered) }) t.Run("default", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - + mock.NewSystemProbe(t) cfg := New() + assert.Equal(t, 100000, cfg.MaxRedisStatsBuffered) }) } @@ -1399,10 +1302,11 @@ func TestNetworkConfigEnabled(t *testing.T) { t.Setenv("DD_SYSTEM_PROBE_SERVICE_MONITORING_ENABLED", strconv.FormatBool(*tc.usmIn)) } - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) + cfg := New() _, err = sysconfig.New(f.Name(), "") require.NoError(t, err) - cfg := New() + assert.Equal(t, tc.npmEnabled, cfg.NPMEnabled, "npm state") assert.Equal(t, tc.usmEnabled, cfg.ServiceMonitoringEnabled, "usm state") }) @@ -1411,200 +1315,184 @@ func TestNetworkConfigEnabled(t *testing.T) { func TestIstioMonitoring(t *testing.T) { t.Run("default value", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + assert.False(t, cfg.EnableIstioMonitoring) }) t.Run("via yaml", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - tls: - istio: - enabled: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.tls.istio.enabled", true) + cfg := New() + assert.True(t, cfg.EnableIstioMonitoring) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_ISTIO_ENABLED", "true") - cfg := New() + assert.True(t, cfg.EnableIstioMonitoring) }) } func TestEnvoyPathConfig(t *testing.T) { t.Run("default value", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + assert.EqualValues(t, cfg.EnvoyPath, "/bin/envoy") }) t.Run("via yaml", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - tls: - istio: - envoy_path: "/test/envoy" -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.tls.istio.envoy_path", "/test/envoy") + cfg := New() + assert.EqualValues(t, "/test/envoy", cfg.EnvoyPath) }) t.Run("value set through env var", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_ISTIO_ENVOY_PATH", "/test/envoy") - cfg := New() + assert.EqualValues(t, "/test/envoy", cfg.EnvoyPath) }) } func TestNodeJSMonitoring(t *testing.T) { t.Run("default value", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + assert.False(t, cfg.EnableNodeJSMonitoring) }) t.Run("via yaml", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - tls: - nodejs: - enabled: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.tls.nodejs.enabled", true) + cfg := New() + assert.True(t, cfg.EnableNodeJSMonitoring) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_NODEJS_ENABLED", "true") - cfg := New() + assert.True(t, cfg.EnableNodeJSMonitoring) }) } func TestUSMEventStream(t *testing.T) { t.Run("default value", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + assert.False(t, cfg.EnableUSMEventStream) }) t.Run("via yaml", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - enable_event_stream: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enable_event_stream", true) + cfg := New() + assert.True(t, cfg.EnableUSMEventStream) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_EVENT_STREAM", "true") - cfg := New() + assert.True(t, cfg.EnableUSMEventStream) }) } func TestMaxUSMConcurrentRequests(t *testing.T) { t.Run("default value", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Assert that if not explicitly set this param defaults to `MaxTrackedConnections` // Note this behavior should be deprecated on 7.50 assert.Equal(t, cfg.MaxTrackedConnections, cfg.MaxUSMConcurrentRequests) }) t.Run("via yaml", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - max_concurrent_requests: 1000 -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.max_concurrent_requests", 1000) + cfg := New() + assert.Equal(t, uint32(1000), cfg.MaxUSMConcurrentRequests) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_MAX_CONCURRENT_REQUESTS", "3000") - cfg := New() + assert.Equal(t, uint32(3000), cfg.MaxUSMConcurrentRequests) }) } func TestUSMTLSNativeEnabled(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -network_config: - enable_https_monitoring: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.enable_https_monitoring", true) + cfg := New() require.True(t, cfg.EnableNativeTLSMonitoring) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLE_HTTPS_MONITORING", "true") - cfg := New() require.True(t, cfg.EnableNativeTLSMonitoring) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - tls: - native: - enabled: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.tls.native.enabled", true) + cfg := New() + require.True(t, cfg.EnableNativeTLSMonitoring) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_NATIVE_ENABLED", "true") - cfg := New() require.True(t, cfg.EnableNativeTLSMonitoring) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLE_HTTPS_MONITORING", "true") t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_NATIVE_ENABLED", "false") - cfg := New() require.False(t, cfg.EnableNativeTLSMonitoring) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLE_HTTPS_MONITORING", "false") t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_NATIVE_ENABLED", "true") - cfg := New() require.True(t, cfg.EnableNativeTLSMonitoring) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value + mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLE_HTTPS_MONITORING", "true") t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_NATIVE_ENABLED", "true") cfg := New() @@ -1613,8 +1501,9 @@ service_monitoring_config: }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.False(t, cfg.EnableNativeTLSMonitoring) }) @@ -1622,67 +1511,57 @@ service_monitoring_config: func TestUSMTLSGoEnabled(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - enable_go_tls_support: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enable_go_tls_support", true) + cfg := New() require.True(t, cfg.EnableGoTLSSupport) }) t.Run("via deprecated ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_GO_TLS_SUPPORT", "true") - cfg := New() require.True(t, cfg.EnableGoTLSSupport) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - tls: - go: - enabled: true -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.tls.go.enabled", true) + cfg := New() + require.True(t, cfg.EnableGoTLSSupport) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_GO_ENABLED", "true") - cfg := New() require.True(t, cfg.EnableGoTLSSupport) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_GO_TLS_SUPPORT", "true") + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_GO_ENABLED", "false") - + t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_GO_TLS_SUPPORT", "true") cfg := New() require.False(t, cfg.EnableGoTLSSupport) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_GO_TLS_SUPPORT", "false") t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_GO_ENABLED", "true") - cfg := New() require.True(t, cfg.EnableGoTLSSupport) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_ENABLE_GO_TLS_SUPPORT", "true") t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_GO_ENABLED", "true") cfg := New() @@ -1691,8 +1570,9 @@ service_monitoring_config: }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.False(t, cfg.EnableGoTLSSupport) }) @@ -1700,28 +1580,25 @@ service_monitoring_config: func TestUSMTLSGoExcludeSelf(t *testing.T) { t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := configurationFromYAML(t, ` -service_monitoring_config: - tls: - go: - exclude_self: false -`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.tls.go.exclude_self", false) + cfg := New() + require.False(t, cfg.GoTLSExcludeSelf) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_GO_EXCLUDE_SELF", "false") - cfg := New() require.False(t, cfg.GoTLSExcludeSelf) }) t.Run("Not disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mock.NewSystemProbe(t) cfg := New() + // Default value. require.True(t, cfg.GoTLSExcludeSelf) }) @@ -1729,103 +1606,83 @@ service_monitoring_config: func TestProcessServiceInference(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := modelCfgFromYAML(t, ` -service_monitoring_config: - enabled: true - process_service_inference: - enabled: true`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.enabled", true) + New() - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.enabled")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.enabled")) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mockSystemProbe := mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLED", "true") t.Setenv("DD_SYSTEM_PROBE_PROCESS_SERVICE_INFERENCE_ENABLED", "true") - cfg := aconfig.SystemProbe() - sysconfig.Adjust(cfg) + New() + sysconfig.Adjust(mockSystemProbe) - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.enabled")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.enabled")) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := modelCfgFromYAML(t, ` -network_config: - enabled: true -system_probe_config: - process_service_inference: - enabled: true`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("network_config.enabled", true) + mockSystemProbe.SetWithoutSource("system_probe_config.process_service_inference.enabled", true) + New() - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.enabled")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.enabled")) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.enabled", true) + mockSystemProbe.SetWithoutSource("system_probe_config.process_service_inference.enabled", false) + New() - cfg := modelCfgFromYAML(t, ` -service_monitoring_config: - enabled: true - process_service_inference: - enabled: true -system_probe_config: - process_service_inference: - enabled: false`) - - require.False(t, cfg.GetBool("system_probe_config.process_service_inference.enabled")) + require.False(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.enabled")) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.enabled", false) + mockSystemProbe.SetWithoutSource("system_probe_config.process_service_inference.enabled", true) - cfg := modelCfgFromYAML(t, ` -service_monitoring_config: - enabled: true - process_service_inference: - enabled: false -system_probe_config: - process_service_inference: - enabled: true`) + New() - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.enabled")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.enabled")) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := modelCfgFromYAML(t, ` -service_monitoring_config: - enabled: true - process_service_inference: - enabled: true -system_probe_config: - process_service_inference: - enabled: true`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.enabled", true) + mockSystemProbe.SetWithoutSource("system_probe_config.process_service_inference.enabled", true) + + New() - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.enabled")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.enabled")) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := modelCfgFromYAML(t, ``) - require.False(t, cfg.GetBool("system_probe_config.process_service_inference.enabled")) + mockSystemProbe := mock.NewSystemProbe(t) + New() + require.False(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.enabled")) }) t.Run("Enabled without net, dsm, sm enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := modelCfgFromYAML(t, ` -system_probe_config: - process_service_inference: - enabled: true`) - require.False(t, cfg.GetBool("system_probe_config.process_service_inference.enabled")) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("system_probe_config.process_service_inference.enabled", true) + New() + require.False(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.enabled")) }) t.Run("test platform specific defaults", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mockSystemProbe := mock.NewSystemProbe(t) // usm or npm must be enabled for the process_service_inference to be enabled - cfg := modelCfgFromYAML(t, ` -service_monitoring_config: - enabled: true`) - sysconfig.Adjust(cfg) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + New() + sysconfig.Adjust(mockSystemProbe) var expected bool if runtime.GOOS == "windows" { @@ -1834,130 +1691,72 @@ service_monitoring_config: expected = false } - require.Equal(t, expected, cfg.GetBool("system_probe_config.process_service_inference.enabled")) + require.Equal(t, expected, mockSystemProbe.GetBool("system_probe_config.process_service_inference.enabled")) }) } func TestProcessServiceInferenceWindows(t *testing.T) { t.Run("via deprecated YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := modelCfgFromYAML(t, ` -service_monitoring_config: - enabled: true - process_service_inference: - use_windows_service_name: true`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.use_windows_service_name", true) + New() - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) }) t.Run("via ENV variable", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mockSystemProbe := mock.NewSystemProbe(t) t.Setenv("DD_SYSTEM_PROBE_NETWORK_ENABLED", "true") t.Setenv("DD_SYSTEM_PROBE_PROCESS_SERVICE_INFERENCE_USE_WINDOWS_SERVICE_NAME", "true") - cfg := aconfig.SystemProbe() - sysconfig.Adjust(cfg) + sysconfig.Adjust(mockSystemProbe) + New() - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) }) t.Run("via YAML", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := modelCfgFromYAML(t, ` -network_config: - enabled: true -system_probe_config: - process_service_inference: - use_windows_service_name: true`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.use_windows_service_name", true) + New() - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) }) t.Run("Deprecated is enabled, new is disabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mockSystemProbe := mock.NewSystemProbe(t) - cfg := modelCfgFromYAML(t, ` -service_monitoring_config: - enabled: true - process_service_inference: - use_windows_service_name: true -system_probe_config: - process_service_inference: - use_windows_service_name: false`) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.use_windows_service_name", true) + mockSystemProbe.SetWithoutSource("system_probe_config.process_service_inference.use_windows_service_name", false) + New() - require.False(t, cfg.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) + require.False(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) }) t.Run("Deprecated is disabled, new is enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.use_windows_service_name", false) + mockSystemProbe.SetWithoutSource("system_probe_config.process_service_inference.use_windows_service_name", true) + New() - cfg := modelCfgFromYAML(t, ` -service_monitoring_config: - enabled: true - process_service_inference: - use_windows_service_name: false -system_probe_config: - process_service_inference: - use_windows_service_name: true`) - - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) }) t.Run("Both enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - // Setting a different value - cfg := modelCfgFromYAML(t, ` -service_monitoring_config: - enabled: true - process_service_inference: - use_windows_service_name: true -system_probe_config: - process_service_inference: - use_windows_service_name: true`) + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.enabled", true) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.use_windows_service_name", true) + mockSystemProbe.SetWithoutSource("system_probe_config.process_service_inference.use_windows_service_name", true) - require.True(t, cfg.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) + require.True(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) }) t.Run("Not enabled", func(t *testing.T) { - aconfig.ResetSystemProbeConfig(t) - cfg := modelCfgFromYAML(t, ` -system_probe_config: - process_service_inference: - use_windows_service_name: false`) - require.False(t, cfg.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) - }) -} + mockSystemProbe := mock.NewSystemProbe(t) + mockSystemProbe.SetWithoutSource("service_monitoring_config.process_service_inference.use_windows_service_name", false) -func configurationFromYAML(t *testing.T, yaml string) *Config { - f, err := os.CreateTemp("", "system-probe.*.yaml") - require.NoError(t, err) - defer os.Remove(f.Name()) - - b := []byte(yaml) - n, err := f.Write(b) - require.NoError(t, err) - require.Equal(t, len(b), n) - f.Sync() - - _, err = sysconfig.New(f.Name(), "") - require.NoError(t, err) - return New() -} - -func modelCfgFromYAML(t *testing.T, yaml string) model.Config { - f, err := os.CreateTemp("", "system-probe.*.yaml") - require.NoError(t, err) - defer os.Remove(f.Name()) - - b := []byte(yaml) - n, err := f.Write(b) - require.NoError(t, err) - require.Equal(t, len(b), n) - f.Sync() - - _, err = sysconfig.New(f.Name(), "") - - require.NoError(t, err) - cfg := aconfig.SystemProbe() - sysconfig.Adjust(cfg) - - return cfg + require.False(t, mockSystemProbe.GetBool("system_probe_config.process_service_inference.use_windows_service_name")) + }) } From d7d0a29c32bc5252706b8202b5aa432ec8e82a02 Mon Sep 17 00:00:00 2001 From: Adel Haj Hassan <41540817+adel121@users.noreply.github.com> Date: Wed, 21 Aug 2024 12:51:48 +0200 Subject: [PATCH 075/245] add missing prefix separator for host tag entity id in readme.md (#28623) --- comp/core/tagger/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comp/core/tagger/README.md b/comp/core/tagger/README.md index 9f675f848d968..88ad5d8229623 100644 --- a/comp/core/tagger/README.md +++ b/comp/core/tagger/README.md @@ -53,7 +53,7 @@ Tagger entities are identified by a string-typed ID, with one of the following f | workloadmeta.KindContainer | `container_id://` | | workloadmeta.KindContainerImageMetadata | `container_image_metadata://` | | workloadmeta.KindECSTask | `ecs_task://` | -| workloadmeta.KindHost | `host` | +| workloadmeta.KindHost | `host://` | | workloadmeta.KindKubernetesDeployment | `deployment:///` | | workloadmeta.KindKubernetesMetadata | `kubernetes_metadata://///` (`` is empty in cluster-scoped objects) | | workloadmeta.KindKubernetesPod | `kubernetes_pod_uid://` | From 4cdca2160c06aeb9c2fb6d1f5723e84e88abf2ca Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Wed, 21 Aug 2024 12:55:50 +0200 Subject: [PATCH 076/245] discovery: Add E2E test to CI (#28611) Co-authored-by: Kevin Fairise <132568982+KevinFairise2@users.noreply.github.com> --- .gitlab-ci.yml | 8 ++++++++ .gitlab/e2e/e2e.yml | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6328a6722166..0d9e588f6a133 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -883,6 +883,14 @@ workflow: - test/new-e2e/tests/npm/**/* compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 +.on_discovery_or_e2e_changes: + - !reference [.on_e2e_main_release_or_rc] + - changes: + paths: + - test/new-e2e/tests/discovery/**/* + - pkg/collector/corechecks/servicediscovery/**/* + compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 + .on_aml_or_e2e_changes: - !reference [.on_e2e_main_release_or_rc] - changes: diff --git a/.gitlab/e2e/e2e.yml b/.gitlab/e2e/e2e.yml index f337ec283ff1a..546263168147d 100644 --- a/.gitlab/e2e/e2e.yml +++ b/.gitlab/e2e/e2e.yml @@ -308,6 +308,18 @@ new-e2e-cws: # Temporary, remove once we made sure the recent changes have no impact on the stability of these tests allow_failure: true +new-e2e-discovery: + extends: .new_e2e_template + needs: + - !reference [.needs_new_e2e_template] + - deploy_deb_testing-a7_x64 + rules: + - !reference [.on_discovery_or_e2e_changes] + - !reference [.manual] + variables: + TARGETS: ./tests/discovery + TEAM: universal-service-monitoring + new-e2e-process: extends: .new_e2e_template needs: From d3a3f5066647f2ee1a70827eef6447d95f9ac408 Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Wed, 21 Aug 2024 13:35:15 +0200 Subject: [PATCH 077/245] [ASCII-2215] Revert "Update e2e test as IMDSv1 is disabled by default (#28060)" (#28615) --- .../hostname/hostname_ec2_nix_test.go | 13 +++++++------ .../agent-subcommands/status/status_nix_test.go | 3 +-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go index 2b72ead2651c9..2437c5a4306ad 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go @@ -6,7 +6,6 @@ package hostname import ( - "strings" "testing" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" @@ -47,11 +46,13 @@ func (v *linuxHostnameSuite) TestAgentConfigPreferImdsv2() { } // https://github.com/DataDog/datadog-agent/blob/main/pkg/util/hostname/README.md#the-current-logic -func (v *linuxHostnameSuite) TestAgentDefaultHostnameIMDSv1() { - v.UpdateEnv(awshost.ProvisionerNoFakeIntake(v.GetOs(), awshost.WithAgentOptions(agentparams.WithAgentConfig("ec2_prefer_imdsv2: false")))) +func (v *linuxHostnameSuite) TestAgentHostnameDefaultsToResourceId() { + v.UpdateEnv(awshost.ProvisionerNoFakeIntake(v.GetOs(), awshost.WithAgentOptions(agentparams.WithAgentConfig("")))) + metadata := client.NewEC2Metadata(v.T(), v.Env().RemoteHost.Host, v.Env().RemoteHost.OSFamily) hostname := v.Env().Agent.Client.Hostname() - osHostname := v.Env().RemoteHost.Host.MustExecute("hostname") - osHostname = strings.TrimSpace(osHostname) - assert.Equal(v.T(), osHostname, hostname) + + // Default configuration of hostname for EC2 instances is the resource-id + resourceID := metadata.Get("instance-id") + assert.Equal(v.T(), resourceID, hostname) } diff --git a/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go b/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go index e1b1b37f2616f..c531e88af0ea4 100644 --- a/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go @@ -21,9 +21,8 @@ type linuxStatusSuite struct { } func TestLinuxStatusSuite(t *testing.T) { - enableImdsv2 := awshost.WithAgentOptions(agentparams.WithAgentConfig("ec2_prefer_imdsv2: true")) t.Parallel() - e2e.Run(t, &linuxStatusSuite{}, e2e.WithProvisioner(awshost.ProvisionerNoFakeIntake(enableImdsv2))) + e2e.Run(t, &linuxStatusSuite{}, e2e.WithProvisioner(awshost.ProvisionerNoFakeIntake())) } func (v *linuxStatusSuite) TestStatusHostname() { From 83be75f9ab597975b53220773aba9493e27d3bc6 Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Wed, 21 Aug 2024 13:43:16 +0200 Subject: [PATCH 078/245] Use linux tags in devcontainer config, lint package level (#28599) --- tasks/devcontainer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/devcontainer.py b/tasks/devcontainer.py index eb820228f423e..c8c738df05112 100644 --- a/tasks/devcontainer.py +++ b/tasks/devcontainer.py @@ -43,7 +43,7 @@ def setup( return build_include = ( - get_default_build_tags(build=target, flavor=flavor) + get_default_build_tags(build=target, flavor=flavor, platform='linux') if build_include is None else filter_incompatible_tags(build_include.split(",")) ) @@ -92,7 +92,7 @@ def setup( "go.buildTags": local_build_tags, "go.testTags": local_build_tags, "go.lintTool": "golangci-lint", - "go.lintOnSave": "file", + "go.lintOnSave": "package", "go.lintFlags": [ "--build-tags", local_build_tags, @@ -106,7 +106,7 @@ def setup( } } devcontainer["postStartCommand"] = ( - f"git config --global --add safe.directory {AGENT_REPOSITORY_PATH} && invoke install-tools && invoke deps" + f"git config --global --add safe.directory {AGENT_REPOSITORY_PATH} && invoke -e install-tools && invoke -e deps" ) devcontainer["remoteEnv"] = {"GITLAB_TOKEN": "${localEnv:GITLAB_TOKEN}"} From c6c17c4d952a671a51a4432c404623b6f76b61f8 Mon Sep 17 00:00:00 2001 From: Alexandre Menasria <47357713+amenasria@users.noreply.github.com> Date: Wed, 21 Aug 2024 13:46:40 +0200 Subject: [PATCH 079/245] [Invoke] Remove the `--custom-golangci-lint` flag from the `install-tools` task (#28621) --- .github/workflows/codeql-analysis.yml | 2 +- tasks/install_tasks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index cb1d0be9db095..5ac98f7c6cd52 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -54,7 +54,7 @@ jobs: - name: Build DataDog agent run: | - invoke install-tools --no-custom-golangci-lint + invoke install-tools invoke deps invoke agent.build --build-exclude=systemd diff --git a/tasks/install_tasks.py b/tasks/install_tasks.py index f557ded6670ea..17371da7ab3fe 100644 --- a/tasks/install_tasks.py +++ b/tasks/install_tasks.py @@ -46,7 +46,7 @@ def download_tools(ctx): @task -def install_tools(ctx: Context, max_retry: int = 3, custom_golangci_lint=True): +def install_tools(ctx: Context, max_retry: int = 3): """Install all Go tools for testing.""" with gitlab_section("Installing Go tools", collapsed=True): with environ({'GO111MODULE': 'on'}): @@ -54,8 +54,8 @@ def install_tools(ctx: Context, max_retry: int = 3, custom_golangci_lint=True): with ctx.cd(path): for tool in tools: run_command_with_retry(ctx, f"go install {tool}", max_retry=max_retry) - if custom_golangci_lint: - install_custom_golanci_lint(ctx) + # Always install the custom golangci-lint not to fail on custom linters run (e.g pkgconfigusage) + install_custom_golanci_lint(ctx) def install_custom_golanci_lint(ctx): From 553bd9a72d398234e49a234f9a0b42f616ba10a9 Mon Sep 17 00:00:00 2001 From: Olivier G <52180542+ogaca-dd@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:29:25 +0200 Subject: [PATCH 080/245] [ASCII-2101]: Flare module `Params` (#28389) --- cmd/agent/subcommands/flare/command.go | 3 +-- cmd/agent/subcommands/run/command.go | 3 +-- cmd/systray/command/command.go | 3 +-- comp/core/flare/component.go | 5 +++-- comp/core/flare/component_mock.go | 17 ----------------- 5 files changed, 6 insertions(+), 25 deletions(-) delete mode 100644 comp/core/flare/component_mock.go diff --git a/cmd/agent/subcommands/flare/command.go b/cmd/agent/subcommands/flare/command.go index 04fb9572349ee..0be7259d3096e 100644 --- a/cmd/agent/subcommands/flare/command.go +++ b/cmd/agent/subcommands/flare/command.go @@ -108,7 +108,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { SysprobeConfigParams: sysprobeconfigimpl.NewParams(sysprobeconfigimpl.WithSysProbeConfFilePath(globalParams.SysProbeConfFilePath), sysprobeconfigimpl.WithFleetPoliciesDirPath(globalParams.FleetPoliciesDirPath)), LogParams: log.ForOneShot(command.LoggerName, "off", false), }), - fx.Supply(flare.NewLocalParams( + flare.Module(flare.NewLocalParams( commonpath.GetDistPath(), commonpath.PyChecksPath, commonpath.DefaultLogFile, @@ -128,7 +128,6 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { workloadmetafx.Module(), taggerimpl.OptionalModule(), autodiscoveryimpl.OptionalModule(), // if forceLocal is true, we will start autodiscovery (loadComponents) later - flare.Module(), fx.Supply(optional.NewNoneOption[collector.Component]()), compressionimpl.Module(), diagnosesendermanagerimpl.Module(), diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go index a9c3abf98056c..1e2a6ca9a582b 100644 --- a/cmd/agent/subcommands/run/command.go +++ b/cmd/agent/subcommands/run/command.go @@ -327,7 +327,7 @@ func run(log log.Component, func getSharedFxOption() fx.Option { return fx.Options( - fx.Supply(flare.NewParams( + flare.Module(flare.NewParams( path.GetDistPath(), path.PyChecksPath, path.DefaultLogFile, @@ -335,7 +335,6 @@ func getSharedFxOption() fx.Option { path.DefaultDogstatsDLogFile, path.DefaultStreamlogsLogFile, )), - flare.Module(), core.Bundle(), fx.Supply(dogstatsdServer.Params{ Serverless: false, diff --git a/cmd/systray/command/command.go b/cmd/systray/command/command.go index fcfc679410f58..cd421597483e1 100644 --- a/cmd/systray/command/command.go +++ b/cmd/systray/command/command.go @@ -98,7 +98,7 @@ func MakeCommand() *cobra.Command { }), core.Bundle(), // flare - fx.Supply(flare.NewParams( + flare.Module(flare.NewParams( path.GetDistPath(), path.PyChecksPath, path.DefaultLogFile, @@ -107,7 +107,6 @@ func MakeCommand() *cobra.Command { path.DefaultStreamlogsLogFile, )), fx.Supply(optional.NewNoneOption[autodiscovery.Component]()), - flare.Module(), fx.Supply(optional.NewNoneOption[workloadmeta.Component]()), fx.Supply(optional.NewNoneOption[collector.Component]()), compressionimpl.Module(), diff --git a/comp/core/flare/component.go b/comp/core/flare/component.go index 1fb8c04e78434..0ac0ef51c87a5 100644 --- a/comp/core/flare/component.go +++ b/comp/core/flare/component.go @@ -26,7 +26,8 @@ type Component interface { } // Module defines the fx options for this component. -func Module() fxutil.Module { +func Module(params Params) fxutil.Module { return fxutil.Component( - fx.Provide(newFlare)) + fx.Provide(newFlare), + fx.Supply(params)) } diff --git a/comp/core/flare/component_mock.go b/comp/core/flare/component_mock.go deleted file mode 100644 index f0c8decbc0925..0000000000000 --- a/comp/core/flare/component_mock.go +++ /dev/null @@ -1,17 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build test - -package flare - -// team: agent-shared-components - -// Mock is the mocked component type. -type Mock interface { - Component - - // no further methods are defined. -} From ecc95d123eb63ebc3f4abc2afebc3e5fb9cfe4bd Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Wed, 21 Aug 2024 14:36:01 +0200 Subject: [PATCH 081/245] (fleet) fix system probe permissions when installing with the installer (#28610) --- pkg/fleet/installer/service/datadog_agent.go | 26 ++++++++++++++++--- .../tests/installer/package_agent_test.go | 8 ++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/pkg/fleet/installer/service/datadog_agent.go b/pkg/fleet/installer/service/datadog_agent.go index cb7ff98380d9b..30a69271d59d9 100644 --- a/pkg/fleet/installer/service/datadog_agent.go +++ b/pkg/fleet/installer/service/datadog_agent.go @@ -15,6 +15,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "github.com/DataDog/datadog-agent/pkg/fleet/installer/repository" "github.com/DataDog/datadog-agent/pkg/fleet/internal/cdn" @@ -59,6 +60,16 @@ var ( } ) +var ( + // matches omnibus/package-scripts/agent-deb/postinst + rootOwnedAgentPaths = []string{ + "embedded/bin/system-probe", + "embedded/bin/security-agent", + "embedded/share/system-probe/ebpf", + "embedded/share/system-probe/java", + } +) + // SetupAgent installs and starts the agent func SetupAgent(ctx context.Context, _ []string) (err error) { span, ctx := tracer.StartSpanFromContext(ctx, "setup_agent") @@ -95,7 +106,7 @@ func SetupAgent(ctx context.Context, _ []string) (err error) { if err = os.Chown("/etc/datadog-agent", ddAgentUID, ddAgentGID); err != nil { return fmt.Errorf("failed to chown /etc/datadog-agent: %v", err) } - if err = chownRecursive("/opt/datadog-packages/datadog-agent/stable/", ddAgentUID, ddAgentGID); err != nil { + if err = chownRecursive("/opt/datadog-packages/datadog-agent/stable/", ddAgentUID, ddAgentGID, rootOwnedAgentPaths); err != nil { return fmt.Errorf("failed to chown /opt/datadog-packages/datadog-agent/stable/: %v", err) } @@ -198,11 +209,20 @@ func stopOldAgentUnits(ctx context.Context) error { return nil } -func chownRecursive(path string, uid int, gid int) error { +func chownRecursive(path string, uid int, gid int, ignorePaths []string) error { return filepath.Walk(path, func(p string, _ os.FileInfo, err error) error { if err != nil { return err } + relPath, err := filepath.Rel(path, p) + if err != nil { + return err + } + for _, ignore := range ignorePaths { + if relPath == ignore || strings.HasPrefix(relPath, ignore+string(os.PathSeparator)) { + return nil + } + } return os.Chown(p, uid, gid) }) } @@ -213,7 +233,7 @@ func StartAgentExperiment(ctx context.Context) error { if err != nil { return fmt.Errorf("error getting dd-agent user and group IDs: %w", err) } - if err = chownRecursive("/opt/datadog-packages/datadog-agent/experiment/", ddAgentUID, ddAgentGID); err != nil { + if err = chownRecursive("/opt/datadog-packages/datadog-agent/experiment/", ddAgentUID, ddAgentGID, rootOwnedAgentPaths); err != nil { return fmt.Errorf("failed to chown /opt/datadog-packages/datadog-agent/experiment/: %v", err) } return startUnit(ctx, agentExp, "--no-block") diff --git a/test/new-e2e/tests/installer/package_agent_test.go b/test/new-e2e/tests/installer/package_agent_test.go index 7cbc0fbbbe09b..02b21a0b4b86b 100644 --- a/test/new-e2e/tests/installer/package_agent_test.go +++ b/test/new-e2e/tests/installer/package_agent_test.go @@ -7,6 +7,7 @@ package installer import ( "fmt" + "path" "path/filepath" "strings" "time" @@ -56,6 +57,13 @@ func (s *packageAgentSuite) TestInstall() { agentDir := fmt.Sprintf("/opt/datadog-packages/datadog-agent/%s", agentVersion) state.AssertDirExists(agentDir, 0755, "dd-agent", "dd-agent") + + state.AssertFileExists(path.Join(agentDir, "embedded/bin/system-probe"), 0755, "root", "root") + state.AssertFileExists(path.Join(agentDir, "embedded/bin/security-agent"), 0755, "root", "root") + state.AssertDirExists(path.Join(agentDir, "embedded/share/system-probe/java"), 0755, "root", "root") + state.AssertDirExists(path.Join(agentDir, "embedded/share/system-probe/ebpf"), 0755, "root", "root") + state.AssertFileExists(path.Join(agentDir, "embedded/share/system-probe/ebpf/dns.o"), 0644, "root", "root") + state.AssertSymlinkExists("/opt/datadog-packages/datadog-agent/stable", agentDir, "root", "root") // FIXME: this file is either dd-agent or root depending on the OS for some reason // state.AssertFileExists("/etc/datadog-agent/install.json", 0644, "dd-agent", "dd-agent") From 0bbbd19d4eec8bc0655e75760b64fc98182c1976 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 13:14:40 +0000 Subject: [PATCH 082/245] Bump github.com/gosnmp/gosnmp from 1.37.1-0.20240115134726-db0c09337869 to 1.38.0 (#28414) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3135c86490b5a..8932919ec6f48 100644 --- a/go.mod +++ b/go.mod @@ -213,7 +213,7 @@ require ( github.com/google/gopacket v1.1.19 github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 github.com/gorilla/mux v1.8.1 - github.com/gosnmp/gosnmp v1.37.1-0.20240115134726-db0c09337869 + github.com/gosnmp/gosnmp v1.38.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/h2non/filetype v1.1.3 diff --git a/go.sum b/go.sum index b3bf2e8247cde..9245dc315cd6e 100644 --- a/go.sum +++ b/go.sum @@ -1592,8 +1592,8 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= -github.com/gosnmp/gosnmp v1.37.1-0.20240115134726-db0c09337869 h1:MphZl80DUq02iPzdDxVffuUD1V24UH6UtmEY6FX+/M4= -github.com/gosnmp/gosnmp v1.37.1-0.20240115134726-db0c09337869/go.mod h1:GDH9vNqpsD7f2HvZhKs5dlqSEcAS6s6Qp099oZRCR+M= +github.com/gosnmp/gosnmp v1.38.0 h1:I5ZOMR8kb0DXAFg/88ACurnuwGwYkXWq3eLpJPHMEYc= +github.com/gosnmp/gosnmp v1.38.0/go.mod h1:FE+PEZvKrFz9afP9ii1W3cprXuVZ17ypCcyyfYuu5LY= github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= From 4ea043e21953aebe94aa1a5f4d02e4ae374c09f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 13:24:22 +0000 Subject: [PATCH 083/245] Bump golang.org/x/net from 0.27.0 to 0.28.0 (#28415) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- LICENSE-3rdparty.csv | 172 +++++++++++++++++++++---------------------- go.mod | 8 +- go.sum | 16 ++-- 3 files changed, 98 insertions(+), 98 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 53e28b42dbd4c..db2138997cbe1 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -2571,37 +2571,37 @@ core,go4.org/netipx,BSD-3-Clause,Alex Willmer | Copyright core,go4.org/unsafe/assume-no-moving-gc,BSD-3-Clause,"Copyright (c) 2020, Brad Fitzpatrick" core,golang.org/x/arch/arm64/arm64asm,BSD-3-Clause,Copyright 2015 The Go Authors core,golang.org/x/arch/x86/x86asm,BSD-3-Clause,Copyright 2015 The Go Authors -core,golang.org/x/crypto/argon2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/bcrypt,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/blake2b,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/blowfish,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/cast5,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/chacha20,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved +core,golang.org/x/crypto/argon2,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/bcrypt,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/blake2b,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/blowfish,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/cast5,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/chacha20,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/crypto/chacha20poly1305,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/cryptobyte,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/cryptobyte/asn1,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/curve25519,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/hkdf,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/internal/alias,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/internal/poly1305,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/nacl/secretbox,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/ocsp,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/openpgp,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/openpgp/armor,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/openpgp/elgamal,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/openpgp/errors,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/openpgp/packet,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/openpgp/s2k,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/pbkdf2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/pkcs12,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/pkcs12/internal/rc2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/salsa20/salsa,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/scrypt,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/sha3,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/ssh,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/ssh/agent,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/ssh/internal/bcrypt_pbkdf,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/crypto/ssh/knownhosts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved +core,golang.org/x/crypto/cryptobyte,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/cryptobyte/asn1,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/curve25519,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/hkdf,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/internal/alias,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/internal/poly1305,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/nacl/secretbox,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/ocsp,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/openpgp,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/openpgp/armor,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/openpgp/elgamal,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/openpgp/errors,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/openpgp/packet,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/openpgp/s2k,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/pbkdf2,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/pkcs12,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/pkcs12/internal/rc2,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/salsa20/salsa,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/scrypt,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/sha3,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/ssh,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/ssh/agent,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/ssh/internal/bcrypt_pbkdf,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/crypto/ssh/knownhosts,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/exp/constraints,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/exp/maps,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/exp/mmap,BSD-3-Clause,Copyright 2009 The Go Authors @@ -2612,29 +2612,29 @@ core,golang.org/x/exp/slog/internal/buffer,BSD-3-Clause,Copyright 2009 The Go Au core,golang.org/x/exp/typeparams,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved | Copyright 2009 The Go Authors core,golang.org/x/lint,BSD-3-Clause,Copyright (c) 2013 The Go Authors. All rights reserved core,golang.org/x/lint/golint,BSD-3-Clause,Copyright (c) 2013 The Go Authors. All rights reserved -core,golang.org/x/net/bpf,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/context,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/html,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/html/atom,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/html/charset,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/http/httpproxy,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/http2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/http2/h2c,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/http2/hpack,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/icmp,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/idna,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/internal/iana,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/internal/socket,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/internal/socks,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/internal/timeseries,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/ipv4,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/ipv6,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/netutil,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/proxy,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/publicsuffix,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/trace,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/net/websocket,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved +core,golang.org/x/net/bpf,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/context,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/html,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/html/atom,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/html/charset,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/http/httpproxy,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/http2,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/http2/h2c,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/http2/hpack,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/icmp,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/idna,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/internal/iana,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/internal/socket,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/internal/socks,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/internal/timeseries,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/ipv4,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/ipv6,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/netutil,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/proxy,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/publicsuffix,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/trace,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/net/websocket,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/oauth2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved core,golang.org/x/oauth2/authhandler,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved core,golang.org/x/oauth2/clientcredentials,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved @@ -2658,39 +2658,39 @@ core,golang.org/x/sys/windows/registry,BSD-3-Clause,Copyright 2009 The Go Author core,golang.org/x/sys/windows/svc,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/sys/windows/svc/eventlog,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/sys/windows/svc/mgr,BSD-3-Clause,Copyright 2009 The Go Authors -core,golang.org/x/term,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/cases,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/charmap,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/htmlindex,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/ianaindex,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/internal,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/internal/identifier,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/japanese,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/korean,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/simplifiedchinese,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/traditionalchinese,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/encoding/unicode,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/feature/plural,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/internal,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/internal/catmsg,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/internal/format,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/internal/language,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/internal/language/compact,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/internal/number,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/internal/stringset,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/internal/tag,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/internal/utf8internal,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/language,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/message,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/message/catalog,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/runes,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/secure/bidirule,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/secure/precis,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/transform,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/unicode/bidi,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/unicode/norm,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang.org/x/text/width,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved +core,golang.org/x/term,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/cases,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/charmap,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/htmlindex,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/ianaindex,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/internal,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/internal/identifier,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/japanese,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/korean,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/simplifiedchinese,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/traditionalchinese,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/encoding/unicode,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/feature/plural,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/internal,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/internal/catmsg,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/internal/format,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/internal/language,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/internal/language/compact,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/internal/number,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/internal/stringset,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/internal/tag,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/internal/utf8internal,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/language,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/message,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/message/catalog,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/runes,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/secure/bidirule,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/secure/precis,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/transform,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/unicode/bidi,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/unicode/norm,BSD-3-Clause,Copyright 2009 The Go Authors +core,golang.org/x/text/width,BSD-3-Clause,Copyright 2009 The Go Authors core,golang.org/x/time/rate,BSD-3-Clause,Copyright 2009 The Go Authors core,golang/go,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved. core,gomodules.xyz/jsonpatch/v2,Apache-2.0,Copyright (c) 2015 The Authors diff --git a/go.mod b/go.mod index 8932919ec6f48..b90909e72de40 100644 --- a/go.mod +++ b/go.mod @@ -300,10 +300,10 @@ require ( go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d golang.org/x/arch v0.9.0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/net v0.27.0 + golang.org/x/net v0.28.0 golang.org/x/sync v0.8.0 golang.org/x/sys v0.23.0 - golang.org/x/text v0.16.0 + golang.org/x/text v0.17.0 golang.org/x/time v0.6.0 golang.org/x/tools v0.23.0 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 @@ -565,10 +565,10 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect go.opentelemetry.io/otel/trace v1.27.0 go.opentelemetry.io/proto/otlp v1.2.0 // indirect - golang.org/x/crypto v0.25.0 // indirect + golang.org/x/crypto v0.26.0 // indirect golang.org/x/mod v0.20.0 golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/term v0.22.0 // indirect + golang.org/x/term v0.23.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/api v0.185.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 9245dc315cd6e..8291470d6cf72 100644 --- a/go.sum +++ b/go.sum @@ -2875,8 +2875,8 @@ golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -3036,8 +3036,8 @@ golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -3240,8 +3240,8 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -3261,8 +3261,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From cdd47872bb9dbaa44b995a75d4ff178d9faaf6fd Mon Sep 17 00:00:00 2001 From: Pierre Gimalac Date: Wed, 21 Aug 2024 15:24:27 +0200 Subject: [PATCH 084/245] Fix package comment in component template (#28542) --- tasks/components_templates/impl/component.go.tmpl | 6 +++--- tasks/components_templates/mock/mock.go.tmpl | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tasks/components_templates/impl/component.go.tmpl b/tasks/components_templates/impl/component.go.tmpl index 8ec5773472910..ba2f8a4fe1cf3 100644 --- a/tasks/components_templates/impl/component.go.tmpl +++ b/tasks/components_templates/impl/component.go.tmpl @@ -1,6 +1,6 @@ ${COPYRIGHT_HEADER} -// Package impl implements the ${COMPONENT_NAME} component interface +// Package ${COMPONENT_NAME}impl implements the ${COMPONENT_NAME} component interface package ${COMPONENT_NAME}impl import ( @@ -22,7 +22,7 @@ type Provides struct { // NewComponent creates a new ${COMPONENT_NAME} component func NewComponent(reqs Requires) (Provides, error) { // TODO: Implement the ${COMPONENT_NAME} component - + provides := Provides{} return provides, nil -} \ No newline at end of file +} diff --git a/tasks/components_templates/mock/mock.go.tmpl b/tasks/components_templates/mock/mock.go.tmpl index b39bfa86c3a5f..f67119a43fed1 100644 --- a/tasks/components_templates/mock/mock.go.tmpl +++ b/tasks/components_templates/mock/mock.go.tmpl @@ -2,11 +2,12 @@ ${COPYRIGHT_HEADER} //go:build test +// Package mock provides a mock for the ${COMPONENT_NAME} component package mock import ( "testing" - + ${COMPONENT_NAME} "github.com/DataDog/datadog-agent/${COMPONENT_PATH}/def" ) From 8ab1a6fbb8aba71c825fb8ccfcbcf29de3331ba7 Mon Sep 17 00:00:00 2001 From: maxime mouial Date: Wed, 21 Aug 2024 15:30:25 +0200 Subject: [PATCH 085/245] Deprecation of SetEnvVarTransformer in favor of typed helpers (#28484) --- pkg/config/model/types.go | 8 ++++++++ pkg/config/model/viper.go | 24 ++++++++++++++++++++++++ pkg/config/model/viper_test.go | 31 +++++++++++++++++++++++++++++++ pkg/config/setup/apm.go | 33 +++++++++++++++------------------ 4 files changed, 78 insertions(+), 18 deletions(-) diff --git a/pkg/config/model/types.go b/pkg/config/model/types.go index bede1d74c4b6e..5613a41dbcef1 100644 --- a/pkg/config/model/types.go +++ b/pkg/config/model/types.go @@ -110,7 +110,15 @@ type Setup interface { SetEnvPrefix(in string) BindEnv(input ...string) SetEnvKeyReplacer(r *strings.Replacer) + + // SetEnvKeyTransformer is deprecated in favor of ParseEnvAs* functions SetEnvKeyTransformer(key string, fn func(string) interface{}) + // The following helpers allow a type to be enforce when parsing environment variables. Most of them exists to + // support historic behavior. Refrain from adding more as it's most likely a sign of poorly design configuration + // layout. + ParseEnvAsStringSlice(key string, fx func(string) []string) + ParseEnvAsMapStringInterface(key string, fx func(string) map[string]interface{}) + ParseEnvAsSliceMapString(key string, fx func(string) []map[string]string) // SetKnown adds a key to the set of known valid config keys SetKnown(key string) diff --git a/pkg/config/model/viper.go b/pkg/config/model/viper.go index 195d68bbc8bf4..9cf12b87e680e 100644 --- a/pkg/config/model/viper.go +++ b/pkg/config/model/viper.go @@ -238,12 +238,36 @@ func (c *safeConfig) GetKnownKeysLowercased() map[string]interface{} { // SetEnvKeyTransformer allows defining a transformer function which decides // how an environment variables value gets assigned to key. +// +// [DEPRECATED] This function will soon be remove. Use one of the ParseEnvAs* helpers instead. func (c *safeConfig) SetEnvKeyTransformer(key string, fn func(string) interface{}) { c.Lock() defer c.Unlock() c.Viper.SetEnvKeyTransformer(key, fn) } +// ParseEnvAsStringSlice registers a transformer function to parse an an environment variables as a []string. +func (c *safeConfig) ParseEnvAsStringSlice(key string, fn func(string) []string) { + c.Lock() + defer c.Unlock() + c.Viper.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) +} + +// ParseEnvAsMapStringInterface registers a transformer function to parse an an environment variables as a +// map[string]interface{}. +func (c *safeConfig) ParseEnvAsMapStringInterface(key string, fn func(string) map[string]interface{}) { + c.Lock() + defer c.Unlock() + c.Viper.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) +} + +// ParseEnvAsSliceMapString registers a transformer function to parse an an environment variables as a []map[string]string. +func (c *safeConfig) ParseEnvAsSliceMapString(key string, fn func(string) []map[string]string) { + c.Lock() + defer c.Unlock() + c.Viper.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) +} + // SetFs wraps Viper for concurrent access func (c *safeConfig) SetFs(fs afero.Fs) { c.Lock() diff --git a/pkg/config/model/viper_test.go b/pkg/config/model/viper_test.go index ea54fe712fc90..83bec5ee7a66b 100644 --- a/pkg/config/model/viper_test.go +++ b/pkg/config/model/viper_test.go @@ -418,3 +418,34 @@ func TestMergeFleetPolicy(t *testing.T) { assert.Equal(t, "baz", config.Get("foo")) assert.Equal(t, SourceFleetPolicies, config.GetSource("foo")) } + +func TestParseEnvAsStringSlice(t *testing.T) { + config := NewConfig("test", "DD", strings.NewReplacer(".", "_")) + + config.BindEnv("slice_of_string") + config.ParseEnvAsStringSlice("slice_of_string", func(string) []string { return []string{"a", "b", "c"} }) + + t.Setenv("DD_SLICE_OF_STRING", "__some_data__") + assert.Equal(t, []string{"a", "b", "c"}, config.Get("slice_of_string")) +} + +func TestParseEnvAsMapStringInterface(t *testing.T) { + config := NewConfig("test", "DD", strings.NewReplacer(".", "_")) + + config.BindEnv("map_of_float") + config.ParseEnvAsMapStringInterface("map_of_float", func(string) map[string]interface{} { return map[string]interface{}{"a": 1.0, "b": 2.0, "c": 3.0} }) + + t.Setenv("DD_MAP_OF_FLOAT", "__some_data__") + assert.Equal(t, map[string]interface{}{"a": 1.0, "b": 2.0, "c": 3.0}, config.Get("map_of_float")) + assert.Equal(t, map[string]interface{}{"a": 1.0, "b": 2.0, "c": 3.0}, config.GetStringMap("map_of_float")) +} + +func TestParseEnvAsSliceMapString(t *testing.T) { + config := NewConfig("test", "DD", strings.NewReplacer(".", "_")) + + config.BindEnv("map") + config.ParseEnvAsSliceMapString("map", func(string) []map[string]string { return []map[string]string{{"a": "a", "b": "b", "c": "c"}} }) + + t.Setenv("DD_MAP", "__some_data__") + assert.Equal(t, []map[string]string{{"a": "a", "b": "b", "c": "c"}}, config.Get("map")) +} diff --git a/pkg/config/setup/apm.go b/pkg/config/setup/apm.go index 769ecef5c64d3..251af766fc0af 100644 --- a/pkg/config/setup/apm.go +++ b/pkg/config/setup/apm.go @@ -154,7 +154,7 @@ func setupAPM(config pkgconfigmodel.Setup) { config.BindEnv("apm_config.obfuscation.credit_cards.luhn", "DD_APM_OBFUSCATION_CREDIT_CARDS_LUHN") config.BindEnvAndSetDefault("apm_config.debug.port", 5012, "DD_APM_DEBUG_PORT") config.BindEnv("apm_config.features", "DD_APM_FEATURES") - config.SetEnvKeyTransformer("apm_config.features", func(s string) interface{} { + config.ParseEnvAsStringSlice("apm_config.features", func(s string) []string { // Either commas or spaces can be used as separators. // Comma takes precedence as it was the only supported separator in the past. // Mixing separators is not supported. @@ -170,7 +170,7 @@ func setupAPM(config pkgconfigmodel.Setup) { return res }) - config.SetEnvKeyTransformer("apm_config.ignore_resources", func(in string) interface{} { + config.ParseEnvAsStringSlice("apm_config.ignore_resources", func(in string) []string { r, err := splitCSVString(in, ',') if err != nil { log.Warnf(`"apm_config.ignore_resources" can not be parsed: %v`, err) @@ -179,15 +179,12 @@ func setupAPM(config pkgconfigmodel.Setup) { return r }) - config.SetEnvKeyTransformer("apm_config.filter_tags.require", parseKVList("apm_config.filter_tags.require")) + config.ParseEnvAsStringSlice("apm_config.filter_tags.require", parseKVList("apm_config.filter_tags.require")) + config.ParseEnvAsStringSlice("apm_config.filter_tags.reject", parseKVList("apm_config.filter_tags.reject")) + config.ParseEnvAsStringSlice("apm_config.filter_tags_regex.require", parseKVList("apm_config.filter_tags_regex.require")) + config.ParseEnvAsStringSlice("apm_config.filter_tags_regex.reject", parseKVList("apm_config.filter_tags_regex.reject")) - config.SetEnvKeyTransformer("apm_config.filter_tags.reject", parseKVList("apm_config.filter_tags.reject")) - - config.SetEnvKeyTransformer("apm_config.filter_tags_regex.require", parseKVList("apm_config.filter_tags_regex.require")) - - config.SetEnvKeyTransformer("apm_config.filter_tags_regex.reject", parseKVList("apm_config.filter_tags_regex.reject")) - - config.SetEnvKeyTransformer("apm_config.replace_tags", func(in string) interface{} { + config.ParseEnvAsSliceMapString("apm_config.replace_tags", func(in string) []map[string]string { var out []map[string]string if err := json.Unmarshal([]byte(in), &out); err != nil { log.Warnf(`"apm_config.replace_tags" can not be parsed: %v`, err) @@ -195,7 +192,7 @@ func setupAPM(config pkgconfigmodel.Setup) { return out }) - config.SetEnvKeyTransformer("apm_config.analyzed_spans", func(in string) interface{} { + config.ParseEnvAsMapStringInterface("apm_config.analyzed_spans", func(in string) map[string]interface{} { out, err := parseAnalyzedSpans(in) if err != nil { log.Errorf(`Bad format for "apm_config.analyzed_spans" it should be of the form \"service_name|operation_name=rate,other_service|other_operation=rate\", error: %v`, err) @@ -204,7 +201,7 @@ func setupAPM(config pkgconfigmodel.Setup) { }) config.BindEnv("apm_config.peer_tags", "DD_APM_PEER_TAGS") - config.SetEnvKeyTransformer("apm_config.peer_tags", func(in string) interface{} { + config.ParseEnvAsStringSlice("apm_config.peer_tags", func(in string) []string { var out []string if err := json.Unmarshal([]byte(in), &out); err != nil { log.Warnf(`"apm_config.peer_tags" can not be parsed: %v`, err) @@ -213,8 +210,8 @@ func setupAPM(config pkgconfigmodel.Setup) { }) } -func parseKVList(key string) func(string) interface{} { - return func(in string) interface{} { +func parseKVList(key string) func(string) []string { + return func(in string) []string { if len(in) == 0 { return []string{} } @@ -254,10 +251,10 @@ func parseNameAndRate(token string) (string, float64, error) { // parseAnalyzedSpans parses the env string to extract a map of spans to be analyzed by service and operation. // the format is: service_name|operation_name=rate,other_service|other_operation=rate -func parseAnalyzedSpans(env string) (analyzedSpans map[string]interface{}, err error) { - analyzedSpans = make(map[string]interface{}) +func parseAnalyzedSpans(env string) (map[string]interface{}, error) { + analyzedSpans := make(map[string]interface{}) if env == "" { - return + return analyzedSpans, nil } tokens := strings.Split(env, ",") for _, token := range tokens { @@ -267,5 +264,5 @@ func parseAnalyzedSpans(env string) (analyzedSpans map[string]interface{}, err e } analyzedSpans[name] = rate } - return + return analyzedSpans, nil } From d7587c242f8afe2dbb24845b1bf0b59d591ef673 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 13:36:11 +0000 Subject: [PATCH 086/245] Bump golang.org/x/tools from 0.23.0 to 0.24.0 (#28417) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b90909e72de40..0ee136c57df2f 100644 --- a/go.mod +++ b/go.mod @@ -305,7 +305,7 @@ require ( golang.org/x/sys v0.23.0 golang.org/x/text v0.17.0 golang.org/x/time v0.6.0 - golang.org/x/tools v0.23.0 + golang.org/x/tools v0.24.0 golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 google.golang.org/genproto v0.0.0-20240617180043-68d350f18fd4 // indirect google.golang.org/grpc v1.65.0 diff --git a/go.sum b/go.sum index 8291470d6cf72..f75ba7d887846 100644 --- a/go.sum +++ b/go.sum @@ -3354,8 +3354,8 @@ golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 2bdcde3d75d34a2b90ec0a1e61ab5cd5d1e6c7ae Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Wed, 21 Aug 2024 15:51:57 +0200 Subject: [PATCH 087/245] discovery: Rewrite Python APM instrumentation detection (#28580) --- .../corechecks/servicediscovery/apm/detect.go | 95 +++++++----------- .../servicediscovery/apm/detect_nix_test.go | 52 ++++------ .../servicediscovery/module/impl_linux.go | 2 +- .../module/impl_linux_test.go | 68 ++++++++++--- pkg/network/usm/ebpf_ssl_test.go | 3 +- .../usm/sharedlibraries/testutil/testutil.go | 19 ++-- .../ddtrace}/build.sh | 0 .../ddtrace}/fakessl.c | 0 .../ddtrace}/libssl.so.amd64 | Bin .../ddtrace}/libssl.so.arm64 | Bin test/new-e2e/tests/discovery/linux_test.go | 20 +++- .../discovery/testdata/provision/provision.sh | 5 +- .../testdata/provision/python/__init__.py | 0 .../testdata/provision/python/instrumented.py | 7 ++ 14 files changed, 155 insertions(+), 116 deletions(-) rename pkg/network/usm/testdata/{libmmap => site-packages/ddtrace}/build.sh (100%) rename pkg/network/usm/testdata/{libmmap => site-packages/ddtrace}/fakessl.c (100%) rename pkg/network/usm/testdata/{libmmap => site-packages/ddtrace}/libssl.so.amd64 (100%) rename pkg/network/usm/testdata/{libmmap => site-packages/ddtrace}/libssl.so.arm64 (100%) create mode 100644 test/new-e2e/tests/discovery/testdata/provision/python/__init__.py create mode 100644 test/new-e2e/tests/discovery/testdata/provision/python/instrumented.py diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect.go b/pkg/collector/corechecks/servicediscovery/apm/detect.go index 47d1947acbdcd..d356ca1952ca1 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect.go @@ -3,21 +3,23 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. +//go:build linux + // Package apm provides functionality to detect the type of APM instrumentation a service is using. package apm import ( - "bytes" + "bufio" "errors" "io" - "io/fs" "os" - "os/exec" "path/filepath" + "strconv" "strings" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/language" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/language/reader" + "github.com/DataDog/datadog-agent/pkg/util/kernel" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -33,7 +35,7 @@ const ( Injected Instrumentation = "injected" ) -type detector func(args []string, envs map[string]string) Instrumentation +type detector func(pid int, args []string, envs map[string]string) Instrumentation var ( detectorMap = map[language.Language]detector{ @@ -41,16 +43,16 @@ var ( language.Java: javaDetector, language.Node: nodeDetector, language.Python: pythonDetector, - language.Ruby: rubyDetector, } // For now, only allow a subset of the above detectors to actually run. allowedLangs = map[language.Language]struct{}{ - language.Java: {}, + language.Java: {}, + language.Python: {}, } ) // Detect attempts to detect the type of APM instrumentation for the given service. -func Detect(args []string, envs map[string]string, lang language.Language) Instrumentation { +func Detect(pid int, args []string, envs map[string]string, lang language.Language) Instrumentation { // first check to see if the DD_INJECTION_ENABLED is set to tracer if isInjected(envs) { return Injected @@ -62,7 +64,7 @@ func Detect(args []string, envs map[string]string, lang language.Language) Instr // different detection for provided instrumentation for each if detect, ok := detectorMap[lang]; ok { - return detect(args, envs) + return detect(pid, args, envs) } return None @@ -80,61 +82,42 @@ func isInjected(envs map[string]string) bool { return false } -func rubyDetector(_ []string, _ map[string]string) Instrumentation { - return None -} +func pythonDetectorFromMapsReader(reader io.Reader) Instrumentation { + scanner := bufio.NewScanner(bufio.NewReader(reader)) + for scanner.Scan() { + line := scanner.Text() -func pythonDetector(args []string, envs map[string]string) Instrumentation { - /* - Check for VIRTUAL_ENV env var - if it's there, use $VIRTUAL_ENV/lib/python{}/site-packages/ and see if ddtrace is inside - if so, return PROVIDED - if it's not there, - exec args[0] -c "import sys; print(':'.join(sys.path))" - split on : - for each part - see if it ends in site-packages - if so, check if ddtrace is inside - if so, return PROVIDED - return NONE - */ - if path, ok := envs["VIRTUAL_ENV"]; ok { - venv := os.DirFS(path) - libContents, err := fs.ReadDir(venv, "lib") - if err == nil { - for _, v := range libContents { - if strings.HasPrefix(v.Name(), "python") && v.IsDir() { - tracedir, err := fs.Stat(venv, "lib/"+v.Name()+"/site-packages/ddtrace") - if err == nil && tracedir.IsDir() { - return Provided - } - } - } + if strings.Contains(line, "/ddtrace/") { + return Provided } - // the virtual env didn't have ddtrace, can exit - return None } - // slow option... - results, err := exec.Command(args[0], `-c`, `"import sys; print(':'.join(sys.path))"`).Output() + + return None +} + +// pythonDetector detects the use of the ddtrace package in the process. Since +// the ddtrace package uses native libraries, the paths of these libraries will +// show up in /proc/$PID/maps. +// +// It looks for the "/ddtrace/" part of the path. It doesn not look for the +// "/site-packages/" part since some environments (such as pyinstaller) may not +// use that exact name. +// +// For example: +// 7aef453fc000-7aef453ff000 rw-p 0004c000 fc:06 7895473 /home/foo/.local/lib/python3.10/site-packages/ddtrace/internal/_encoding.cpython-310-x86_64-linux-gnu.so +// 7aef45400000-7aef45459000 r--p 00000000 fc:06 7895588 /home/foo/.local/lib/python3.10/site-packages/ddtrace/internal/datadog/profiling/libdd_wrapper.so +func pythonDetector(pid int, _ []string, _ map[string]string) Instrumentation { + mapsPath := kernel.HostProc(strconv.Itoa(pid), "maps") + mapsFile, err := os.Open(mapsPath) if err != nil { - log.Warn("Failed to execute command", err) return None } + defer mapsFile.Close() - results = bytes.TrimSpace(results) - parts := strings.Split(string(results), ":") - for _, v := range parts { - if strings.HasSuffix(v, "/site-packages") { - _, err := os.Stat(v + "/ddtrace") - if err == nil { - return Provided - } - } - } - return None + return pythonDetectorFromMapsReader(mapsFile) } -func nodeDetector(_ []string, envs map[string]string) Instrumentation { +func nodeDetector(_ int, _ []string, envs map[string]string) Instrumentation { // check package.json, see if it has dd-trace in it. // first find it wd := "" @@ -179,7 +162,7 @@ func nodeDetector(_ []string, envs map[string]string) Instrumentation { return None } -func javaDetector(args []string, envs map[string]string) Instrumentation { +func javaDetector(_ int, args []string, envs map[string]string) Instrumentation { ignoreArgs := map[string]bool{ "-version": true, "-Xshare:dump": true, @@ -228,7 +211,7 @@ func findFile(fileName string) (io.ReadCloser, bool) { const datadogDotNetInstrumented = "Datadog.Trace.ClrProfiler.Native" -func dotNetDetector(args []string, envs map[string]string) Instrumentation { +func dotNetDetector(_ int, args []string, envs map[string]string) Instrumentation { // if it's just the word `dotnet` by itself, don't instrument if len(args) == 1 && args[0] == "dotnet" { return None diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go index 56562c6d92ec4..c4592debc2573 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go @@ -3,12 +3,11 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !windows +//go:build linux package apm import ( - "os" "strings" "testing" @@ -86,7 +85,7 @@ func Test_javaDetector(t *testing.T) { } for _, d := range data { t.Run(d.name, func(t *testing.T) { - result := javaDetector(d.args, d.envs) + result := javaDetector(0, d.args, d.envs) if result != d.result { t.Errorf("expected %s got %s", d.result, result) } @@ -95,51 +94,38 @@ func Test_javaDetector(t *testing.T) { } func Test_pythonDetector(t *testing.T) { - tmpDir := t.TempDir() - err := os.MkdirAll(tmpDir+"/lib/python3.11/site-packages/ddtrace", 0700) - if err != nil { - t.Fatal(err) - } - tmpDir2 := t.TempDir() - err = os.MkdirAll(tmpDir2+"/lib/python3.11/site-packages/notddtrace", 0700) - if err != nil { - t.Fatal(err) - } data := []struct { name string - args []string - envs map[string]string + maps string result Instrumentation }{ { - name: "venv_provided", - args: []string{"./echoer.sh", "nope"}, - envs: map[string]string{"VIRTUAL_ENV": tmpDir}, - result: Provided, + name: "empty maps", + maps: "", + result: None, }, { - name: "venv_none", - args: []string{"./testdata/echoer.sh", "nope"}, - envs: map[string]string{"VIRTUAL_ENV": tmpDir2}, + name: "not in maps", + maps: ` +79f6cd47d000-79f6cd47f000 r--p 00000000 fc:04 793163 /usr/lib/python3.10/lib-dynload/_bz2.cpython-310-x86_64-linux-gnu.so +79f6cd479000-79f6cd47a000 r-xp 00001000 fc:06 5507018 /home/foo/.local/lib/python3.10/site-packages/ddtrace_fake/md.cpython-310-x86_64-linux-gnu.so + `, result: None, }, { - name: "cmd_provided", - args: []string{"./testdata/cmd_works.sh"}, + name: "in maps", + maps: ` +79f6cd47d000-79f6cd47f000 r--p 00000000 fc:04 793163 /usr/lib/python3.10/lib-dynload/_bz2.cpython-310-x86_64-linux-gnu.so +79f6cd482000-79f6cd484000 r--p 00005000 fc:04 793163 /usr/lib/python3.10/lib-dynload/_bz2.cpython-310-x86_64-linux-gnu.so +79f6cd438000-79f6cd441000 r-xp 00004000 fc:06 7895596 /home/foo/.local/lib/python3.10/site-packages-internal/ddtrace/internal/datadog/profiling/crashtracker/_crashtracker.cpython-310-x86_64-linux-gnu.so + `, result: Provided, }, - { - name: "cmd_none", - args: []string{"./testdata/cmd_fails.sh"}, - result: None, - }, } for _, d := range data { t.Run(d.name, func(t *testing.T) { - result := pythonDetector(d.args, d.envs) - if result != d.result { - t.Errorf("expected %s got %s", d.result, result) - } + result := pythonDetectorFromMapsReader(strings.NewReader(d.maps)) + assert.Equal(t, d.result, result) }) } } diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go index d739c4e938b83..508dc9fc6d898 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux.go @@ -214,7 +214,7 @@ func (s *discovery) getServiceInfo(proc *process.Process) (*serviceInfo, error) name := servicediscovery.GetServiceName(cmdline, envs) language := language.FindInArgs(cmdline) - apmInstrumentation := apm.Detect(cmdline, envs, language) + apmInstrumentation := apm.Detect(int(proc.Pid), cmdline, envs, language) return &serviceInfo{name: name, apmInstrumentation: apmInstrumentation}, nil } diff --git a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go index 7989e35067823..e4fd8437272a1 100644 --- a/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/module/impl_linux_test.go @@ -40,6 +40,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" protocolUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" + fileopener "github.com/DataDog/datadog-agent/pkg/network/usm/sharedlibraries/testutil" usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/util/optional" ) @@ -333,30 +334,36 @@ func TestAPMInstrumentationInjected(t *testing.T) { require.Equal(t, string(apm.Injected), portMap[pid].APMInstrumentation) } +func makeAlias(t *testing.T, alias string, serverBin string) string { + binDir := filepath.Dir(serverBin) + aliasPath := filepath.Join(binDir, alias) + + target, err := os.Readlink(aliasPath) + if err == nil && target == serverBin { + return aliasPath + } + + os.Remove(aliasPath) + err = os.Symlink(serverBin, aliasPath) + require.NoError(t, err) + + return aliasPath +} + func buildFakeServer(t *testing.T) string { curDir, err := testutil.CurDir() require.NoError(t, err) serverBin, err := usmtestutil.BuildGoBinaryWrapper(filepath.Join(curDir, "testutil"), "fake_server") require.NoError(t, err) - binDir := filepath.Dir(serverBin) for _, alias := range []string{"java"} { - aliasPath := filepath.Join(binDir, alias) - - target, err := os.Readlink(aliasPath) - if err == nil && target == serverBin { - continue - } - - os.Remove(aliasPath) - err = os.Symlink(serverBin, aliasPath) - require.NoError(t, err) + makeAlias(t, alias, serverBin) } - return binDir + return filepath.Dir(serverBin) } -func TestAPMInstrumentationProvided(t *testing.T) { +func TestAPMInstrumentationProvidedJava(t *testing.T) { serverDir := buildFakeServer(t) url := setupDiscoveryModule(t) @@ -377,6 +384,41 @@ func TestAPMInstrumentationProvided(t *testing.T) { }, 30*time.Second, 100*time.Millisecond) } +func TestAPMInstrumentationProvidedPython(t *testing.T) { + curDir, err := testutil.CurDir() + require.NoError(t, err) + + fmapper := fileopener.BuildFmapper(t) + fakePython := makeAlias(t, "python", fmapper) + + // We need the process to map something in a directory called + // "site-packages/ddtrace". The actual mapped file does not matter. + ddtrace := filepath.Join(curDir, "..", "..", "..", "..", "network", "usm", "testdata", "site-packages", "ddtrace") + lib := filepath.Join(ddtrace, fmt.Sprintf("libssl.so.%s", runtime.GOARCH)) + + // Give the process a listening socket + listener, err := net.Listen("tcp", "") + require.NoError(t, err) + f, err := listener.(*net.TCPListener).File() + listener.Close() + require.NoError(t, err) + t.Cleanup(func() { f.Close() }) + disableCloseOnExec(t, f) + + cmd, err := fileopener.OpenFromProcess(t, fakePython, lib) + require.NoError(t, err) + + url := setupDiscoveryModule(t) + + pid := cmd.Process.Pid + require.EventuallyWithT(t, func(collect *assert.CollectT) { + portMap := getServicesMap(t, url) + assert.Contains(collect, portMap, pid) + fmt.Println(portMap[pid]) + assert.Equal(collect, string(apm.Provided), portMap[pid].APMInstrumentation) + }, 30*time.Second, 100*time.Millisecond) +} + // Check that we can get listening processes in other namespaces. func TestNamespaces(t *testing.T) { url := setupDiscoveryModule(t) diff --git a/pkg/network/usm/ebpf_ssl_test.go b/pkg/network/usm/ebpf_ssl_test.go index 617dd25912ecb..99d8dcf37fdfb 100644 --- a/pkg/network/usm/ebpf_ssl_test.go +++ b/pkg/network/usm/ebpf_ssl_test.go @@ -32,7 +32,8 @@ func testArch(t *testing.T, arch string) { curDir, err := testutil.CurDir() require.NoError(t, err) - libmmap := filepath.Join(curDir, "testdata", "libmmap") + // Named site-packages/ddtrace since it is used from servicediscovery tests too. + libmmap := filepath.Join(curDir, "testdata", "site-packages", "ddtrace") lib := filepath.Join(libmmap, fmt.Sprintf("libssl.so.%s", arch)) monitor := setupUSMTLSMonitor(t, cfg) diff --git a/pkg/network/usm/sharedlibraries/testutil/testutil.go b/pkg/network/usm/sharedlibraries/testutil/testutil.go index a8123133ba614..1cf1bcee23e3a 100644 --- a/pkg/network/usm/sharedlibraries/testutil/testutil.go +++ b/pkg/network/usm/sharedlibraries/testutil/testutil.go @@ -26,10 +26,9 @@ import ( // mutex protecting build process var mux sync.Mutex -// OpenFromAnotherProcess launches an external file that holds active handler to the given paths. -func OpenFromAnotherProcess(t *testing.T, paths ...string) (*exec.Cmd, error) { - programExecutable := build(t) - +// OpenFromProcess launches the specified external program which holds an active +// handle to the given paths. +func OpenFromProcess(t *testing.T, programExecutable string, paths ...string) (*exec.Cmd, error) { cmd := exec.Command(programExecutable, paths...) patternScanner := protocolstestutil.NewScanner(regexp.MustCompile("awaiting signal"), make(chan struct{}, 1)) cmd.Stdout = patternScanner @@ -56,8 +55,16 @@ func OpenFromAnotherProcess(t *testing.T, paths ...string) (*exec.Cmd, error) { } } -// build only gets executed when running tests locally -func build(t *testing.T) string { +// OpenFromAnotherProcess launches an external program that holds an active +// handle to the given paths. +func OpenFromAnotherProcess(t *testing.T, paths ...string) (*exec.Cmd, error) { + programExecutable := BuildFmapper(t) + return OpenFromProcess(t, programExecutable, paths...) +} + +// BuildFmapper builds the external program which is used to hold references to +// shared libraries for testing. +func BuildFmapper(t *testing.T) string { mux.Lock() defer mux.Unlock() diff --git a/pkg/network/usm/testdata/libmmap/build.sh b/pkg/network/usm/testdata/site-packages/ddtrace/build.sh similarity index 100% rename from pkg/network/usm/testdata/libmmap/build.sh rename to pkg/network/usm/testdata/site-packages/ddtrace/build.sh diff --git a/pkg/network/usm/testdata/libmmap/fakessl.c b/pkg/network/usm/testdata/site-packages/ddtrace/fakessl.c similarity index 100% rename from pkg/network/usm/testdata/libmmap/fakessl.c rename to pkg/network/usm/testdata/site-packages/ddtrace/fakessl.c diff --git a/pkg/network/usm/testdata/libmmap/libssl.so.amd64 b/pkg/network/usm/testdata/site-packages/ddtrace/libssl.so.amd64 similarity index 100% rename from pkg/network/usm/testdata/libmmap/libssl.so.amd64 rename to pkg/network/usm/testdata/site-packages/ddtrace/libssl.so.amd64 diff --git a/pkg/network/usm/testdata/libmmap/libssl.so.arm64 b/pkg/network/usm/testdata/site-packages/ddtrace/libssl.so.arm64 similarity index 100% rename from pkg/network/usm/testdata/libmmap/libssl.so.arm64 rename to pkg/network/usm/testdata/site-packages/ddtrace/libssl.so.arm64 diff --git a/test/new-e2e/tests/discovery/linux_test.go b/test/new-e2e/tests/discovery/linux_test.go index 7ccd32b4a33da..0709e0cc41b82 100644 --- a/test/new-e2e/tests/discovery/linux_test.go +++ b/test/new-e2e/tests/discovery/linux_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" @@ -75,18 +76,25 @@ func (s *linuxTestSuite) TestServiceDiscoveryCheck() { payloads, err := client.GetServiceDiscoveries() require.NoError(t, err) - found := false + foundMap := make(map[string]*aggregator.ServiceDiscoveryPayload) for _, p := range payloads { name := p.Payload.ServiceName t.Log("RequestType", p.RequestType, "ServiceName", name) - if p.RequestType == "start-service" && name == "python" { - found = true - break + if p.RequestType == "start-service" { + foundMap[name] = p } } - assert.True(c, found, "service not found") + found := foundMap["python.server"] + if assert.NotNil(c, found) { + assert.Equal(c, "none", found.Payload.APMInstrumentation) + } + + found = foundMap["python.instrumented"] + if assert.NotNil(c, found) { + assert.Equal(c, "provided", found.Payload.APMInstrumentation) + } }, 3*time.Minute, 10*time.Second) } @@ -124,8 +132,10 @@ func (s *linuxTestSuite) provisionServer() { func (s *linuxTestSuite) startServices() { s.Env().RemoteHost.MustExecute("sudo systemctl start python-svc") + s.Env().RemoteHost.MustExecute("sudo systemctl start python-instrumented") } func (s *linuxTestSuite) stopServices() { + s.Env().RemoteHost.MustExecute("sudo systemctl stop python-instrumented") s.Env().RemoteHost.MustExecute("sudo systemctl stop python-svc") } diff --git a/test/new-e2e/tests/discovery/testdata/provision/provision.sh b/test/new-e2e/tests/discovery/testdata/provision/provision.sh index 43e7138ea4f4c..99584fe7ffd4a 100755 --- a/test/new-e2e/tests/discovery/testdata/provision/provision.sh +++ b/test/new-e2e/tests/discovery/testdata/provision/provision.sh @@ -3,7 +3,9 @@ set -e apt-get update -apt-get install -y ca-certificates curl gnupg python3 +apt-get install -y ca-certificates curl gnupg python3 python3-pip + +pip install ddtrace # Install our own services install_systemd_unit () { @@ -31,6 +33,7 @@ EOM } install_systemd_unit "python-svc" "/usr/bin/python3 /home/ubuntu/e2e-test/python/server.py" "8082" +install_systemd_unit "python-instrumented" "/usr/bin/python3 /home/ubuntu/e2e-test/python/instrumented.py" "8083" systemctl daemon-reload diff --git a/test/new-e2e/tests/discovery/testdata/provision/python/__init__.py b/test/new-e2e/tests/discovery/testdata/provision/python/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/test/new-e2e/tests/discovery/testdata/provision/python/instrumented.py b/test/new-e2e/tests/discovery/testdata/provision/python/instrumented.py new file mode 100644 index 0000000000000..0d9f5f2906d18 --- /dev/null +++ b/test/new-e2e/tests/discovery/testdata/provision/python/instrumented.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import ddtrace.auto # type: ignore # noqa: F401 +import server + +if __name__ == '__main__': + server.run() From a9be38591fc31bae65a89d881202fdfa3d0eab6f Mon Sep 17 00:00:00 2001 From: Adel Haj Hassan <41540817+adel121@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:45:27 +0200 Subject: [PATCH 088/245] refactor tagger entity id prefixes to its own type (#28626) --- comp/core/tagger/common/prefixes.go | 42 +++++++++++++++++++ .../collectors/workloadmeta_extract.go | 19 ++++----- comp/core/tagger/types/types.go | 13 ++++++ 3 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 comp/core/tagger/common/prefixes.go diff --git a/comp/core/tagger/common/prefixes.go b/comp/core/tagger/common/prefixes.go new file mode 100644 index 0000000000000..6988fcb1c8675 --- /dev/null +++ b/comp/core/tagger/common/prefixes.go @@ -0,0 +1,42 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +// Package common provides common constants and methods for the tagger component and implementation +package common + +import "github.com/DataDog/datadog-agent/comp/core/tagger/types" + +const ( + // ContainerID is the prefix `container_id` + ContainerID types.EntityIDPrefix = "container_id" + // ContainerImageMetadata is the prefix `container_image_metadata` + ContainerImageMetadata types.EntityIDPrefix = "container_image_metadata" + // ECSTask is the prefix `ecs_task` + ECSTask types.EntityIDPrefix = "ecs_task" + // Host is the prefix `host` + Host types.EntityIDPrefix = "host" + // KubernetesDeployment is the prefix `deployment` + KubernetesDeployment types.EntityIDPrefix = "deployment" + // KubernetesMetadata is the prefix `kubernetes_metadata` + KubernetesMetadata types.EntityIDPrefix = "kubernetes_metadata" + // KubernetesPodUID is the prefix `kubernetes_pod_uid` + KubernetesPodUID types.EntityIDPrefix = "kubernetes_pod_uid" + // Process is the prefix `process` + Process types.EntityIDPrefix = "process" +) + +// AllPrefixesSet returns a set of all possible entity id prefixes that can be used in the tagger +func AllPrefixesSet() map[types.EntityIDPrefix]struct{} { + return map[types.EntityIDPrefix]struct{}{ + ContainerID: {}, + ContainerImageMetadata: {}, + ECSTask: {}, + Host: {}, + KubernetesDeployment: {}, + KubernetesMetadata: {}, + KubernetesPodUID: {}, + Process: {}, + } +} diff --git a/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go b/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go index 29c95f150bd61..aa4208829ca7e 100644 --- a/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go +++ b/comp/core/tagger/taggerimpl/collectors/workloadmeta_extract.go @@ -11,15 +11,14 @@ import ( "fmt" "strings" + "github.com/DataDog/datadog-agent/comp/core/tagger/common" k8smetadata "github.com/DataDog/datadog-agent/comp/core/tagger/k8s_metadata" "github.com/DataDog/datadog-agent/comp/core/tagger/taglist" "github.com/DataDog/datadog-agent/comp/core/tagger/tags" "github.com/DataDog/datadog-agent/comp/core/tagger/types" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/config" - "github.com/DataDog/datadog-agent/pkg/util/containers" "github.com/DataDog/datadog-agent/pkg/util/kubernetes" - "github.com/DataDog/datadog-agent/pkg/util/kubernetes/kubelet" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -822,21 +821,21 @@ func (c *WorkloadMetaCollector) addOpenTelemetryStandardTags(container *workload func buildTaggerEntityID(entityID workloadmeta.EntityID) string { switch entityID.Kind { case workloadmeta.KindContainer: - return containers.BuildTaggerEntityName(entityID.ID) + return common.ContainerID.ToUID(entityID.ID) case workloadmeta.KindKubernetesPod: - return kubelet.PodUIDToTaggerEntityName(entityID.ID) + return common.KubernetesPodUID.ToUID(entityID.ID) case workloadmeta.KindECSTask: - return fmt.Sprintf("ecs_task://%s", entityID.ID) + return common.ECSTask.ToUID(entityID.ID) case workloadmeta.KindContainerImageMetadata: - return fmt.Sprintf("container_image_metadata://%s", entityID.ID) + return common.ContainerImageMetadata.ToUID(entityID.ID) case workloadmeta.KindProcess: - return fmt.Sprintf("process://%s", entityID.ID) + return common.Process.ToUID(entityID.ID) case workloadmeta.KindKubernetesDeployment: - return fmt.Sprintf("deployment://%s", entityID.ID) + return common.KubernetesDeployment.ToUID(entityID.ID) case workloadmeta.KindHost: - return fmt.Sprintf("host://%s", entityID.ID) + return common.Host.ToUID(entityID.ID) case workloadmeta.KindKubernetesMetadata: - return fmt.Sprintf("kubernetes_metadata://%s", entityID.ID) + return common.KubernetesMetadata.ToUID(entityID.ID) default: log.Errorf("can't recognize entity %q with kind %q; trying %s://%s as tagger entity", entityID.ID, entityID.Kind, entityID.ID, entityID.Kind) diff --git a/comp/core/tagger/types/types.go b/comp/core/tagger/types/types.go index 5cc3df3487001..cbe90462ce2b3 100644 --- a/comp/core/tagger/types/types.go +++ b/comp/core/tagger/types/types.go @@ -164,3 +164,16 @@ type EntityEvent struct { EventType EventType Entity Entity } + +// EntityIDPrefix represents the prefix of a TagEntity id +type EntityIDPrefix string + +// ToUID builds a unique id from the passed id +// if the passed id is empty, an empty string is returned +// else it returns `{entityPrefix}://{id}` +func (e EntityIDPrefix) ToUID(id string) string { + if id == "" { + return "" + } + return fmt.Sprintf("%s://%s", e, id) +} From f438bdd5420a59628c4c943f75759afb6369def8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:48:34 +0000 Subject: [PATCH 089/245] Bump golang.org/x/sys from 0.23.0 to 0.24.0 (#28413) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0ee136c57df2f..69728809ccb15 100644 --- a/go.mod +++ b/go.mod @@ -302,7 +302,7 @@ require ( golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/net v0.28.0 golang.org/x/sync v0.8.0 - golang.org/x/sys v0.23.0 + golang.org/x/sys v0.24.0 golang.org/x/text v0.17.0 golang.org/x/time v0.6.0 golang.org/x/tools v0.24.0 diff --git a/go.sum b/go.sum index f75ba7d887846..5dae9c140fc35 100644 --- a/go.sum +++ b/go.sum @@ -3222,8 +3222,8 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From f91e5d9106f0e768e2287f021905243015acbe55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20L=C3=B3pez=20de=20Heredia?= Date: Wed, 21 Aug 2024 17:32:03 +0200 Subject: [PATCH 090/245] [APM] Fix potential integer overflow on conversion (#28572) --- pkg/trace/filters/replacer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/trace/filters/replacer.go b/pkg/trace/filters/replacer.go index 2616b0c46326d..7d243b21b4133 100644 --- a/pkg/trace/filters/replacer.go +++ b/pkg/trace/filters/replacer.go @@ -86,7 +86,7 @@ func (f Replacer) ReplaceStatsGroup(b *pb.ClientGroupedStats) { b.Resource = re.ReplaceAllString(b.Resource, str) fallthrough case "http.status_code": - strcode := re.ReplaceAllString(strconv.Itoa(int(b.HTTPStatusCode)), str) + strcode := re.ReplaceAllString(strconv.FormatUint(uint64(b.HTTPStatusCode), 10), str) if code, err := strconv.ParseUint(strcode, 10, 32); err == nil { b.HTTPStatusCode = uint32(code) } From 89da69e6868fa0119f10b972b53e4d813c56d8e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 15:57:31 +0000 Subject: [PATCH 091/245] Bump actions/setup-go from 5.0.0 to 5.0.2 (#27573) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/buildimages-update.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/cws-btfhub-sync.yml | 4 ++-- .github/workflows/go_mod_tidy.yml | 2 +- .github/workflows/gohai.yml | 2 +- .github/workflows/serverless-benchmarks.yml | 6 +++--- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/buildimages-update.yml b/.github/workflows/buildimages-update.yml index e3fc6d7bc2e15..9a04aceed38e4 100644 --- a/.github/workflows/buildimages-update.yml +++ b/.github/workflows/buildimages-update.yml @@ -58,7 +58,7 @@ jobs: python-version: 3.11 cache: "pip" - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: # use the go version from the input, not from the .go-version file # in case it's a Go update PR diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5ac98f7c6cd52..15b7721faf5ce 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,7 +37,7 @@ jobs: echo "CGO_LDFLAGS= -L${GITHUB_WORKSPACE}/rtloader/build/rtloader -ldl " >> $GITHUB_ENV echo "CGO_CFLAGS= -I${GITHUB_WORKSPACE}/rtloader/include -I${GITHUB_WORKSPACE}/rtloader/common " >> $GITHUB_ENV - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: ".go-version" diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml index 8003a7d2ca0f0..829361101dd01 100644 --- a/.github/workflows/cws-btfhub-sync.yml +++ b/.github/workflows/cws-btfhub-sync.yml @@ -69,7 +69,7 @@ jobs: - run: pip install -r requirements.txt - name: Install go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: '.go-version' @@ -105,7 +105,7 @@ jobs: - run: pip install -r requirements.txt - name: Install go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: '.go-version' diff --git a/.github/workflows/go_mod_tidy.yml b/.github/workflows/go_mod_tidy.yml index 002a454c08ad0..d90caf056fb82 100644 --- a/.github/workflows/go_mod_tidy.yml +++ b/.github/workflows/go_mod_tidy.yml @@ -26,7 +26,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Install go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: ".go-version" - name: Install python diff --git a/.github/workflows/gohai.yml b/.github/workflows/gohai.yml index 19da9e4695b7e..bb20f0e0104df 100644 --- a/.github/workflows/gohai.yml +++ b/.github/workflows/gohai.yml @@ -31,7 +31,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version-file: ${{ matrix.go-file }} - name: Test diff --git a/.github/workflows/serverless-benchmarks.yml b/.github/workflows/serverless-benchmarks.yml index 1b1388da42007..5dc1739754b84 100644 --- a/.github/workflows/serverless-benchmarks.yml +++ b/.github/workflows/serverless-benchmarks.yml @@ -27,7 +27,7 @@ jobs: ref: ${{ github.base_ref }} - name: Install Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version: stable @@ -63,7 +63,7 @@ jobs: ref: ${{ github.sha }} - name: Install Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version: stable @@ -92,7 +92,7 @@ jobs: steps: - name: Install Go - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version: stable cache: false From b95bc0051af0b3f16952de3051381bf13047008b Mon Sep 17 00:00:00 2001 From: "Duong (Yoon)" <47346352+DDuongNguyen@users.noreply.github.com> Date: Wed, 21 Aug 2024 12:00:58 -0400 Subject: [PATCH 092/245] [AGENT-11735] Fix Nil Pointer Dereference in Tailer DidRotate Functions (#28502) --- pkg/logs/tailers/file/rotate_nix.go | 4 ++-- pkg/logs/tailers/file/rotate_windows.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/logs/tailers/file/rotate_nix.go b/pkg/logs/tailers/file/rotate_nix.go index db4ac6a66023e..72f1240e6de0e 100644 --- a/pkg/logs/tailers/file/rotate_nix.go +++ b/pkg/logs/tailers/file/rotate_nix.go @@ -22,9 +22,9 @@ import ( // - removed and recreated // - truncated func (t *Tailer) DidRotate() (bool, error) { - f, err := filesystem.OpenShared(t.osFile.Name()) + f, err := filesystem.OpenShared(t.fullpath) if err != nil { - return false, fmt.Errorf("open %q: %w", t.osFile.Name(), err) + return false, fmt.Errorf("open %q: %w", t.fullpath, err) } defer f.Close() lastReadOffset := t.lastReadOffset.Load() diff --git a/pkg/logs/tailers/file/rotate_windows.go b/pkg/logs/tailers/file/rotate_windows.go index 37853d190299a..59ac517a3c240 100644 --- a/pkg/logs/tailers/file/rotate_windows.go +++ b/pkg/logs/tailers/file/rotate_windows.go @@ -21,7 +21,7 @@ import ( func (t *Tailer) DidRotate() (bool, error) { f, err := filesystem.OpenShared(t.fullpath) if err != nil { - return false, fmt.Errorf("open %q: %w", t.osFile.Name(), err) + return false, fmt.Errorf("open %q: %w", t.fullpath, err) } defer f.Close() offset := t.lastReadOffset.Load() From c55933a4d9485ea71e2e323d89869f31977c25a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillermo=20Juli=C3=A1n?= Date: Wed, 21 Aug 2024 18:08:55 +0200 Subject: [PATCH 093/245] [EBPF-554] Ensure kmt compiler image is the correct version (#28454) Co-authored-by: Bryce Kahle --- tasks/kernel_matrix_testing/compiler.py | 42 +++++++++++++++++++++---- tasks/kmt.py | 8 +++-- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/tasks/kernel_matrix_testing/compiler.py b/tasks/kernel_matrix_testing/compiler.py index b330bb5ff2e05..e2f949d62f8d1 100644 --- a/tasks/kernel_matrix_testing/compiler.py +++ b/tasks/kernel_matrix_testing/compiler.py @@ -30,7 +30,16 @@ def get_build_image_suffix_and_version() -> tuple[str, str]: ci_config = yaml.load(f, Loader=GitlabYamlLoader()) ci_vars = ci_config['variables'] - return ci_vars['DATADOG_AGENT_BUILDIMAGES_SUFFIX'], ci_vars['DATADOG_AGENT_BUILDIMAGES'] + return ci_vars['DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX'], ci_vars['DATADOG_AGENT_SYSPROBE_BUILDIMAGES'] + + +def get_docker_image_name(ctx: Context, container: str) -> str: + res = ctx.run(f"docker inspect \"{container}\"", hide=True) + if res is None or not res.ok: + raise ValueError(f"Could not get {container} info") + + data = json.loads(res.stdout) + return data[0]["Config"]["Image"] def has_ddtool_helpers() -> bool: @@ -65,17 +74,25 @@ def image(self): return f"{image_base}_{self.arch.ci_arch}{suffix}:{version}" - @property - def is_running(self): + def _check_container_exists(self, allow_stopped=False): if self.ctx.config.run["dry"]: warn(f"[!] Dry run, not checking if compiler {self.name} is running") return True - res = self.ctx.run(f"docker ps -aqf \"name={self.name}\"", hide=True) + args = "a" if allow_stopped else "" + res = self.ctx.run(f"docker ps -{args}qf \"name={self.name}\"", hide=True) if res is not None and res.ok: return res.stdout.rstrip() != "" return False + @property + def is_running(self): + return self._check_container_exists(allow_stopped=False) + + @property + def is_loaded(self): + return self._check_container_exists(allow_stopped=True) + def ensure_running(self): if not self.is_running: info(f"[*] Compiler for {self.arch} not running, starting it...") @@ -84,6 +101,15 @@ def ensure_running(self): except Exception as e: raise Exit(f"Failed to start compiler for {self.arch}: {e}") from e + def ensure_version(self): + if not self.is_loaded: + return # Nothing to do if the container is not loaded + + image_used = get_docker_image_name(self.ctx, self.name) + if image_used != self.image: + warn(f"[!] Running compiler image {image_used} is different from the expected {self.image}, will restart") + self.start() + def exec(self, cmd: str, user="compiler", verbose=True, run_dir: PathOrStr | None = None, allow_fail=False): if run_dir: cmd = f"cd {run_dir} && {cmd}" @@ -102,7 +128,7 @@ def stop(self) -> Result: return cast('Result', res) # Avoid mypy error about res being None def start(self) -> None: - if self.is_running: + if self.is_loaded: self.stop() # Check if the image exists @@ -202,4 +228,8 @@ def prepare_for_cross_compile(self): def get_compiler(ctx: Context): - return CompilerImage(ctx, Arch.local()) + cc = CompilerImage(ctx, Arch.local()) + cc.ensure_version() + cc.ensure_running() + + return cc diff --git a/tasks/kmt.py b/tasks/kmt.py index 6d38c64af6a91..578b95755ddfc 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -675,10 +675,12 @@ def prepare( raise Exit( f"Architecture {arch} (inferred {arch_obj}) is not supported. Supported architectures are amd64 and arm64" ) - cc = get_compiler(ctx) - if arch_obj.is_cross_compiling(): - cc.ensure_ready_for_cross_compile() + if not ci: + cc = get_compiler(ctx) + + if arch_obj.is_cross_compiling(): + cc.ensure_ready_for_cross_compile() pkgs = "" if packages: From 026cc1017f0faf1c3b981d848091ff6a0a012a1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:40:35 +0000 Subject: [PATCH 094/245] Bump golang.org/x/sys from 0.23.0 to 0.24.0 in /pkg/gohai (#28427) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pkg/gohai/go.mod | 2 +- pkg/gohai/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/gohai/go.mod b/pkg/gohai/go.mod index 75633a855ce39..10685f7af8e2f 100644 --- a/pkg/gohai/go.mod +++ b/pkg/gohai/go.mod @@ -10,7 +10,7 @@ require ( github.com/moby/sys/mountinfo v0.7.2 github.com/shirou/gopsutil/v3 v3.24.5 github.com/stretchr/testify v1.9.0 - golang.org/x/sys v0.23.0 + golang.org/x/sys v0.24.0 ) require ( diff --git a/pkg/gohai/go.sum b/pkg/gohai/go.sum index 555fa36effac6..f8710927d8496 100644 --- a/pkg/gohai/go.sum +++ b/pkg/gohai/go.sum @@ -47,8 +47,8 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 19f4609f4e65daa05ed4f3d04a7394deb019d19d Mon Sep 17 00:00:00 2001 From: Derek Brown Date: Wed, 21 Aug 2024 10:18:24 -0700 Subject: [PATCH 095/245] [windows][usm] load apm tags along with usm service tags (#28495) --- cmd/agent/common/path/go.mod | 1 - cmd/agent/common/path/go.sum | 2 - comp/core/secrets/go.mod | 1 - comp/core/secrets/go.sum | 2 - .../protocols/http/driver_interface.go | 3 + .../protocols/http/etw_http_service.go | 37 ++-- pkg/network/protocols/http/model_windows.go | 36 +++- pkg/util/winutil/iisconfig/apmtags.go | 116 +++++++++++ pkg/util/winutil/iisconfig/apmtags_test.go | 120 ++++++++++++ pkg/util/winutil/{ => iisconfig}/iisconfig.go | 16 +- .../winutil/{ => iisconfig}/iisconfig_test.go | 24 ++- pkg/util/winutil/iisconfig/tagtree.go | 184 ++++++++++++++++++ .../iisconfig/testdata/app1/datadog.json | 5 + .../iisconfig/testdata/app2/datadog.json | 5 + .../iisconfig/testdata/app3/datadog.json | 5 + .../iisconfig/testdata/app4/datadog.json | 5 + .../iisconfig/testdata/app_1.config.xml | 9 + .../winutil/iisconfig/testdata/apptest.xml | 46 +++++ .../{ => iisconfig}/testdata/iisconfig.xml | 6 +- 19 files changed, 594 insertions(+), 29 deletions(-) create mode 100644 pkg/util/winutil/iisconfig/apmtags.go create mode 100644 pkg/util/winutil/iisconfig/apmtags_test.go rename pkg/util/winutil/{ => iisconfig}/iisconfig.go (91%) rename pkg/util/winutil/{ => iisconfig}/iisconfig_test.go (58%) create mode 100644 pkg/util/winutil/iisconfig/tagtree.go create mode 100644 pkg/util/winutil/iisconfig/testdata/app1/datadog.json create mode 100644 pkg/util/winutil/iisconfig/testdata/app2/datadog.json create mode 100644 pkg/util/winutil/iisconfig/testdata/app3/datadog.json create mode 100644 pkg/util/winutil/iisconfig/testdata/app4/datadog.json create mode 100644 pkg/util/winutil/iisconfig/testdata/app_1.config.xml create mode 100644 pkg/util/winutil/iisconfig/testdata/apptest.xml rename pkg/util/winutil/{ => iisconfig}/testdata/iisconfig.xml (99%) diff --git a/cmd/agent/common/path/go.mod b/cmd/agent/common/path/go.mod index cbc72e9e1cfbc..97ffc05c7fa1a 100644 --- a/cmd/agent/common/path/go.mod +++ b/cmd/agent/common/path/go.mod @@ -19,7 +19,6 @@ require ( require ( github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect go.uber.org/atomic v1.11.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/cmd/agent/common/path/go.sum b/cmd/agent/common/path/go.sum index e34b1d17f405f..cff4a17f6553a 100644 --- a/cmd/agent/common/path/go.sum +++ b/cmd/agent/common/path/go.sum @@ -2,8 +2,6 @@ github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/comp/core/secrets/go.mod b/comp/core/secrets/go.mod index b8f6a20dc49f4..b2eb8c6fbc42f 100644 --- a/comp/core/secrets/go.mod +++ b/comp/core/secrets/go.mod @@ -40,7 +40,6 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect diff --git a/comp/core/secrets/go.sum b/comp/core/secrets/go.sum index d9cd59ebe7aae..e963d7263bf06 100644 --- a/comp/core/secrets/go.sum +++ b/comp/core/secrets/go.sum @@ -9,8 +9,6 @@ github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= diff --git a/pkg/network/protocols/http/driver_interface.go b/pkg/network/protocols/http/driver_interface.go index 9063cdcd64d6f..0bca6d58c2543 100644 --- a/pkg/network/protocols/http/driver_interface.go +++ b/pkg/network/protocols/http/driver_interface.go @@ -20,6 +20,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/config" "github.com/DataDog/datadog-agent/pkg/network/driver" "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/DataDog/datadog-agent/pkg/util/winutil/iisconfig" "golang.org/x/sys/windows" ) @@ -45,6 +46,8 @@ type WinHttpTransaction struct { SiteName string // HeaderLength uint32 // ContentLength uint32 + TagsFromJson iisconfig.APMTags + TagsFromConfig iisconfig.APMTags } //nolint:revive // TODO(WKIT) Fix revive linter diff --git a/pkg/network/protocols/http/etw_http_service.go b/pkg/network/protocols/http/etw_http_service.go index 162a8fd16e562..ee913ae4b7c8d 100644 --- a/pkg/network/protocols/http/etw_http_service.go +++ b/pkg/network/protocols/http/etw_http_service.go @@ -140,6 +140,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "unsafe" @@ -148,6 +149,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/driver" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/util/winutil" + "github.com/DataDog/datadog-agent/pkg/util/winutil/iisconfig" ) //nolint:revive // TODO(WKIT) Fix revive linter @@ -200,7 +202,8 @@ type HttpConnLink struct { http WinHttpTransaction - url string + url string + urlPath string // list of etw notifications, in order, that this transaction has been seen // this is for internal debugging; is not surfaced anywhere. @@ -274,7 +277,7 @@ var ( lastSummaryTime time.Time - iisConfig *winutil.DynamicIISConfig + iisConfig atomic.Pointer[iisconfig.DynamicIISConfig] ) func init() { @@ -742,6 +745,8 @@ func httpCallbackOnHTTPRequestTraceTaskParse(eventInfo *etw.DDEventRecord) { if len(urlParsed.Path) == 0 { urlParsed.Path = "/" } + httpConnLink.urlPath = urlParsed.Path + // httpConnLink.http.RequestFragment[0] = 32 is done to simulate // func getPath(reqFragment, buffer []byte) []byte // which expects something like "GET /foo?var=bar HTTP/1.1" @@ -830,7 +835,11 @@ func httpCallbackOnHTTPRequestTraceTaskDeliver(eventInfo *etw.DDEventRecord) { httpConnLink.http.AppPool = appPool httpConnLink.http.SiteID = userData.GetUint32(16) - httpConnLink.http.SiteName = iisConfig.GetSiteNameFromID(httpConnLink.http.SiteID) + cfg := iisConfig.Load() + if cfg != nil { + httpConnLink.http.SiteName = cfg.GetSiteNameFromID(httpConnLink.http.SiteID) + httpConnLink.http.TagsFromJson, httpConnLink.http.TagsFromConfig = cfg.GetAPMTags(httpConnLink.http.SiteID, httpConnLink.urlPath) + } // Parse url if urlOffset > userData.Length() { @@ -977,7 +986,10 @@ func httpCallbackOnHTTPRequestTraceTaskSrvdFrmCache(eventInfo *etw.DDEventRecord // <<>> httpConnLink.http.SiteID = cacheEntry.http.SiteID - httpConnLink.http.SiteName = iisConfig.GetSiteNameFromID(cacheEntry.http.SiteID) + cfg := iisConfig.Load() + if cfg != nil { + httpConnLink.http.SiteName = cfg.GetSiteNameFromID(cacheEntry.http.SiteID) + } completeReqRespTracking(eventInfo, httpConnLink) servedFromCache++ @@ -1427,26 +1439,29 @@ func (hei *EtwInterface) OnStart() { initializeEtwHttpServiceSubscription() httpServiceSubscribed = true var err error - iisConfig, err = winutil.NewDynamicIISConfig() + config, err := iisconfig.NewDynamicIISConfig() if err != nil { log.Warnf("Failed to create iis config %v", err) - iisConfig = nil + config = nil } else { - err = iisConfig.Start() + err = config.Start() if err != nil { log.Warnf("Failed to start iis config %v", err) - iisConfig = nil + config = nil } } + if config != nil { + iisConfig.Store(config) + } } //nolint:revive // TODO(WKIT) Fix revive linter func (hei *EtwInterface) OnStop() { httpServiceSubscribed = false initializeEtwHttpServiceSubscription() - if iisConfig != nil { - iisConfig.Stop() - iisConfig = nil + cfg := iisConfig.Swap(nil) + if cfg != nil { + cfg.Stop() } } func ipAndPortFromTup(tup driver.ConnTupleType, local bool) ([16]uint8, uint16) { diff --git a/pkg/network/protocols/http/model_windows.go b/pkg/network/protocols/http/model_windows.go index 9c21bb1170fd1..685060030600a 100644 --- a/pkg/network/protocols/http/model_windows.go +++ b/pkg/network/protocols/http/model_windows.go @@ -106,14 +106,40 @@ func (tx *WinHttpTransaction) StaticTags() uint64 { // //nolint:revive // TODO(WKIT) Fix revive linter func (tx *WinHttpTransaction) DynamicTags() []string { + tags := make([]string, 0, 6) + if len(tx.AppPool) != 0 || len(tx.SiteName) != 0 { - return []string{ - fmt.Sprintf("http.iis.app_pool:%v", tx.AppPool), - fmt.Sprintf("http.iis.site:%v", tx.SiteID), - fmt.Sprintf("http.iis.sitename:%v", tx.SiteName), + tags = append(tags, fmt.Sprintf("http.iis.site:%v", tx.SiteID)) + if (len(tx.AppPool)) > 0 { + tags = append(tags, fmt.Sprintf("http.iis.app_pool:%v", tx.AppPool)) + } + if (len(tx.SiteName)) > 0 { + tags = append(tags, fmt.Sprintf("http.iis.sitename:%v", tx.SiteName)) } } - return nil + + // tag precedence is web.config -> datadog.json + if (len(tx.TagsFromConfig.DDEnv)) > 0 { + tags = append(tags, fmt.Sprintf("env:%v", tx.TagsFromConfig.DDEnv)) + } else if (len(tx.TagsFromJson.DDEnv)) > 0 { + tags = append(tags, fmt.Sprintf("env:%v", tx.TagsFromJson.DDEnv)) + } + + if (len(tx.TagsFromConfig.DDService)) > 0 { + tags = append(tags, fmt.Sprintf("service:%v", tx.TagsFromConfig.DDService)) + } else if (len(tx.TagsFromJson.DDService)) > 0 { + tags = append(tags, fmt.Sprintf("service:%v", tx.TagsFromJson.DDService)) + } + + if (len(tx.TagsFromConfig.DDVersion)) > 0 { + tags = append(tags, fmt.Sprintf("version:%v", tx.TagsFromConfig.DDVersion)) + } else if (len(tx.TagsFromJson.DDVersion)) > 0 { + tags = append(tags, fmt.Sprintf("version:%v", tx.TagsFromJson.DDVersion)) + } + if len(tags) == 0 { + return nil + } + return tags } //nolint:revive // TODO(WKIT) Fix revive linter diff --git a/pkg/util/winutil/iisconfig/apmtags.go b/pkg/util/winutil/iisconfig/apmtags.go new file mode 100644 index 0000000000000..ee940ac2b7f5e --- /dev/null +++ b/pkg/util/winutil/iisconfig/apmtags.go @@ -0,0 +1,116 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. +//go:build windows + +// Package iisconfig manages iis configuration +package iisconfig + +/* +the file datadog.json can be located anywhere; it is path-relative to a .net application +give the path name, read the json and return it as a map of string/string +*/ + +import ( + "encoding/json" + "encoding/xml" + "os" + + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// APMTags holds the APM tags +type APMTags struct { + DDService string + DDEnv string + DDVersion string +} + +// ReadDatadogJSON reads a datadog.json file and returns the APM tags +func ReadDatadogJSON(datadogJSONPath string) (APMTags, error) { + var datadogJSON map[string]string + var apmtags APMTags + + file, err := os.Open(datadogJSONPath) + if err != nil { + return apmtags, err + } + defer file.Close() + + decoder := json.NewDecoder(file) + err = decoder.Decode(&datadogJSON) + if err != nil { + return apmtags, err + } + apmtags.DDService = datadogJSON["DD_SERVICE"] + apmtags.DDEnv = datadogJSON["DD_ENV"] + apmtags.DDVersion = datadogJSON["DD_VERSION"] + return apmtags, nil +} + +type iisAppSetting struct { + Key string `xml:"key,attr"` + Value string `xml:"value,attr"` +} +type iisAppSettings struct { + XMLName xml.Name `xml:"appSettings"` + Adds []iisAppSetting `xml:"add"` +} + +type appConfiguration struct { + XMLName xml.Name `xml:"configuration"` + AppSettings iisAppSettings +} + +var ( + errorlogcount = 0 +) + +// ReadDotNetConfig reads an iis config file(xml) and returns the APM tags +func ReadDotNetConfig(cfgpath string) (APMTags, error) { //(APMTags, error) { + var newcfg appConfiguration + var apmtags APMTags + var chasedatadogJSON string + f, err := os.ReadFile(cfgpath) + if err != nil { + return apmtags, err + } + err = xml.Unmarshal(f, &newcfg) + if err != nil { + return apmtags, err + } + for _, setting := range newcfg.AppSettings.Adds { + switch setting.Key { + case "DD_SERVICE": + apmtags.DDService = setting.Value + case "DD_ENV": + apmtags.DDEnv = setting.Value + case "DD_VERSION": + apmtags.DDVersion = setting.Value + case "DD_TRACE_CONFIG_FILE": + chasedatadogJSON = setting.Value + } + } + if len(chasedatadogJSON) > 0 { + ddjson, err := ReadDatadogJSON(chasedatadogJSON) + if err == nil { + if len(ddjson.DDService) > 0 { + apmtags.DDService = ddjson.DDService + } + if len(ddjson.DDEnv) > 0 { + apmtags.DDEnv = ddjson.DDEnv + } + if len(ddjson.DDVersion) > 0 { + apmtags.DDVersion = ddjson.DDVersion + } + } else { + // only log every 1000 occurrences because if this is misconfigured, it could flood the log + if errorlogcount%1000 == 0 { + log.Warnf("Error reading configured datadog.json file %s: %v", chasedatadogJSON, err) + } + errorlogcount++ + } + } + return apmtags, nil +} diff --git a/pkg/util/winutil/iisconfig/apmtags_test.go b/pkg/util/winutil/iisconfig/apmtags_test.go new file mode 100644 index 0000000000000..2844d5c515715 --- /dev/null +++ b/pkg/util/winutil/iisconfig/apmtags_test.go @@ -0,0 +1,120 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. +//go:build windows + +package iisconfig + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAppConfig(t *testing.T) { + path, err := os.Getwd() + require.Nil(t, err) + + apppath := filepath.Join(path, "testdata", "app_1.config.xml") + //apppath := filepath.Join(path, "testdata", "iisconfig.xml") + iisCfg, err := ReadDotNetConfig(apppath) + assert.Nil(t, err) + assert.NotNil(t, iisCfg) + + assert.Equal(t, iisCfg.DDService, "service1") + assert.Equal(t, iisCfg.DDEnv, "false") + assert.Equal(t, iisCfg.DDVersion, "1.0-prerelease") + +} + +func TestPathSplitting(t *testing.T) { + + t.Run("Test Root path", func(t *testing.T) { + sp := splitPaths("/") + assert.Equal(t, 0, len(sp)) + }) + + t.Run("Test path depth 3", func(t *testing.T) { + sp := splitPaths("/path/to/app") + assert.Equal(t, 3, len(sp)) + assert.Equal(t, "path", sp[0]) + assert.Equal(t, "to", sp[1]) + assert.Equal(t, "app", sp[2]) + }) + + t.Run("Test path depth 3 with trailing slash", func(t *testing.T) { + sp := splitPaths("/path/to/app/") + assert.Equal(t, 3, len(sp)) + assert.Equal(t, "path", sp[0]) + assert.Equal(t, "to", sp[1]) + assert.Equal(t, "app", sp[2]) + }) + +} + +func TestAPMTags(t *testing.T) { + path, err := os.Getwd() + require.Nil(t, err) + + iisCfgPath = filepath.Join(path, "testdata", "apptest.xml") + testroot := filepath.Join(path, "testdata") + os.Setenv("TESTROOTDIR", testroot) + defer os.Unsetenv("TESTROOTDIR") + + iisCfg, err := NewDynamicIISConfig() + assert.Nil(t, err) + assert.NotNil(t, iisCfg) + + err = iisCfg.Start() + assert.Nil(t, err) + + t.Run("Test simple root path", func(t *testing.T) { + tags, _ := iisCfg.GetAPMTags(2, "/") + assert.Equal(t, "app1", tags.DDService) + }) + t.Run("Test deeper path from top app", func(t *testing.T) { + tags, _ := iisCfg.GetAPMTags(2, "/path/to/app") + assert.Equal(t, "app1", tags.DDService) + }) + + t.Run("test top level app2", func(t *testing.T) { + tags, _ := iisCfg.GetAPMTags(2, "/app2") + assert.Equal(t, "app2", tags.DDService) + }) + t.Run("test deeper app2", func(t *testing.T) { + tags, _ := iisCfg.GetAPMTags(2, "/app2/some/path") + assert.Equal(t, "app2", tags.DDService) + }) + + t.Run("test app3 nested in app2", func(t *testing.T) { + tags, _ := iisCfg.GetAPMTags(2, "/app2/app3") + assert.Equal(t, "app3", tags.DDService) + }) + t.Run("test app3 nested in app2 with path", func(t *testing.T) { + tags, _ := iisCfg.GetAPMTags(2, "/app2/app3/some/path") + assert.Equal(t, "app3", tags.DDService) + }) + + t.Run("test secondary site", func(t *testing.T) { + tags, _ := iisCfg.GetAPMTags(3, "/") + assert.Equal(t, "app1", tags.DDService) + }) + t.Run("test secondary site app 3", func(t *testing.T) { + // this should still be app1 because the root path on the + // second site is different + tags, _ := iisCfg.GetAPMTags(3, "/app2/app3") + assert.Equal(t, "app1", tags.DDService) + }) + t.Run("test secondary site actual app3", func(t *testing.T) { + tags, _ := iisCfg.GetAPMTags(3, "/siteapp2/siteapp3") + assert.Equal(t, "app3", tags.DDService) + }) + t.Run("test secondary site actual app3 with file", func(t *testing.T) { + tags, _ := iisCfg.GetAPMTags(3, "/siteapp2/siteapp3/somefile") + assert.Equal(t, "app3", tags.DDService) + }) +} diff --git a/pkg/util/winutil/iisconfig.go b/pkg/util/winutil/iisconfig/iisconfig.go similarity index 91% rename from pkg/util/winutil/iisconfig.go rename to pkg/util/winutil/iisconfig/iisconfig.go index be6eb091baded..61af8bb534645 100644 --- a/pkg/util/winutil/iisconfig.go +++ b/pkg/util/winutil/iisconfig/iisconfig.go @@ -4,7 +4,7 @@ // Copyright 2016-present Datadog, Inc. //go:build windows -package winutil +package iisconfig import ( "encoding/xml" @@ -34,6 +34,7 @@ type DynamicIISConfig struct { stopChannel chan bool xmlcfg *iisConfiguration siteIDToName map[uint32]string + pathTrees map[uint32]*pathTreeEntry } // NewDynamicIISConfig creates a new DynamicIISConfig @@ -114,18 +115,20 @@ type iisApplication struct { VirtualDirs []iisVirtualDirectory `xml:"virtualDirectory"` } type iisSite struct { - Name string `xml:"name,attr"` - SiteID string `xml:"id,attr"` - Application iisApplication - Bindings []iisBinding `xml:"bindings>binding"` + Name string `xml:"name,attr"` + SiteID string `xml:"id,attr"` + Applications []iisApplication `xml:"application"` + Bindings []iisBinding `xml:"bindings>binding"` } type iisSystemApplicationHost struct { XMLName xml.Name `xml:"system.applicationHost"` Sites []iisSite `xml:"sites>site"` } + type iisConfiguration struct { XMLName xml.Name `xml:"configuration"` ApplicationHost iisSystemApplicationHost + AppSettings iisAppSettings } func (iiscfg *DynamicIISConfig) readXMLConfig() error { @@ -147,10 +150,13 @@ func (iiscfg *DynamicIISConfig) readXMLConfig() error { } idmap[uint32(id)] = site.Name } + + pt := buildPathTagTree(&newcfg) iiscfg.mux.Lock() defer iiscfg.mux.Unlock() iiscfg.xmlcfg = &newcfg iiscfg.siteIDToName = idmap + iiscfg.pathTrees = pt return nil } diff --git a/pkg/util/winutil/iisconfig_test.go b/pkg/util/winutil/iisconfig/iisconfig_test.go similarity index 58% rename from pkg/util/winutil/iisconfig_test.go rename to pkg/util/winutil/iisconfig/iisconfig_test.go index 9ae9c5127fdbe..1757e9b7e9e88 100644 --- a/pkg/util/winutil/iisconfig_test.go +++ b/pkg/util/winutil/iisconfig/iisconfig_test.go @@ -4,7 +4,7 @@ // Copyright 2016-present Datadog, Inc. //go:build windows -package winutil +package iisconfig import ( "os" @@ -35,3 +35,25 @@ func TestConfigLoading(t *testing.T) { assert.Equal(t, name, "TestSite") iisCfg.Stop() } + +func TestUninitializedConfig(t *testing.T) { + path, err := os.Getwd() + require.Nil(t, err) + + iisCfgPath = filepath.Join(path, "testdata", "iisconfig.xml") + iisCfg, err := NewDynamicIISConfig() + + // by not calling start, this will simulate either a caller trying to use w/o calling + // start, or a race where the start is still "in progress" when the caller tries to use it + + assert.Nil(t, err) + assert.NotNil(t, iisCfg) + + name := iisCfg.GetSiteNameFromID(0) + assert.Equal(t, name, "") + + atags, cfgtags := iisCfg.GetAPMTags(0, "/") + assert.Equal(t, atags.DDService, "") + assert.Equal(t, cfgtags.DDService, "") + +} diff --git a/pkg/util/winutil/iisconfig/tagtree.go b/pkg/util/winutil/iisconfig/tagtree.go new file mode 100644 index 0000000000000..79d77557a16d9 --- /dev/null +++ b/pkg/util/winutil/iisconfig/tagtree.go @@ -0,0 +1,184 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. +//go:build windows + +package iisconfig + +import ( + "os" + "path/filepath" + "strconv" + + "golang.org/x/sys/windows/registry" +) + +/* + IIS can have multiple sites. Each site can have multiple applications. + each application can have its own config. + + Such as + + + + + + + + + + + + + in the above, there each application should be treated separately. + + so if theURL is /app2/app3/appx, then we look in d:\source + /app2/app4/appx, then we look in d:\tmp + /app3 then we look in app1 + + pathTreeEntry implements a search tree to simplify the search for matching paths. +*/ +type pathTreeEntry struct { + nodes map[string]*pathTreeEntry + ddjson APMTags + appconfig APMTags +} + +func splitPaths(path string) []string { + path = filepath.Clean(path) + if path == "/" || path == "\\" { + return make([]string, 0) + } + a, b := filepath.Split(path) + s := splitPaths(a) + return append(s, b) +} + +func findInPathTree(pathTrees map[uint32]*pathTreeEntry, siteID uint32, urlpath string) (APMTags, APMTags) { + // urlpath will come in as something like + // /path/to/app + + // break down the path + pathparts := splitPaths(urlpath) + + if _, ok := pathTrees[siteID]; !ok { + return APMTags{}, APMTags{} + } + if len(pathparts) == 0 { + return pathTrees[siteID].ddjson, pathTrees[siteID].appconfig + } + + currNode := pathTrees[siteID] + + for _, part := range pathparts { + if _, ok := currNode.nodes[part]; !ok { + return currNode.ddjson, currNode.appconfig + } + currNode = currNode.nodes[part] + } + return currNode.ddjson, currNode.appconfig +} + +func addToPathTree(pathTrees map[uint32]*pathTreeEntry, siteID string, urlpath string, ddjson, appconfig APMTags) { + + intid, err := strconv.Atoi(siteID) + if err != nil { + return + } + id := uint32(intid) + // urlpath will come in as something like + // /path/to/app + // need to build the tree all the way down + + // break down the path + pathparts := splitPaths(urlpath) + + if _, ok := pathTrees[id]; !ok { + pathTrees[id] = &pathTreeEntry{ + nodes: make(map[string]*pathTreeEntry), + } + } + if len(pathparts) == 0 { + pathTrees[id].ddjson = ddjson + pathTrees[id].appconfig = appconfig + return + } + + currNode := pathTrees[id] + + for _, part := range pathparts { + if _, ok := currNode.nodes[part]; !ok { + currNode.nodes[part] = &pathTreeEntry{ + nodes: make(map[string]*pathTreeEntry), + } + } + currNode = currNode.nodes[part] + } + currNode.ddjson = ddjson + currNode.appconfig = appconfig +} + +// GetAPMTags returns the APM tags for the given siteID and URL path +func (iiscfg *DynamicIISConfig) GetAPMTags(siteID uint32, urlpath string) (APMTags, APMTags) { + iiscfg.mux.Lock() + defer iiscfg.mux.Unlock() + return findInPathTree(iiscfg.pathTrees, siteID, urlpath) +} + +func buildPathTagTree(xmlcfg *iisConfiguration) map[uint32]*pathTreeEntry { + pathTrees := make(map[uint32]*pathTreeEntry) + + for _, site := range xmlcfg.ApplicationHost.Sites { + for _, app := range site.Applications { + for _, vdir := range app.VirtualDirs { + if vdir.Path != "/" { + // assume that non `/` virtual paths mean that + // it's a virtual directory and not an application + // the application root will always be at / + continue + } + + // check to see if the datadog.json or web.config exists + ppath := vdir.PhysicalPath + ppath, err := registry.ExpandString(ppath) + if err != nil { + ppath = vdir.PhysicalPath + } + + ddjsonpath := filepath.Join(ppath, "datadog.json") + webcfg := filepath.Join(ppath, "web.config") + hasddjson := false + haswebcfg := false + var ddjson APMTags + var appconfig APMTags + + if _, err := os.Stat(ddjsonpath); err == nil { + hasddjson = true + } + if _, err := os.Stat(webcfg); err == nil { + haswebcfg = true + } + + if hasddjson { + ddjson, err = ReadDatadogJSON(ddjsonpath) + if err != nil { + hasddjson = false + } + } + if haswebcfg { + appconfig, err = ReadDotNetConfig(webcfg) + if err != nil { + haswebcfg = false + } + } + if !hasddjson && !haswebcfg { + continue + } + + addToPathTree(pathTrees, site.SiteID, app.Path, ddjson, appconfig) + } + } + } + return pathTrees +} diff --git a/pkg/util/winutil/iisconfig/testdata/app1/datadog.json b/pkg/util/winutil/iisconfig/testdata/app1/datadog.json new file mode 100644 index 0000000000000..deb9f3e52db63 --- /dev/null +++ b/pkg/util/winutil/iisconfig/testdata/app1/datadog.json @@ -0,0 +1,5 @@ +{ + "DD_SERVICE": "app1", + "DD_ENV": "staging", + "DD_VERSION": "1.0-prerelease" + } \ No newline at end of file diff --git a/pkg/util/winutil/iisconfig/testdata/app2/datadog.json b/pkg/util/winutil/iisconfig/testdata/app2/datadog.json new file mode 100644 index 0000000000000..e1a8718f32fec --- /dev/null +++ b/pkg/util/winutil/iisconfig/testdata/app2/datadog.json @@ -0,0 +1,5 @@ +{ + "DD_SERVICE": "app2", + "DD_ENV": "staging", + "DD_VERSION": "1.0-prerelease" + } \ No newline at end of file diff --git a/pkg/util/winutil/iisconfig/testdata/app3/datadog.json b/pkg/util/winutil/iisconfig/testdata/app3/datadog.json new file mode 100644 index 0000000000000..ee2153f9835f3 --- /dev/null +++ b/pkg/util/winutil/iisconfig/testdata/app3/datadog.json @@ -0,0 +1,5 @@ +{ + "DD_SERVICE": "app3", + "DD_ENV": "staging", + "DD_VERSION": "1.0-prerelease" + } \ No newline at end of file diff --git a/pkg/util/winutil/iisconfig/testdata/app4/datadog.json b/pkg/util/winutil/iisconfig/testdata/app4/datadog.json new file mode 100644 index 0000000000000..5dd98fd3f7fea --- /dev/null +++ b/pkg/util/winutil/iisconfig/testdata/app4/datadog.json @@ -0,0 +1,5 @@ +{ + "DD_SERVICE": "app4", + "DD_ENV": "staging", + "DD_VERSION": "1.0-prerelease" + } \ No newline at end of file diff --git a/pkg/util/winutil/iisconfig/testdata/app_1.config.xml b/pkg/util/winutil/iisconfig/testdata/app_1.config.xml new file mode 100644 index 0000000000000..4bab99ded3c6c --- /dev/null +++ b/pkg/util/winutil/iisconfig/testdata/app_1.config.xml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/pkg/util/winutil/iisconfig/testdata/apptest.xml b/pkg/util/winutil/iisconfig/testdata/apptest.xml new file mode 100644 index 0000000000000..29a90bfb0b8a8 --- /dev/null +++ b/pkg/util/winutil/iisconfig/testdata/apptest.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pkg/util/winutil/testdata/iisconfig.xml b/pkg/util/winutil/iisconfig/testdata/iisconfig.xml similarity index 99% rename from pkg/util/winutil/testdata/iisconfig.xml rename to pkg/util/winutil/iisconfig/testdata/iisconfig.xml index 7e0dda430c94b..a059929881090 100644 --- a/pkg/util/winutil/testdata/iisconfig.xml +++ b/pkg/util/winutil/iisconfig/testdata/iisconfig.xml @@ -11,7 +11,11 @@ --> - + + + + +