diff --git a/.circleci/config.yml b/.circleci/config.yml index 7506f2f4587d8..d733dab5e32df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ experimental: templates: job_template: &job_template docker: - - image: gcr.io/datadoghq/agent-circleci-runner:v45979046-89a73a40 + - image: gcr.io/datadoghq/agent-circleci-runner:v46542806-c7a4a6be environment: USE_SYSTEM_LIBS: "1" working_directory: /go/src/github.com/DataDog/datadog-agent diff --git a/.copyright-overrides.yml b/.copyright-overrides.yml index 9fbeffdec2351..8051687b44906 100644 --- a/.copyright-overrides.yml +++ b/.copyright-overrides.yml @@ -373,3 +373,5 @@ github.com/vibrantbyte/go-antpath/extend: vibrantbyte |suchao github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp: Copyright 2024 gRPC authors. github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp/grpc_gcp: Copyright 2024 gRPC authors. github.com/GoogleCloudPlatform/grpc-gcp-go/grpcgcp/multiendpoint: Copyright 2024 gRPC authors. + +github.com/NVIDIA/go-nvml/*: Copyright 2023 NVIDIA CORPORATION diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 815b90c7adaf0..9f3de6186a47d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -270,6 +270,7 @@ /docs/cloud-workload-security/ @DataDog/documentation @DataDog/agent-security /docs/public/components/ @DataDog/documentation @DataDog/agent-shared-components +/docs/public/hostname/ @DataDog/documentation @DataDog/agent-shared-components /docs/public/architecture/dogstatsd/ @DataDog/documentation @DataDog/agent-metrics-logs /docs/public/guidelines/deprecated-components-documentation/ @DataDog/documentation @DataDog/agent-shared-components @@ -371,6 +372,7 @@ /pkg/trace/traceutil/otel*.go @DataDog/opentelemetry /pkg/trace/stats/ @DataDog/agent-apm @DataDog/opentelemetry /pkg/trace/telemetry/ @DataDog/apm-trace-storage +/pkg/trace/transform/ @DataDog/opentelemetry /comp/core/autodiscovery/listeners/ @DataDog/container-platform /comp/core/autodiscovery/listeners/cloudfoundry*.go @DataDog/platform-integrations /comp/core/autodiscovery/listeners/snmp*.go @DataDog/network-device-monitoring @@ -570,7 +572,6 @@ /comp/core/workloadmeta/collectors/internal/cloudfoundry @DataDog/platform-integrations /pkg/sbom/ @DataDog/container-integrations @DataDog/agent-security /pkg/internaltelemetry @DataDog/windows-kernel-integrations @DataDog/fleet -/pkg-config/ @DataDog/agent-delivery /pkg/networkpath/ @DataDog/network-device-monitoring @DataDog/Networks /pkg/collector/corechecks/networkpath/ @DataDog/network-device-monitoring @DataDog/Networks @@ -666,6 +667,7 @@ /test/new-e2e/tests/remote-config @DataDog/remote-config /test/new-e2e/tests/updater @DataDog/fleet @DataDog/windows-agent /test/new-e2e/tests/installer @DataDog/fleet @DataDog/windows-agent +/test/new-e2e/tests/gpu @Datadog/ebpf-platform /test/otel/ @DataDog/opentelemetry /test/system/ @DataDog/agent-shared-components /test/system/dogstatsd/ @DataDog/agent-metrics-logs diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml index 9240bb666b3f9..4440e2127e3c6 100644 --- a/.github/workflows/serverless-binary-size.yml +++ b/.github/workflows/serverless-binary-size.yml @@ -159,6 +159,5 @@ jobs: if: steps.should.outputs.should_run == 'true' with: header: serverless-binary-size - hide_and_recreate: true - hide_classify: "RESOLVED" + recreate: true path: ${{ steps.write.outputs.filename }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d243069ce3123..b33c2e8612e09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,6 @@ include: - .gitlab/integration_test/include.yml - .gitlab/internal_image_deploy/internal_image_deploy.yml - .gitlab/internal_kubernetes_deploy/include.yml - - .gitlab/junit_upload/junit_upload.yml - .gitlab/kitchen_cleanup/include.yml - .gitlab/kitchen_deploy/kitchen_deploy.yml - .gitlab/kitchen_testing/include.yml @@ -173,15 +172,15 @@ variables: # To use images from datadog-agent-buildimages dev branches, set the corresponding # SUFFIX variable to _test_only DATADOG_AGENT_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_BUILDIMAGES: v45979046-89a73a40 + DATADOG_AGENT_BUILDIMAGES: v46542806-c7a4a6be DATADOG_AGENT_WINBUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_WINBUILDIMAGES: v45979046-89a73a40 + DATADOG_AGENT_WINBUILDIMAGES: v46542806-c7a4a6be DATADOG_AGENT_ARMBUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_ARMBUILDIMAGES: v45979046-89a73a40 + DATADOG_AGENT_ARMBUILDIMAGES: v46542806-c7a4a6be DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v45979046-89a73a40 + DATADOG_AGENT_SYSPROBE_BUILDIMAGES: v46542806-c7a4a6be DATADOG_AGENT_BTF_GEN_BUILDIMAGES_SUFFIX: "" - DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v45979046-89a73a40 + DATADOG_AGENT_BTF_GEN_BUILDIMAGES: v46542806-c7a4a6be DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded DEB_GPG_KEY_ID: c0962c7d @@ -257,15 +256,20 @@ variables: VCPKG_BLOB_SAS_URL: ci.datadog-agent-buildimages.vcpkg_blob_sas_url # windows-agent WINGET_PAT: ci.datadog-agent.winget_pat # windows-agent # End aws ssm variables + # Start vault variables + AGENT_API_KEY_ORG2: agent-api-key-org-2 # agent-devx-infra + AGENT_APP_KEY_ORG2: agent-ci-app-key-org-2 # agent-devx-infra + AGENT_GITHUB_APP: agent-github-app # agent-devx-infra + AGENT_QA_E2E: agent-qa-e2e # agent-devx-loops + ATLASSIAN_WRITE: atlassian-write # agent-devx-infra + DOCKER_REGISTRY_RO: dockerhub-readonly # agent-delivery + GITLAB_TOKEN: gitlab-token # agent-devx-infra + INSTALL_SCRIPT_API_KEY_ORG2: install-script-api-key-org-2 # agent-devx-infra + MACOS_GITHUB_APP_1: macos-github-app-one # agent-devx-infra + MACOS_GITHUB_APP_2: macos-github-app-two # agent-devx-infra # End vault variables - ATLASSIAN_WRITE: atlassian-write - AGENT_GITHUB_APP: agent-github-app - MACOS_GITHUB_APP_1: macos-github-app-one - MACOS_GITHUB_APP_2: macos-github-app-two - DOCKER_REGISTRY_RO: dockerhub-readonly - DD_PKG_VERSION: "latest" # Job stage attempts (see https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts) @@ -771,19 +775,6 @@ workflow: paths: *system_probe_change_paths compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 -.on_e2e_changes_or_manual: - - <<: *if_main_branch - - !reference [.except_mergequeue] - - changes: - paths: - - test/new-e2e/pkg/**/* - - test/new-e2e/test-infra-definition/* - - test/new-e2e/go.mod - compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 - when: on_success - - when: manual - allow_failure: true - # New E2E related rules .on_e2e_main_release_or_rc: # This rule is used as a base for all new-e2e rules @@ -800,6 +791,7 @@ workflow: when: on_success - changes: paths: + - .gitlab/e2e/e2e.yml - test/new-e2e/pkg/**/* - test/new-e2e/go.mod - flakes.yaml @@ -1196,3 +1188,12 @@ workflow: compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 - when: manual allow_failure: true + +.on_gpu_or_e2e_changes: + - !reference [.on_e2e_main_release_or_rc] + - changes: + paths: + - pkg/gpu/**/* + - test/new-e2e/tests/gpu/**/* + - pkg/collector/corechecks/gpu/**/* + compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 diff --git a/.gitlab/.ci-linters.yml b/.gitlab/.ci-linters.yml index db38acacdd029..b888dff051b73 100644 --- a/.gitlab/.ci-linters.yml +++ b/.gitlab/.ci-linters.yml @@ -109,7 +109,6 @@ job-owners: - trace-agent-v05-2cpus-normal_load-fixed_sps - trace-agent-v05-2cpus-stress_load-fixed_sps - trigger-flakes-finder - - unit_tests_arm64_junit_upload - unit_tests_notify - update_rc_build_links - validate_modules diff --git a/.gitlab/.pre/cancel-prev-pipelines.yml b/.gitlab/.pre/cancel-prev-pipelines.yml index 48b5170248759..9a23b472e6bfa 100644 --- a/.gitlab/.pre/cancel-prev-pipelines.yml +++ b/.gitlab/.pre/cancel-prev-pipelines.yml @@ -14,5 +14,5 @@ cancel-prev-pipelines: when: never - when: on_success script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - inv pipeline.auto-cancel-previous-pipelines diff --git a/.gitlab/.pre/gitlab_configuration.yml b/.gitlab/.pre/gitlab_configuration.yml index fd814462e2499..3aea26e5dfb2b 100644 --- a/.gitlab/.pre/gitlab_configuration.yml +++ b/.gitlab/.pre/gitlab_configuration.yml @@ -6,7 +6,7 @@ test_gitlab_configuration: - !reference [.except_mergequeue] - !reference [.on_gitlab_changes] script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - inv -e linter.gitlab-ci - inv -e linter.job-change-path - inv -e linter.gitlab-change-paths @@ -20,7 +20,7 @@ test_gitlab_compare_to: - !reference [.except_mergequeue] - !reference [.on_gitlab_changes] script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - !reference [.setup_agent_github_app] - pip install -r tasks/requirements.txt - inv pipeline.compare-to-itself @@ -37,7 +37,7 @@ compute_gitlab_ci_config: - git checkout main - git checkout - script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - mkdir -p artifacts - inv -e gitlab.compute-gitlab-ci-config --before-file artifacts/before.gitlab-ci.yml --after-file artifacts/after.gitlab-ci.yml --diff-file artifacts/diff.gitlab-ci.yml artifacts: diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS index 4b07782561abf..4cd44753a62dd 100644 --- a/.gitlab/JOBOWNERS +++ b/.gitlab/JOBOWNERS @@ -161,6 +161,7 @@ new-e2e-ndm* @DataDog/network-device-monitoring new-e2e-npm* @DataDog/Networks new-e2e-cws* @DataDog/agent-security new-e2e-orchestrator* @DataDog/container-app +new-e2e-otel* @DataDog/opentelemetry e2e_pre_test* @DataDog/agent-devx-loops new-e2e-remote-config* @DataDog/remote-config new-e2e-installer* @DataDog/fleet @@ -170,6 +171,7 @@ new-e2e-windows-systemprobe @DataDog/windows-kernel-integrations new-e2e-windows-security-agent @DataDog/windows-kernel-integrations new-e2e_windows_powershell_module_test @DataDog/windows-kernel-integrations new-e2e-eks-cleanup-on-failure @DataDog/agent-devx-loops +new-e2e-gpu* @DataDog/ebpf-platform # Kernel matrix testing upload_dependencies* @DataDog/ebpf-platform diff --git a/.gitlab/choco_deploy/choco_deploy.yml b/.gitlab/choco_deploy/choco_deploy.yml index 56e66603fe9af..6d7f8959a554d 100644 --- a/.gitlab/choco_deploy/choco_deploy.yml +++ b/.gitlab/choco_deploy/choco_deploy.yml @@ -12,7 +12,7 @@ publish_choco_7_x64: before_script: - $tmpfile = [System.IO.Path]::GetTempFileName() - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:CHOCOLATEY_API_KEY" -tempFile "$tmpfile") - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } + - If ($lastExitCode -ne "0") { exit "$lastExitCode" } - $chocolateyApiKey=$(cat "$tmpfile") - Remove-Item "$tmpfile" script: diff --git a/.gitlab/common/container_publish_job_templates.yml b/.gitlab/common/container_publish_job_templates.yml index 19786f15fa81d..1472699787add 100644 --- a/.gitlab/common/container_publish_job_templates.yml +++ b/.gitlab/common/container_publish_job_templates.yml @@ -13,7 +13,7 @@ IMG_VARIABLES: "" IMG_SIGNING: "" script: # We can't use the 'trigger' keyword on manual jobs, otherwise they can't be run if the pipeline fails and is retried - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - | if [[ "$BUCKET_BRANCH" == "nightly" && ( "$IMG_SOURCES" =~ "$SRC_AGENT" || "$IMG_SOURCES" =~ "$SRC_DCA" || "$IMG_SOURCES" =~ "$SRC_CWS_INSTRUMENTATION" || "$IMG_VARIABLES" =~ "$SRC_AGENT" || "$IMG_VARIABLES" =~ "$SRC_DCA" || "$IMG_VARIABLES" =~ "$SRC_CWS_INSTRUMENTATION" ) ]]; then export ECR_RELEASE_SUFFIX="-nightly" diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index fdd381dca9893..5316c7e28cf7d 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -4,4 +4,4 @@ variables: # and check the job creating the image to make sure you have the right SHA prefix TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 7e55b9e3279a + TEST_INFRA_DEFINITIONS_BUILDIMAGES: f694c4dc33e4 diff --git a/.gitlab/container_build/docker_linux.yml b/.gitlab/container_build/docker_linux.yml index 4b0ff59a4369b..99bec497d7b18 100644 --- a/.gitlab/container_build/docker_linux.yml +++ b/.gitlab/container_build/docker_linux.yml @@ -2,7 +2,6 @@ .docker_build_job_definition: stage: container_build script: - - aws s3 sync --only-show-errors $S3_ARTIFACTS_URI $BUILD_CONTEXT - TAG_SUFFIX=${TAG_SUFFIX:-} - BUILD_ARG=${BUILD_ARG:-} - | @@ -24,6 +23,18 @@ retry: 2 timeout: 30m +# Base template for jobs that don't rely on the packaging job artifacts but +# rather from binaries stored in the 'dd-ci-artefacts-build-stable' bucket +.docker_build_s3: + before_script: + - aws s3 sync --only-show-errors $S3_ARTIFACTS_URI $BUILD_CONTEXT + +# Base template to move the packaged artifact from gitlab into the build +# context +.docker_build_artifact: + before_script: + - mv $OMNIBUS_PACKAGE_DIR/*.deb $BUILD_CONTEXT + .docker_build_job_definition_amd64: extends: .docker_build_job_definition image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/docker_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES @@ -40,13 +51,12 @@ # build agent7 image docker_build_agent7: - extends: .docker_build_job_definition_amd64 + extends: [.docker_build_job_definition_amd64, .docker_build_artifact] rules: - !reference [.except_mergequeue] - when: on_success needs: - job: agent_deb-x64-a7 - artifacts: false variables: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent @@ -67,13 +77,12 @@ single_machine_performance-amd64-a7: IMG_DESTINATIONS: 08450328-agent:${CI_COMMIT_SHA}-7-amd64 docker_build_agent7_arm64: - extends: .docker_build_job_definition_arm64 + extends: [.docker_build_job_definition_arm64, .docker_build_artifact] rules: - !reference [.except_mergequeue] - when: on_success needs: - job: agent_deb-arm64-a7 - artifacts: false variables: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent @@ -82,13 +91,12 @@ docker_build_agent7_arm64: # build agent7 jmx image docker_build_agent7_jmx: - extends: .docker_build_job_definition_amd64 + extends: [.docker_build_job_definition_amd64, .docker_build_artifact] rules: - !reference [.except_mergequeue] - when: on_success needs: - job: agent_deb-x64-a7 - artifacts: false variables: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent @@ -96,13 +104,12 @@ docker_build_agent7_jmx: BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-agent_7*_amd64.deb docker_build_agent7_jmx_arm64: - extends: .docker_build_job_definition_arm64 + extends: [.docker_build_job_definition_arm64, .docker_build_artifact] rules: - !reference [.except_mergequeue] - when: on_success needs: - job: agent_deb-arm64-a7 - artifacts: false variables: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent @@ -111,13 +118,12 @@ docker_build_agent7_jmx_arm64: # build agent7 UA image docker_build_ot_agent7: - extends: .docker_build_job_definition_amd64 + extends: [.docker_build_job_definition_amd64, .docker_build_artifact] rules: - !reference [.except_mergequeue] - when: on_success needs: - job: ot_agent_deb-x64-a7 - artifacts: false variables: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent @@ -125,13 +131,12 @@ docker_build_ot_agent7: BUILD_ARG: --target test --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-ot-agent_7*_amd64.deb docker_build_ot_agent7_arm64: - extends: .docker_build_job_definition_arm64 + extends: [.docker_build_job_definition_arm64, .docker_build_artifact] rules: - !reference [.except_mergequeue] - when: on_success needs: - job: ot_agent_deb-arm64-a7 - artifacts: false variables: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent @@ -140,13 +145,12 @@ docker_build_ot_agent7_arm64: # build agent7 jmx image docker_build_ot_agent7_jmx: - extends: .docker_build_job_definition_amd64 + extends: [.docker_build_job_definition_amd64, .docker_build_artifact] rules: - !reference [.except_mergequeue] - when: on_success needs: - job: ot_agent_deb-x64-a7 - artifacts: false variables: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent @@ -154,13 +158,12 @@ docker_build_ot_agent7_jmx: BUILD_ARG: --target test --build-arg WITH_JMX=true --build-arg PYTHON_VERSION=3 --build-arg DD_AGENT_ARTIFACT=datadog-ot-agent_7*_amd64.deb docker_build_ot_agent7_jmx_arm64: - extends: .docker_build_job_definition_arm64 + extends: [.docker_build_job_definition_arm64, .docker_build_artifact] rules: - !reference [.except_mergequeue] - when: on_success needs: - job: ot_agent_deb-arm64-a7 - artifacts: false variables: IMAGE: registry.ddbuild.io/ci/datadog-agent/agent BUILD_CONTEXT: Dockerfiles/agent @@ -169,7 +172,7 @@ docker_build_ot_agent7_jmx_arm64: # build the cluster-agent image docker_build_cluster_agent_amd64: - extends: .docker_build_job_definition_amd64 + extends: [.docker_build_job_definition_amd64, .docker_build_s3] rules: !reference [.on_tag_or_a7] needs: - job: cluster_agent-build_amd64 @@ -182,10 +185,11 @@ docker_build_cluster_agent_amd64: IMAGE: registry.ddbuild.io/ci/datadog-agent/cluster-agent BUILD_CONTEXT: Dockerfiles/cluster-agent before_script: + - !reference [.docker_build_s3, before_script] - cp -Rvf Dockerfiles/agent/nosys-seccomp Dockerfiles/cluster-agent/ docker_build_cluster_agent_arm64: - extends: .docker_build_job_definition_arm64 + extends: [.docker_build_job_definition_arm64, .docker_build_s3] rules: !reference [.on_tag_or_a7] needs: - job: cluster_agent-build_arm64 @@ -198,11 +202,12 @@ docker_build_cluster_agent_arm64: IMAGE: registry.ddbuild.io/ci/datadog-agent/cluster-agent BUILD_CONTEXT: Dockerfiles/cluster-agent before_script: + - !reference [.docker_build_s3, before_script] - cp -Rvf Dockerfiles/agent/nosys-seccomp Dockerfiles/cluster-agent/ # build the cws-instrumentation image docker_build_cws_instrumentation_amd64: - extends: .docker_build_job_definition_amd64 + extends: [.docker_build_job_definition_amd64, .docker_build_s3] rules: !reference [.on_tag_or_a7] needs: - job: cws_instrumentation-build_amd64 @@ -212,7 +217,7 @@ docker_build_cws_instrumentation_amd64: BUILD_CONTEXT: Dockerfiles/cws-instrumentation docker_build_cws_instrumentation_arm64: - extends: .docker_build_job_definition_arm64 + extends: [.docker_build_job_definition_arm64, .docker_build_s3] rules: !reference [.on_tag_or_a7] needs: - job: cws_instrumentation-build_arm64 @@ -223,7 +228,7 @@ docker_build_cws_instrumentation_arm64: # build the dogstatsd image docker_build_dogstatsd_amd64: - extends: .docker_build_job_definition_amd64 + extends: [.docker_build_job_definition_amd64, .docker_build_s3] rules: - !reference [.except_mergequeue] - when: on_success @@ -237,7 +242,7 @@ docker_build_dogstatsd_amd64: # build the dogstatsd image docker_build_dogstatsd_arm64: - extends: .docker_build_job_definition_arm64 + extends: [.docker_build_job_definition_arm64, .docker_build_s3] rules: - !reference [.except_mergequeue] - when: on_success diff --git a/.gitlab/container_build/docker_windows.yml b/.gitlab/container_build/docker_windows.yml index dada869f2a0d3..c9b67e24ef86b 100644 --- a/.gitlab/container_build/docker_windows.yml +++ b/.gitlab/container_build/docker_windows.yml @@ -34,8 +34,8 @@ -v "$(Get-Location):C:\mnt" -v \\.\pipe\docker_engine:\\.\pipe\docker_engine 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_${Env:VARIANT}_x64${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} powershell - -C C:\mnt\tools\ci\docker-login.ps1 - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } + -File C:\mnt\tools\ci\docker-login.ps1 + - If ($lastExitCode -ne "0") { exit "$lastExitCode" } - powershell -Command "$(Get-Location)\tools\ci\retry.ps1 docker build --no-cache --build-arg GENERAL_ARTIFACTS_CACHE_BUCKET_URL=${GENERAL_ARTIFACTS_CACHE_BUCKET_URL} ${BUILD_ARG} --pull --file ${BUILD_CONTEXT}/windows/amd64/Dockerfile --tag ${TARGET_TAG} ${BUILD_CONTEXT}" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - powershell -Command "$(Get-Location)\tools\ci\retry.ps1 docker push ${TARGET_TAG}" diff --git a/.gitlab/deploy_packages/winget.yml b/.gitlab/deploy_packages/winget.yml index f28f946b1fb0c..54261a8d1a867 100644 --- a/.gitlab/deploy_packages/winget.yml +++ b/.gitlab/deploy_packages/winget.yml @@ -12,7 +12,7 @@ publish_winget_7_x64: before_script: - $tmpfile = [System.IO.Path]::GetTempFileName() - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:WINGET_PAT" -tempFile "$tmpfile") - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } + - If ($lastExitCode -ne "0") { exit "$lastExitCode" } - $wingetPat=$(cat "$tmpfile") - Remove-Item "$tmpfile" script: diff --git a/.gitlab/e2e/e2e.yml b/.gitlab/e2e/e2e.yml index 05446446a02bb..e63a63070b18c 100644 --- a/.gitlab/e2e/e2e.yml +++ b/.gitlab/e2e/e2e.yml @@ -11,11 +11,11 @@ - !reference [.retrieve_linux_go_e2e_deps] # Setup AWS Credentials - mkdir -p ~/.aws - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config || exit $? - export AWS_PROFILE=agent-qa-ci # Now all `aws` commands target the agent-qa profile - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_PUBLIC_KEY_RSA > $E2E_PUBLIC_KEY_PATH || exit $? - - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_KEY_RSA > $E2E_PRIVATE_KEY_PATH || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_PUBLIC_KEY_PATH || exit $? + - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key_rsa > $E2E_PRIVATE_KEY_PATH || exit $? # Use S3 backend - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" # Setup Azure credentials. https://www.pulumi.com/registry/packages/azure-native/installation-configuration/#set-configuration-using-pulumi-config @@ -485,6 +485,41 @@ new-e2e-windows-security-agent: TARGETS: ./tests/security-agent-functional TEAM: windows-kernel-integrations +new-e2e-otel-eks-init: + stage: e2e_init + extends: .new_e2e_template + rules: + - !reference [.on_otel_or_e2e_changes] + - !reference [.manual] + needs: + - !reference [.needs_new_e2e_template] + - qa_dca + - qa_agent + - qa_agent_ot + variables: + TARGETS: ./tests/otel + TEAM: otel + EXTRA_PARAMS: --run TestOTelAgentIAEKS + E2E_INIT_ONLY: "true" + allow_failure: true + +new-e2e-otel-eks: + extends: .new_e2e_template + rules: + - !reference [.on_otel_or_e2e_changes] + - !reference [.manual] + needs: + - !reference [.needs_new_e2e_template] + - qa_dca + - qa_agent + - qa_agent_ot + - new-e2e-otel-eks-init + variables: + TARGETS: ./tests/otel + EXTRA_PARAMS: --run TestOTelAgentIAEKS + TEAM: otel + E2E_PRE_INITIALIZED: "true" + new-e2e-otel: extends: .new_e2e_template rules: @@ -497,6 +532,7 @@ new-e2e-otel: - qa_agent_ot variables: TARGETS: ./tests/otel + EXTRA_PARAMS: --skip TestOTelAgentIAEKS TEAM: otel .new-e2e_package_signing: @@ -533,6 +569,19 @@ new-e2e-cspm: TEAM: cspm timeout: 35m +new-e2e-gpu: + extends: .new_e2e_template + rules: + - !reference [.on_gpu_or_e2e_changes] + - !reference [.manual] + variables: + TARGETS: ./tests/gpu # the target path where tests are + TEAM: ebpf-platform + needs: # list of required jobs. By default gitlab waits for any previous jobs. + - !reference [.needs_new_e2e_template] + - deploy_deb_testing-a7_x64 # agent 7 debian package + + generate-flakes-finder-pipeline: image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES stage: e2e @@ -541,6 +590,7 @@ generate-flakes-finder-pipeline: - !reference [.on_deploy_nightly_repo_branch] - !reference [.manual] needs: + - compute_gitlab_ci_config - deploy_deb_testing-a7_arm64 - deploy_deb_testing-a7_x64 - deploy_rpm_testing-a7_arm64 @@ -557,7 +607,6 @@ generate-flakes-finder-pipeline: - qa_agent_ot tags: ["arch:amd64"] script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN - inv -e testwasher.generate-flake-finder-pipeline artifacts: paths: @@ -587,11 +636,11 @@ new-e2e-eks-cleanup-on-failure: script: # Setup AWS Credentials - mkdir -p ~/.aws - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config || exit $? - export AWS_PROFILE=agent-qa-ci # Now all `aws` commands target the agent-qa profile - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_PUBLIC_KEY_RSA > $E2E_PUBLIC_KEY_PATH || exit $? - - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_KEY_RSA > $E2E_PRIVATE_KEY_PATH || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_public_key_rsa > $E2E_PUBLIC_KEY_PATH || exit $? + - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key_rsa > $E2E_PRIVATE_KEY_PATH || exit $? # Use S3 backend - PULUMI_CONFIG_PASSPHRASE=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $E2E_PULUMI_CONFIG_PASSPHRASE) || exit $?; export PULUMI_CONFIG_PASSPHRASE - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" @@ -600,6 +649,8 @@ new-e2e-eks-cleanup-on-failure: E2E_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub E2E_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key E2E_PIPELINE_ID: $CI_PIPELINE_ID - when: always + rules: + - !reference [.except_mergequeue] + - when: always allow_failure: true diff --git a/.gitlab/e2e_install_packages/common.yml b/.gitlab/e2e_install_packages/common.yml index 1bb6c813d84cc..64abe046a1135 100644 --- a/.gitlab/e2e_install_packages/common.yml +++ b/.gitlab/e2e_install_packages/common.yml @@ -28,7 +28,7 @@ - START_MAJOR_VERSION: [5, 6, 7] END_MAJOR_VERSION: [7] script: - - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY ) || exit $?; export DATADOG_AGENT_API_KEY + - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token ) || exit $?; export DATADOG_AGENT_API_KEY - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version $START_MAJOR_VERSION --dest-agent-version $END_MAJOR_VERSION --test-washer .new-e2e_script_upgrade_persisting_integrations: @@ -38,7 +38,7 @@ TEAM: agent-delivery EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --arch $E2E_ARCH --flavor $FLAVOR script: - - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY ) || exit $?; export DATADOG_AGENT_API_KEY + - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token) || exit $?; export DATADOG_AGENT_API_KEY - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version 7 --test-washer .new-e2e_rpm: @@ -48,5 +48,5 @@ TEAM: agent-delivery EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --arch $E2E_ARCH script: - - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY) || exit $?; export DATADOG_AGENT_API_KEY + - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token) || exit $?; export DATADOG_AGENT_API_KEY - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --test-washer diff --git a/.gitlab/functional_test/regression_detector.yml b/.gitlab/functional_test/regression_detector.yml index c288fe0824f42..ec44f3d9fc9ed 100644 --- a/.gitlab/functional_test/regression_detector.yml +++ b/.gitlab/functional_test/regression_detector.yml @@ -128,7 +128,7 @@ single-machine-performance-regression_detector: # invoke task has additional logic that does not seem to apply well to SMP's # JUnit XML. Agent CI seems to use `datadog-agent` as the service name when # uploading JUnit XML, so the upload command below respects that convention. - - DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$API_KEY_ORG2")" || exit $?; export DATADOG_API_KEY + - DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_API_KEY_ORG2" token)" || exit $?; export DATADOG_API_KEY - datadog-ci junit upload --service datadog-agent outputs/junit.xml # Finally, exit 1 if the job signals a regression else 0. - RUST_LOG="${RUST_LOG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE} diff --git a/.gitlab/install_script_testing/install_script_testing.yml b/.gitlab/install_script_testing/install_script_testing.yml index 8578cba0e42a1..53ee9c17d1335 100644 --- a/.gitlab/install_script_testing/install_script_testing.yml +++ b/.gitlab/install_script_testing/install_script_testing.yml @@ -5,7 +5,7 @@ test_install_script: tags: ["arch:amd64"] script: - set +x - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - export TESTING_APT_URL=$DEB_TESTING_S3_BUCKET - export TESTING_YUM_URL=$RPM_TESTING_S3_BUCKET - export TEST_PIPELINE_ID=$CI_PIPELINE_ID diff --git a/.gitlab/integration_test/windows.yml b/.gitlab/integration_test/windows.yml index 2ddf04fddb997..39e24c348f56e 100644 --- a/.gitlab/integration_test/windows.yml +++ b/.gitlab/integration_test/windows.yml @@ -9,7 +9,7 @@ before_script: - $tmpfile = [System.IO.Path]::GetTempFileName() - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:VCPKG_BLOB_SAS_URL" -tempFile "$tmpfile") - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } + - If ($lastExitCode -ne "0") { exit "$lastExitCode" } - $vcpkgBlobSaSUrl=$(cat "$tmpfile") - Remove-Item "$tmpfile" script: diff --git a/.gitlab/internal_image_deploy/internal_image_deploy.yml b/.gitlab/internal_image_deploy/internal_image_deploy.yml index 6d0cc3245e3b5..58bfebfa2bd87 100644 --- a/.gitlab/internal_image_deploy/internal_image_deploy.yml +++ b/.gitlab/internal_image_deploy/internal_image_deploy.yml @@ -22,7 +22,7 @@ docker_trigger_internal: TMPL_SRC_REPO: ci/datadog-agent/agent RELEASE_STAGING: "true" script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - | if [ "$BUCKET_BRANCH" = "nightly" ]; then @@ -67,7 +67,7 @@ docker_trigger_internal-ot: TMPL_SRC_REPO: ci/datadog-agent/agent RELEASE_STAGING: "true" script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - | if [ "$BUCKET_BRANCH" = "nightly" ]; then @@ -113,7 +113,7 @@ docker_trigger_cluster_agent_internal: RELEASE_STAGING: "true" RELEASE_PROD: "true" script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - | if [ "$BUCKET_BRANCH" = "nightly" ]; then @@ -159,7 +159,7 @@ docker_trigger_cws_instrumentation_internal: RELEASE_STAGING: "true" RELEASE_PROD: "true" script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - if [ "$BUCKET_BRANCH" = "beta" ] || [ "$BUCKET_BRANCH" = "stable" ]; then TMPL_SRC_REPO="${TMPL_SRC_REPO}-release"; fi - | if [ "$BUCKET_BRANCH" = "nightly" ]; then diff --git a/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml index 605ac0def4114..792b7e727b537 100644 --- a/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml +++ b/.gitlab/internal_kubernetes_deploy/internal_kubernetes_deploy.yml @@ -36,7 +36,7 @@ internal_kubernetes_deploy_experimental: EXPLICIT_WORKFLOWS: "//workflows:beta_builds.agents_nightly.staging-deploy.publish,//workflows:beta_builds.agents_nightly.staging-validate.publish,//workflows:beta_builds.agents_nightly.prod-wait-business-hours.publish,//workflows:beta_builds.agents_nightly.prod-deploy.publish,//workflows:beta_builds.agents_nightly.prod-validate.publish,//workflows:beta_builds.agents_nightly.publish-image-confirmation.publish" BUNDLE_VERSION_OVERRIDE: "v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}" script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - "inv pipeline.trigger-child-pipeline --project-name DataDog/k8s-datadog-agent-ops --git-ref main --variable OPTION_AUTOMATIC_ROLLOUT --variable EXPLICIT_WORKFLOWS diff --git a/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml b/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml index 067ca517fdba9..3f880e8cb0ac6 100644 --- a/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml +++ b/.gitlab/internal_kubernetes_deploy/rc_kubernetes_deploy.yml @@ -22,7 +22,7 @@ rc_kubernetes_deploy: EXPLICIT_WORKFLOWS: "//workflows:deploy_rc.agents_rc" AGENT_IMAGE_TAG: $CI_COMMIT_REF_NAME script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - "inv pipeline.trigger-child-pipeline --project-name DataDog/k8s-datadog-agent-ops --git-ref main --variable OPTION_AUTOMATIC_ROLLOUT --variable EXPLICIT_WORKFLOWS diff --git a/.gitlab/junit_upload/junit_upload.yml b/.gitlab/junit_upload/junit_upload.yml deleted file mode 100644 index 3ec6b8b35e709..0000000000000 --- a/.gitlab/junit_upload/junit_upload.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -unit_tests_arm64_junit_upload: - stage: junit_upload - rules: - - !reference [.except_mergequeue] - - when: always - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - allow_failure: true - needs: - - tests_deb-arm64-py3 - - tests_rpm-arm64-py3 - script: - - $CI_PROJECT_DIR/tools/ci/junit_upload.sh - -# The corresponding upload for the x86 tests is in the test jobs themselves, we have to upload -# the ARM64 test results here as we need an x86 image to run the upload. -kmt_arm64_junit_upload: - stage: junit_upload - rules: - - !reference [.except_mergequeue] - - when: always - image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - allow_failure: true - needs: - - job: kmt_run_sysprobe_tests_arm64 - optional: true - - job: kmt_run_secagent_tests_arm64 - optional: true - script: - - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "$DD_AGENT_TESTING_DIR/junit-*.tar.gz" diff --git a/.gitlab/kernel_matrix_testing/common.yml b/.gitlab/kernel_matrix_testing/common.yml index b2acf5a35d422..9768897240102 100644 --- a/.gitlab/kernel_matrix_testing/common.yml +++ b/.gitlab/kernel_matrix_testing/common.yml @@ -29,7 +29,7 @@ .write_ssh_key_file: - touch $AWS_EC2_SSH_KEY_FILE && chmod 600 $AWS_EC2_SSH_KEY_FILE - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SSH_KEY > $AWS_EC2_SSH_KEY_FILE || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E ssh_key > $AWS_EC2_SSH_KEY_FILE || exit $? # Without the newline ssh silently fails and moves on to try other auth methods - echo "" >> $AWS_EC2_SSH_KEY_FILE - chmod 600 $AWS_EC2_SSH_KEY_FILE @@ -47,7 +47,7 @@ .kmt_new_profile: - mkdir -p ~/.aws - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config || exit $? - export AWS_PROFILE=agent-qa-ci .define_if_collect_complexity: @@ -60,7 +60,7 @@ - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}" .collect_outcomes_kmt: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET))) | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output) # Collect setup-ddvm systemd service logs - mkdir -p $CI_PROJECT_DIR/logs @@ -114,7 +114,7 @@ scp $DD_AGENT_TESTING_DIR/kmt-dockers-$ARCH.tar.gz metal_instance:/opt/kernel-version-testing fi after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] variables: AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key @@ -144,7 +144,7 @@ VMCONFIG_FILE: "${CI_PROJECT_DIR}/vmconfig-${CI_PIPELINE_ID}-${ARCH}.json" EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json before_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.retrieve_linux_go_deps] - !reference [.kmt_new_profile] - !reference [.write_ssh_key_file] @@ -159,7 +159,7 @@ - jq "." $CI_PROJECT_DIR/stack.output - pulumi logout after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - export AWS_PROFILE=agent-qa-ci - !reference [.shared_filters_and_queries] - mkdir -p $CI_PROJECT_DIR/libvirt/log/$ARCH $CI_PROJECT_DIR/libvirt/xml $CI_PROJECT_DIR/libvirt/qemu $CI_PROJECT_DIR/libvirt/dnsmasq @@ -200,7 +200,7 @@ image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES tags: ["arch:amd64"] before_script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN - !reference [.kmt_new_profile] script: - !reference [.shared_filters_and_queries] @@ -217,7 +217,7 @@ aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}" fi after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] # Manual cleanup jobs, these will be used to cleanup the instances after the tests @@ -247,7 +247,7 @@ RETRY: 2 EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json before_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.kmt_new_profile] - !reference [.write_ssh_key_file] - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt @@ -335,6 +335,6 @@ notify_ebpf_complexity_changes: - python3 -m pip install tabulate # Required for printing the tables - python3 -m pip install -r tasks/libs/requirements-github.txt - !reference [.setup_agent_github_app] - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_FULL_API_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN script: - inv -e ebpf.generate-complexity-summary-for-pr diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index c75f78c5449d8..64929e7b335d3 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -32,7 +32,7 @@ kmt_setup_env_secagent_arm64: AMI_ID_ARG: "--arm-ami-id=$KERNEL_MATRIX_TESTING_ARM_AMI_ID" LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-arm TEST_COMPONENT: security-agent - TEST_SETS: all_tests + TEST_SETS: cws_host kmt_setup_env_secagent_x64: extends: @@ -45,7 +45,7 @@ kmt_setup_env_secagent_x64: AMI_ID_ARG: "--x86-ami-id=$KERNEL_MATRIX_TESTING_X86_AMI_ID" LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-x86 TEST_COMPONENT: security-agent - TEST_SETS: all_tests + TEST_SETS: cws_host,cws_docker .upload_secagent_tests: stage: kernel_matrix_testing_prepare @@ -72,7 +72,7 @@ kmt_setup_env_secagent_x64: # upload connector to metal instance - scp $CI_PROJECT_DIR/connector-${ARCH} metal_instance:/home/ubuntu/connector after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] variables: AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key @@ -148,11 +148,30 @@ kmt_run_secagent_tests_x64: - "opensuse_15.3" - "opensuse_15.5" - "suse_12.5" - TEST_SET: [all_tests] + TEST_SET: [cws_host] after_script: - !reference [.collect_outcomes_kmt] - !reference [.upload_junit_kmt] +kmt_run_secagent_tests_x64_docker: + extends: + - .kmt_run_secagent_tests + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES + tags: ["arch:amd64"] + needs: + - kmt_setup_env_secagent_x64 + - upload_dependencies_secagent_x64 + - upload_secagent_tests_x64 + variables: + ARCH: "x86_64" + parallel: + matrix: + - TAG: + - "ubuntu_24.04" + TEST_SET: [cws_docker] + after_script: + - !reference [.collect_outcomes_kmt] + - !reference [.upload_junit_kmt] kmt_run_secagent_tests_arm64: extends: @@ -183,9 +202,10 @@ kmt_run_secagent_tests_arm64: - "rocky_8.5" - "rocky_9.3" - "opensuse_15.5" - TEST_SET: ["all_tests"] + TEST_SET: ["cws_host"] after_script: - !reference [.collect_outcomes_kmt] + - !reference [.upload_junit_kmt] .kmt_secagent_cleanup: extends: diff --git a/.gitlab/kernel_matrix_testing/system_probe.yml b/.gitlab/kernel_matrix_testing/system_probe.yml index da9c47e167702..7b16017409d4b 100644 --- a/.gitlab/kernel_matrix_testing/system_probe.yml +++ b/.gitlab/kernel_matrix_testing/system_probe.yml @@ -35,7 +35,7 @@ upload_dependencies_sysprobe_arm64: - mkdir $KMT_DOCKERS - inv -e system-probe.save-test-dockers --use-crane --output-dir $KMT_DOCKERS --arch $ARCH after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] artifacts: expire_in: 1 day @@ -82,7 +82,7 @@ pull_test_dockers_arm64: - !reference [.setup_ssh_config] - scp $CI_PROJECT_DIR/kmt-deps/ci/$ARCH/$ARCHIVE_NAME metal_instance:/opt/kernel-version-testing/ after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] variables: DEPENDENCIES: $CI_PROJECT_DIR/kmt-deps/ci/$ARCH/btfs @@ -161,7 +161,7 @@ kmt_setup_env_sysprobe_x64: # upload connector to metal instance - scp $CI_PROJECT_DIR/connector-${ARCH} metal_instance:/home/ubuntu/connector after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] variables: AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key @@ -270,6 +270,7 @@ kmt_run_sysprobe_tests_arm64: TEST_SET: ["only_usm", "no_usm"] after_script: - !reference [.collect_outcomes_kmt] + - !reference [.upload_junit_kmt] .kmt_sysprobe_cleanup: extends: diff --git a/.gitlab/notify/notify.yml b/.gitlab/notify/notify.yml index d148fb9729438..3db4bf284d339 100644 --- a/.gitlab/notify/notify.yml +++ b/.gitlab/notify/notify.yml @@ -25,8 +25,8 @@ notify: resource_group: notification timeout: 15 minutes # Added to prevent a stuck job blocking the resource_group defined above script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_READ_API_TOKEN) || exit $?; export GITLAB_TOKEN - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt - | # Do not send notifications if this is a child pipeline of another repo @@ -53,8 +53,8 @@ send_pipeline_stats: when: always dependencies: [] script: - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_READ_API_TOKEN) || exit $?; export GITLAB_TOKEN - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - invoke -e notify.send-stats notify_github: @@ -115,8 +115,8 @@ notify_gitlab_ci_changes: .failure_summary_setup: - SLACK_API_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SLACK_AGENT_CI_TOKEN) || exit $?; export SLACK_API_TOKEN - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_READ_API_TOKEN) || exit $?; export GITLAB_TOKEN - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN read_api) || exit $?; export GITLAB_TOKEN + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt # Upload failure summary data to S3 at the end of each main pipeline @@ -172,8 +172,8 @@ close_failing_tests_stale_issues: echo "This script is run weekly on Fridays" exit fi - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY - - DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $APP_KEY_ORG2) || exit $?; export DD_APP_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY + - DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_APP_KEY_ORG2 token) || exit $?; export DD_APP_KEY - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE token) || exit $?; export ATLASSIAN_PASSWORD - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME - python3 -m pip install -r requirements.txt -r tasks/requirements_release_tasks.txt # For Atlassian / Jira dependencies diff --git a/.gitlab/package_build/heroku.yml b/.gitlab/package_build/heroku.yml index c6002c5acd820..a0244f105c80b 100644 --- a/.gitlab/package_build/heroku.yml +++ b/.gitlab/package_build/heroku.yml @@ -31,8 +31,6 @@ - inv -e omnibus.build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --base-dir $OMNIBUS_BASE_DIR ${USE_S3_CACHING} --skip-deps --go-mod-cache="$GOPATH/pkg/mod" --system-probe-bin=/tmp/system-probe --flavor heroku - ls -la $OMNIBUS_PACKAGE_DIR - !reference [.lint_linux_packages] - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-heroku-agent_*_${PACKAGE_ARCH}.deb $S3_ARTIFACTS_URI/$DESTINATION_DEB - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-heroku-agent-dbg_*_${PACKAGE_ARCH}.deb $S3_ARTIFACTS_URI/$DESTINATION_DBG_DEB - !reference [.upload_sbom_artifacts] variables: KUBERNETES_MEMORY_REQUEST: "32Gi" diff --git a/.gitlab/packaging/deb.yml b/.gitlab/packaging/deb.yml index 1bed3c58a89cc..1e672a00de1e4 100644 --- a/.gitlab/packaging/deb.yml +++ b/.gitlab/packaging/deb.yml @@ -7,7 +7,6 @@ - !reference [.setup_deb_signing_key] - inv -e omnibus.build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --target-project ${DD_PROJECT} ${OMNIBUS_EXTRA_ARGS} - !reference [.lint_linux_packages] - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-${DD_PROJECT}_*_${PACKAGE_ARCH}.deb $S3_ARTIFACTS_URI/$DESTINATION_DEB artifacts: expire_in: 2 weeks paths: @@ -68,7 +67,6 @@ agent_deb-arm64-a7: - !reference [.setup_deb_signing_key] - inv -e omnibus.build --release-version "$RELEASE_VERSION" --major-version "$AGENT_MAJOR_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --target-project ${DD_PROJECT} --flavor ot ${OMNIBUS_EXTRA_ARGS} - !reference [.lint_linux_packages] - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-ot-agent_*_${PACKAGE_ARCH}.deb $S3_ARTIFACTS_URI/$DESTINATION_DEB ot_agent_deb-x64-a7: extends: [.package_ot_deb_common, .package_deb_x86, .package_deb_agent_7] @@ -126,7 +124,6 @@ installer_deb-arm64: - !reference [.setup_deb_signing_key] - inv -e omnibus.build --release-version "$RELEASE_VERSION" --base-dir $OMNIBUS_BASE_DIR --skip-deps --flavor iot - !reference [.lint_linux_packages] - - $S3_CP_CMD $OMNIBUS_PACKAGE_DIR/datadog-*_${PACKAGE_ARCH}.deb $S3_ARTIFACTS_URI/$DESTINATION_DEB artifacts: expire_in: 2 weeks paths: diff --git a/.gitlab/packaging/oci.yml b/.gitlab/packaging/oci.yml index 0efb48a3e6e0e..62519011d319b 100644 --- a/.gitlab/packaging/oci.yml +++ b/.gitlab/packaging/oci.yml @@ -59,9 +59,7 @@ - ls -l ${OUTPUT_DIR}/ - datadog-package merge ${OUTPUT_DIR}/*.tar # We need to propagate the exact version in the pipeline artifact - - cp merged.tar ${OMNIBUS_PACKAGE_DIR}/${MERGED_FILE} - # Only the major version is needed in the S3 bucket - - $S3_CP_CMD merged.tar $S3_ARTIFACTS_URI/${OCI_PRODUCT}_7_oci.tar + - mv merged.tar ${OMNIBUS_PACKAGE_DIR}/${MERGED_FILE} artifacts: paths: - ${OMNIBUS_PACKAGE_DIR} diff --git a/.gitlab/pkg_metrics/pkg_metrics.yml b/.gitlab/pkg_metrics/pkg_metrics.yml index 7001321d669c4..c2927f8e4ac27 100644 --- a/.gitlab/pkg_metrics/pkg_metrics.yml +++ b/.gitlab/pkg_metrics/pkg_metrics.yml @@ -45,7 +45,7 @@ send_pkg_size: optional: true script: # Get API key to send metrics - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY # Allow failures: some packages are not always built, and therefore stats cannot be sent for them - set +e diff --git a/.gitlab/setup/setup.yml b/.gitlab/setup/setup.yml index 28779481845c8..8321782a850ea 100644 --- a/.gitlab/setup/setup.yml +++ b/.gitlab/setup/setup.yml @@ -21,12 +21,12 @@ github_rate_limit_info: - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 key_b64) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 app_id) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 1 # Send stats for app 2 - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 key_b64) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 app_id) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 2 allow_failure: true diff --git a/.gitlab/source_test/golang_deps_diff.yml b/.gitlab/source_test/golang_deps_diff.yml index 6d156dd6f4963..b12d9b3ee1e2d 100644 --- a/.gitlab/source_test/golang_deps_diff.yml +++ b/.gitlab/source_test/golang_deps_diff.yml @@ -15,7 +15,7 @@ golang_deps_diff: - !reference [.retrieve_linux_go_deps] script: # Get API key to send metrics - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - inv -e diff.go-deps --report-file=deps-report.md --report-metrics --git-ref "${CI_COMMIT_REF_NAME}" artifacts: paths: @@ -64,7 +64,7 @@ golang_deps_send_count_metrics: - !reference [.retrieve_linux_go_deps] script: # Get API key to send metrics - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - inv -e go-deps.send-count-metrics --git-sha "${CI_COMMIT_SHA}" --git-ref "${CI_COMMIT_REF_NAME}" golang_deps_test: diff --git a/.gitlab/source_test/linux.yml b/.gitlab/source_test/linux.yml index 31bcf7767b5b1..50e121aea3f7b 100644 --- a/.gitlab/source_test/linux.yml +++ b/.gitlab/source_test/linux.yml @@ -124,6 +124,7 @@ tests_deb-arm64-py3: - .rtloader_tests - .linux_tests after_script: + - !reference [.upload_junit_source] - !reference [.upload_coverage] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:arm64"] @@ -136,6 +137,7 @@ tests_rpm-arm64-py3: - .rtloader_tests - .linux_tests after_script: + - !reference [.upload_junit_source] - !reference [.upload_coverage] image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/rpm_arm64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:arm64"] @@ -172,7 +174,7 @@ new-e2e-unit-tests: - !reference [.retrieve_linux_go_e2e_deps] # Setup AWS Credentials - mkdir -p ~/.aws - - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_PROFILE >> ~/.aws/config || exit $? + - $CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_QA_E2E profile >> ~/.aws/config || exit $? - export AWS_PROFILE=agent-qa-ci # Use S3 backend - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE" diff --git a/.gitlab/source_test/windows.yml b/.gitlab/source_test/windows.yml index fa4b857241a24..96f3fa43806e6 100644 --- a/.gitlab/source_test/windows.yml +++ b/.gitlab/source_test/windows.yml @@ -46,7 +46,7 @@ -e COVERAGE_CACHE_FLAG="${COVERAGE_CACHE_FLAG}" 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} c:\mnt\tasks\winbuildscripts\unittests.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } + - If ($lastExitCode -ne "0") { exit "$lastExitCode" } variables: TEST_OUTPUT_FILE: test_output.json artifacts: diff --git a/.gitlab/trigger_release/trigger_release.yml b/.gitlab/trigger_release/trigger_release.yml index c0c67e2d50d49..1993d598ff476 100644 --- a/.gitlab/trigger_release/trigger_release.yml +++ b/.gitlab/trigger_release/trigger_release.yml @@ -19,7 +19,7 @@ # agent-release-management creates pipeline for both Agent 6 and Agent 7 # when triggered with major version 7 - RELEASE_VERSION="$(inv agent.version --major-version 7 --url-safe --omnibus-format)-1" || exit $?; export RELEASE_VERSION - - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_SCHEDULER_TOKEN) || exit $?; export GITLAB_TOKEN + - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_TOKEN write_api) || exit $?; export GITLAB_TOKEN - 'inv pipeline.trigger-child-pipeline --project-name "DataDog/agent-release-management" --git-ref "main" --variable ACTION --variable AUTO_RELEASE diff --git a/.mockery.yaml b/.mockery.yaml index 9135bad6ef9c3..50c2336b1f844 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -29,6 +29,8 @@ packages: github.com/DataDog/datadog-agent/pkg/process/net: interfaces: SysProbeUtil: + config: + with-expecter: true github.com/DataDog/datadog-agent/pkg/process/procutil: interfaces: Probe: @@ -40,5 +42,5 @@ packages: config: mock-build-tags: test replace-type: - # https://github.com/vektra/mockery/issues/331 - - github.com/DataDog/datadog-agent/pkg/serializer/types.stubMessageBody=github.com/DataDog/datadog-agent/pkg/serializer/types.ProcessMessageBody + # https://github.com/vektra/mockery/issues/331 + - github.com/DataDog/datadog-agent/pkg/serializer/types.stubMessageBody=github.com/DataDog/datadog-agent/pkg/serializer/types.ProcessMessageBody diff --git a/Dockerfiles/agent/entrypoint.d/otel-agent b/Dockerfiles/agent/entrypoint.d/otel-agent index ed58d8b7f6fdc..f3f0809ef3195 100755 --- a/Dockerfiles/agent/entrypoint.d/otel-agent +++ b/Dockerfiles/agent/entrypoint.d/otel-agent @@ -1,4 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +if [[ -x /opt/datadog-agent/embedded/bin/otel-agent ]]; then exec /opt/datadog-agent/embedded/bin/otel-agent --config /etc/datadog-agent/otel-config.yaml +fi diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 97b3b358c57fc..adb14f7d9d7a1 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -243,6 +243,8 @@ core,github.com/Microsoft/hcsshim/internal/wclayer,MIT,Copyright (c) 2015 Micros core,github.com/Microsoft/hcsshim/internal/winapi,MIT,Copyright (c) 2015 Microsoft | Copyright (c) 2018 Microsoft Corp. All rights reserved core,github.com/Microsoft/hcsshim/osversion,MIT,Copyright (c) 2015 Microsoft | Copyright (c) 2018 Microsoft Corp. All rights reserved core,github.com/Microsoft/hcsshim/pkg/ociwclayer,MIT,Copyright (c) 2015 Microsoft | Copyright (c) 2018 Microsoft Corp. All rights reserved +core,github.com/NVIDIA/go-nvml/pkg/dl,Apache-2.0,Copyright 2023 NVIDIA CORPORATION +core,github.com/NVIDIA/go-nvml/pkg/nvml,Apache-2.0,Copyright 2023 NVIDIA CORPORATION core,github.com/NYTimes/gziphandler,Apache-2.0,Copyright 2016-2017 The New York Times Company core,github.com/OneOfOne/xxhash,Apache-2.0,Copyright (c) 2014 Ahmed W. (OneOfOne) core,github.com/ProtonMail/go-crypto/bitcurves,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved diff --git a/cmd/agent/subcommands/flare/command.go b/cmd/agent/subcommands/flare/command.go index 8c159e673cacd..79311e31c29c4 100644 --- a/cmd/agent/subcommands/flare/command.go +++ b/cmd/agent/subcommands/flare/command.go @@ -11,8 +11,11 @@ import ( "encoding/json" "errors" "fmt" + "net" + "net/url" "os" "path" + "strconv" "time" "github.com/fatih/color" @@ -49,7 +52,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/config/settings" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" - "github.com/DataDog/datadog-agent/pkg/process/net" + procnet "github.com/DataDog/datadog-agent/pkg/process/net" "github.com/DataDog/datadog-agent/pkg/serializer" "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" @@ -75,6 +78,7 @@ type cliParams struct { profileBlocking bool profileBlockingRate int withStreamLogs time.Duration + providerTimeout time.Duration } // Commands returns a slice of subcommands for the 'agent' command. @@ -153,6 +157,7 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { flareCmd.Flags().BoolVarP(&cliParams.profileBlocking, "profile-blocking", "B", false, "Add gorouting blocking profile to the performance data in the flare") flareCmd.Flags().IntVarP(&cliParams.profileBlockingRate, "profile-blocking-rate", "", 10000, "Set the fraction of goroutine blocking events that are reported in the blocking profile") flareCmd.Flags().DurationVarP(&cliParams.withStreamLogs, "with-stream-logs", "L", 0*time.Second, "Add stream-logs data to the flare. It will collect logs for the amount of seconds passed to the flag") + flareCmd.Flags().DurationVarP(&cliParams.providerTimeout, "provider-timeout", "t", 0*time.Second, "Timeout to run each flare provider in seconds. This is not a global timeout for the flare creation process.") flareCmd.SetArgs([]string{"caseID"}) return []*cobra.Command{flareCmd} @@ -244,9 +249,9 @@ func readProfileData(seconds int) (flare.ProfileData, error) { } if pkgconfigsetup.SystemProbe().GetBool("system_probe_config.enabled") { - probeUtil, probeUtilErr := net.GetRemoteSystemProbeUtil(pkgconfigsetup.SystemProbe().GetString("system_probe_config.sysprobe_socket")) + probeUtil, probeUtilErr := procnet.GetRemoteSystemProbeUtil(pkgconfigsetup.SystemProbe().GetString("system_probe_config.sysprobe_socket")) - if !errors.Is(probeUtilErr, net.ErrNotImplemented) { + if !errors.Is(probeUtilErr, procnet.ErrNotImplemented) { sysProbeGet := func() pprofGetter { return func(path string) ([]byte, error) { if probeUtilErr != nil { @@ -350,9 +355,9 @@ func makeFlare(flareComp flare.Component, var filePath string if cliParams.forceLocal { - filePath, err = createArchive(flareComp, profile, nil) + filePath, err = createArchive(flareComp, profile, cliParams.providerTimeout, nil) } else { - filePath, err = requestArchive(flareComp, profile) + filePath, err = requestArchive(flareComp, profile, cliParams.providerTimeout) } if err != nil { @@ -382,21 +387,33 @@ func makeFlare(flareComp flare.Component, return nil } -func requestArchive(flareComp flare.Component, pdata flare.ProfileData) (string, error) { +func requestArchive(flareComp flare.Component, pdata flare.ProfileData, providerTimeout time.Duration) (string, error) { fmt.Fprintln(color.Output, color.BlueString("Asking the agent to build the flare archive.")) c := util.GetClient(false) // FIX: get certificates right then make this true ipcAddress, err := pkgconfigsetup.GetIPCAddress(pkgconfigsetup.Datadog()) if err != nil { fmt.Fprintln(color.Output, color.RedString(fmt.Sprintf("Error getting IPC address for the agent: %s", err))) - return createArchive(flareComp, pdata, err) + return createArchive(flareComp, pdata, providerTimeout, err) } - urlstr := fmt.Sprintf("https://%v:%v/agent/flare", ipcAddress, pkgconfigsetup.Datadog().GetInt("cmd_port")) + cmdport := pkgconfigsetup.Datadog().GetInt("cmd_port") + url := &url.URL{ + Scheme: "https", + Host: net.JoinHostPort(ipcAddress, strconv.Itoa(cmdport)), + Path: "/agent/flare", + } + if providerTimeout > 0 { + q := url.Query() + q.Set("provider_timeout", strconv.FormatInt(int64(providerTimeout), 10)) + url.RawQuery = q.Encode() + } + + urlstr := url.String() // Set session token if err = util.SetAuthToken(pkgconfigsetup.Datadog()); err != nil { fmt.Fprintln(color.Output, color.RedString(fmt.Sprintf("Error: %s", err))) - return createArchive(flareComp, pdata, err) + return createArchive(flareComp, pdata, providerTimeout, err) } p, err := json.Marshal(pdata) @@ -414,15 +431,15 @@ func requestArchive(flareComp flare.Component, pdata flare.ProfileData) (string, fmt.Fprintln(color.Output, color.RedString("The agent was unable to make the flare. (is it running?)")) err = fmt.Errorf("Error getting flare from running agent: %w", err) } - return createArchive(flareComp, pdata, err) + return createArchive(flareComp, pdata, providerTimeout, err) } return string(r), nil } -func createArchive(flareComp flare.Component, pdata flare.ProfileData, ipcError error) (string, error) { +func createArchive(flareComp flare.Component, pdata flare.ProfileData, providerTimeout time.Duration, ipcError error) (string, error) { fmt.Fprintln(color.Output, color.YellowString("Initiating flare locally.")) - filePath, err := flareComp.Create(pdata, ipcError) + filePath, err := flareComp.Create(pdata, providerTimeout, ipcError) if err != nil { fmt.Printf("The flare zipfile failed to be created: %s\n", err) return "", err diff --git a/cmd/agent/subcommands/run/command.go b/cmd/agent/subcommands/run/command.go index c1cf43618fe79..261cb79e7a72e 100644 --- a/cmd/agent/subcommands/run/command.go +++ b/cmd/agent/subcommands/run/command.go @@ -478,7 +478,7 @@ func startAgent( _ sysprobeconfig.Component, server dogstatsdServer.Component, wmeta workloadmeta.Component, - _ tagger.Component, + tagger tagger.Component, ac autodiscovery.Component, rcclient rcclient.Component, _ optional.Option[logsAgent.Component], @@ -579,7 +579,7 @@ func startAgent( jmxfetch.RegisterWith(ac) // Set up check collector - commonchecks.RegisterChecks(wmeta, cfg, telemetry) + commonchecks.RegisterChecks(wmeta, tagger, cfg, telemetry) ac.AddScheduler("check", pkgcollector.InitCheckScheduler(optional.NewOption(collector), demultiplexer, logReceiver), true) demultiplexer.AddAgentStartupTelemetry(version.AgentVersion) diff --git a/cmd/agent/subcommands/snmp/command.go b/cmd/agent/subcommands/snmp/command.go index 46f44826765f7..49e56e1248d1b 100644 --- a/cmd/agent/subcommands/snmp/command.go +++ b/cmd/agent/subcommands/snmp/command.go @@ -417,7 +417,7 @@ func scanDevice(connParams *snmpConnectionParams, args argsType, snmpScanner snm namespace := conf.GetString("network_devices.namespace") - err = snmpScanner.RunDeviceScan(snmp, namespace) + err = snmpScanner.RunDeviceScan(snmp, namespace, connParams.IPAddress) if err != nil { return fmt.Errorf("unable to perform device scan: %v", err) } diff --git a/cmd/cluster-agent/subcommands/start/command.go b/cmd/cluster-agent/subcommands/start/command.go index 37b1305a9ec9d..b768b277fce03 100644 --- a/cmd/cluster-agent/subcommands/start/command.go +++ b/cmd/cluster-agent/subcommands/start/command.go @@ -372,7 +372,7 @@ func start(log log.Component, common.LoadComponents(secretResolver, wmeta, ac, config.GetString("confd_path")) // Set up check collector - registerChecks(wmeta, config) + registerChecks(wmeta, taggerComp, config) ac.AddScheduler("check", pkgcollector.InitCheckScheduler(optional.NewOption(collector), demultiplexer, logReceiver), true) // start the autoconfig, this will immediately run any configured check @@ -577,7 +577,7 @@ func initializeRemoteConfigClient(rcService rccomp.Component, config config.Comp return rcClient, nil } -func registerChecks(wlm workloadmeta.Component, cfg config.Component) { +func registerChecks(wlm workloadmeta.Component, tagger tagger.Component, cfg config.Component) { // Required checks corecheckLoader.RegisterCheck(cpu.CheckName, cpu.Factory()) corecheckLoader.RegisterCheck(memory.CheckName, memory.Factory()) @@ -586,7 +586,7 @@ func registerChecks(wlm workloadmeta.Component, cfg config.Component) { corecheckLoader.RegisterCheck(filehandles.CheckName, filehandles.Factory()) // Flavor specific checks - corecheckLoader.RegisterCheck(kubernetesapiserver.CheckName, kubernetesapiserver.Factory()) + corecheckLoader.RegisterCheck(kubernetesapiserver.CheckName, kubernetesapiserver.Factory(tagger)) corecheckLoader.RegisterCheck(ksm.CheckName, ksm.Factory()) corecheckLoader.RegisterCheck(helm.CheckName, helm.Factory()) corecheckLoader.RegisterCheck(disk.CheckName, disk.Factory()) diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index ae281fbeddc83..2a612cfe1750c 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -259,7 +259,7 @@ func bootstrapCommand() *cobra.Command { return bootstrapper.Bootstrap(ctx, b.env) }, } - cmd.Flags().DurationVarP(&timeout, "timeout", "T", 3*time.Minute, "timeout to bootstrap with") + cmd.Flags().DurationVarP(&timeout, "timeout", "T", 10*time.Minute, "timeout to bootstrap with") return cmd } @@ -277,7 +277,7 @@ func setupCommand() *cobra.Command { return installer.Setup(ctx, cmd.env) }, } - cmd.Flags().DurationVarP(&timeout, "timeout", "T", 3*time.Minute, "timeout to install with") + cmd.Flags().DurationVarP(&timeout, "timeout", "T", 10*time.Minute, "timeout to install with") return cmd } diff --git a/comp/core/flare/component.go b/comp/core/flare/component.go index 0ac0ef51c87a5..9921e8e32d766 100644 --- a/comp/core/flare/component.go +++ b/comp/core/flare/component.go @@ -10,9 +10,12 @@ package flare import ( + "time" + + "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/core/flare/helpers" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "go.uber.org/fx" ) // team: agent-shared-components @@ -20,7 +23,9 @@ import ( // Component is the component type. type Component interface { // Create creates a new flare locally and returns the path to the flare file. - Create(pdata ProfileData, ipcError error) (string, error) + // + // If providerTimeout is 0 or negative, the timeout from the configuration will be used. + Create(pdata ProfileData, providerTimeout time.Duration, ipcError error) (string, error) // Send sends a flare archive to Datadog. Send(flarePath string, caseID string, email string, source helpers.FlareSource) (string, error) } diff --git a/comp/core/flare/flare.go b/comp/core/flare/flare.go index b142c143770dd..6983272a95137 100644 --- a/comp/core/flare/flare.go +++ b/comp/core/flare/flare.go @@ -13,6 +13,7 @@ import ( "path/filepath" "reflect" "runtime" + "strconv" "time" "go.uber.org/fx" @@ -113,7 +114,7 @@ func (f *flare) onAgentTaskEvent(taskType rcclienttypes.TaskType, task rcclientt return true, fmt.Errorf("User handle was not provided in the flare agent task") } - filePath, err := f.Create(nil, nil) + filePath, err := f.Create(nil, 0, nil) if err != nil { return true, err } @@ -140,14 +141,25 @@ func (f *flare) createAndReturnFlarePath(w http.ResponseWriter, r *http.Request) } } + var providerTimeout time.Duration + + queryProviderTimeout := r.URL.Query().Get("provider_timeout") + if queryProviderTimeout != "" { + givenTimeout, err := strconv.ParseInt(queryProviderTimeout, 10, 64) + if err == nil && givenTimeout > 0 { + providerTimeout = time.Duration(givenTimeout) + } else { + f.log.Warnf("provider_timeout query parameter must be a positive integer, but was %s, using configuration value", queryProviderTimeout) + } + } + // Reset the `server_timeout` deadline for this connection as creating a flare can take some time conn := apiutils.GetConnection(r) _ = conn.SetDeadline(time.Time{}) var filePath string - var err error f.log.Infof("Making a flare") - filePath, err = f.Create(profile, nil) + filePath, err := f.Create(profile, providerTimeout, nil) if err != nil || filePath == "" { if err != nil { @@ -168,7 +180,13 @@ func (f *flare) Send(flarePath string, caseID string, email string, source helpe } // Create creates a new flare and returns the path to the final archive file. -func (f *flare) Create(pdata ProfileData, ipcError error) (string, error) { +// +// If providerTimeout is 0 or negative, the timeout from the configuration will be used. +func (f *flare) Create(pdata ProfileData, providerTimeout time.Duration, ipcError error) (string, error) { + if providerTimeout <= 0 { + providerTimeout = f.config.GetDuration("flare_provider_timeout") + } + fb, err := helpers.NewFlareBuilder(f.params.local) if err != nil { return "", err @@ -189,14 +207,13 @@ func (f *flare) Create(pdata ProfileData, ipcError error) (string, error) { fb.AddFileWithoutScrubbing(filepath.Join("profiles", name), data) //nolint:errcheck } - f.runProviders(fb) + f.runProviders(fb, providerTimeout) return fb.Save() } -func (f *flare) runProviders(fb types.FlareBuilder) { - flareStepTimeout := f.config.GetDuration("flare_provider_timeout") * time.Second - timer := time.NewTimer(flareStepTimeout) +func (f *flare) runProviders(fb types.FlareBuilder, providerTimeout time.Duration) { + timer := time.NewTimer(providerTimeout) defer timer.Stop() for _, p := range f.providers { @@ -226,10 +243,10 @@ func (f *flare) runProviders(fb types.FlareBuilder) { <-timer.C } case <-timer.C: - err := f.log.Warnf("flare provider '%s' skipped after %s", providerName, flareStepTimeout) + err := f.log.Warnf("flare provider '%s' skipped after %s", providerName, providerTimeout) _ = fb.Logf("%s", err.Error()) } - timer.Reset(flareStepTimeout) + timer.Reset(providerTimeout) } f.log.Info("All flare providers have been run, creating archive...") diff --git a/comp/core/flare/flare_test.go b/comp/core/flare/flare_test.go index 4ffe83274c8cd..b85b9de27311d 100644 --- a/comp/core/flare/flare_test.go +++ b/comp/core/flare/flare_test.go @@ -25,7 +25,6 @@ import ( "github.com/DataDog/datadog-agent/comp/core/secrets/secretsimpl" nooptelemetry "github.com/DataDog/datadog-agent/comp/core/telemetry/noopsimpl" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" - "github.com/DataDog/datadog-agent/pkg/config/model" "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/optional" ) @@ -104,14 +103,14 @@ func TestRunProviders(t *testing.T) { fx.ResultTags(`group:"flare"`), )), ) - deps.Config.Set("flare_provider_timeout", 1, model.SourceAgentRuntime) + + cliProviderTimeout := time.Nanosecond f := newFlare(deps) fb, err := helpers.NewFlareBuilder(false) require.NoError(t, err) - flare := f.Comp.(*flare) - flare.runProviders(fb) + f.Comp.(*flare).runProviders(fb, cliProviderTimeout) require.True(t, firstRan.Load()) require.True(t, secondRan.Load()) diff --git a/comp/core/flare/flareimpl/mock.go b/comp/core/flare/flareimpl/mock.go index 906dfe77dd333..1bf408abf69df 100644 --- a/comp/core/flare/flareimpl/mock.go +++ b/comp/core/flare/flareimpl/mock.go @@ -10,12 +10,14 @@ package flareimpl import ( "net/http" + "time" + + "go.uber.org/fx" api "github.com/DataDog/datadog-agent/comp/api/api/def" "github.com/DataDog/datadog-agent/comp/core/flare" "github.com/DataDog/datadog-agent/comp/core/flare/helpers" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "go.uber.org/fx" ) // MockModule defines the fx options for the mock component. @@ -42,7 +44,7 @@ func (fc *MockFlare) handlerFunc(w http.ResponseWriter, _ *http.Request) { } // Create mocks the flare create function -func (fc *MockFlare) Create(_ flare.ProfileData, _ error) (string, error) { +func (fc *MockFlare) Create(_ flare.ProfileData, _ time.Duration, _ error) (string, error) { return "a string", nil } diff --git a/comp/core/flare/helpers/send_flare.go b/comp/core/flare/helpers/send_flare.go index 5246ee5306469..c5854529d17a1 100644 --- a/comp/core/flare/helpers/send_flare.go +++ b/comp/core/flare/helpers/send_flare.go @@ -34,8 +34,9 @@ var ( // any modification to this struct should also be applied to datadog-agent/test/fakeintake/server/body.go type flareResponse struct { - CaseID int `json:"case_id,omitempty"` - Error string `json:"error,omitempty"` + CaseID int `json:"case_id,omitempty"` + Error string `json:"error,omitempty"` + RequestUUID string `json:"request_uuid,omitempty"` } // FlareSource has metadata about why the flare was sent @@ -184,7 +185,11 @@ func analyzeResponse(r *http.Response, apiKey string) (string, error) { } if res.Error != "" { - response := fmt.Sprintf("An error occurred while uploading the flare: %s. Please contact support by email.", res.Error) + var uuidReport string + if res.RequestUUID != "" { + uuidReport = fmt.Sprintf(" and facilitate the request uuid: `%s`", res.RequestUUID) + } + response := fmt.Sprintf("An error occurred while uploading the flare: %s. Please contact support by email%s.", res.Error, uuidReport) return response, errors.New(res.Error) } diff --git a/comp/core/flare/helpers/send_flare_test.go b/comp/core/flare/helpers/send_flare_test.go index 759971ad5d0d2..9e6d373f64bac 100644 --- a/comp/core/flare/helpers/send_flare_test.go +++ b/comp/core/flare/helpers/send_flare_test.go @@ -127,6 +127,19 @@ func TestAnalyzeResponse(t *testing.T) { }) t.Run("error-from-server", func(t *testing.T) { + r := &http.Response{ + StatusCode: 200, + Header: http.Header{"Content-Type": []string{"application/json; charset=UTF-8"}}, + Body: io.NopCloser(bytes.NewBuffer([]byte("{\"case_id\": 1234, \"error\": \"uhoh\", \"request_uuid\": \"1dd9a912-843f-4987-9007-b915edb3d047\"}"))), + } + resstr, reserr := analyzeResponse(r, "abcdef") + require.Equal(t, errors.New("uhoh"), reserr) + require.Equal(t, + "An error occurred while uploading the flare: uhoh. Please contact support by email and facilitate the request uuid: `1dd9a912-843f-4987-9007-b915edb3d047`.", + resstr) + }) + + t.Run("error-from-server-with-no-request_uuid", func(t *testing.T) { r := &http.Response{ StatusCode: 200, Header: http.Header{"Content-Type": []string{"application/json; charset=UTF-8"}}, diff --git a/comp/core/gui/guiimpl/agent.go b/comp/core/gui/guiimpl/agent.go index 222697759a122..eedbd1a374e84 100644 --- a/comp/core/gui/guiimpl/agent.go +++ b/comp/core/gui/guiimpl/agent.go @@ -149,7 +149,7 @@ func makeFlare(w http.ResponseWriter, r *http.Request, flare flare.Component) { return } - filePath, e := flare.Create(nil, nil) + filePath, e := flare.Create(nil, 0, nil) if e != nil { w.Write([]byte("Error creating flare zipfile: " + e.Error())) log.Errorf("Error creating flare zipfile: %s", e.Error()) diff --git a/comp/core/log/impl-trace/go.sum b/comp/core/log/impl-trace/go.sum index 90e32e1facb5c..dcb66361555ea 100644 --- a/comp/core/log/impl-trace/go.sum +++ b/comp/core/log/impl-trace/go.sum @@ -223,16 +223,16 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= diff --git a/comp/core/tagger/global.go b/comp/core/tagger/global.go index d807a67edd837..87fd6a1884734 100644 --- a/comp/core/tagger/global.go +++ b/comp/core/tagger/global.go @@ -60,14 +60,6 @@ func GetEntityHash(entityID string, cardinality types.TagCardinality) string { return "" } -// StandardTags is an interface function that queries taggerclient singleton -func StandardTags(entityID string) ([]string, error) { - if globalTagger == nil { - return nil, fmt.Errorf("a global tagger must be set before calling StandardTags") - } - return globalTagger.Standard(entityID) -} - // AgentTags is an interface function that queries taggerclient singleton func AgentTags(cardinality types.TagCardinality) ([]string, error) { if globalTagger == nil { @@ -92,11 +84,6 @@ func List() types.TaggerListResponse { return types.TaggerListResponse{} } -// GetTaggerInstance returns the global Tagger instance -func GetTaggerInstance() Component { - return globalTagger -} - // SetNewCaptureTagger will set capture tagger in global tagger instance by using provided capture tagger func SetNewCaptureTagger(newCaptureTagger Component) { if globalTagger != nil { diff --git a/comp/core/tagger/taggerimpl/tagger.go b/comp/core/tagger/taggerimpl/tagger.go index 9b200209ef473..2fb661469c18d 100644 --- a/comp/core/tagger/taggerimpl/tagger.go +++ b/comp/core/tagger/taggerimpl/tagger.go @@ -114,10 +114,9 @@ type TaggerClient struct { log log.Component } -func createTaggerClient(defaultTagger taggerComp.Component, captureTagger taggerComp.Component, l log.Component) *TaggerClient { +func createTaggerClient(defaultTagger taggerComp.Component, l log.Component) *TaggerClient { return &TaggerClient{ defaultTagger: defaultTagger, - captureTagger: captureTagger, log: l, } } @@ -135,24 +134,24 @@ func newTaggerClient(deps dependencies) provides { if err != nil { deps.Log.Errorf("unable to deps.Configure the remote tagger: %s", err) - taggerClient = createTaggerClient(local.NewFakeTagger(deps.Config, telemetryStore), nil, deps.Log) + taggerClient = createTaggerClient(local.NewFakeTagger(deps.Config, telemetryStore), deps.Log) } else if options.Disabled { deps.Log.Errorf("remote tagger is disabled in clc runner.") - taggerClient = createTaggerClient(local.NewFakeTagger(deps.Config, telemetryStore), nil, deps.Log) + taggerClient = createTaggerClient(local.NewFakeTagger(deps.Config, telemetryStore), deps.Log) } else { filter := types.NewFilterBuilder().Exclude(types.KubernetesPodUID).Build(types.HighCardinality) - taggerClient = createTaggerClient(remote.NewTagger(options, deps.Config, telemetryStore, filter), nil, deps.Log) + taggerClient = createTaggerClient(remote.NewTagger(options, deps.Config, telemetryStore, filter), deps.Log) } case taggerComp.NodeRemoteTaggerAgent: options, _ := remote.NodeAgentOptions(deps.Config) - taggerClient = createTaggerClient(remote.NewTagger(options, deps.Config, telemetryStore, types.NewMatchAllFilter()), nil, deps.Log) + taggerClient = createTaggerClient(remote.NewTagger(options, deps.Config, telemetryStore, types.NewMatchAllFilter()), deps.Log) case taggerComp.LocalTaggerAgent: - taggerClient = createTaggerClient(local.NewTagger(deps.Config, deps.Wmeta, telemetryStore), nil, deps.Log) + taggerClient = createTaggerClient(local.NewTagger(deps.Config, deps.Wmeta, telemetryStore), deps.Log) case taggerComp.FakeTagger: // all binaries are expected to provide their own tagger at startup. we // provide a fake tagger for testing purposes, as calling the global // tagger without proper initialization is very common there. - taggerClient = createTaggerClient(local.NewFakeTagger(deps.Config, telemetryStore), nil, deps.Log) + taggerClient = createTaggerClient(local.NewFakeTagger(deps.Config, telemetryStore), deps.Log) } if taggerClient != nil { diff --git a/comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector_test.go b/comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector_test.go index 71e47ded6be31..f8d639ad39298 100644 --- a/comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector_test.go +++ b/comp/core/workloadmeta/collectors/internal/remote/processcollector/process_collector_test.go @@ -38,13 +38,28 @@ import ( const dummySubscriber = "dummy-subscriber" +func newMockServer(ctx context.Context, responses []*pbgo.ProcessStreamResponse, errorResponse bool) *mockServer { + ctx, cancelFunc := context.WithCancel(ctx) + return &mockServer{ + ctx: ctx, + cancelFunc: cancelFunc, + responses: responses, + errorResponse: errorResponse, + } +} + type mockServer struct { pbgo.UnimplementedProcessEntityStreamServer - + ctx context.Context + cancelFunc context.CancelFunc responses []*pbgo.ProcessStreamResponse errorResponse bool // first response is an error } +func (s *mockServer) stop() { + s.cancelFunc() +} + // StreamEntities sends the responses back to the client func (s *mockServer) StreamEntities(_ *pbgo.ProcessStreamEntitiesRequest, out pbgo.ProcessEntityStream_StreamEntitiesServer) error { // Handle error response for the first request @@ -60,6 +75,7 @@ func (s *mockServer) StreamEntities(_ *pbgo.ProcessStreamEntitiesRequest, out pb } } + <-s.ctx.Done() return nil } @@ -254,11 +270,12 @@ func TestCollection(t *testing.T) { time.Sleep(time.Second) + ctx := context.Background() + // remote process collector server (process agent) - server := &mockServer{ - responses: test.serverResponses, - errorResponse: test.errorResponse, - } + server := newMockServer(ctx, test.serverResponses, test.errorResponse) + defer server.stop() + grpcServer := grpc.NewServer() pbgo.RegisterProcessEntityStreamServer(grpcServer, server) @@ -286,13 +303,12 @@ func TestCollection(t *testing.T) { mockStore.Notify(test.preEvents) - ctx, cancel := context.WithCancel(context.TODO()) - // Subscribe to the mockStore ch := mockStore.Subscribe(dummySubscriber, workloadmeta.NormalPriority, nil) + collectorCtx, cancelCollectorCtxFunc := context.WithCancel(ctx) // Collect process data - err = collector.Start(ctx, mockStore) + err = collector.Start(collectorCtx, mockStore) require.NoError(t, err) // Number of events expected. Each response can hold multiple events, either Set or Unset @@ -319,7 +335,7 @@ func TestCollection(t *testing.T) { mockStore.Unsubscribe(ch) grpcServer.Stop() - cancel() + cancelCollectorCtxFunc() // Verify final state for i := range test.expectedProcesses { diff --git a/comp/metadata/inventoryagent/README.md b/comp/metadata/inventoryagent/README.md index 07afed9f121b7..8d542dfa36eeb 100644 --- a/comp/metadata/inventoryagent/README.md +++ b/comp/metadata/inventoryagent/README.md @@ -83,7 +83,6 @@ The payload is a JSON dict with the following fields - `feature_usm_kafka_enabled` - **bool**: True if Kafka monitoring is enabled for Universal Service Monitoring (see: `service_monitoring_config.enable_kafka_monitoring` config option in `system-probe.yaml`) - `feature_usm_postgres_enabled` - **bool**: True if Postgres monitoring is enabled for Universal Service Monitoring (see: `service_monitoring_config.enable_postgres_monitoring` config option in `system-probe.yaml`) - `feature_usm_redis_enabled` - **bool**: True if Redis monitoring is enabled for Universal Service Monitoring (see: `service_monitoring_config.enable_redis_monitoring` config option in `system-probe.yaml`) - - `feature_usm_java_tls_enabled` - **bool**: True if HTTPS monitoring through java TLS is enabled for Universal Service Monitoring (see: `service_monitoring_config.tls.java.enabled` config option in `system-probe.yaml`). - `feature_usm_go_tls_enabled` - **bool**: True if HTTPS monitoring through GoTLS is enabled for Universal Service Monitoring (see: `service_monitoring_config.tls.go.enabled` config option in `system-probe.yaml`). - `feature_discovery_enabled` - **bool**: True if discovery module is enabled (see: `discovery.enabled` config option). - `feature_dynamic_instrumentation_enabled` - **bool**: True if dynamic instrumentation module is enabled (see: `dynamic_instrumentation.enabled` config option). diff --git a/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent.go b/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent.go index 72e94be45a615..095c6e9effce9 100644 --- a/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent.go +++ b/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent.go @@ -299,7 +299,6 @@ func (ia *inventoryagent) fetchSystemProbeMetadata() { ia.data["feature_usm_kafka_enabled"] = sysProbeConf.GetBool("service_monitoring_config.enable_kafka_monitoring") ia.data["feature_usm_postgres_enabled"] = sysProbeConf.GetBool("service_monitoring_config.enable_postgres_monitoring") ia.data["feature_usm_redis_enabled"] = sysProbeConf.GetBool("service_monitoring_config.enable_redis_monitoring") - ia.data["feature_usm_java_tls_enabled"] = sysProbeConf.GetBool("service_monitoring_config.tls.java.enabled") ia.data["feature_usm_http2_enabled"] = sysProbeConf.GetBool("service_monitoring_config.enable_http2_monitoring") ia.data["feature_usm_istio_enabled"] = sysProbeConf.GetBool("service_monitoring_config.tls.istio.enabled") ia.data["feature_usm_http_by_status_code_enabled"] = sysProbeConf.GetBool("service_monitoring_config.enable_http_stats_by_status_code") diff --git a/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent_test.go b/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent_test.go index 3a3b279ae777a..3038e093ff093 100644 --- a/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent_test.go +++ b/comp/metadata/inventoryagent/inventoryagentimpl/inventoryagent_test.go @@ -216,7 +216,6 @@ func TestInitData(t *testing.T) { "feature_usm_kafka_enabled": true, "feature_usm_postgres_enabled": true, "feature_usm_redis_enabled": true, - "feature_usm_java_tls_enabled": true, "feature_usm_http2_enabled": true, "feature_usm_istio_enabled": true, "feature_usm_http_by_status_code_enabled": true, @@ -490,7 +489,6 @@ func TestFetchSystemProbeAgent(t *testing.T) { assert.False(t, ia.data["feature_usm_kafka_enabled"].(bool)) assert.False(t, ia.data["feature_usm_postgres_enabled"].(bool)) assert.False(t, ia.data["feature_usm_redis_enabled"].(bool)) - assert.False(t, ia.data["feature_usm_java_tls_enabled"].(bool)) assert.False(t, ia.data["feature_usm_http2_enabled"].(bool)) assert.False(t, ia.data["feature_usm_istio_enabled"].(bool)) assert.True(t, ia.data["feature_usm_http_by_status_code_enabled"].(bool)) @@ -543,7 +541,6 @@ func TestFetchSystemProbeAgent(t *testing.T) { assert.False(t, ia.data["feature_usm_enabled"].(bool)) assert.False(t, ia.data["feature_usm_kafka_enabled"].(bool)) assert.False(t, ia.data["feature_usm_postgres_enabled"].(bool)) - assert.False(t, ia.data["feature_usm_java_tls_enabled"].(bool)) assert.False(t, ia.data["feature_usm_http2_enabled"].(bool)) assert.False(t, ia.data["feature_usm_istio_enabled"].(bool)) assert.False(t, ia.data["feature_usm_http_by_status_code_enabled"].(bool)) @@ -644,7 +641,6 @@ dynamic_instrumentation: assert.True(t, ia.data["feature_usm_kafka_enabled"].(bool)) assert.True(t, ia.data["feature_usm_postgres_enabled"].(bool)) assert.True(t, ia.data["feature_usm_redis_enabled"].(bool)) - assert.True(t, ia.data["feature_usm_java_tls_enabled"].(bool)) assert.True(t, ia.data["feature_usm_http2_enabled"].(bool)) assert.True(t, ia.data["feature_usm_istio_enabled"].(bool)) assert.True(t, ia.data["feature_usm_http_by_status_code_enabled"].(bool)) diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index b2d80d31b7e7a..4280cf75bd9d5 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -242,7 +242,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.15 // indirect + github.com/DataDog/go-sqllexer v0.0.16 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect @@ -440,7 +440,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.27.0 // indirect go.opentelemetry.io/contrib/zpages v0.52.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/bridge/opencensus v1.27.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.27.0 // indirect @@ -450,10 +450,10 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.2.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.18.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum index 051600118ccd0..7a284d7db11ec 100644 --- a/comp/otelcol/ddflareextension/impl/go.sum +++ b/comp/otelcol/ddflareextension/impl/go.sum @@ -66,8 +66,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= -github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.16 h1:RoSUMS6MECyB3gTUIdydzXwK5NhEhv6GMJkS7ptsgRA= +github.com/DataDog/go-sqllexer v0.0.16/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJKaNItsdzcrk4YqILNDZlXPTNt4k= @@ -911,8 +911,8 @@ go.opentelemetry.io/contrib/propagators/b3 v1.27.0 h1:IjgxbomVrV9za6bRi8fWCNXENs go.opentelemetry.io/contrib/propagators/b3 v1.27.0/go.mod h1:Dv9obQz25lCisDvvs4dy28UPh974CxkahRDUPsY7y9E= go.opentelemetry.io/contrib/zpages v0.52.0 h1:MPgkMy0Cp3O5EdfVXP0ss3ujhEibysTM4eszx7E7d+E= go.opentelemetry.io/contrib/zpages v0.52.0/go.mod h1:fqG5AFdoYru3A3DnhibVuaaEfQV2WKxE7fYE1jgDRwk= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/bridge/opencensus v1.27.0 h1:ao9aGGHd+G4YfjBpGs6vbkvt5hoC67STlJA9fCnOAcs= go.opentelemetry.io/otel/bridge/opencensus v1.27.0/go.mod h1:uRvWtAAXzyVOST0WMPX5JHGBaAvBws+2F8PcC5gMnTk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 h1:bFgvUr3/O4PHj3VQcFEuYKvRZJX1SJDQ+11JXuSB3/w= @@ -931,14 +931,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0 h1:/jlt1Y8gXWiHG9 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0/go.mod h1:bmToOGOBZ4hA9ghphIc1PAf66VA8KOtsuy3+ScStG20= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index d8a69876d51dd..344dedb5abddc 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -117,8 +117,8 @@ require ( go.opentelemetry.io/collector/exporter v0.104.0 go.opentelemetry.io/collector/featuregate v1.11.0 go.opentelemetry.io/collector/pdata v1.11.0 - go.opentelemetry.io/otel/metric v1.30.0 - go.opentelemetry.io/otel/trace v1.30.0 + go.opentelemetry.io/otel/metric v1.31.0 + go.opentelemetry.io/otel/trace v1.31.0 go.uber.org/zap v1.27.0 google.golang.org/protobuf v1.34.2 ) @@ -191,7 +191,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.56.0-rc.3 // indirect github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.15 // indirect + github.com/DataDog/go-sqllexer v0.0.16 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/logs v0.20.0 // indirect @@ -288,7 +288,7 @@ require ( go.opentelemetry.io/collector/receiver v0.104.0 // indirect go.opentelemetry.io/collector/semconv v0.104.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index 249e64737379b..71a0a814b244b 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= -github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.16 h1:RoSUMS6MECyB3gTUIdydzXwK5NhEhv6GMJkS7ptsgRA= +github.com/DataDog/go-sqllexer v0.0.16/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= @@ -433,18 +433,18 @@ go.opentelemetry.io/collector/semconv v0.104.0 h1:dUvajnh+AYJLEW/XOPk0T0BlwltSdi go.opentelemetry.io/collector/semconv v0.104.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/prometheus v0.49.0 h1:Er5I1g/YhfYv9Affk9nJLfH/+qCCVVg1f2R9AbJfqDQ= go.opentelemetry.io/otel/exporters/prometheus v0.49.0/go.mod h1:KfQ1wpjf3zsHjzP149P4LyAwWRupc6c7t1ZJ9eXpKQM= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= diff --git a/comp/otelcol/otlp/components/metricsclient/go.mod b/comp/otelcol/otlp/components/metricsclient/go.mod index 83c152fac7f32..334fd4792cf43 100644 --- a/comp/otelcol/otlp/components/metricsclient/go.mod +++ b/comp/otelcol/otlp/components/metricsclient/go.mod @@ -8,8 +8,8 @@ require ( github.com/DataDog/datadog-agent/pkg/trace v0.56.0-rc.3 github.com/DataDog/datadog-go/v5 v5.5.0 github.com/stretchr/testify v1.9.0 - go.opentelemetry.io/otel v1.30.0 - go.opentelemetry.io/otel/metric v1.30.0 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/metric v1.31.0 go.opentelemetry.io/otel/sdk/metric v1.27.0 ) @@ -20,7 +20,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/sync v0.8.0 // indirect diff --git a/comp/otelcol/otlp/components/metricsclient/go.sum b/comp/otelcol/otlp/components/metricsclient/go.sum index 190fe8a1e96ad..18f900c41cc26 100644 --- a/comp/otelcol/otlp/components/metricsclient/go.sum +++ b/comp/otelcol/otlp/components/metricsclient/go.sum @@ -30,16 +30,16 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index d6dde46f3558f..65b3c209ef8d5 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -38,7 +38,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect - github.com/DataDog/go-sqllexer v0.0.15 // indirect + github.com/DataDog/go-sqllexer v0.0.16 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect @@ -82,11 +82,11 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.104.0 // indirect go.opentelemetry.io/collector/semconv v0.104.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index be59ca525ee43..a201b6217a5a0 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= -github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.16 h1:RoSUMS6MECyB3gTUIdydzXwK5NhEhv6GMJkS7ptsgRA= +github.com/DataDog/go-sqllexer v0.0.16/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 h1:fKv05WFWHCXQmUTehW1eEZvXJP65Qv00W4V01B1EqSA= @@ -168,18 +168,18 @@ go.opentelemetry.io/collector/processor v0.104.0 h1:KSvMDu4DWmK1/k2z2rOzMtTvAa00 go.opentelemetry.io/collector/processor v0.104.0/go.mod h1:qU2/xCCYdvVORkN6aq0H/WUWkvo505VGYg2eOwPvaTg= go.opentelemetry.io/collector/semconv v0.104.0 h1:dUvajnh+AYJLEW/XOPk0T0BlwltSdi3vrjO7nSOos3k= go.opentelemetry.io/collector/semconv v0.104.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/prometheus v0.49.0 h1:Er5I1g/YhfYv9Affk9nJLfH/+qCCVVg1f2R9AbJfqDQ= go.opentelemetry.io/otel/exporters/prometheus v0.49.0/go.mod h1:KfQ1wpjf3zsHjzP149P4LyAwWRupc6c7t1ZJ9eXpKQM= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= diff --git a/comp/snmpscan/def/component.go b/comp/snmpscan/def/component.go index 30f9860c27447..b81d86b2ea28f 100644 --- a/comp/snmpscan/def/component.go +++ b/comp/snmpscan/def/component.go @@ -15,6 +15,6 @@ import ( // Component is the component type. type Component interface { // Triggers a device scan - RunDeviceScan(snmpConection *gosnmp.GoSNMP, deviceNamespace string) error + RunDeviceScan(snmpConection *gosnmp.GoSNMP, deviceNamespace string, deviceIPAddress string) error RunSnmpWalk(snmpConection *gosnmp.GoSNMP, firstOid string) error } diff --git a/comp/snmpscan/impl/devicescan.go b/comp/snmpscan/impl/devicescan.go index 74878b83aa6bc..cef73d6aee845 100644 --- a/comp/snmpscan/impl/devicescan.go +++ b/comp/snmpscan/impl/devicescan.go @@ -16,13 +16,13 @@ import ( "github.com/gosnmp/gosnmp" ) -func (s snmpScannerImpl) RunDeviceScan(snmpConnection *gosnmp.GoSNMP, deviceNamespace string) error { +func (s snmpScannerImpl) RunDeviceScan(snmpConnection *gosnmp.GoSNMP, deviceNamespace string, deviceIPAddress string) error { pdus, err := gatherPDUs(snmpConnection) if err != nil { return err } - deviceID := deviceNamespace + ":" + snmpConnection.LocalAddr + deviceID := deviceNamespace + ":" + deviceIPAddress var deviceOids []*metadata.DeviceOID for _, pdu := range pdus { record, err := metadata.DeviceOIDFromPDU(deviceID, pdu) diff --git a/comp/snmpscan/impl/snmpscan_test.go b/comp/snmpscan/impl/snmpscan_test.go index 3d7caa2fa9019..5f14967a1012c 100644 --- a/comp/snmpscan/impl/snmpscan_test.go +++ b/comp/snmpscan/impl/snmpscan_test.go @@ -40,9 +40,9 @@ func TestSnmpScanComp(t *testing.T) { snmpConnection.LocalAddr = "127.0.0.1" snmpConnection.Port = 0 - err = snmpScanner.Comp.RunDeviceScan(snmpConnection, "default") + err = snmpScanner.Comp.RunDeviceScan(snmpConnection, "default", "127.0.0.1") assert.ErrorContains(t, err, "&GoSNMP.Conn is missing. Provide a connection or use Connect()") - err = snmpScanner.Comp.RunDeviceScan(snmpConnection, "default") + err = snmpScanner.Comp.RunDeviceScan(snmpConnection, "default", "127.0.0.1") assert.ErrorContains(t, err, "&GoSNMP.Conn is missing. Provide a connection or use Connect()") } diff --git a/comp/snmpscan/mock/mock.go b/comp/snmpscan/mock/mock.go index ec4b34e6b36bb..3804870a33f40 100644 --- a/comp/snmpscan/mock/mock.go +++ b/comp/snmpscan/mock/mock.go @@ -32,7 +32,7 @@ func New(*testing.T) Provides { } } -func (m mock) RunDeviceScan(_ *gosnmp.GoSNMP, _ string) error { +func (m mock) RunDeviceScan(_ *gosnmp.GoSNMP, _ string, _ string) error { return nil } func (m mock) RunSnmpWalk(_ *gosnmp.GoSNMP, _ string) error { diff --git a/comp/systray/systray/systrayimpl/doflare.go b/comp/systray/systray/systrayimpl/doflare.go index 229f1f113f71c..b6748933c5dde 100644 --- a/comp/systray/systray/systrayimpl/doflare.go +++ b/comp/systray/systray/systrayimpl/doflare.go @@ -196,7 +196,7 @@ func requestFlare(s *systrayImpl, caseID, customerEmail string) (response string } s.log.Debug("Initiating flare locally.") - filePath, e = s.flare.Create(nil, e) + filePath, e = s.flare.Create(nil, 0, e) if e != nil { s.log.Errorf("The flare zipfile failed to be created: %s\n", e) return diff --git a/docs/dev/agent_build.md b/docs/dev/agent_build.md index d4490f71e1c18..fa05758d550f7 100644 --- a/docs/dev/agent_build.md +++ b/docs/dev/agent_build.md @@ -49,9 +49,8 @@ Also note that the trace agent needs to be built and run separately. For more in ## Additional details -We use `pkg-config` to make compilers and linkers aware of Python. If you need -to adjust the build for your specific configuration, add or edit the files within -the `pkg-config` folder. +We use `pkg-config` to make compilers and linkers aware of Python. The required .pc files are +provided automatically when building python through omnibus. By default, the Agent combines multiple functionalities into a single binary to reduce the space used on disk. The `DD_BUNDLED_AGENT` environment variable is used to select diff --git a/docs/public/hostname/hostname_force_config_as_canonical.md b/docs/public/hostname/hostname_force_config_as_canonical.md new file mode 100644 index 0000000000000..28d7a5d254602 --- /dev/null +++ b/docs/public/hostname/hostname_force_config_as_canonical.md @@ -0,0 +1,21 @@ +# Config-provided hostname starting with `ip-` or `domu` + +## Description of the issue + +In v6 and v7 Agents, if `hostname` is set in `datadog.yaml` (or through the `DD_HOSTNAME` env var) and its value starts with `ip-` or `domu`, the hostname is not used in-app as the canonical hostname, even if it is a valid hostname. +More information about what a canonical hostname is can be found at [How does Datadog determine the Agent hostname?](https://docs.datadoghq.com/agent/faq/how-datadog-agent-determines-the-hostname/?tab=agentv6v7#agent-versions). + +To know if your Agents are affected, starting with v6.16.0 and v7.16.0, the Agent logs the following warning if it detects a situation where the config-provided hostname is a valid hostname but will not be accepted as the canonical hostname in-app: +`Hostname '' defined in configuration are not used as the in-app hostname. For more information: https://dtdg.co/agent-hostname-force-config-as-canonical` + +If this warning is logged, you have the following options: + +- If you are satisfied with the in-app hostname: unset the configured `hostname` from `datadog.yaml` (or the `DD_HOSTNAME` env var) and restart the Agent; or +- If you are not satisfied with the in-app hostname, and want the configured hostname to appear as the in-app hostname, follow the instructions below + +## Allowing Agent in-app hostnames to start with `ip-` or `domu` + +Starting with Agent v6.16.0 and v7.16.0, the Agent supports the config option `hostname_force_config_as_canonical` (default: `false`). When set to `true`, a configuration-provided hostname starting with `ip-` or `domu` is accepted as the canonical hostname in-app: + +- For new hosts, enabling this option works immediately. +- For hosts that already report to Datadog, after enabling this option, contact Datadog support at support@datadoghq.com so that the in-app hostname can be changed to your configuration-provided hostname. diff --git a/go.mod b/go.mod index 045c0eb7a1c36..9b4978e72091d 100644 --- a/go.mod +++ b/go.mod @@ -558,15 +558,15 @@ require ( go.opentelemetry.io/collector/semconv v0.104.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.27.0 // indirect - go.opentelemetry.io/otel v1.30.0 + go.opentelemetry.io/otel v1.31.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/otel/sdk v1.29.0 go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 + go.opentelemetry.io/otel/trace v1.31.0 go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/mod v0.21.0 @@ -602,6 +602,7 @@ require ( github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/impl v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/pkg/config/structure v0.60.0-devel github.com/DataDog/datadog-agent/pkg/util/defaultpaths v0.0.0-00010101000000-000000000000 + github.com/NVIDIA/go-nvml v0.12.4-0 github.com/containerd/containerd/api v1.7.19 github.com/containerd/errdefs v0.1.0 github.com/distribution/reference v0.6.0 diff --git a/go.sum b/go.sum index b1bf94b1b5b64..08658fcc4533c 100644 --- a/go.sum +++ b/go.sum @@ -779,6 +779,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Microsoft/hcsshim v0.12.7 h1:MP6R1spmjxTE4EU4J3YsrTxn8CjvN9qwjTKJXldFaRg= github.com/Microsoft/hcsshim v0.12.7/go.mod h1:HPbAuJ9BvQYYZbB4yEQcyGIsTP5L4yHKeO9XO149AEM= +github.com/NVIDIA/go-nvml v0.12.4-0 h1:4tkbB3pT1O77JGr0gQ6uD8FrsUPqP1A/EOEm2wI1TUg= +github.com/NVIDIA/go-nvml v0.12.4-0/go.mod h1:8Llmj+1Rr+9VGGwZuRer5N/aCjxGuR5nPb/9ebBiIEQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= @@ -2731,8 +2733,8 @@ go.opentelemetry.io/contrib/propagators/b3 v1.27.0 h1:IjgxbomVrV9za6bRi8fWCNXENs go.opentelemetry.io/contrib/propagators/b3 v1.27.0/go.mod h1:Dv9obQz25lCisDvvs4dy28UPh974CxkahRDUPsY7y9E= go.opentelemetry.io/contrib/zpages v0.52.0 h1:MPgkMy0Cp3O5EdfVXP0ss3ujhEibysTM4eszx7E7d+E= go.opentelemetry.io/contrib/zpages v0.52.0/go.mod h1:fqG5AFdoYru3A3DnhibVuaaEfQV2WKxE7fYE1jgDRwk= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/bridge/opencensus v1.27.0 h1:ao9aGGHd+G4YfjBpGs6vbkvt5hoC67STlJA9fCnOAcs= go.opentelemetry.io/otel/bridge/opencensus v1.27.0/go.mod h1:uRvWtAAXzyVOST0WMPX5JHGBaAvBws+2F8PcC5gMnTk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 h1:bFgvUr3/O4PHj3VQcFEuYKvRZJX1SJDQ+11JXuSB3/w= @@ -2751,14 +2753,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0 h1:/jlt1Y8gXWiHG9 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0/go.mod h1:bmToOGOBZ4hA9ghphIc1PAf66VA8KOtsuy3+ScStG20= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= go.opentelemetry.io/otel/sdk/metric v1.29.0 h1:K2CfmJohnRgvZ9UAj2/FhIf/okdWcNdBwe1m8xFXiSY= go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= diff --git a/omnibus/config/projects/agent.rb b/omnibus/config/projects/agent.rb index c2421a99bffb6..0fb7b138432d6 100644 --- a/omnibus/config/projects/agent.rb +++ b/omnibus/config/projects/agent.rb @@ -47,7 +47,6 @@ # dir will be determined by the Windows installer. This path must not contain # spaces because Omnibus doesn't quote the Git commands it launches. INSTALL_DIR = 'C:/opt/datadog-agent/' - PYTHON_2_EMBEDDED_DIR = format('%s/embedded2', INSTALL_DIR) PYTHON_3_EMBEDDED_DIR = format('%s/embedded3', INSTALL_DIR) else INSTALL_DIR = ENV["INSTALL_DIR"] || '/opt/datadog-agent' @@ -56,7 +55,6 @@ install_dir INSTALL_DIR if windows_target? - python_2_embedded PYTHON_2_EMBEDDED_DIR python_3_embedded PYTHON_3_EMBEDDED_DIR maintainer 'Datadog Inc.' # Windows doesn't want our e-mail address :( else @@ -232,14 +230,7 @@ dependency 'datadog-agent-mac-app' end - if with_python_runtime? "2" - dependency 'pylint2' - dependency 'datadog-agent-integrations-py2' - end - - if with_python_runtime? "3" - dependency 'datadog-agent-integrations-py3' - end + dependency 'datadog-agent-integrations-py3' if linux_target? dependency 'datadog-security-agent-policies' @@ -345,14 +336,6 @@ "#{install_dir}\\bin\\agent\\ddtray.exe", "#{install_dir}\\bin\\agent\\libdatadog-agent-three.dll" ] - if with_python_runtime? "2" - BINARIES_TO_SIGN.concat([ - "#{install_dir}\\bin\\agent\\libdatadog-agent-two.dll", - "#{install_dir}\\embedded2\\python.exe", - "#{install_dir}\\embedded2\\python27.dll", - "#{install_dir}\\embedded2\\pythonw.exe" - ]) - end BINARIES_TO_SIGN.each do |bin| sign_file bin diff --git a/omnibus/config/software/datadog-agent-dependencies.rb b/omnibus/config/software/datadog-agent-dependencies.rb index fe4a6332d361f..9b0ba32e5e80c 100644 --- a/omnibus/config/software/datadog-agent-dependencies.rb +++ b/omnibus/config/software/datadog-agent-dependencies.rb @@ -5,13 +5,6 @@ # Used for memory profiling with the `status py` agent subcommand dependency 'pympler' -if with_python_runtime? "2" - dependency 'pylint2' - dependency 'datadog-agent-integrations-py2-dependencies' -end - -if with_python_runtime? "3" - dependency 'datadog-agent-integrations-py3-dependencies' -end +dependency 'datadog-agent-integrations-py3-dependencies' dependency "systemd" if linux_target? diff --git a/omnibus/config/software/datadog-agent-finalize.rb b/omnibus/config/software/datadog-agent-finalize.rb index da54ee837006f..18156227f1d58 100644 --- a/omnibus/config/software/datadog-agent-finalize.rb +++ b/omnibus/config/software/datadog-agent-finalize.rb @@ -64,27 +64,16 @@ if linux_target? || osx_target? # Setup script aliases, e.g. `/opt/datadog-agent/embedded/bin/pip` will # default to `pip2` if the default Python runtime is Python 2. - if with_python_runtime? "2" - delete "#{install_dir}/embedded/bin/pip" - link "#{install_dir}/embedded/bin/pip2", "#{install_dir}/embedded/bin/pip" + delete "#{install_dir}/embedded/bin/pip" + link "#{install_dir}/embedded/bin/pip3", "#{install_dir}/embedded/bin/pip" + + delete "#{install_dir}/embedded/bin/python" + link "#{install_dir}/embedded/bin/python3", "#{install_dir}/embedded/bin/python" + + # Used in https://docs.datadoghq.com/agent/guide/python-3/ + delete "#{install_dir}/embedded/bin/2to3" + link "#{install_dir}/embedded/bin/2to3-3.12", "#{install_dir}/embedded/bin/2to3" - # Used in https://docs.datadoghq.com/agent/guide/python-3/ - delete "#{install_dir}/embedded/bin/2to3" - link "#{install_dir}/embedded/bin/2to3-2.7", "#{install_dir}/embedded/bin/2to3" - # Setup script aliases, e.g. `/opt/datadog-agent/embedded/bin/pip` will - # default to `pip3` if the default Python runtime is Python 3 (Agent 7.x). - # Caution: we don't want to do this for Agent 6.x - elsif with_python_runtime? "3" - delete "#{install_dir}/embedded/bin/pip" - link "#{install_dir}/embedded/bin/pip3", "#{install_dir}/embedded/bin/pip" - - delete "#{install_dir}/embedded/bin/python" - link "#{install_dir}/embedded/bin/python3", "#{install_dir}/embedded/bin/python" - - # Used in https://docs.datadoghq.com/agent/guide/python-3/ - delete "#{install_dir}/embedded/bin/2to3" - link "#{install_dir}/embedded/bin/2to3-3.12", "#{install_dir}/embedded/bin/2to3" - end delete "#{install_dir}/embedded/lib/config_guess" # Delete .pc files which aren't needed after building diff --git a/omnibus/config/software/datadog-agent-integrations-py2-dependencies.rb b/omnibus/config/software/datadog-agent-integrations-py2-dependencies.rb deleted file mode 100644 index 5f9592c9d97f6..0000000000000 --- a/omnibus/config/software/datadog-agent-integrations-py2-dependencies.rb +++ /dev/null @@ -1,10 +0,0 @@ -name 'datadog-agent-integrations-py2-dependencies' - -dependency 'pip2' - -if linux_target? - # add nfsiostat script - dependency 'nfsiostat' - # needed for glusterfs - dependency 'gstatus' -end diff --git a/omnibus/config/software/datadog-agent-integrations-py2.rb b/omnibus/config/software/datadog-agent-integrations-py2.rb deleted file mode 100644 index ee05b6d366b32..0000000000000 --- a/omnibus/config/software/datadog-agent-integrations-py2.rb +++ /dev/null @@ -1,252 +0,0 @@ -# Unless explicitly stated otherwise all files in this repository are licensed -# under the Apache License Version 2.0. -# This product includes software developed at Datadog (https:#www.datadoghq.com/). -# Copyright 2016-present Datadog, Inc. - -require './lib/ostools.rb' -require 'json' - -name 'datadog-agent-integrations-py2' - -license "BSD-3-Clause" -license_file "./LICENSE" - -dependency 'datadog-agent-integrations-py2-dependencies' - -relative_path 'integrations-core' -whitelist_file "embedded/lib/python2.7/site-packages/.libsaerospike" -whitelist_file "embedded/lib/python2.7/site-packages/psycopg2" -whitelist_file "embedded/lib/python2.7/site-packages/wrapt" -whitelist_file "embedded/lib/python2.7/site-packages/pymqi" - -source git: 'https://github.com/DataDog/integrations-core.git' - -always_build true - -integrations_core_version = ENV['INTEGRATIONS_CORE_VERSION'] -if integrations_core_version.nil? || integrations_core_version.empty? - integrations_core_version = 'master' -end -default_version integrations_core_version - -# folder names containing integrations from -core that won't be packaged with the Agent -excluded_folders = [ - 'datadog_checks_base', # namespacing package for wheels (NOT AN INTEGRATION) - 'datadog_checks_dev', # Development package, (NOT AN INTEGRATION) - 'datadog_checks_tests_helper', # Testing and Development package, (NOT AN INTEGRATION) - 'docker_daemon', # Agent v5 only -] - -if osx_target? - # Temporarily exclude Aerospike until builder supports new dependency - excluded_folders.push('aerospike') -end - -if arm_target? - # This doesn't build on ARM - excluded_folders.push('ibm_mq') -end - -final_constraints_file = 'final_constraints-py2.txt' -agent_requirements_file = 'agent_requirements-py2.txt' -filtered_agent_requirements_in = 'agent_requirements-py2.in' -agent_requirements_in = 'agent_requirements.in' - -build do - # The dir for confs - if osx_target? - conf_dir = "#{install_dir}/etc/conf.d" - else - conf_dir = "#{install_dir}/etc/datadog-agent/conf.d" - end - mkdir conf_dir - - # aliases for pip - if windows_target? - pip = "#{windows_safe_path(python_2_embedded)}\\Scripts\\pip.exe" - python = "#{windows_safe_path(python_2_embedded)}\\python.exe" - else - pip = "#{install_dir}/embedded/bin/pip2" - python = "#{install_dir}/embedded/bin/python2" - end - - # If a python_mirror was set, it's passed through a pip config file so that we're not leaking the API key in the CI Output - # Else the pip config file so pip will act casually - pip_config_file = ENV['PIP_CONFIG_FILE'] - pre_build_env = { - "PIP_CONFIG_FILE" => "#{pip_config_file}" - } - - # Install dependencies - python_version = "py2" - lockfile_name = case - when linux_target? - arm_target? ? "linux-aarch64" : "linux-x86_64" - when osx_target? - "macos-x86_64" - when windows_target? - "windows-x86_64" - end + "_#{python_version}.txt" - lockfile = windows_safe_path(project_dir, ".deps", "resolved", lockfile_name) - command "#{python} -m pip install --require-hashes --only-binary=:all: --no-deps -r #{lockfile}" - - # Prepare build env for integrations - wheel_build_dir = windows_safe_path(project_dir, ".wheels") - build_deps_dir = windows_safe_path(project_dir, ".build_deps") - # We download build dependencies to make them available without an index when installing integrations - command "#{pip} download --dest #{build_deps_dir} hatchling==0.25.1", :env => pre_build_env - command "#{pip} download --dest #{build_deps_dir} setuptools==40.9.0", :env => pre_build_env # Version from ./setuptools2.rb - command "#{pip} install wheel==0.37.1", :env => pre_build_env # Pin to the last version that supports Python 2 - build_env = { - "PIP_FIND_LINKS" => build_deps_dir, - "PIP_CONFIG_FILE" => pip_config_file, - } - - # Install base package - cwd_base = windows_safe_path(project_dir, "datadog_checks_base") - command "#{python} -m pip wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => build_env, :cwd => cwd_base - command "#{python} -m pip install datadog_checks_base --no-deps --no-index --find-links=#{wheel_build_dir}" - - # - # Install Core integrations - # - - # Create a constraint file after installing all the core dependencies and before any integration - # This is then used as a constraint file by the integration command to avoid messing with the agent's python environment - command "#{pip} freeze > #{install_dir}/#{final_constraints_file}" - - if windows_target? - cached_wheels_dir = "#{windows_safe_path(wheel_build_dir)}\\.cached" - else - cached_wheels_dir = "#{wheel_build_dir}/.cached" - end - - block "Install integrations" do - tasks_dir_in = windows_safe_path(Dir.pwd) - # Collect integrations to install - checks_to_install = ( - shellout! "inv agent.collect-integrations #{project_dir} 2 #{os} #{excluded_folders.join(',')}", - :cwd => tasks_dir_in - ).stdout.split() - - # Retrieving integrations from cache - cache_bucket = ENV.fetch('INTEGRATION_WHEELS_CACHE_BUCKET', '') - cache_branch = (shellout! "inv release.get-release-json-value base_branch", cwd: File.expand_path('..', tasks_dir_in)).stdout.strip - if cache_bucket != '' - mkdir cached_wheels_dir - shellout! "inv -e agent.get-integrations-from-cache " \ - "--python 2 --bucket #{cache_bucket} " \ - "--branch #{cache_branch || 'main'} " \ - "--integrations-dir #{windows_safe_path(project_dir)} " \ - "--target-dir #{cached_wheels_dir} " \ - "--integrations #{checks_to_install.join(',')}", - :cwd => tasks_dir_in - - # install all wheels from cache in one pip invocation to speed things up - if windows_target? - shellout! "#{python} -m pip install --no-deps --no-index " \ - "--find-links #{windows_safe_path(cached_wheels_dir)} -r #{windows_safe_path(cached_wheels_dir)}\\found.txt" - else - shellout! "#{python} -m pip install --no-deps --no-index " \ - "--find-links #{cached_wheels_dir} -r #{cached_wheels_dir}/found.txt" - end - end - - # get list of integration wheels already installed from cache - installed_list = Array.new - if cache_bucket != '' - installed_out = `#{python} -m pip list --format json` - if $?.exitstatus == 0 - installed = JSON.parse(installed_out) - installed.each do |package| - package.each do |key, value| - if key == "name" && value.start_with?("datadog-") - installed_list.push(value["datadog-".length..-1]) - end - end - end - else - raise "Failed to list pip installed packages" - end - end - - checks_to_install.each do |check| - check_dir = File.join(project_dir, check) - check_conf_dir = "#{conf_dir}/#{check}.d" - - # For each conf file, if it already exists, that means the `datadog-agent` software def - # wrote it first. In that case, since the agent's confs take precedence, skip the conf - conf_files = ["conf.yaml.example", "conf.yaml.default", "metrics.yaml", "auto_conf.yaml"] - - conf_files.each do |filename| - src = windows_safe_path(check_dir,"datadog_checks", check, "data", filename) - dest = check_conf_dir - if File.exist?(src) and !File.exist?(windows_safe_path(dest, filename)) - FileUtils.mkdir_p(dest) - FileUtils.cp_r(src, dest) - end - end - - # Copy SNMP profiles - profile_folders = ['profiles', 'default_profiles'] - profile_folders.each do |profile_folder| - folder_path = "#{check_dir}/datadog_checks/#{check}/data/#{profile_folder}" - if File.exist? folder_path - FileUtils.cp_r folder_path, "#{check_conf_dir}/" - end - end - - # pip < 21.2 replace underscores by dashes in package names per https://pip.pypa.io/en/stable/news/#v21-2 - # whether or not this might switch back in the future is not guaranteed, so we check for both name - # with dashes and underscores - if installed_list.include?(check) || installed_list.include?(check.gsub('_', '-')) - next - end - - if windows_target? - shellout! "#{python} -m pip wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => build_env, :cwd => "#{windows_safe_path(project_dir)}\\#{check}" - shellout! "#{python} -m pip install datadog-#{check} --no-deps --no-index --find-links=#{wheel_build_dir}" - else - shellout! "#{pip} wheel . --no-deps --no-index --wheel-dir=#{wheel_build_dir}", :env => build_env, :cwd => "#{project_dir}/#{check}" - shellout! "#{pip} install datadog-#{check} --no-deps --no-index --find-links=#{wheel_build_dir}" - end - if cache_bucket != '' && ENV.fetch('INTEGRATION_WHEELS_SKIP_CACHE_UPLOAD', '') == '' && cache_branch != nil - shellout! "inv -e agent.upload-integration-to-cache " \ - "--python 2 --bucket #{cache_bucket} " \ - "--branch #{cache_branch} " \ - "--integrations-dir #{windows_safe_path(project_dir)} " \ - "--build-dir #{wheel_build_dir} " \ - "--integration #{check}", - :cwd => tasks_dir_in - end - end - end - - # Patch applies to only one file: set it explicitly as a target, no need for -p - if windows_target? - patch :source => "create-regex-at-runtime.patch", :target => "#{python_2_embedded}/Lib/site-packages/yaml/reader.py" - patch :source => "remove-maxfile-maxpath-psutil.patch", :target => "#{python_2_embedded}/Lib/site-packages/psutil/__init__.py" - else - patch :source => "create-regex-at-runtime.patch", :target => "#{install_dir}/embedded/lib/python2.7/site-packages/yaml/reader.py" - patch :source => "remove-maxfile-maxpath-psutil.patch", :target => "#{install_dir}/embedded/lib/python2.7/site-packages/psutil/__init__.py" - end - - # Run pip check to make sure the agent's python environment is clean, all the dependencies are compatible - if windows_target? - command "#{python} -m pip check" - else - command "#{pip} check" - end - - # Removing tests that don't need to be shipped in the embedded folder - if windows_target? - delete "#{python_2_embedded}/Lib/site-packages/Cryptodome/SelfTest/" - else - delete "#{install_dir}/embedded/lib/python2.7/site-packages/Cryptodome/SelfTest/" - end - - # Ship `requirements-agent-release.txt` file containing the versions of every check shipped with the agent - # Used by the `datadog-agent integration` command to prevent downgrading a check to a version - # older than the one shipped in the agent - copy "#{project_dir}/requirements-agent-release.txt", "#{install_dir}/" -end diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb index 7098a19562567..ee030c17aacca 100644 --- a/omnibus/config/software/datadog-agent.rb +++ b/omnibus/config/software/datadog-agent.rb @@ -11,8 +11,7 @@ # creates required build directories dependency 'datadog-agent-prepare' -dependency "python2" if with_python_runtime? "2" -dependency "python3" if with_python_runtime? "3" +dependency "python3" dependency "openscap" if linux_target? and !arm7l_target? and !heroku_target? # Security-agent dependency, not needed for Heroku @@ -43,24 +42,17 @@ env = { 'GOPATH' => gopath.to_path, 'PATH' => "#{gopath.to_path}/bin:#{ENV['PATH']}", - "Python2_ROOT_DIR" => "#{windows_safe_path(python_2_embedded)}", - "Python3_ROOT_DIR" => "#{windows_safe_path(python_3_embedded)}", - "CMAKE_INSTALL_PREFIX" => "#{windows_safe_path(python_2_embedded)}", } major_version_arg = "%MAJOR_VERSION%" - py_runtimes_arg = "%PY_RUNTIMES%" else env = { 'GOPATH' => gopath.to_path, 'PATH' => "#{gopath.to_path}/bin:#{ENV['PATH']}", - "Python2_ROOT_DIR" => "#{install_dir}/embedded", - "Python3_ROOT_DIR" => "#{install_dir}/embedded", "LDFLAGS" => "-Wl,-rpath,#{install_dir}/embedded/lib -L#{install_dir}/embedded/lib", "CGO_CFLAGS" => "-I. -I#{install_dir}/embedded/include", "CGO_LDFLAGS" => "-Wl,-rpath,#{install_dir}/embedded/lib -L#{install_dir}/embedded/lib" } major_version_arg = "$MAJOR_VERSION" - py_runtimes_arg = "$PY_RUNTIMES" end unless ENV["OMNIBUS_GOMODCACHE"].nil? || ENV["OMNIBUS_GOMODCACHE"].empty? @@ -83,13 +75,13 @@ do_windows_sysprobe = "--windows-sysprobe" end command "inv -e rtloader.clean" - command "inv -e rtloader.make --python-runtimes #{py_runtimes_arg} --install-prefix \"#{windows_safe_path(python_2_embedded)}\" --cmake-options \"-G \\\"Unix Makefiles\\\" \\\"-DPython3_EXECUTABLE=#{windows_safe_path(python_3_embedded)}\\python.exe\"\"", :env => env + command "inv -e rtloader.make --install-prefix \"#{windows_safe_path(python_2_embedded)}\" --cmake-options \"-G \\\"Unix Makefiles\\\" \\\"-DPython3_EXECUTABLE=#{windows_safe_path(python_3_embedded)}\\python.exe\"\"", :env => env command "mv rtloader/bin/*.dll #{install_dir}/bin/agent/" - command "inv -e agent.build --exclude-rtloader --python-runtimes #{py_runtimes_arg} --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded #{do_windows_sysprobe} --flavor #{flavor_arg}", env: env + command "inv -e agent.build --exclude-rtloader --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded #{do_windows_sysprobe} --flavor #{flavor_arg}", env: env command "inv -e systray.build --major-version #{major_version_arg} --rebuild", env: env else command "inv -e rtloader.clean" - command "inv -e rtloader.make --python-runtimes #{py_runtimes_arg} --install-prefix \"#{install_dir}/embedded\" --cmake-options '-DCMAKE_CXX_FLAGS:=\"-D_GLIBCXX_USE_CXX11_ABI=0\" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_FIND_FRAMEWORK:STRING=NEVER -DPython#{py_runtimes_arg}_EXECUTABLE=#{install_dir}/embedded/bin/python#{py_runtimes_arg}'", :env => env + command "inv -e rtloader.make --install-prefix \"#{install_dir}/embedded\" --cmake-options '-DCMAKE_CXX_FLAGS:=\"-D_GLIBCXX_USE_CXX11_ABI=0\" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_FIND_FRAMEWORK:STRING=NEVER -DPython3_EXECUTABLE=#{install_dir}/embedded/bin/python3'", :env => env command "inv -e rtloader.install" bundle_arg = bundled_agents ? bundled_agents.map { |k| "--bundle #{k}" }.join(" ") : "--bundle agent" @@ -97,10 +89,10 @@ if linux_target? include_sds = "--include-sds" # we only support SDS on Linux targets for now end - command "inv -e agent.build --exclude-rtloader #{include_sds} --python-runtimes #{py_runtimes_arg} --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{default_install_dir}/embedded --python-home-2=#{default_install_dir}/embedded --python-home-3=#{default_install_dir}/embedded --flavor #{flavor_arg} #{bundle_arg}", env: env + command "inv -e agent.build --exclude-rtloader #{include_sds} --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{default_install_dir}/embedded --python-home-2=#{default_install_dir}/embedded --python-home-3=#{default_install_dir}/embedded --flavor #{flavor_arg} #{bundle_arg}", env: env if heroku_target? - command "inv -e agent.build --exclude-rtloader --python-runtimes #{py_runtimes_arg} --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded --python-home-2=#{install_dir}/embedded --python-home-3=#{install_dir}/embedded --flavor #{flavor_arg} --agent-bin=bin/agent/core-agent --bundle agent", env: env + command "inv -e agent.build --exclude-rtloader --major-version #{major_version_arg} --rebuild --no-development --install-path=#{install_dir} --embedded-path=#{install_dir}/embedded --python-home-2=#{install_dir}/embedded --python-home-3=#{install_dir}/embedded --flavor #{flavor_arg} --agent-bin=bin/agent/core-agent --bundle agent", env: env end end @@ -131,7 +123,7 @@ if not bundled_agents.include? "trace-agent" platform = windows_arch_i386? ? "x86" : "x64" - command "invoke trace-agent.build --python-runtimes #{py_runtimes_arg} --install-path=#{install_dir} --major-version #{major_version_arg} --flavor #{flavor_arg}", :env => env + command "invoke trace-agent.build --install-path=#{install_dir} --major-version #{major_version_arg} --flavor #{flavor_arg}", :env => env end if windows_target? @@ -142,7 +134,7 @@ # Process agent if not bundled_agents.include? "process-agent" - command "invoke -e process-agent.build --python-runtimes #{py_runtimes_arg} --install-path=#{install_dir} --major-version #{major_version_arg} --flavor #{flavor_arg} --no-bundle", :env => env + command "invoke -e process-agent.build --install-path=#{install_dir} --major-version #{major_version_arg} --flavor #{flavor_arg} --no-bundle", :env => env end if windows_target? diff --git a/omnibus/config/software/datadog-cf-finalize.rb b/omnibus/config/software/datadog-cf-finalize.rb index 32714e87f86b0..5de3c3d423c46 100644 --- a/omnibus/config/software/datadog-cf-finalize.rb +++ b/omnibus/config/software/datadog-cf-finalize.rb @@ -34,9 +34,6 @@ copy "#{install_dir}/bin/agent/agent.exe", "#{cf_bin_root_bin}" copy "#{install_dir}/bin/agent/libdatadog-agent-three.dll", "#{cf_bin_root_bin}" - if with_python_runtime? "2" - copy "#{install_dir}/bin/agent/libdatadog-agent-two.dll", "#{cf_bin_root_bin}" - end copy "#{install_dir}/bin/agent/process-agent.exe", "#{cf_bin_root_bin}/agent" copy "#{install_dir}/bin/agent/trace-agent.exe", "#{cf_bin_root_bin}/agent" diff --git a/omnibus/config/software/datadog-iot-agent.rb b/omnibus/config/software/datadog-iot-agent.rb index ac2b4887928bc..afabbd0835188 100644 --- a/omnibus/config/software/datadog-iot-agent.rb +++ b/omnibus/config/software/datadog-iot-agent.rb @@ -33,15 +33,13 @@ if windows_target? major_version_arg = "%MAJOR_VERSION%" - py_runtimes_arg = "%PY_RUNTIMES%" else major_version_arg = "$MAJOR_VERSION" - py_runtimes_arg = "$PY_RUNTIMES" env['CGO_CFLAGS'] = "-I#{install_dir}/embedded/include" end if linux_target? - command "invoke agent.build --flavor iot --rebuild --no-development --python-runtimes #{py_runtimes_arg} --major-version #{major_version_arg}", env: env + command "invoke agent.build --flavor iot --rebuild --no-development --major-version #{major_version_arg}", env: env mkdir "#{install_dir}/bin" mkdir "#{install_dir}/run/" @@ -63,7 +61,7 @@ mkdir conf_dir mkdir "#{install_dir}/bin/agent" - command "inv agent.build --flavor iot --rebuild --no-development --python-runtimes #{py_runtimes_arg} --major-version #{major_version_arg}", env: env + command "inv agent.build --flavor iot --rebuild --no-development --major-version #{major_version_arg}", env: env # move around bin and config files move 'bin/agent/dist/datadog.yaml', "#{conf_dir}/datadog.yaml.example" diff --git a/omnibus/config/software/kubernetes-py2.rb b/omnibus/config/software/kubernetes-py2.rb deleted file mode 100644 index afd3a55f87369..0000000000000 --- a/omnibus/config/software/kubernetes-py2.rb +++ /dev/null @@ -1,18 +0,0 @@ -name "kubernetes-py2" -default_version "18.20.0" - -dependency "pip2" -dependency "pyyaml-py2" - -source :url => "https://files.pythonhosted.org/packages/9c/f8/0eb10c6939b77788c10449d47d85a4740bb4a5608e1a504807fcdb5babd0/kubernetes-#{version}.tar.gz", - :sha256 => "0c72d00e7883375bd39ae99758425f5e6cb86388417cf7cc84305c211b2192cf", - :extract => :seven_zip - -relative_path "kubernetes-#{version}" - -build do - license "Apache-2.0" - license_file "./LICENSE.txt" - - command "#{install_dir}/embedded/bin/pip2 install ." -end diff --git a/omnibus/config/software/pip2.rb b/omnibus/config/software/pip2.rb deleted file mode 100644 index e05ceb8530227..0000000000000 --- a/omnibus/config/software/pip2.rb +++ /dev/null @@ -1,34 +0,0 @@ -name "pip2" -default_version "20.3.3" - -dependency "setuptools2" - - -source :url => "https://github.com/pypa/pip/archive/#{version}.tar.gz", - :sha256 => "016f8d509871b72fb05da911db513c11059d8a99f4591dda3050a3cf83a29a79", - :extract => :seven_zip - -relative_path "pip-#{version}" - -build do - license "MIT" - license_file "https://raw.githubusercontent.com/pypa/pip/main/LICENSE.txt" - - patch :source => "remove-python27-deprecation-warning.patch", :target => "src/pip/_internal/cli/base_command.py" - - if ohai["platform"] == "windows" - python_bin = "#{windows_safe_path(python_2_embedded)}\\python.exe" - python_prefix = "#{windows_safe_path(python_2_embedded)}" - else - python_bin = "#{install_dir}/embedded/bin/python2" - python_prefix = "#{install_dir}/embedded" - end - - command "#{python_bin} setup.py install --prefix=#{python_prefix}" - - if ohai["platform"] != "windows" - block do - FileUtils.rm_f(Dir.glob("#{install_dir}/embedded/lib/python2.7/site-packages/pip-*-py2.7.egg/pip/_vendor/distlib/*.exe")) - end - end -end diff --git a/omnibus/config/software/pylint2.rb b/omnibus/config/software/pylint2.rb deleted file mode 100644 index 4e5fdabfe34ef..0000000000000 --- a/omnibus/config/software/pylint2.rb +++ /dev/null @@ -1,37 +0,0 @@ -name "pylint2" -# Ship 1.x as 2.x only supports python 3 -default_version "1.9.5" - -dependency "pip2" - -build do - # pylint is only called in a subprocess by the Agent, so the Agent doesn't have to be GPL as well - license "GPL-2.0" - - # aliases for the pips - if windows_target? - pip2 = "#{windows_safe_path(python_2_embedded)}\\Scripts\\pip.exe" - python2 = "#{windows_safe_path(python_2_embedded)}\\python.exe" - else - pip2 = "#{install_dir}/embedded/bin/pip2" - python2 = "#{install_dir}/embedded/bin/python2" - end - - # If a python_mirror was set, it's passed through a pip config file so that we're not leaking the API key in the CI Output - # Else the pip config file so pip will act casually - pip_config_file = ENV['PIP_CONFIG_FILE'] - build_env = { - "PIP_CONFIG_FILE" => "#{pip_config_file}" - } - - # pin 2 dependencies of pylint: - # - configparser: later versions (up to v3.7.1) are broken - # - lazy-object-proxy 1.7.0 broken on python 2 https://github.com/ionelmc/python-lazy-object-proxy/issues/61 - if windows_target? - command "#{python2} -m pip install configparser==3.5.0 lazy-object-proxy==1.6.0", :env => build_env - command "#{python2} -m pip install pylint==#{version}", :env => build_env - else - command "#{pip2} install configparser==3.5.0 lazy-object-proxy==1.6.0", :env => build_env - command "#{pip2} install pylint==#{version}", :env => build_env - end -end diff --git a/omnibus/config/software/pympler.rb b/omnibus/config/software/pympler.rb index 367a8d182e059..fdfcdd4d00f0d 100644 --- a/omnibus/config/software/pympler.rb +++ b/omnibus/config/software/pympler.rb @@ -9,34 +9,17 @@ name 'pympler' default_version "0.7" -if with_python_runtime? "3" - dependency 'pip3' - dependency 'setuptools3' -end - -if with_python_runtime? "2" - dependency 'pip2' -end +dependency 'pip3' +dependency 'setuptools3' pympler_requirement = "pympler==#{default_version}" build do - if with_python_runtime? "3" - if windows_target? - python = "#{windows_safe_path(python_3_embedded)}\\python.exe" - else - python = "#{install_dir}/embedded/bin/python3" - end - command "#{python} -m pip install #{pympler_requirement}" - end - - if with_python_runtime? "2" - if windows_target? - python = "#{windows_safe_path(python_2_embedded)}\\python.exe" - else - python = "#{install_dir}/embedded/bin/python2" - end - command "#{python} -m pip install #{pympler_requirement}" + if windows_target? + python = "#{windows_safe_path(python_3_embedded)}\\python.exe" + else + python = "#{install_dir}/embedded/bin/python3" end + command "#{python} -m pip install #{pympler_requirement}" end diff --git a/omnibus/config/software/python2.rb b/omnibus/config/software/python2.rb deleted file mode 100644 index f35a7275e1cd6..0000000000000 --- a/omnibus/config/software/python2.rb +++ /dev/null @@ -1,101 +0,0 @@ -# -# Copyright:: Copyright (c) 2013-2014 Chef Software, Inc. -# License:: Apache License, Version 2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name "python2" - -if ohai["platform"] != "windows" - default_version "2.7.18" - - dependency "libxcrypt" - dependency "ncurses" - dependency "zlib" - dependency "openssl" - dependency "bzip2" - dependency "libsqlite3" - dependency "libyaml" - - source :url => "http://python.org/ftp/python/#{version}/Python-#{version}.tgz", - :sha256 => "da3080e3b488f648a3d7a4560ddee895284c3380b11d6de75edb986526b9a814" - - relative_path "Python-#{version}" - - env = with_standard_compiler_flags(with_embedded_path) - env['CFLAGS'] << ' -fPIC' - - python_configure_options = ["--with-ensurepip=no"] # pip is installed separately by its own software def - - if mac_os_x? - python_configure_options.push("--enable-ipv6", - "--with-universal-archs=intel", - "--enable-shared", - "--without-gcc", - "CC=clang") - elsif linux_target? - python_configure_options.push("--enable-unicode=ucs4", - "--enable-shared") - end - - build do - # 2.0 is the license version here, not the python version - license "Python-2.0" - - patch :source => "avoid-allocating-thunks-in-ctypes.patch" if linux_target? - patch :source => "fix-platform-ubuntu.diff" if linux_target? - patch :source => "0001-disable-multiarch.patch" if linux_target? - # security patches backported by the debian community - # see: http://deb.debian.org/debian/pool/main/p/python2.7/python2.7_2.7.18-6.diff.gz - patch :source => "python2.7_2.7.18-cve-2019-20907.diff" unless windows_target? - patch :source => "python2.7_2.7.18-cve-2020-8492.diff" unless windows_target? - patch :source => "python2.7_2.7.18-cve-2021-3177.diff" unless windows_target? - - configure(*python_configure_options, :env => env) - command "make -j #{workers}", :env => env - command "make install", :env => env - delete "#{install_dir}/embedded/lib/python2.7/test" - - move "#{install_dir}/embedded/bin/2to3", "#{install_dir}/embedded/bin/2to3-2.7" - - block do - FileUtils.rm_f(Dir.glob("#{install_dir}/embedded/lib/python2.7/lib-dynload/readline.*")) - FileUtils.rm_f(Dir.glob("#{install_dir}/embedded/lib/python2.7/lib-dynload/gdbm.so")) - FileUtils.rm_f(Dir.glob("#{install_dir}/embedded/lib/python2.7/lib-dynload/dbm.so")) - FileUtils.rm_f(Dir.glob("#{install_dir}/embedded/lib/python2.7/distutils/command/wininst-*.exe")) - end - end - -else - default_version "2.7.18-8829519" - dependency "vc_redist" - - if windows_arch_i386? - source :url => "https://dd-agent-omnibus.s3.amazonaws.com/python-windows-#{version}-x86.zip", - :sha256 => "295F16FB166AC26624AE9CBA08666DB437E0B8DDBB8D8D987F0598B71E4B6B24".downcase, - :extract => :seven_zip - else - source :url => "https://dd-agent-omnibus.s3.amazonaws.com/python-windows-#{version}-x64.zip", - :sha256 => "58424EEB272E5678E732402CAF150124CD583B81F5DA442C911CE71A63ECD339".downcase, - :extract => :seven_zip - end - build do - # 2.0 is the license version here, not the python version - license "Python-2.0" - - # - # expand python zip into the embedded directory - command "XCOPY /YEHIR *.* \"#{windows_safe_path(python_2_embedded)}\"" - end -end diff --git a/omnibus/config/software/pyyaml-py2.rb b/omnibus/config/software/pyyaml-py2.rb deleted file mode 100644 index c4ee7f6e9735b..0000000000000 --- a/omnibus/config/software/pyyaml-py2.rb +++ /dev/null @@ -1,19 +0,0 @@ -name "pyyaml-py2" -default_version "5.4.1" - -dependency "pip2" - -source :url => "https://github.com/yaml/pyyaml/archive/refs/tags/#{version}.tar.gz", - :sha256 => "75f966559c5f262dfc44da0f958cc2aa18953ae5021f2c3657b415c5a370045f", - :extract => :seven_zip - -relative_path "pyyaml-#{version}" - -build do - license "MIT" - license_file "./LICENSE.txt" - - command "sed -i 's/Cython/Cython<3.0.0/g' pyproject.toml" - - command "#{install_dir}/embedded/bin/pip2 install ." -end diff --git a/omnibus/config/software/setuptools2.rb b/omnibus/config/software/setuptools2.rb deleted file mode 100644 index a02111346eb8d..0000000000000 --- a/omnibus/config/software/setuptools2.rb +++ /dev/null @@ -1,28 +0,0 @@ -name "setuptools2" -default_version "40.9.0" - -skip_transitive_dependency_licensing true - -dependency "python2" - -relative_path "setuptools-#{version}" - -source :url => "https://github.com/pypa/setuptools/archive/v#{version}.tar.gz", - :sha256 => "9ef6623c057d6e46ada8156bb48dc72ef6dbe721768720cc66966cca4097061c", - :extract => :seven_zip - -build do - # 2.0 is the license version here, not the python version - license "Python-2.0" - - if ohai["platform"] == "windows" - python_bin = "#{windows_safe_path(python_2_embedded)}\\python.exe" - python_prefix = "#{windows_safe_path(python_2_embedded)}" - else - python_bin = "#{install_dir}/embedded/bin/python2" - python_prefix = "#{install_dir}/embedded" - end - - command "#{python_bin} bootstrap.py" - command "#{python_bin} setup.py install --prefix=#{python_prefix}" -end diff --git a/omnibus/lib/ostools.rb b/omnibus/lib/ostools.rb index cc5722a9e8b6b..13d8d26e06e67 100644 --- a/omnibus/lib/ostools.rb +++ b/omnibus/lib/ostools.rb @@ -63,7 +63,3 @@ def os end end -def with_python_runtime?(runtime) - python_runtimes = ENV['PY_RUNTIMES'].nil? ? ['3'] : ENV['PY_RUNTIMES'].split(',') - return python_runtimes.include? runtime -end diff --git a/pkg-config/README.md b/pkg-config/README.md deleted file mode 100644 index 5a7bd31b52ba7..0000000000000 --- a/pkg-config/README.md +++ /dev/null @@ -1,24 +0,0 @@ -## pkg-config modules - -This directory contains the `pkg-config` module files of the dependencies of the project, by platform and -"build type" (i.e. using either system libraries or embedded libraries). - -During a build, pkg-config will search for a module file in these directories, in this order: -1. `/embedded/` (unless `USE_SYSTEM_LIBS` is passed to the rake build command) -2. `/system/` -3. default pkg-config paths of the environment - -The "embedded/" module files define the flags to compile and link against the libraries -provided in the datadog-agent package. - -The "system/" files should only be present for dependencies that don't provide pkg-config files -for the platform. They define the flags to compile and link against the libraries provided by the system. - -`cgo` uses `pkg-config` to determine which compiler and linker flags to use when the following directive -is present in the go source file: - -``` -// #cgo pkg-config -``` - -(see https://golang.org/cmd/cgo/ for more details on `cgo` usage) diff --git a/pkg-config/darwin/embedded/python-2.7.pc b/pkg-config/darwin/embedded/python-2.7.pc deleted file mode 100644 index b180204361c84..0000000000000 --- a/pkg-config/darwin/embedded/python-2.7.pc +++ /dev/null @@ -1,13 +0,0 @@ -prefix=/opt/datadog-agent/embedded -exec_prefix=${prefix} -libdir=${exec_prefix}/lib -includedir=${prefix}/include -osx_include=/System/Library/Frameworks/Python.framework/Headers - -Name: Python -Description: Python library -Requires: -Version: 2.7 -Libs.private: -ldl -Libs: -L${libdir} -lpython2.7 -Cflags: -I${includedir}/python2.7 diff --git a/pkg-config/linux/embedded/python-2.7.pc b/pkg-config/linux/embedded/python-2.7.pc deleted file mode 100644 index 44c19be883ecf..0000000000000 --- a/pkg-config/linux/embedded/python-2.7.pc +++ /dev/null @@ -1,11 +0,0 @@ -prefix=/opt/datadog-agent/embedded -exec_prefix=${prefix} -libdir=${exec_prefix}/lib -includedir=${prefix}/include - -Name: Python -Description: Python library -Requires: -Version: 2.7 -Libs: -L${libdir} -lpython2.7 -lpthread -ldl -lutil -lm -Cflags: -I${includedir}/python2.7 diff --git a/pkg-config/windows/embedded/python-2.7.pc b/pkg-config/windows/embedded/python-2.7.pc deleted file mode 100644 index b6f11bd55b0ad..0000000000000 --- a/pkg-config/windows/embedded/python-2.7.pc +++ /dev/null @@ -1,11 +0,0 @@ -prefix=C:/opt/datadog-agent/embedded -exec_prefix=${prefix} -libdir=${exec_prefix}/libs -includedir=${prefix}/include - -Name: Python -Description: Python library -Requires: -Version: 2.7 -Libs: -L${libdir} -lpython27 -lpthread -lm -Cflags: -I${includedir} -DMS_WIN64 diff --git a/pkg-config/windows/system/python-2.7.pc b/pkg-config/windows/system/python-2.7.pc deleted file mode 100644 index 31415a879aad3..0000000000000 --- a/pkg-config/windows/system/python-2.7.pc +++ /dev/null @@ -1,11 +0,0 @@ -prefix=C:/python27-x64 -exec_prefix=${prefix} -libdir=${exec_prefix}/libs -includedir=${prefix}/include - -Name: Python -Description: Python library -Requires: -Version: 2.7 -Libs: -L${libdir} -lpython27 -lpthread -lm -Cflags: -I${includedir} -DMS_WIN64 diff --git a/pkg/cli/subcommands/check/command.go b/pkg/cli/subcommands/check/command.go index a9788a143c7f1..56f224389b455 100644 --- a/pkg/cli/subcommands/check/command.go +++ b/pkg/cli/subcommands/check/command.go @@ -254,7 +254,7 @@ func run( cliParams *cliParams, demultiplexer demultiplexer.Component, wmeta workloadmeta.Component, - _ tagger.Component, + tagger tagger.Component, ac autodiscovery.Component, secretResolver secrets.Component, agentAPI internalAPI.Component, @@ -289,7 +289,7 @@ func run( // TODO: (components) - Until the checks are components we set there context so they can depends on components. check.InitializeInventoryChecksContext(invChecks) pkgcollector.InitPython(common.GetPythonPaths()...) - commonchecks.RegisterChecks(wmeta, config, telemetry) + commonchecks.RegisterChecks(wmeta, tagger, config, telemetry) common.LoadComponents(secretResolver, wmeta, ac, pkgconfigsetup.Datadog().GetString("confd_path")) ac.LoadAndRun(context.Background()) diff --git a/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_apiserver.go b/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_apiserver.go index 5bb3ec879312b..37113ed5d1ae0 100644 --- a/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_apiserver.go +++ b/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_apiserver.go @@ -120,6 +120,7 @@ type KubeASCheck struct { eventCollection eventCollection ac *apiserver.APIClient oshiftAPILevel apiserver.OpenShiftAPILevel + tagger tagger.Component } func (c *KubeASConfig) parse(data []byte) error { @@ -133,20 +134,25 @@ func (c *KubeASConfig) parse(data []byte) error { } // NewKubeASCheck returns a new KubeASCheck -func NewKubeASCheck(base core.CheckBase, instance *KubeASConfig) *KubeASCheck { +func NewKubeASCheck(base core.CheckBase, instance *KubeASConfig, tagger tagger.Component) *KubeASCheck { return &KubeASCheck{ CheckBase: base, instance: instance, + tagger: tagger, } } // Factory creates a new check factory -func Factory() optional.Option[func() check.Check] { - return optional.NewOption(newCheck) +func Factory(tagger tagger.Component) optional.Option[func() check.Check] { + return optional.NewOption( + func() check.Check { + return newCheck(tagger) + }, + ) } -func newCheck() check.Check { - return NewKubeASCheck(core.NewCheckBase(CheckName), &KubeASConfig{}) +func newCheck(tagger tagger.Component) check.Check { + return NewKubeASCheck(core.NewCheckBase(CheckName), &KubeASConfig{}, tagger) } // Configure parses the check configuration and init the check. @@ -188,9 +194,9 @@ func (k *KubeASCheck) Configure(senderManager sender.SenderManager, _ uint64, co } if k.instance.UnbundleEvents { - k.eventCollection.Transformer = newUnbundledTransformer(clusterName, tagger.GetTaggerInstance(), k.instance.CollectedEventTypes, k.instance.BundleUnspecifiedEvents, k.instance.FilteringEnabled) + k.eventCollection.Transformer = newUnbundledTransformer(clusterName, k.tagger, k.instance.CollectedEventTypes, k.instance.BundleUnspecifiedEvents, k.instance.FilteringEnabled) } else { - k.eventCollection.Transformer = newBundledTransformer(clusterName, tagger.GetTaggerInstance(), k.instance.CollectedEventTypes, k.instance.FilteringEnabled) + k.eventCollection.Transformer = newBundledTransformer(clusterName, k.tagger, k.instance.CollectedEventTypes, k.instance.FilteringEnabled) } return nil diff --git a/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_apiserver_test.go b/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_apiserver_test.go index a345aaaedcae4..ca421c88b97cb 100644 --- a/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_apiserver_test.go +++ b/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_apiserver_test.go @@ -14,6 +14,7 @@ import ( v1 "k8s.io/api/core/v1" obj "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" core "github.com/DataDog/datadog-agent/pkg/collector/corechecks" "github.com/DataDog/datadog-agent/pkg/metrics/servicecheck" @@ -82,8 +83,10 @@ func TestParseComponentStatus(t *testing.T) { Items: nil, } + tagger := taggerimpl.SetupFakeTagger(t) + // FIXME: use the factory instead - kubeASCheck := NewKubeASCheck(core.NewCheckBase(CheckName), &KubeASConfig{}) + kubeASCheck := NewKubeASCheck(core.NewCheckBase(CheckName), &KubeASConfig{}, tagger) mocked := mocksender.NewMockSender(kubeASCheck.ID()) mocked.On("ServiceCheck", "kube_apiserver_controlplane.up", servicecheck.ServiceCheckOK, "", []string{"component:Zookeeper"}, "imok") diff --git a/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_openshift_test.go b/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_openshift_test.go index d410ec83c7724..9145bd72f3b24 100644 --- a/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_openshift_test.go +++ b/pkg/collector/corechecks/cluster/kubernetesapiserver/kubernetes_openshift_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/comp/core/tagger/taggerimpl" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -33,9 +34,11 @@ func TestReportClusterQuotas(t *testing.T) { pkgconfigsetup.Datadog().SetWithoutSource("cluster_name", "test-cluster-name") defer pkgconfigsetup.Datadog().SetWithoutSource("cluster_name", prevClusterName) + tagger := taggerimpl.SetupFakeTagger(t) + instanceCfg := []byte("") initCfg := []byte("") - kubeASCheck := newCheck().(*KubeASCheck) + kubeASCheck := newCheck(tagger).(*KubeASCheck) err = kubeASCheck.Configure(aggregator.NewNoOpSenderManager(), integration.FakeConfigHash, instanceCfg, initCfg, "test") require.NoError(t, err) diff --git a/pkg/collector/corechecks/cluster/kubernetesapiserver/stub.go b/pkg/collector/corechecks/cluster/kubernetesapiserver/stub.go index 8e34be6187ced..c43a33bfd3275 100644 --- a/pkg/collector/corechecks/cluster/kubernetesapiserver/stub.go +++ b/pkg/collector/corechecks/cluster/kubernetesapiserver/stub.go @@ -9,6 +9,7 @@ package kubernetesapiserver import ( + "github.com/DataDog/datadog-agent/comp/core/tagger" "github.com/DataDog/datadog-agent/pkg/collector/check" "github.com/DataDog/datadog-agent/pkg/util/optional" ) @@ -19,6 +20,6 @@ const ( ) // Factory creates a new check factory -func Factory() optional.Option[func() check.Check] { +func Factory(_ tagger.Component) optional.Option[func() check.Check] { return optional.NewNoneOption[func() check.Check]() } diff --git a/pkg/collector/corechecks/ebpf/ebpf.go b/pkg/collector/corechecks/ebpf/ebpf.go index 4e9e6fe65a3cf..c7c2eb17d80a4 100644 --- a/pkg/collector/corechecks/ebpf/ebpf.go +++ b/pkg/collector/corechecks/ebpf/ebpf.go @@ -39,7 +39,7 @@ type EBPFCheckConfig struct { // EBPFCheck grabs eBPF map/program/perf buffer metrics type EBPFCheck struct { config *EBPFCheckConfig - sysProbeUtil *processnet.RemoteSysProbeUtil + sysProbeUtil processnet.SysProbeUtil core.CheckBase } diff --git a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go index d3cd6aab4cf12..231d7c2e027c4 100644 --- a/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go +++ b/pkg/collector/corechecks/ebpf/probe/ebpfcheck/probe.go @@ -20,6 +20,7 @@ import ( "unsafe" manager "github.com/DataDog/ebpf-manager" + "github.com/cihub/seelog" "github.com/cilium/ebpf" "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/features" @@ -305,9 +306,11 @@ func (k *Probe) getProgramStats(stats *model.EBPFStats) error { stats.Programs = append(stats.Programs, ps) } - log.Tracef("found %d programs", len(stats.Programs)) - for _, ps := range stats.Programs { - log.Tracef("name=%s prog_id=%d type=%s", ps.Name, ps.ID, ps.Type.String()) + if log.ShouldLog(seelog.TraceLvl) { + log.Tracef("found %d programs", len(stats.Programs)) + for _, ps := range stats.Programs { + log.Tracef("name=%s prog_id=%d type=%s", ps.Name, ps.ID, ps.Type.String()) + } } return nil diff --git a/pkg/collector/corechecks/gpu/config.go b/pkg/collector/corechecks/gpu/config.go new file mode 100644 index 0000000000000..d534be2f0ba2d --- /dev/null +++ b/pkg/collector/corechecks/gpu/config.go @@ -0,0 +1,16 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +// Package gpu defines the agent corecheck for +// the GPU integration +package gpu + +// CheckName defines the name of the +// GPU check +const CheckName = "gpu" + +// CheckConfig holds the configuration for the GPU check +type CheckConfig struct { +} diff --git a/pkg/collector/corechecks/gpu/gpu.go b/pkg/collector/corechecks/gpu/gpu.go new file mode 100644 index 0000000000000..5785acdbd50a7 --- /dev/null +++ b/pkg/collector/corechecks/gpu/gpu.go @@ -0,0 +1,212 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +package gpu + +import ( + "fmt" + "time" + + "gopkg.in/yaml.v2" + + "github.com/NVIDIA/go-nvml/pkg/nvml" + + sysconfig "github.com/DataDog/datadog-agent/cmd/system-probe/config" + "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" + "github.com/DataDog/datadog-agent/pkg/aggregator/sender" + "github.com/DataDog/datadog-agent/pkg/collector/check" + core "github.com/DataDog/datadog-agent/pkg/collector/corechecks" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu/model" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + processnet "github.com/DataDog/datadog-agent/pkg/process/net" + sectime "github.com/DataDog/datadog-agent/pkg/security/resolvers/time" + "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/DataDog/datadog-agent/pkg/util/optional" +) + +// Check represents the GPU check that will be periodically executed via the Run() function +type Check struct { + core.CheckBase + config *CheckConfig // config for the check + sysProbeUtil processnet.SysProbeUtil // sysProbeUtil is used to communicate with system probe + lastCheckTime time.Time // lastCheckTime is the time the last check was done, used to compute GPU average utilization + timeResolver *sectime.Resolver // TODO: Move resolver to a common package, see EBPF-579 + statProcessors map[uint32]*StatsProcessor // statProcessors is a map of processors, one per pid + gpuDevices []gpuDevice // gpuDevices is a list of GPU devices found in the host +} + +// Factory creates a new check factory +func Factory() optional.Option[func() check.Check] { + return optional.NewOption(newCheck) +} + +func newCheck() check.Check { + return &Check{ + CheckBase: core.NewCheckBase(CheckName), + config: &CheckConfig{}, + statProcessors: make(map[uint32]*StatsProcessor), + } +} + +// Cancel cancels the check +func (m *Check) Cancel() { + m.CheckBase.Cancel() + + ret := nvml.Shutdown() + if ret != nvml.SUCCESS && ret != nvml.ERROR_UNINITIALIZED { + log.Warnf("Failed to shutdown NVML: %v", nvml.ErrorString(ret)) + } +} + +// Configure parses the check configuration and init the check +func (m *Check) Configure(senderManager sender.SenderManager, _ uint64, config, initConfig integration.Data, source string) error { + if err := m.CommonConfigure(senderManager, initConfig, config, source); err != nil { + return err + } + if err := yaml.Unmarshal(config, m.config); err != nil { + return fmt.Errorf("invalid gpu check config: %w", err) + } + + return nil +} + +func (m *Check) ensureInitialized() error { + var err error + + if m.sysProbeUtil == nil { + m.sysProbeUtil, err = processnet.GetRemoteSystemProbeUtil( + pkgconfigsetup.SystemProbe().GetString("system_probe_config.sysprobe_socket"), + ) + if err != nil { + return fmt.Errorf("sysprobe connection: %w", err) + } + } + + if m.timeResolver == nil { + m.timeResolver, err = sectime.NewResolver() + if err != nil { + return fmt.Errorf("cannot create time resolver: %w", err) + } + } + + if m.gpuDevices == nil { + m.gpuDevices, err = getGPUDevices() + if err != nil { + return fmt.Errorf("cannot get GPU devices: %w", err) + } + } + + return nil +} + +// ensureProcessor ensures that there is a stats processor for the given key +func (m *Check) ensureProcessor(key *model.StreamKey, snd sender.Sender, gpuThreads int, checkDuration time.Duration) { + if _, ok := m.statProcessors[key.Pid]; !ok { + m.statProcessors[key.Pid] = &StatsProcessor{ + key: key, + } + } + + m.statProcessors[key.Pid].totalThreadSecondsUsed = 0 + m.statProcessors[key.Pid].sender = snd + m.statProcessors[key.Pid].gpuMaxThreads = gpuThreads + m.statProcessors[key.Pid].measuredInterval = checkDuration + m.statProcessors[key.Pid].timeResolver = m.timeResolver + m.statProcessors[key.Pid].lastCheck = m.lastCheckTime +} + +// Run executes the check +func (m *Check) Run() error { + if err := m.ensureInitialized(); err != nil { + return err + } + if len(m.gpuDevices) == 0 { + log.Warnf("No GPU devices found") + return nil + } + + sysprobeData, err := m.sysProbeUtil.GetCheck(sysconfig.GPUMonitoringModule) + if err != nil { + return fmt.Errorf("cannot get data from system-probe: %w", err) + } + now := time.Now() + + var checkDuration time.Duration + // mark the check duration as close to the actual check as possible + if !m.lastCheckTime.IsZero() { + checkDuration = now.Sub(m.lastCheckTime) + } + + snd, err := m.GetSender() + if err != nil { + return fmt.Errorf("get metric sender: %w", err) + } + + // Commit the metrics even in case of an error + defer snd.Commit() + + stats, ok := sysprobeData.(model.GPUStats) + if !ok { + return log.Errorf("gpu check raw data has incorrect type: %T", stats) + } + + // TODO: Multiple GPUs are not supported yet + gpuThreads, err := m.gpuDevices[0].GetMaxThreads() + if err != nil { + return fmt.Errorf("get GPU device threads: %w", err) + } + + for _, data := range stats.CurrentData { + m.ensureProcessor(&data.Key, snd, gpuThreads, checkDuration) + m.statProcessors[data.Key.Pid].processCurrentData(data) + } + + for _, data := range stats.PastData { + m.ensureProcessor(&data.Key, snd, gpuThreads, checkDuration) + m.statProcessors[data.Key.Pid].processPastData(data) + } + + m.configureNormalizationFactor() + + for _, processor := range m.statProcessors { + if processor.hasPendingData { + err := processor.markInterval() + if err != nil { + return fmt.Errorf("mark interval: %w", err) + } + } else { + err := processor.finish(now) + // delete even in an error case, as we don't want to keep the processor around + delete(m.statProcessors, processor.key.Pid) + if err != nil { + return fmt.Errorf("finish processor: %w", err) + } + } + } + + m.lastCheckTime = now + + return nil +} + +func (m *Check) configureNormalizationFactor() { + // As we compute the utilization based on the number of threads launched by the kernel, we need to + // normalize the utilization if we get above 100%, as the GPU can enqueue threads. + totalGPUUtilization := 0.0 + for _, processor := range m.statProcessors { + // Only consider processors that received data this interval + if processor.hasPendingData { + totalGPUUtilization += processor.getGPUUtilization() + } + } + + normFactor := max(1.0, totalGPUUtilization) + + for _, processor := range m.statProcessors { + processor.setGPUUtilizationNormalizationFactor(normFactor) + } +} diff --git a/pkg/collector/corechecks/gpu/gpu_stub.go b/pkg/collector/corechecks/gpu/gpu_stub.go new file mode 100644 index 0000000000000..5aefc602175b9 --- /dev/null +++ b/pkg/collector/corechecks/gpu/gpu_stub.go @@ -0,0 +1,18 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build !linux + +package gpu + +import ( + "github.com/DataDog/datadog-agent/pkg/collector/check" + "github.com/DataDog/datadog-agent/pkg/util/optional" +) + +// Factory creates a new check factory +func Factory() optional.Option[func() check.Check] { + return optional.NewNoneOption[func() check.Check]() +} diff --git a/pkg/collector/corechecks/gpu/nvml.go b/pkg/collector/corechecks/gpu/nvml.go new file mode 100644 index 0000000000000..425dd52bd9572 --- /dev/null +++ b/pkg/collector/corechecks/gpu/nvml.go @@ -0,0 +1,89 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +// Package gpu defines the agent corecheck for +// the GPU integration +package gpu + +import ( + "errors" + "fmt" + "sync" + + "github.com/NVIDIA/go-nvml/pkg/nvml" +) + +var initOnce sync.Once + +type gpuDevice struct { + nvml.Device +} + +func wrapNvmlError(ret nvml.Return) error { + if ret == nvml.SUCCESS { + return nil + } + + return errors.New(nvml.ErrorString(ret)) +} + +func ensureNvmlInit() error { + var err error + initOnce.Do(func() { + err = wrapNvmlError(nvml.Init()) + }) + + return err +} + +func getGPUDevices() ([]gpuDevice, error) { + err := ensureNvmlInit() + if err != nil { + return nil, err + } + + count, ret := nvml.DeviceGetCount() + if err := wrapNvmlError(ret); err != nil { + return nil, fmt.Errorf("cannot get number of GPU devices: %w", err) + } + + var devices []gpuDevice + + for i := 0; i < count; i++ { + device, ret := nvml.DeviceGetHandleByIndex(i) + if err := wrapNvmlError(ret); err != nil { + return nil, fmt.Errorf("cannot get handle for GPU device %d: %w", i, err) + } + + devices = append(devices, gpuDevice{device}) + } + + return devices, nil +} + +func (d *gpuDevice) GetNumMultiprocessors() (int, error) { + devProps, ret := d.GetAttributes() + if err := wrapNvmlError(ret); err != nil { + return 0, fmt.Errorf("cannot get device attributes: %w", err) + } + + return int(devProps.MultiprocessorCount), nil +} + +// GetMaxThreads returns the maximum number of threads that can be run on the +// GPU. Each GPU core runs a thread, so this is the number of cores. Do not +// confuse the number of cores with the number of streaming multiprocessors +// (SM): the number of cores is equal to the number of SMs multiplied by the +// number of cores per SM. +func (d *gpuDevice) GetMaxThreads() (int, error) { + cores, ret := d.GetNumGpuCores() + if err := wrapNvmlError(ret); err != nil { + return 0, fmt.Errorf("cannot get number of GPU cores: %w", err) + } + + return int(cores), nil +} diff --git a/pkg/collector/corechecks/gpu/stats.go b/pkg/collector/corechecks/gpu/stats.go new file mode 100644 index 0000000000000..c055dcab74f83 --- /dev/null +++ b/pkg/collector/corechecks/gpu/stats.go @@ -0,0 +1,237 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux + +package gpu + +import ( + "fmt" + "math" + "time" + + "github.com/DataDog/datadog-agent/pkg/aggregator/sender" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu/model" + sectime "github.com/DataDog/datadog-agent/pkg/security/resolvers/time" +) + +const ( + gpuMetricsNs = "gpu." + metricNameMemory = gpuMetricsNs + "memory" + metricNameUtil = gpuMetricsNs + "utilization" + metricNameMaxMem = gpuMetricsNs + "max_memory" +) + +// StatsProcessor is responsible for processing the data from the GPU eBPF probe and generating metrics from it +type StatsProcessor struct { + // key is the key of the stream this processor is processing + key *model.StreamKey + + // totalThreadSecondsUsed is the total amount of thread-seconds used by the GPU in the current interval + totalThreadSecondsUsed float64 + + // sender is the sender to use to send metrics + sender sender.Sender + + // gpuMaxThreads is the maximum number of threads the GPU can run in parallel, for utilization computations + gpuMaxThreads int + + // lastCheck is the last time the processor was checked + lastCheck time.Time + + // lastMemPoint is the time of the last memory timeseries point sent + lastMemPointEpoch uint64 + + // measuredInterval is the interval between the last two checks + measuredInterval time.Duration + + // timeResolver is the time resolver to use to resolve timestamps + timeResolver *sectime.Resolver + + // currentAllocs is the list of current memory allocations + currentAllocs []*model.MemoryAllocation + + // pastAllocs is the list of past memory allocations + pastAllocs []*model.MemoryAllocation + + // lastKernelEnd is the timestamp of the last kernel end + lastKernelEnd time.Time + + // firstKernelStart is the timestamp of the first kernel start + firstKernelStart time.Time + + // sentEvents is the number of events sent by this processor + sentEvents int + + // maxTimestampLastMetric is the maximum timestamp of the last metric sent + maxTimestampLastMetric time.Time + + // utilizationNormFactor is the factor to normalize the utilization by, to account for the fact that we might have more kernels enqueued than the GPU can run in parallel. This factor + // allows distributing the utilization over all the streams that are enqueued + utilizationNormFactor float64 + + // hasPendingData is true if there is data pending to be sent + hasPendingData bool +} + +// processKernelSpan processes a kernel span +func (sp *StatsProcessor) processKernelSpan(span *model.KernelSpan) { + tsStart := sp.timeResolver.ResolveMonotonicTimestamp(span.StartKtime) + tsEnd := sp.timeResolver.ResolveMonotonicTimestamp(span.EndKtime) + + if sp.firstKernelStart.IsZero() || tsStart.Before(sp.firstKernelStart) { + sp.firstKernelStart = tsStart + } + + // we only want to consider data that was not already processed in the previous interval + if sp.lastCheck.After(tsStart) { + tsStart = sp.lastCheck + } + duration := tsEnd.Sub(tsStart) + sp.totalThreadSecondsUsed += duration.Seconds() * float64(min(span.AvgThreadCount, uint64(sp.gpuMaxThreads))) // we can't use more threads than the GPU has + if tsEnd.After(sp.lastKernelEnd) { + sp.lastKernelEnd = tsEnd + } +} + +func (sp *StatsProcessor) processPastData(data *model.StreamData) { + for _, span := range data.Spans { + sp.processKernelSpan(span) + } + + sp.pastAllocs = append(sp.pastAllocs, data.Allocations...) + sp.hasPendingData = true +} + +func (sp *StatsProcessor) processCurrentData(data *model.StreamData) { + for _, span := range data.Spans { + sp.processKernelSpan(span) + } + + sp.currentAllocs = data.Allocations + sp.hasPendingData = true +} + +// getTags returns the tags to use for the metrics +func (sp *StatsProcessor) getTags() []string { + return []string{ + fmt.Sprintf("pid:%d", sp.key.Pid), + } +} + +func (sp *StatsProcessor) getGPUUtilization() float64 { + intervalSecs := sp.measuredInterval.Seconds() + if intervalSecs > 0 { + availableThreadSeconds := float64(sp.gpuMaxThreads) * intervalSecs + return sp.totalThreadSecondsUsed / availableThreadSeconds + } + + return 0 +} + +func (sp *StatsProcessor) setGPUUtilizationNormalizationFactor(factor float64) { + sp.utilizationNormFactor = factor +} + +func (sp *StatsProcessor) markInterval() error { + intervalSecs := sp.measuredInterval.Seconds() + if intervalSecs > 0 { + utilization := sp.getGPUUtilization() / sp.utilizationNormFactor + + // if this is the first event, we need to send it with the timestamp of the first kernel start so that we have a complete series + if sp.sentEvents == 0 { + err := sp.sender.GaugeWithTimestamp(metricNameUtil, utilization, "", sp.getTags(), float64(sp.firstKernelStart.Unix())) + if err != nil { + return fmt.Errorf("cannot send metric: %w", err) + } + } + + // aftewards, we only need to update the utilization at the point of the last kernel end + err := sp.sender.GaugeWithTimestamp(metricNameUtil, utilization, "", sp.getTags(), float64(sp.lastKernelEnd.Unix())) + if err != nil { + return fmt.Errorf("cannot send metric: %w", err) + } + + if sp.lastKernelEnd.After(sp.maxTimestampLastMetric) { + sp.maxTimestampLastMetric = sp.lastKernelEnd + } + } + + var memTsBuilder tseriesBuilder + + firstUnfreedAllocKTime := uint64(math.MaxUint64) + + for _, alloc := range sp.currentAllocs { + firstUnfreedAllocKTime = min(firstUnfreedAllocKTime, alloc.StartKtime) + } + + for _, alloc := range sp.pastAllocs { + // Only build the timeseries up until the point of the first unfreed allocation. After that, the timeseries is still incomplete + // until all those allocations are freed. + if alloc.EndKtime < firstUnfreedAllocKTime { + startEpoch := sp.timeResolver.ResolveMonotonicTimestamp(alloc.StartKtime).Unix() + endEpoch := sp.timeResolver.ResolveMonotonicTimestamp(alloc.EndKtime).Unix() + memTsBuilder.AddEvent(uint64(startEpoch), uint64(endEpoch), int64(alloc.Size)) + } + } + + points, maxValue := memTsBuilder.Build() + tags := sp.getTags() + sentPoints := false + + for _, point := range points { + // Do not send points that are before the last check, those have been already sent + // Also do not send points that are 0, unless we have already sent some points, in which case + // we need them to close the series + if point.time > sp.lastMemPointEpoch && (point.value > 0 || sentPoints) { + err := sp.sender.GaugeWithTimestamp(metricNameMemory, float64(point.value), "", tags, float64(point.time)) + if err != nil { + return fmt.Errorf("cannot send metric: %w", err) + } + + if int64(point.time) > sp.maxTimestampLastMetric.Unix() { + sp.maxTimestampLastMetric = time.Unix(int64(point.time), 0) + } + + sentPoints = true + } + } + + if len(points) > 0 { + sp.lastMemPointEpoch = points[len(points)-1].time + } + + sp.sender.Gauge(metricNameMaxMem, float64(maxValue), "", tags) + sp.sentEvents++ + + sp.hasPendingData = false + + return nil +} + +// finish ensures that all metrics sent by this processor are properly closed with a 0 value +func (sp *StatsProcessor) finish(now time.Time) error { + lastTs := now + + // Don't mark events as lasting more than what they should. + if !sp.maxTimestampLastMetric.IsZero() { + lastTs = sp.maxTimestampLastMetric.Add(time.Second) + } + + err := sp.sender.GaugeWithTimestamp(metricNameMemory, 0, "", sp.getTags(), float64(lastTs.Unix())) + if err != nil { + return fmt.Errorf("cannot send metric: %w", err) + } + err = sp.sender.GaugeWithTimestamp(metricNameMaxMem, 0, "", sp.getTags(), float64(lastTs.Unix())) + if err != nil { + return fmt.Errorf("cannot send metric: %w", err) + } + err = sp.sender.GaugeWithTimestamp(metricNameUtil, 0, "", sp.getTags(), float64(lastTs.Unix())) + if err != nil { + return fmt.Errorf("cannot send metric: %w", err) + } + + return nil +} diff --git a/pkg/collector/corechecks/gpu/tseries.go b/pkg/collector/corechecks/gpu/tseries.go new file mode 100644 index 0000000000000..5abb42bfb3be8 --- /dev/null +++ b/pkg/collector/corechecks/gpu/tseries.go @@ -0,0 +1,68 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package gpu + +import ( + "slices" +) + +// tseriesBuilder is a helper to build a time series of events with duration +type tseriesBuilder struct { + points []tsPoint +} + +type tsPoint struct { + time uint64 + value int64 +} + +func (b *tseriesBuilder) AddEvent(startTime, endTime uint64, value int64) { + b.points = append(b.points, tsPoint{time: startTime, value: value}) + b.points = append(b.points, tsPoint{time: endTime, value: -value}) +} + +func (b *tseriesBuilder) AddEventStart(startTime uint64, value int64) { + b.points = append(b.points, tsPoint{time: startTime, value: value}) +} + +// buildTseries builds the time series, returning a slice of points and the max value in the interval +func (b *tseriesBuilder) Build() ([]tsPoint, int64) { + // sort by timestamp. Stable so that events that start and end at the same time are processed in the order they were added + slices.SortStableFunc(b.points, func(p1, p2 tsPoint) int { + return int(p1.time - p2.time) + }) + + maxValue := int64(0) + currentValue := int64(0) + + // Now we build the time series by doing a cumulative sum of the values at each point, accounting for the unit factor. + // Multiple points can end up at the same rounded timestamp. + tseries := make([]tsPoint, 0) + for i := range b.points { + // Check if we need to add a new point + currTime := b.points[i].time + if i > 0 { + prevTime := tseries[len(tseries)-1].time + + // We advanced past the last timeseries point, so create a new one + if currTime != prevTime { + tseries = append(tseries, tsPoint{time: currTime, value: 0}) + } + } else if i == 0 { + // Always add the first point + tseries = append(tseries, tsPoint{time: uint64(currTime), value: 0}) + } + + // Update the current value for this point + currentValue += b.points[i].value + + // assign it to the current point and update the maximum + tseries[len(tseries)-1].value = currentValue + maxValue = max(maxValue, currentValue) + } + + return tseries, maxValue +} diff --git a/pkg/collector/corechecks/gpu/tseries_test.go b/pkg/collector/corechecks/gpu/tseries_test.go new file mode 100644 index 0000000000000..3e534bed55172 --- /dev/null +++ b/pkg/collector/corechecks/gpu/tseries_test.go @@ -0,0 +1,72 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package gpu + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTseriesBuilder(t *testing.T) { + events := []struct { + start uint64 + end uint64 + value int64 + }{ + // Test overlapping events + {0, 10, 1}, + {5, 15, 2}, + + // Test events with gaps + {20, 30, 10}, + // Test events with same end + {28, 30, 3}, + // Test events with same start + {31, 33, 4}, + {31, 34, 5}, + + // Test gap plus events with no end + {40, 50, 8}, + } + + onlystarts := []struct { + start uint64 + value int64 + }{ + {35, 7}, + } + + builder := tseriesBuilder{} + for _, e := range events { + builder.AddEvent(e.start, e.end, e.value) + } + for _, s := range onlystarts { + builder.AddEventStart(s.start, s.value) + } + + tseries, max := builder.Build() + assert.Equal(t, max, int64(15)) // From event [40,50]=8 and onlystarts [35,inf]=7a + + expected := []tsPoint{ + {0, 1}, + {5, 3}, + {10, 2}, + {15, 0}, + {20, 10}, + {28, 13}, + {30, 0}, + {31, 9}, + {33, 5}, + {34, 0}, + {35, 7}, + {40, 15}, + {50, 7}, + } + + require.ElementsMatch(t, expected, tseries) +} diff --git a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go index 76b83d4c25e9e..efe3b291b289b 100644 --- a/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go +++ b/pkg/collector/corechecks/servicediscovery/apm/detect_nix_test.go @@ -241,6 +241,9 @@ func TestDotNetDetector(t *testing.T) { } func TestGoDetector(t *testing.T) { + if os.Getenv("CI") == "" && os.Getuid() != 0 { + t.Skip("skipping test; requires root privileges") + } curDir, err := testutil.CurDir() require.NoError(t, err) serverBinWithSymbols, err := usmtestutil.BuildGoBinaryWrapper(filepath.Join(curDir, "testutil"), "instrumented") diff --git a/pkg/collector/corechecks/servicediscovery/envs/envs.go b/pkg/collector/corechecks/servicediscovery/envs/envs.go index cdd80971f24f9..09c4ce0c366af 100644 --- a/pkg/collector/corechecks/servicediscovery/envs/envs.go +++ b/pkg/collector/corechecks/servicediscovery/envs/envs.go @@ -30,23 +30,12 @@ var targets = map[string]struct{}{ // Variables - collected of targeted environment variables. type Variables struct { - Vars map[string]string -} - -// NewVariables returns a new [Variables] to collect env. variables. -func NewVariables(vars map[string]string) Variables { - return Variables{ - Vars: vars, - } + vars map[string]string } // Get returns an environment variable if it is present in the collection func (ev *Variables) Get(name string) (string, bool) { - if _, ok := targets[name]; !ok { - return "", false - } - - val, ok := ev.Vars[name] + val, ok := ev.vars[name] return val, ok } @@ -66,10 +55,10 @@ func (ev *Variables) Set(name, val string) bool { if _, ok := targets[name]; !ok { return false } - if ev.Vars == nil { - ev.Vars = make(map[string]string) + if ev.vars == nil { + ev.vars = make(map[string]string) } - ev.Vars[name] = val + ev.vars[name] = val return true } diff --git a/pkg/collector/corechecks/servicediscovery/envs/envs_testutils.go b/pkg/collector/corechecks/servicediscovery/envs/envs_testutils.go index 55b4a7182a642..ead8e3d10e3e6 100644 --- a/pkg/collector/corechecks/servicediscovery/envs/envs_testutils.go +++ b/pkg/collector/corechecks/servicediscovery/envs/envs_testutils.go @@ -9,6 +9,19 @@ package envs import "fmt" +// NewVariables returns a new [Variables] +// for unit tests to verify that the input map has only target variables. +func NewVariables(vars map[string]string) Variables { + for env := range vars { + if _, ok := targets[env]; !ok { + return Variables{} + } + } + return Variables{ + vars: vars, + } +} + // GetExpectedEnvs - return list of expected env. variables for testing. func GetExpectedEnvs() []string { expectedEnvs := make([]string, 0, len(targets)) diff --git a/pkg/collector/corechecks/servicediscovery/impl_linux.go b/pkg/collector/corechecks/servicediscovery/impl_linux.go index b994e0f23b0a6..a8c737762c249 100644 --- a/pkg/collector/corechecks/servicediscovery/impl_linux.go +++ b/pkg/collector/corechecks/servicediscovery/impl_linux.go @@ -25,7 +25,7 @@ func init() { } type linuxImpl struct { - getSysProbeClient func() (systemProbeClient, error) + getSysProbeClient processnet.SysProbeUtilGetter time timer ignoreCfg map[string]bool @@ -38,7 +38,7 @@ type linuxImpl struct { func newLinuxImpl(ignoreCfg map[string]bool, containerProvider proccontainers.ContainerProvider) (osImpl, error) { return &linuxImpl{ - getSysProbeClient: getSysProbeClient, + getSysProbeClient: processnet.GetRemoteSystemProbeUtil, time: realTime{}, ignoreCfg: ignoreCfg, containerProvider: containerProvider, @@ -49,7 +49,8 @@ func newLinuxImpl(ignoreCfg map[string]bool, containerProvider proccontainers.Co } func (li *linuxImpl) DiscoverServices() (*discoveredServices, error) { - sysProbe, err := li.getSysProbeClient() + socket := pkgconfigsetup.SystemProbe().GetString("system_probe_config.sysprobe_socket") + sysProbe, err := li.getSysProbeClient(socket) if err != nil { return nil, errWithCode{ err: err, @@ -182,13 +183,3 @@ func (li *linuxImpl) getServiceInfo(service model.Service) serviceInfo { LastHeartbeat: li.time.Now(), } } - -type systemProbeClient interface { - GetDiscoveryServices() (*model.ServicesResponse, error) -} - -func getSysProbeClient() (systemProbeClient, error) { - return processnet.GetRemoteSystemProbeUtil( - pkgconfigsetup.SystemProbe().GetString("system_probe_config.sysprobe_socket"), - ) -} diff --git a/pkg/collector/corechecks/servicediscovery/impl_linux_mock.go b/pkg/collector/corechecks/servicediscovery/impl_linux_mock.go deleted file mode 100644 index 2022d4bcce6c9..0000000000000 --- a/pkg/collector/corechecks/servicediscovery/impl_linux_mock.go +++ /dev/null @@ -1,57 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build linux - -// Code generated by MockGen. DO NOT EDIT. -// Source: impl_linux.go - -// Package servicediscovery is a generated GoMock package. -package servicediscovery - -import ( - reflect "reflect" - - model "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" - gomock "github.com/golang/mock/gomock" -) - -// MocksystemProbeClient is a mock of systemProbeClient interface. -type MocksystemProbeClient struct { - ctrl *gomock.Controller - recorder *MocksystemProbeClientMockRecorder -} - -// MocksystemProbeClientMockRecorder is the mock recorder for MocksystemProbeClient. -type MocksystemProbeClientMockRecorder struct { - mock *MocksystemProbeClient -} - -// NewMocksystemProbeClient creates a new mock instance. -func NewMocksystemProbeClient(ctrl *gomock.Controller) *MocksystemProbeClient { - mock := &MocksystemProbeClient{ctrl: ctrl} - mock.recorder = &MocksystemProbeClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MocksystemProbeClient) EXPECT() *MocksystemProbeClientMockRecorder { - return m.recorder -} - -// GetDiscoveryListeners mocks base method. -func (m *MocksystemProbeClient) GetDiscoveryServices() (*model.ServicesResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetDiscoveryServices") - ret0, _ := ret[0].(*model.ServicesResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetDiscoveryServices indicates an expected call of GetDiscoveryServices. -func (mr *MocksystemProbeClientMockRecorder) GetDiscoveryServices() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDiscoveryServices", reflect.TypeOf((*MocksystemProbeClient)(nil).GetDiscoveryServices)) -} diff --git a/pkg/collector/corechecks/servicediscovery/impl_linux_test.go b/pkg/collector/corechecks/servicediscovery/impl_linux_test.go index 06bfe2c24a135..91bbc71c59458 100644 --- a/pkg/collector/corechecks/servicediscovery/impl_linux_test.go +++ b/pkg/collector/corechecks/servicediscovery/impl_linux_test.go @@ -22,6 +22,8 @@ import ( "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/apm" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" + "github.com/DataDog/datadog-agent/pkg/process/net" + netmocks "github.com/DataDog/datadog-agent/pkg/process/net/mocks" ) type testProc struct { @@ -573,7 +575,7 @@ func Test_linuxImpl(t *testing.T) { require.NotNil(t, check.os) for _, cr := range tc.checkRun { - mSysProbe := NewMocksystemProbeClient(ctrl) + mSysProbe := netmocks.NewSysProbeUtil(t) mSysProbe.EXPECT().GetDiscoveryServices(). Return(cr.servicesResp, nil). Times(1) @@ -584,7 +586,7 @@ func Test_linuxImpl(t *testing.T) { mTimer.EXPECT().Now().Return(cr.time).AnyTimes() // set mocks - check.os.(*linuxImpl).getSysProbeClient = func() (systemProbeClient, error) { + check.os.(*linuxImpl).getSysProbeClient = func(_ string) (net.SysProbeUtil, error) { return mSysProbe, nil } check.os.(*linuxImpl).time = mTimer diff --git a/pkg/collector/corechecks/servicediscovery/module/envs.go b/pkg/collector/corechecks/servicediscovery/module/envs.go index 2363b85e261b7..0c841b3d108c5 100644 --- a/pkg/collector/corechecks/servicediscovery/module/envs.go +++ b/pkg/collector/corechecks/servicediscovery/module/envs.go @@ -190,7 +190,7 @@ func newEnvReader(proc *process.Process) (*EnvReader, error) { return &EnvReader{ file: file, scanner: scanner, - envs: envs.NewVariables(nil), + envs: envs.Variables{}, }, nil } @@ -212,30 +212,30 @@ func (er *EnvReader) add() { // getTargetEnvs reads the environment variables of interest from the /proc//environ file. func getTargetEnvs(proc *process.Process) (envs.Variables, error) { - er, err := newEnvReader(proc) + reader, err := newEnvReader(proc) defer func() { - if er != nil { - er.close() + if reader != nil { + reader.close() } }() if err != nil { - return envs.NewVariables(nil), err + return envs.Variables{}, err } - for er.scanner.Scan() { - er.add() + for reader.scanner.Scan() { + reader.add() } injectionMeta, ok := getInjectionMeta(proc) if !ok { - return er.envs, nil + return reader.envs, nil } for _, env := range injectionMeta.InjectedEnv { name, val, found := strings.Cut(string(env), "=") if found { - er.envs.Set(name, val) + reader.envs.Set(name, val) } } - return er.envs, nil + return reader.envs, nil } diff --git a/pkg/collector/corechecks/servicediscovery/module/envs_test.go b/pkg/collector/corechecks/servicediscovery/module/envs_test.go index f6a0adbe2d2d5..31cb3c8816fe7 100644 --- a/pkg/collector/corechecks/servicediscovery/module/envs_test.go +++ b/pkg/collector/corechecks/servicediscovery/module/envs_test.go @@ -119,12 +119,34 @@ func TestTargetEnvs(t *testing.T) { require.NoError(t, err) expectedMap := envs.GetExpectedMap() - require.Equal(t, vars.Vars, expectedMap) + for k, v := range expectedMap { + val, ok := vars.Get(k) + require.True(t, ok) + require.Equal(t, val, v) + } // check unexpected env variables - require.NotContains(t, vars.Vars, "HOME") - require.NotContains(t, vars.Vars, "PATH") - require.NotContains(t, vars.Vars, "SHELL") + val, ok := vars.Get("HOME") + require.Empty(t, val) + require.False(t, ok) + val, ok = vars.Get("PATH") + require.Empty(t, val) + require.False(t, ok) + val, ok = vars.Get("SHELL") + require.Empty(t, val) + require.False(t, ok) + + // check that non-target variables return an empty map. + vars = envs.NewVariables(map[string]string{ + "NON_TARGET1": "some", + "NON_TARGET2": "some", + }) + val, ok = vars.Get("NON_TARGET1") + require.Empty(t, val) + require.False(t, ok) + val, ok = vars.Get("NON_TARGET2") + require.Empty(t, val) + require.False(t, ok) } // BenchmarkGetEnvs benchmarks reading of all environment variables from /proc//environ. diff --git a/pkg/collector/corechecks/system/wincrashdetect/probe/crashparse.go b/pkg/collector/corechecks/system/wincrashdetect/probe/crashparse.go index c4fb188d2e5d2..38fe8119ba9cd 100644 --- a/pkg/collector/corechecks/system/wincrashdetect/probe/crashparse.go +++ b/pkg/collector/corechecks/system/wincrashdetect/probe/crashparse.go @@ -23,6 +23,7 @@ import ( // allow us to change for testing var readfn = doReadCrashDump +var parseCrashDump = parseWinCrashDump type logCallbackContext struct { loglines []string @@ -107,14 +108,14 @@ func doReadCrashDump(filename string, ctx *logCallbackContext, exterr *uint32) e return nil } -func parseCrashDump(wcs *WinCrashStatus) { +func parseWinCrashDump(wcs *WinCrashStatus) { var ctx logCallbackContext var extendedError uint32 err := readfn(wcs.FileName, &ctx, &extendedError) if err != nil { - wcs.Success = false + wcs.StatusCode = WinCrashStatusCodeFailed wcs.ErrString = fmt.Sprintf("Failed to load crash dump file %v %x", err, extendedError) log.Errorf("Failed to open crash dump %s: %v %x", wcs.FileName, err, extendedError) return @@ -122,7 +123,7 @@ func parseCrashDump(wcs *WinCrashStatus) { if len(ctx.loglines) < 2 { wcs.ErrString = fmt.Sprintf("Invalid crash dump file %s", wcs.FileName) - wcs.Success = false + wcs.StatusCode = WinCrashStatusCodeFailed return } @@ -190,5 +191,5 @@ func parseCrashDump(wcs *WinCrashStatus) { wcs.Offender = callsite break } - wcs.Success = true + wcs.StatusCode = WinCrashStatusCodeSuccess } diff --git a/pkg/collector/corechecks/system/wincrashdetect/probe/crashparse_test.go b/pkg/collector/corechecks/system/wincrashdetect/probe/crashparse_test.go index 161b3b6068119..2efd8d5f21781 100644 --- a/pkg/collector/corechecks/system/wincrashdetect/probe/crashparse_test.go +++ b/pkg/collector/corechecks/system/wincrashdetect/probe/crashparse_test.go @@ -51,12 +51,11 @@ func TestCrashParser(t *testing.T) { FileName: "testdata/crashsample1.txt", } // first read in the sample data - - readfn = testCrashReader + OverrideCrashDumpReader(testCrashReader) parseCrashDump(wcs) - assert.True(t, wcs.Success) + assert.Equal(t, WinCrashStatusCodeSuccess, wcs.StatusCode) assert.Empty(t, wcs.ErrString) assert.Equal(t, "Mon Jun 26 20:44:49.742 2023 (UTC - 7:00)", wcs.DateString) before, _, _ := strings.Cut(wcs.Offender, "+") @@ -72,11 +71,11 @@ func TestCrashParserWithLineSplits(t *testing.T) { } // first read in the sample data - readfn = testCrashReaderWithLineSplits + OverrideCrashDumpReader(testCrashReaderWithLineSplits) parseCrashDump(wcs) - assert.True(t, wcs.Success) + assert.Equal(t, WinCrashStatusCodeSuccess, wcs.StatusCode) assert.Empty(t, wcs.ErrString) assert.Equal(t, "Mon Jun 26 20:44:49.742 2023 (UTC - 7:00)", wcs.DateString) before, _, _ := strings.Cut(wcs.Offender, "+") diff --git a/pkg/collector/corechecks/system/wincrashdetect/probe/win_crash_types.go b/pkg/collector/corechecks/system/wincrashdetect/probe/win_crash_types.go index 5fac5c9853b17..a1a36691be08a 100644 --- a/pkg/collector/corechecks/system/wincrashdetect/probe/win_crash_types.go +++ b/pkg/collector/corechecks/system/wincrashdetect/probe/win_crash_types.go @@ -29,10 +29,26 @@ const ( DumpTypeAutomatic = int(7) // automatic ) +const ( + // WinCrashStatusCodeUnknown indicates an invalid or corrupted code. + WinCrashStatusCodeUnknown = int(-1) + + // WinCrashStatusCodeSuccess indicates that crash dump processing succeeded + // or no crash dump was found. + WinCrashStatusCodeSuccess = int(0) + + // WinCrashStatusCodeBusy indicates that crash dump processing is still busy + // and no result is yet available. + WinCrashStatusCodeBusy = int(1) + + // WinCrashStatusCodeFailed indicates that crash dump processing failed or had an error. + WinCrashStatusCodeFailed = int(2) +) + // WinCrashStatus defines all of the information returned from the system // probe to the caller type WinCrashStatus struct { - Success bool `json:"success"` + StatusCode int `json:"statuscode"` ErrString string `json:"errstring"` FileName string `json:"filename"` Type int `json:"dumptype"` diff --git a/pkg/collector/corechecks/system/wincrashdetect/probe/wincrash_testutil.go b/pkg/collector/corechecks/system/wincrashdetect/probe/wincrash_testutil.go new file mode 100644 index 0000000000000..c2f52b0071589 --- /dev/null +++ b/pkg/collector/corechecks/system/wincrashdetect/probe/wincrash_testutil.go @@ -0,0 +1,26 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build test && windows + +package probe + +type readCrashDumpType func(filename string, ctx *logCallbackContext, _ *uint32) error +type parseCrashDumpType func(wcs *WinCrashStatus) + +// SetCachedSettings sets the settings used for tests without reading the Registry. +func (p *WinCrashProbe) SetCachedSettings(wcs *WinCrashStatus) { + p.status = wcs +} + +// OverrideCrashDumpReader relpaces the crash dump reading function for tests. +func OverrideCrashDumpReader(customCrashReader readCrashDumpType) { + readfn = customCrashReader +} + +// OverrideCrashDumpParser relpaces the crash dump parsing function for tests. +func OverrideCrashDumpParser(customParseCrashDump parseCrashDumpType) { + parseCrashDump = customParseCrashDump +} diff --git a/pkg/collector/corechecks/system/wincrashdetect/probe/wincrashprobe.go b/pkg/collector/corechecks/system/wincrashdetect/probe/wincrashprobe.go index 1d75c514142df..533cfcc2c431b 100644 --- a/pkg/collector/corechecks/system/wincrashdetect/probe/wincrashprobe.go +++ b/pkg/collector/corechecks/system/wincrashdetect/probe/wincrashprobe.go @@ -11,38 +11,126 @@ import ( "fmt" "os" "path/filepath" + "sync" sysconfigtypes "github.com/DataDog/datadog-agent/cmd/system-probe/config/types" "github.com/DataDog/datadog-agent/pkg/util/winutil" "golang.org/x/sys/windows/registry" ) +type probeState uint32 + +const ( + // Idle indicates that the probe is waiting for a request + idle probeState = iota + + // Busy indicates that the probe is currently processing a crash dump + busy + + // Completed indicates that the probe finished processing a crash dump. + completed + + // Failed indicates that the probe failed to process a crash dump. + failed +) + // WinCrashProbe has no stored state. type WinCrashProbe struct { + state probeState + status *WinCrashStatus + mu sync.Mutex } // NewWinCrashProbe returns an initialized WinCrashProbe func NewWinCrashProbe(_ *sysconfigtypes.Config) (*WinCrashProbe, error) { - return &WinCrashProbe{}, nil + return &WinCrashProbe{ + state: idle, + status: nil, + }, nil +} + +// Handles crash dump parsing in a separate thread since this may take very long. +func (p *WinCrashProbe) parseCrashDumpAsync() { + if p.status == nil { + p.state = failed + return + } + + parseCrashDump(p.status) + + p.mu.Lock() + defer p.mu.Unlock() + p.state = completed } // Get returns the current crash, if any func (p *WinCrashProbe) Get() *WinCrashStatus { wcs := &WinCrashStatus{} - err := wcs.getCurrentCrashSettings() - if err != nil { - wcs.ErrString = err.Error() - wcs.Success = false - return wcs - } + // Nothing in this method should take long. + p.mu.Lock() + defer p.mu.Unlock() + + switch p.state { + case idle: + if p.status == nil { + // This is a new request. + err := wcs.getCurrentCrashSettings() + if err != nil { + wcs.ErrString = err.Error() + wcs.StatusCode = WinCrashStatusCodeFailed + } + } else { + // Use cached settings, set by tests. + // Make a copy to avoid side-effect modifications. + *wcs = *(p.status) + } - if len(wcs.FileName) == 0 { - // no filename means no crash dump - wcs.Success = true // we succeeded - return wcs + // Transition to the next state. + if wcs.StatusCode == WinCrashStatusCodeFailed { + // Only try once and cache the failure. + p.status = wcs + p.state = failed + } else if len(wcs.FileName) == 0 { + // No filename means no crash dump + p.status = wcs + p.state = completed + wcs.StatusCode = WinCrashStatusCodeSuccess + } else { + // Kick off the crash dump processing asynchronously. + // The crash dump may be very large and we should not block for a response. + p.state = busy + wcs.StatusCode = WinCrashStatusCodeBusy + + // Make a new copy of the wcs for async processing while returning "Busy" + // for the current response. + p.status = &WinCrashStatus{ + FileName: wcs.FileName, + Type: wcs.Type, + } + + go p.parseCrashDumpAsync() + } + + case busy: + // The crash dump processing is not done yet. Reply busy. + if p.status != nil { + wcs.FileName = p.status.FileName + wcs.Type = p.status.Type + } + wcs.StatusCode = WinCrashStatusCodeBusy + + case failed: + fallthrough + case completed: + // The crash dump processing was done, return the result. + if p.status != nil { + // This result is cached for all subsequent queries. + wcs = p.status + } else { + wcs.StatusCode = WinCrashStatusCodeFailed + } } - parseCrashDump(wcs) return wcs } diff --git a/pkg/collector/corechecks/system/wincrashdetect/wincrashdetect_windows_test.go b/pkg/collector/corechecks/system/wincrashdetect/wincrashdetect_windows_test.go index ffb9ce3647dbc..ff1cdb0ced1c8 100644 --- a/pkg/collector/corechecks/system/wincrashdetect/wincrashdetect_windows_test.go +++ b/pkg/collector/corechecks/system/wincrashdetect/wincrashdetect_windows_test.go @@ -10,11 +10,10 @@ package wincrashdetect import ( "net" "net/http" - - //"strings" + "sync" "testing" + "time" - //"github.com/stretchr/testify/require" "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/cmd/system-probe/utils" @@ -108,7 +107,7 @@ func TestWinCrashReporting(t *testing.T) { // set the return value handled in the check handler above p = &probe.WinCrashStatus{ - Success: true, + StatusCode: probe.WinCrashStatusCodeSuccess, } check := newCheck() @@ -128,7 +127,7 @@ func TestWinCrashReporting(t *testing.T) { testSetup(t) defer testCleanup() p = &probe.WinCrashStatus{ - Success: true, + StatusCode: probe.WinCrashStatusCodeSuccess, FileName: `c:\windows\memory.dmp`, Type: probe.DumpTypeAutomatic, DateString: `Fri Jun 30 15:33:05.086 2023 (UTC - 7:00)`, @@ -201,3 +200,163 @@ func TestWinCrashReporting(t *testing.T) { mock.AssertNumberOfCalls(t, "Commit", 2) }) } + +func TestCrashReportingStates(t *testing.T) { + var crashStatus *probe.WinCrashStatus + + listener, closefunc := createSystemProbeListener() + defer closefunc() + + pkgconfigsetup.InitSystemProbeConfig(pkgconfigsetup.SystemProbe()) + + mux := http.NewServeMux() + server := http.Server{ + Handler: mux, + } + defer server.Close() + + cp, err := probe.NewWinCrashProbe(nil) + assert.NotNil(t, cp) + assert.Nil(t, err) + + wg := sync.WaitGroup{} + + // This will artificially delay the "parsing" to ensure the first check gets a "busy" status. + delayedCrashDumpParser := func(wcs *probe.WinCrashStatus) { + time.Sleep(4 * time.Second) + + assert.Equal(t, `c:\windows\memory.dmp`, wcs.FileName) + assert.Equal(t, probe.DumpTypeAutomatic, wcs.Type) + + wcs.StatusCode = probe.WinCrashStatusCodeSuccess + wcs.ErrString = crashStatus.ErrString + wcs.DateString = crashStatus.DateString + wcs.Offender = crashStatus.Offender + wcs.BugCheck = crashStatus.BugCheck + + // Signal that the artificial delay is done. + wg.Done() + } + + // This ensures that no crash dump parsing should happen. + noCrashDumpParser := func(_ *probe.WinCrashStatus) { + assert.FailNow(t, "Should not parse") + } + + mux.Handle("/windows_crash_detection/check", http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) { + results := cp.Get() + utils.WriteAsJSON(rw, results) + })) + mux.Handle("/debug/stats", http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) { + })) + go server.Serve(listener) + + t.Run("test reporting a crash with a busy intermediate state", func(t *testing.T) { + testSetup(t) + defer testCleanup() + + check := newCheck() + crashCheck := check.(*WinCrashDetect) + mock := mocksender.NewMockSender(crashCheck.ID()) + err := crashCheck.Configure(mock.GetSenderManager(), 0, nil, nil, "") + assert.NoError(t, err) + + crashStatus = &probe.WinCrashStatus{ + StatusCode: probe.WinCrashStatusCodeSuccess, + FileName: `c:\windows\memory.dmp`, + Type: probe.DumpTypeAutomatic, + ErrString: "", + DateString: `Fri Jun 30 15:33:05.086 2023 (UTC - 7:00)`, + Offender: `somedriver.sys`, + BugCheck: "0x00000007", + } + + // Test the 2-check response from crash reporting. + cp.SetCachedSettings(crashStatus) + probe.OverrideCrashDumpParser(delayedCrashDumpParser) + + // First run should be "busy" and not return an event yet. + wg.Add(1) + err = crashCheck.Run() + assert.Nil(t, err) + mock.AssertNumberOfCalls(t, "Gauge", 0) + mock.AssertNumberOfCalls(t, "Rate", 0) + mock.AssertNumberOfCalls(t, "Event", 0) + mock.AssertNumberOfCalls(t, "Commit", 0) + + // Wait for the artificial delay to finish, plus a small time buffer. + wg.Wait() + time.Sleep(4 * time.Second) + + expected := event.Event{ + Priority: event.PriorityNormal, + SourceTypeName: CheckName, + EventType: CheckName, + AlertType: event.AlertTypeError, + Title: formatTitle(crashStatus), + Text: formatText(crashStatus), + } + + mock.On("Event", expected).Return().Times(1) + mock.On("Commit").Return().Times(1) + + // The result should be available now. + err = crashCheck.Run() + assert.Nil(t, err) + mock.AssertNumberOfCalls(t, "Gauge", 0) + mock.AssertNumberOfCalls(t, "Rate", 0) + mock.AssertNumberOfCalls(t, "Event", 1) + mock.AssertNumberOfCalls(t, "Commit", 1) + }) + + t.Run("test that no crash is reported", func(t *testing.T) { + testSetup(t) + defer testCleanup() + + check := newCheck() + crashCheck := check.(*WinCrashDetect) + mock := mocksender.NewMockSender(crashCheck.ID()) + err := crashCheck.Configure(mock.GetSenderManager(), 0, nil, nil, "") + assert.NoError(t, err) + + noCrashStatus := &probe.WinCrashStatus{ + StatusCode: probe.WinCrashStatusCodeSuccess, + FileName: "", + } + + // Test finding no crashes. The response should be immediate. + cp.SetCachedSettings(noCrashStatus) + probe.OverrideCrashDumpParser(noCrashDumpParser) + err = crashCheck.Run() + assert.Nil(t, err) + mock.AssertNumberOfCalls(t, "Gauge", 0) + mock.AssertNumberOfCalls(t, "Rate", 0) + mock.AssertNumberOfCalls(t, "Event", 0) + mock.AssertNumberOfCalls(t, "Commit", 0) + }) + + t.Run("test failure on reading crash settings", func(t *testing.T) { + testSetup(t) + defer testCleanup() + + check := newCheck() + crashCheck := check.(*WinCrashDetect) + mock := mocksender.NewMockSender(crashCheck.ID()) + err := crashCheck.Configure(mock.GetSenderManager(), 0, nil, nil, "") + assert.NoError(t, err) + + failedStatus := &probe.WinCrashStatus{ + StatusCode: probe.WinCrashStatusCodeFailed, + ErrString: "Mocked failure", + } + + // Test having a failure reading setings. The response should be immediate. + cp.SetCachedSettings(failedStatus) + probe.OverrideCrashDumpParser(noCrashDumpParser) + err = crashCheck.Run() + assert.NotNil(t, err) + mock.AssertNumberOfCalls(t, "Rate", 0) + mock.AssertNumberOfCalls(t, "Event", 0) + mock.AssertNumberOfCalls(t, "Commit", 0) + }) +} diff --git a/pkg/collector/python/init.go b/pkg/collector/python/init.go index 7188952a425a5..3bbd1102c2bc6 100644 --- a/pkg/collector/python/init.go +++ b/pkg/collector/python/init.go @@ -205,7 +205,6 @@ var ( PythonVersion = "" // The pythonHome variable typically comes from -ldflags // it's needed in case the agent was built using embedded libs - pythonHome2 = "" pythonHome3 = "" // PythonHome contains the computed value of the Python Home path once the // intepreter is created. It might be empty in case the interpreter wasn't @@ -303,7 +302,7 @@ func pathToBinary(name string, ignoreErrors bool) (string, error) { return absPath, nil } -func resolvePythonExecPath(pythonVersion string, ignoreErrors bool) (string, error) { +func resolvePythonExecPath(ignoreErrors bool) (string, error) { // Since the install location can be set by the user on Windows we use relative import if runtime.GOOS == "windows" { _here, err := executable.Folder() @@ -317,18 +316,12 @@ func resolvePythonExecPath(pythonVersion string, ignoreErrors bool) (string, err } log.Debugf("Executable folder is %v", _here) - embeddedPythonHome2 := filepath.Join(_here, "..", "embedded2") embeddedPythonHome3 := filepath.Join(_here, "..", "embedded3") // We want to use the path-relative embedded2/3 directories above by default. // They will be correct for normal installation on Windows. However, if they // are not present for cases like running unit tests, fall back to the compile // time values. - if _, err := os.Stat(embeddedPythonHome2); os.IsNotExist(err) { - log.Warnf("Relative embedded directory not found for Python 2. Using default: %s", pythonHome2) - } else { - pythonHome2 = embeddedPythonHome2 - } if _, err := os.Stat(embeddedPythonHome3); os.IsNotExist(err) { log.Warnf("Relative embedded directory not found for Python 3. Using default: %s", pythonHome3) } else { @@ -336,11 +329,7 @@ func resolvePythonExecPath(pythonVersion string, ignoreErrors bool) (string, err } } - if pythonVersion == "2" { - PythonHome = pythonHome2 - } else if pythonVersion == "3" { - PythonHome = pythonHome3 - } + PythonHome = pythonHome3 log.Infof("Using '%s' as Python home", PythonHome) @@ -361,7 +350,7 @@ func resolvePythonExecPath(pythonVersion string, ignoreErrors bool) (string, err // don't want to use the default version (aka "python") but rather "python2" or // "python3" based on the configuration. Also on some Python3 platforms there // are no "python" aliases either. - interpreterBasename := "python" + pythonVersion + interpreterBasename := "python3" // If we are in a development env or just the ldflags haven't been set, the PythonHome // variable won't be set so what we do here is to just find out where our current @@ -393,7 +382,7 @@ func Initialize(paths ...string) error { } // Note: pythonBinPath is a module-level var - pythonBinPath, err := resolvePythonExecPath(pythonVersion, allowPathHeuristicsFailure) + pythonBinPath, err := resolvePythonExecPath(allowPathHeuristicsFailure) if err != nil { return err } @@ -407,10 +396,7 @@ func Initialize(paths ...string) error { csPythonExecPath := TrackedCString(pythonBinPath) defer C._free(unsafe.Pointer(csPythonExecPath)) - if pythonVersion == "2" { - log.Infof("Initializing rtloader with Python 2 %s", PythonHome) - rtloader = C.make2(csPythonHome, csPythonExecPath, &pyErr) - } else if pythonVersion == "3" { + if pythonVersion == "3" { log.Infof("Initializing rtloader with Python 3 %s", PythonHome) rtloader = C.make3(csPythonHome, csPythonExecPath, &pyErr) } else { diff --git a/pkg/commonchecks/corechecks.go b/pkg/commonchecks/corechecks.go index e436e773d3377..f4719c58ba0e6 100644 --- a/pkg/commonchecks/corechecks.go +++ b/pkg/commonchecks/corechecks.go @@ -8,6 +8,7 @@ package commonchecks import ( "github.com/DataDog/datadog-agent/comp/core/config" + "github.com/DataDog/datadog-agent/comp/core/tagger" "github.com/DataDog/datadog-agent/comp/core/telemetry" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" corecheckLoader "github.com/DataDog/datadog-agent/pkg/collector/corechecks" @@ -27,6 +28,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/collector/corechecks/ebpf/tcpqueuelength" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/embed/apm" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/embed/process" + "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/net/network" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/net/ntp" ciscosdwan "github.com/DataDog/datadog-agent/pkg/collector/corechecks/network-devices/cisco-sdwan" @@ -53,7 +55,7 @@ import ( ) // RegisterChecks registers all core checks -func RegisterChecks(store workloadmeta.Component, cfg config.Component, telemetry telemetry.Component) { +func RegisterChecks(store workloadmeta.Component, tagger tagger.Component, cfg config.Component, telemetry telemetry.Component) { // Required checks corecheckLoader.RegisterCheck(cpu.CheckName, cpu.Factory()) corecheckLoader.RegisterCheck(memory.CheckName, memory.Factory()) @@ -70,11 +72,12 @@ func RegisterChecks(store workloadmeta.Component, cfg config.Component, telemetr // Flavor specific checks corecheckLoader.RegisterCheck(load.CheckName, load.Factory()) - corecheckLoader.RegisterCheck(kubernetesapiserver.CheckName, kubernetesapiserver.Factory()) + corecheckLoader.RegisterCheck(kubernetesapiserver.CheckName, kubernetesapiserver.Factory(tagger)) corecheckLoader.RegisterCheck(ksm.CheckName, ksm.Factory()) corecheckLoader.RegisterCheck(helm.CheckName, helm.Factory()) corecheckLoader.RegisterCheck(pod.CheckName, pod.Factory(store, cfg)) corecheckLoader.RegisterCheck(ebpf.CheckName, ebpf.Factory()) + corecheckLoader.RegisterCheck(gpu.CheckName, gpu.Factory()) corecheckLoader.RegisterCheck(ecs.CheckName, ecs.Factory(store)) corecheckLoader.RegisterCheck(oomkill.CheckName, oomkill.Factory()) corecheckLoader.RegisterCheck(tcpqueuelength.CheckName, tcpqueuelength.Factory()) diff --git a/pkg/config/model/go.mod b/pkg/config/model/go.mod index d6a513a9faef0..693b2d2a57260 100644 --- a/pkg/config/model/go.mod +++ b/pkg/config/model/go.mod @@ -13,7 +13,6 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 github.com/DataDog/viper v1.13.5 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 - github.com/spf13/afero v1.1.2 github.com/stretchr/testify v1.9.0 golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 ) @@ -28,6 +27,7 @@ require ( github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/pelletier/go-toml v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/afero v1.1.2 // indirect github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.3 // indirect diff --git a/pkg/config/model/types.go b/pkg/config/model/types.go index 2f36fe7f4a409..1d3de47a8952f 100644 --- a/pkg/config/model/types.go +++ b/pkg/config/model/types.go @@ -11,7 +11,6 @@ import ( "time" "github.com/DataDog/viper" - "github.com/spf13/afero" ) // Proxy represents the configuration for proxies in the agent @@ -105,7 +104,6 @@ type Setup interface { // API implemented by viper.Viper SetDefault(key string, value interface{}) - SetFs(fs afero.Fs) SetEnvPrefix(in string) BindEnv(key string, envvars ...string) diff --git a/pkg/config/model/viper.go b/pkg/config/model/viper.go index 7aa9b2720c67a..ff66d1db00543 100644 --- a/pkg/config/model/viper.go +++ b/pkg/config/model/viper.go @@ -22,7 +22,6 @@ import ( "github.com/DataDog/viper" "github.com/mohae/deepcopy" - "github.com/spf13/afero" "golang.org/x/exp/slices" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -279,13 +278,6 @@ func (c *safeConfig) ParseEnvAsSlice(key string, fn func(string) []interface{}) c.Viper.SetEnvKeyTransformer(key, func(data string) interface{} { return fn(data) }) } -// SetFs wraps Viper for concurrent access -func (c *safeConfig) SetFs(fs afero.Fs) { - c.Lock() - defer c.Unlock() - c.Viper.SetFs(fs) -} - // IsSet wraps Viper for concurrent access func (c *safeConfig) IsSet(key string) bool { c.RLock() diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index 8eeba294628d4..e8b932262da4e 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -80,7 +80,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/winutil v0.56.0-rc.3 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/go-libddwaf/v3 v3.3.0 // indirect - github.com/DataDog/go-sqllexer v0.0.15 // indirect + github.com/DataDog/go-sqllexer v0.0.16 // indirect github.com/DataDog/sketches-go v1.4.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index 662e9a017393b..1c40a8aabd49d 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -13,8 +13,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/go-libddwaf/v3 v3.3.0 h1:jS72fuQpFgJZEdEJDmHJCPAgNTEMZoz1EUvimPUOiJ4= github.com/DataDog/go-libddwaf/v3 v3.3.0/go.mod h1:Bz/0JkpGf689mzbUjKJeheJINqsyyhM8p9PDuHdK2Ec= -github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= -github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.16 h1:RoSUMS6MECyB3gTUIdydzXwK5NhEhv6GMJkS7ptsgRA= +github.com/DataDog/go-sqllexer v0.0.16/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 79026d025aea8..c805ca0e2a58a 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -354,7 +354,7 @@ func InitConfig(config pkgconfigmodel.Setup) { config.BindEnvAndSetDefault("scrubber.additional_keys", []string{}) // flare configs - config.BindEnvAndSetDefault("flare_provider_timeout", 10) + config.BindEnvAndSetDefault("flare_provider_timeout", 10*time.Second) // Docker config.BindEnvAndSetDefault("docker_query_timeout", int64(5)) @@ -1882,6 +1882,9 @@ func findUnknownEnvVars(config pkgconfigmodel.Config, environ []string, addition "DD_POD_NAME": {}, // this variable is used by tracers "DD_INSTRUMENTATION_TELEMETRY_ENABLED": {}, + // these variables are used by source code integration + "DD_GIT_COMMIT_SHA": {}, + "DD_GIT_REPOSITORY_URL": {}, } for _, key := range config.GetEnvVars() { knownVars[key] = struct{}{} diff --git a/pkg/config/teeconfig/go.mod b/pkg/config/teeconfig/go.mod index 3eef4f16c3ba7..c0a7e97877f75 100644 --- a/pkg/config/teeconfig/go.mod +++ b/pkg/config/teeconfig/go.mod @@ -11,7 +11,6 @@ replace ( require ( github.com/DataDog/datadog-agent/pkg/config/model v0.0.0-00010101000000-000000000000 github.com/DataDog/viper v1.13.5 - github.com/spf13/afero v1.11.0 ) require ( @@ -24,6 +23,7 @@ require ( github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/pelletier/go-toml v1.2.0 // indirect + github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.3 // indirect diff --git a/pkg/config/teeconfig/teeconfig.go b/pkg/config/teeconfig/teeconfig.go index bc0bbb39da4eb..ed490fd127fe5 100644 --- a/pkg/config/teeconfig/teeconfig.go +++ b/pkg/config/teeconfig/teeconfig.go @@ -12,7 +12,6 @@ import ( "time" "github.com/DataDog/viper" - "github.com/spf13/afero" "github.com/DataDog/datadog-agent/pkg/config/model" ) @@ -104,12 +103,6 @@ func (t *teeConfig) ParseEnvAsSlice(key string, fn func(string) []interface{}) { t.compare.ParseEnvAsSlice(key, fn) } -// SetFs wraps Viper for concurrent access -func (t *teeConfig) SetFs(fs afero.Fs) { - t.baseline.SetFs(fs) - t.compare.SetFs(fs) -} - // IsSet wraps Viper for concurrent access func (t *teeConfig) IsSet(key string) bool { return t.baseline.IsSet(key) diff --git a/pkg/ebpf/uprobes/attacher.go b/pkg/ebpf/uprobes/attacher.go index 1c465ccbf05fc..de71ff62a00a6 100644 --- a/pkg/ebpf/uprobes/attacher.go +++ b/pkg/ebpf/uprobes/attacher.go @@ -176,11 +176,10 @@ type AttacherConfig struct { EbpfConfig *ebpf.Config // PerformInitialScan defines if the attacher should perform an initial scan of the processes before starting the monitor + // Note that if processMonitor is being used (i.e., rules are targeting executables), the ProcessMonitor itself + // will perform an initial scan in its Initialize method. PerformInitialScan bool - // ProcessMonitorEventStream defines whether the process monitor is using the event stream - ProcessMonitorEventStream bool - // EnableDetailedLogging makes the attacher log why it's attaching or not attaching to a process // This is useful for debugging purposes, do not enable in production. EnableDetailedLogging bool @@ -383,11 +382,6 @@ func (ua *UprobeAttacher) handlesExecutables() bool { func (ua *UprobeAttacher) Start() error { var cleanupExec, cleanupExit func() procMonitor := monitor.GetProcessMonitor() - err := procMonitor.Initialize(ua.config.ProcessMonitorEventStream) - if err != nil { - return fmt.Errorf("error initializing process monitor: %w", err) - } - if ua.handlesExecutables() { cleanupExec = procMonitor.SubscribeExec(ua.handleProcessStart) } diff --git a/pkg/ebpf/uprobes/attacher_test.go b/pkg/ebpf/uprobes/attacher_test.go index 6ef68c910f514..4297686e0c47b 100644 --- a/pkg/ebpf/uprobes/attacher_test.go +++ b/pkg/ebpf/uprobes/attacher_test.go @@ -273,13 +273,14 @@ func TestMonitor(t *testing.T) { return } + launchProcessMonitor(t, false) + config := AttacherConfig{ Rules: []*AttachRule{{ LibraryNameRegex: regexp.MustCompile(`libssl.so`), Targets: AttachToExecutable | AttachToSharedLibraries, }}, - ProcessMonitorEventStream: false, - EbpfConfig: ebpfCfg, + EbpfConfig: ebpfCfg, } ua, err := NewUprobeAttacher("mock", config, &MockManager{}, nil, nil) require.NoError(t, err) @@ -654,6 +655,8 @@ func TestUprobeAttacher(t *testing.T) { return } + launchProcessMonitor(t, false) + buf, err := bytecode.GetReader(ebpfCfg.BPFDir, "uprobe_attacher-test.o") require.NoError(t, err) t.Cleanup(func() { buf.Close() }) @@ -821,8 +824,10 @@ func (s *SharedLibrarySuite) TestSingleFile() { func() bool { return methodHasBeenCalledTimes(mockRegistry, "Register", 1) }, - func() { - if cmd != nil && cmd.Process != nil { + func(testSuccess bool) { + // Only kill the process if the test failed, if it succeeded we want to kill it later + // to check if the Unregister call was done correctly + if !testSuccess && cmd != nil && cmd.Process != nil { cmd.Process.Kill() } }, diff --git a/pkg/ebpf/uprobes/testutil.go b/pkg/ebpf/uprobes/testutil.go index a7f21c3586de6..0c91c4ef3912f 100644 --- a/pkg/ebpf/uprobes/testutil.go +++ b/pkg/ebpf/uprobes/testutil.go @@ -181,18 +181,24 @@ func checkIfEventually(condition func() bool, checkInterval time.Duration, check } } -// waitAndRetryIfFail is basically a way to do require.Eventually with multiple retries. -// In each retry, it will run the setupFunc, then wait until the condition defined by testFunc is met or the timeout is reached, and then run the retryCleanup function. -// If the condition is met, it will return, otherwise it will retry the same thing again. -// If the condition is not met after maxRetries, it will fail the test. -func waitAndRetryIfFail(t *testing.T, setupFunc func(), testFunc func() bool, retryCleanup func(), maxRetries int, checkInterval time.Duration, maxSingleCheckTime time.Duration, msgAndArgs ...interface{}) { +// waitAndRetryIfFail is basically a way to do require.Eventually with multiple +// retries. In each retry, it will run the setupFunc, then wait until the +// condition defined by testFunc is met or the timeout is reached, and then run +// the retryCleanup function. The retryCleanup function is useful to clean up +// any state that was set up in the setupFunc. It will receive a boolean +// indicating if the test was successful or not, in case the cleanup needs to be +// different depending on the test result (e.g., if the test didn't fail we +// might want to keep some state). If the condition is met, it will return, +// otherwise it will retry the same thing again. If the condition is not met +// after maxRetries, it will fail the test. +func waitAndRetryIfFail(t *testing.T, setupFunc func(), testFunc func() bool, retryCleanup func(testSuccess bool), maxRetries int, checkInterval time.Duration, maxSingleCheckTime time.Duration, msgAndArgs ...interface{}) { for i := 0; i < maxRetries; i++ { if setupFunc != nil { setupFunc() } result := checkIfEventually(testFunc, checkInterval, maxSingleCheckTime) if retryCleanup != nil { - retryCleanup() + retryCleanup(result) } if result { diff --git a/pkg/eventmonitor/eventmonitor.go b/pkg/eventmonitor/eventmonitor.go index a0b4a84e2cc9f..ce93880272057 100644 --- a/pkg/eventmonitor/eventmonitor.go +++ b/pkg/eventmonitor/eventmonitor.go @@ -11,7 +11,6 @@ package eventmonitor import ( "context" "fmt" - "net" "slices" "sync" "time" @@ -55,7 +54,6 @@ type EventMonitor struct { cancelFnc context.CancelFunc sendStatsChan chan chan bool eventConsumers []EventConsumerInterface - netListener net.Listener wg sync.WaitGroup } @@ -108,8 +106,6 @@ func (m *EventMonitor) Start() error { return fmt.Errorf("unable to register event monitoring module: %w", err) } - m.netListener = ln - m.wg.Add(1) go func() { defer m.wg.Done() @@ -169,17 +165,17 @@ func (m *EventMonitor) Close() { m.GRPCServer.Stop() } - if m.netListener != nil { - m.netListener.Close() + if err := m.cleanup(); err != nil { + seclog.Errorf("failed to cleanup event monitor: %v", err) } - m.cleanup() - m.cancelFnc() m.wg.Wait() // all the go routines should be stopped now we can safely call close the probe and remove the eBPF programs - m.Probe.Close() + if err := m.Probe.Close(); err != nil { + seclog.Errorf("failed to close event monitor probe: %v", err) + } } // SendStats send stats diff --git a/pkg/eventmonitor/eventmonitor_linux.go b/pkg/eventmonitor/eventmonitor_linux.go index cae6dff3153df..5ef2576dce7db 100644 --- a/pkg/eventmonitor/eventmonitor_linux.go +++ b/pkg/eventmonitor/eventmonitor_linux.go @@ -26,10 +26,15 @@ func (m *EventMonitor) getListener() (net.Listener, error) { func (m *EventMonitor) init() error { // force socket cleanup of previous socket not cleanup - os.Remove(m.Config.SocketPath) + if err := os.Remove(m.Config.SocketPath); err != nil && !os.IsNotExist(err) { + return err + } return nil } -func (m *EventMonitor) cleanup() { - os.Remove(m.Config.SocketPath) +func (m *EventMonitor) cleanup() error { + if err := os.Remove(m.Config.SocketPath); err != nil && !os.IsNotExist(err) { + return err + } + return nil } diff --git a/pkg/eventmonitor/eventmonitor_windows.go b/pkg/eventmonitor/eventmonitor_windows.go index e6263e7b9a7f1..84d88bf4e4f56 100644 --- a/pkg/eventmonitor/eventmonitor_windows.go +++ b/pkg/eventmonitor/eventmonitor_windows.go @@ -19,4 +19,6 @@ func (m *EventMonitor) init() error { return nil } -func (m *EventMonitor) cleanup() {} +func (m *EventMonitor) cleanup() error { + return nil +} diff --git a/pkg/fleet/daemon/remote_config.go b/pkg/fleet/daemon/remote_config.go index 24289098f5c58..1b34301343cf1 100644 --- a/pkg/fleet/daemon/remote_config.go +++ b/pkg/fleet/daemon/remote_config.go @@ -36,7 +36,7 @@ func newRemoteConfig(rcFetcher client.ConfigFetcher) (*remoteConfig, error) { client, err := client.NewClient( rcFetcher, client.WithUpdater(), - client.WithProducts(state.ProductUpdaterCatalogDD, state.ProductUpdaterTask), + client.WithProducts(state.ProductUpdaterCatalogDD), client.WithoutTufVerification(), ) if err != nil { diff --git a/pkg/fleet/installer/service/apm_inject.go b/pkg/fleet/installer/service/apm_inject.go index 8899c17af85f6..f1c8028c803c8 100644 --- a/pkg/fleet/installer/service/apm_inject.go +++ b/pkg/fleet/installer/service/apm_inject.go @@ -153,6 +153,9 @@ func (a *apmInjectorInstaller) Setup(ctx context.Context) error { return err } } + if err := setupAppArmor(ctx); err != nil { + return err + } // Create mandatory dirs err = os.Mkdir("/var/log/datadog/dotnet", 0777) @@ -239,6 +242,8 @@ func (a *apmInjectorInstaller) Uninstrument(ctx context.Context) error { dockerErr := a.uninstrumentDocker(ctx) errs = append(errs, dockerErr) } + appArmorErr := removeAppArmor(ctx) + errs = append(errs, appArmorErr) return multierr.Combine(errs...) } diff --git a/pkg/fleet/installer/service/app_armor.go b/pkg/fleet/installer/service/app_armor.go new file mode 100644 index 0000000000000..fceb30c9171a8 --- /dev/null +++ b/pkg/fleet/installer/service/app_armor.go @@ -0,0 +1,91 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !windows + +// Package service provides a way to interact with os services +package service + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/DataDog/datadog-agent/pkg/util/log" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" +) + +const ( + appArmorConfigPath = "/etc/apparmor.d/abstractions/base.d" + appArmorProfile = `/opt/datadog-packages/** rix, +/proc/@{pid}/** rix,` +) + +var datadogProfilePath = filepath.Join(appArmorConfigPath, "datadog") + +func setupAppArmor(ctx context.Context) (err error) { + _, err = exec.LookPath("aa-status") + if err != nil { + // no-op if apparmor is not installed + return nil + } + span, _ := tracer.StartSpanFromContext(ctx, "setup_app_armor") + defer func() { span.Finish(tracer.WithError(err)) }() + if err = os.MkdirAll(appArmorConfigPath, 0755); err != nil { + return fmt.Errorf("failed to create %s: %w", appArmorConfigPath, err) + } + // unfortunately this isn't an atomic change. All files in that directory can be interpreted + // and I did not implement finding a safe directory to write to in the same partition, to run an atomic move. + // This shouldn't be a problem as we reload app armor right after writing the file. + if err = os.WriteFile(datadogProfilePath, []byte(appArmorProfile), 0644); err != nil { + return err + } + if err = reloadAppArmor(); err != nil { + if rollbackErr := os.Remove(datadogProfilePath); rollbackErr != nil { + log.Warnf("failed to remove apparmor profile: %v", rollbackErr) + } + return err + } + return nil +} + +func removeAppArmor(ctx context.Context) (err error) { + _, err = os.Stat(datadogProfilePath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + span, _ := tracer.StartSpanFromContext(ctx, "remove_app_armor") + defer span.Finish(tracer.WithError(err)) + if err = os.Remove(datadogProfilePath); err != nil { + return err + } + return reloadAppArmor() +} + +func reloadAppArmor() error { + if !isAppArmorRunning() { + return nil + } + if running, err := isSystemdRunning(); err != nil { + return err + } else if running { + return exec.Command("systemctl", "reload", "apparmor").Run() + } + return exec.Command("service", "apparmor", "reload").Run() +} + +func isAppArmorRunning() bool { + data, err := os.ReadFile("/sys/module/apparmor/parameters/enabled") + if err != nil { + return false + } + return strings.TrimSpace(string(data)) == "Y" +} diff --git a/pkg/gpu/probe_test.go b/pkg/gpu/probe_test.go index 15cda3b7e76cc..9b0e560fd36c6 100644 --- a/pkg/gpu/probe_test.go +++ b/pkg/gpu/probe_test.go @@ -15,6 +15,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/gpu/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" + "github.com/DataDog/datadog-agent/pkg/process/monitor" "github.com/DataDog/datadog-agent/pkg/util/kernel" ) @@ -42,6 +43,10 @@ func TestProbeCanReceiveEvents(t *testing.T) { t.Skipf("minimum kernel version %s not met, read %s", minimumKernelVersion, kver) } + procMon := monitor.GetProcessMonitor() + require.NotNil(t, procMon) + require.NoError(t, procMon.Initialize(false)) + cfg := NewConfig() cfg.InitialProcessSync = false cfg.BPFDebug = true diff --git a/pkg/gpu/stream.go b/pkg/gpu/stream.go index db993b9600e81..a3e494031346a 100644 --- a/pkg/gpu/stream.go +++ b/pkg/gpu/stream.go @@ -139,8 +139,10 @@ func (sh *StreamHandler) getCurrentData(now uint64) *model.StreamData { return nil } - data := &model.StreamData{ - Spans: []*model.KernelSpan{sh.getCurrentKernelSpan(now)}, + data := &model.StreamData{} + span := sh.getCurrentKernelSpan(now) + if span != nil { + data.Spans = append(data.Spans, span) } for _, alloc := range sh.memAllocEvents { diff --git a/pkg/logs/launchers/integration/launcher.go b/pkg/logs/launchers/integration/launcher.go index b822e90eabad7..de88863114936 100644 --- a/pkg/logs/launchers/integration/launcher.go +++ b/pkg/logs/launchers/integration/launcher.go @@ -50,7 +50,7 @@ type Launcher struct { // fileInfo stores information about each file that is needed in order to keep // track of the combined and overall disk usage by the logs files type fileInfo struct { - filename string + fileWithPath string lastModified time.Time size int64 } @@ -149,7 +149,7 @@ func (s *Launcher) run() { s.integrationToFile[cfg.IntegrationID] = logFile } - filetypeSource := s.makeFileSource(source, logFile.filename) + filetypeSource := s.makeFileSource(source, logFile.fileWithPath) s.sources.AddSource(filetypeSource) } } @@ -179,8 +179,17 @@ func (s *Launcher) receiveLogs(log integrations.IntegrationLog) { // Ensure the individual file doesn't exceed integrations_logs_files_max_size // Add 1 because we write the \n at the end as well logSize := int64(len(log.Log)) + 1 + + if logSize > s.fileSizeMax { + ddLog.Warnf("Individual log size (%d bytes) is larger than maximum allowable file size (%d bytes), skipping writing to log file: %s", logSize, s.fileSizeMax, log.Log) + return + } else if logSize > s.combinedUsageMax { + ddLog.Warnf("Individual log size (%d bytes) is larger than maximum allowable file size (%d bytes), skipping writing to log file: %s", logSize, s.combinedUsageMax, log.Log) + return + } + if fileToUpdate.size+logSize > s.fileSizeMax { - file, err := os.Create(fileToUpdate.filename) + file, err := os.Create(fileToUpdate.fileWithPath) if err != nil { ddLog.Error("Failed to delete and remake oversize file:", err) return @@ -211,7 +220,7 @@ func (s *Launcher) receiveLogs(log integrations.IntegrationLog) { return } - file, err := os.Create(leastRecentlyModifiedFile.filename) + file, err := os.Create(leastRecentlyModifiedFile.fileWithPath) if err != nil { ddLog.Error("Error creating log file:", err) continue @@ -223,7 +232,7 @@ func (s *Launcher) receiveLogs(log integrations.IntegrationLog) { } } - err := s.writeLogToFileFunction(filepath.Join(s.runPath, fileToUpdate.filename), log.Log) + err := s.writeLogToFileFunction(fileToUpdate.fileWithPath, log.Log) if err != nil { ddLog.Warn("Error writing log to file:", err) return @@ -236,12 +245,11 @@ func (s *Launcher) receiveLogs(log integrations.IntegrationLog) { } func (s *Launcher) deleteFile(file *fileInfo) error { - filename := filepath.Join(s.runPath, file.filename) - err := os.Remove(filename) + err := os.Remove(file.fileWithPath) if err != nil { return err } - ddLog.Info("Successfully deleted log file:", filename) + ddLog.Info("Successfully deleted log file:", file.fileWithPath) s.combinedUsageSize -= file.size @@ -321,7 +329,7 @@ func (s *Launcher) createFile(source string) (*fileInfo, error) { } fileInfo := &fileInfo{ - filename: filepath, + fileWithPath: filepath, lastModified: time.Now(), size: 0, } @@ -349,8 +357,8 @@ func computeMaxDiskUsage(runPath string, logsTotalUsageSetting int64, usageRatio diskReserved := float64(usage.Total) * (1 - usageRatio) diskAvailable := int64(usage.Available) - int64(math.Ceil(diskReserved)) - if diskAvailable < 0 { - ddLog.Warn("Available disk calculated as less than 0: ", diskAvailable, ". Disk reserved:", diskReserved) + if diskAvailable <= 0 { + ddLog.Warnf("Available disk calculated as %d bytes, disk reserved is %f bytes. Check %s and make sure there is enough free space on disk", diskAvailable, diskReserved, "integrations_logs_disk_ratio") diskAvailable = 0 } @@ -370,12 +378,12 @@ func (s *Launcher) scanInitialFiles(dir string) error { } fileInfo := &fileInfo{ - filename: info.Name(), + fileWithPath: filepath.Join(dir, info.Name()), size: info.Size(), lastModified: info.ModTime(), } - integrationID := fileNameToID(fileInfo.filename) + integrationID := fileNameToID(fileInfo.fileWithPath) s.integrationToFile[integrationID] = fileInfo s.combinedUsageSize += info.Size() diff --git a/pkg/logs/launchers/integration/launcher_test.go b/pkg/logs/launchers/integration/launcher_test.go index 21d5c293ff860..652f6479019de 100644 --- a/pkg/logs/launchers/integration/launcher_test.go +++ b/pkg/logs/launchers/integration/launcher_test.go @@ -93,7 +93,7 @@ func (suite *LauncherTestSuite) TestSendLog() { assert.Equal(suite.T(), foundSource.Config.Type, config.FileType) assert.Equal(suite.T(), foundSource.Config.Source, "foo") assert.Equal(suite.T(), foundSource.Config.Service, "bar") - expectedPath := filepath.Join(suite.s.runPath, suite.s.integrationToFile[id].filename) + expectedPath := suite.s.integrationToFile[id].fileWithPath assert.Equal(suite.T(), logSample, <-fileLogChan) assert.Equal(suite.T(), expectedPath, <-filepathChan) @@ -113,8 +113,8 @@ func (suite *LauncherTestSuite) TestZeroCombinedUsageMaxFileCreated() { suite.s.combinedUsageMax = 0 filename := "sample_integration_123.log" - filepath := filepath.Join(suite.s.runPath, filename) - file, err := os.Create(filepath) + fileWithPath := filepath.Join(suite.s.runPath, filename) + file, err := os.Create(fileWithPath) assert.Nil(suite.T(), err) file.Close() @@ -143,11 +143,11 @@ func (suite *LauncherTestSuite) TestZeroCombinedUsageMaxFileNotCreated() { } func (suite *LauncherTestSuite) TestSmallCombinedUsageMax() { - suite.s.combinedUsageMax = 10 + suite.s.combinedUsageMax = 15 filename := "sample_integration_123.log" - filepath := filepath.Join(suite.s.runPath, filename) - file, err := os.Create(filepath) + fileWithPath := filepath.Join(suite.s.runPath, filename) + file, err := os.Create(fileWithPath) assert.Nil(suite.T(), err) file.Close() @@ -155,32 +155,41 @@ func (suite *LauncherTestSuite) TestSmallCombinedUsageMax() { suite.s.Start(nil, nil, nil, nil) // Launcher should write this log - writtenLog := "sample" + shortLog := "sample" integrationLog := integrations.IntegrationLog{ - Log: writtenLog, + Log: shortLog, IntegrationID: "sample_integration:123", } suite.s.receiveLogs(integrationLog) - fileStat, err := os.Stat(filepath) + fileStat, err := os.Stat(fileWithPath) assert.Nil(suite.T(), err) - assert.Equal(suite.T(), fileStat.Size(), int64(len(writtenLog)+1)) + assert.Equal(suite.T(), fileStat.Size(), int64(len(shortLog)+1)) - // Launcher should delete file for this log - unwrittenLog := "sample log two" + // Launcher should delete and remake the log file for this log since it would break combinedUsageMax threshold + longLog := "sample log two" integrationLogTwo := integrations.IntegrationLog{ - Log: unwrittenLog, + Log: longLog, IntegrationID: "sample_integration:123", } suite.s.receiveLogs(integrationLogTwo) + _, err = os.Stat(fileWithPath) + assert.Nil(suite.T(), err) - _, err = os.Stat(filepath) - assert.True(suite.T(), os.IsNotExist(err)) + // Launcher should skip writing this log since it's larger than combinedUsageMax + unwrittenLog := "this log is too long" + unwrittenIntegrationLog := integrations.IntegrationLog{ + Log: unwrittenLog, + IntegrationID: "sample_integration:123", + } + suite.s.receiveLogs(unwrittenIntegrationLog) + _, err = os.Stat(fileWithPath) + assert.Nil(suite.T(), err) // Remake the file suite.s.receiveLogs(integrationLog) - fileStat, err = os.Stat(filepath) + fileStat, err = os.Stat(fileWithPath) assert.Nil(suite.T(), err) - assert.Equal(suite.T(), fileStat.Size(), int64(len(writtenLog)+1)) + assert.Equal(suite.T(), fileStat.Size(), int64(len(shortLog)+1)) } func (suite *LauncherTestSuite) TestWriteLogToFile() { @@ -215,12 +224,12 @@ func (suite *LauncherTestSuite) TestWriteMultipleLogsToFile() { // TestDeleteFile tests that deleteFile properly deletes the correct file func (suite *LauncherTestSuite) TestDeleteFile() { filename := "testfile.log" - filepath := filepath.Join(suite.s.runPath, filename) - file, err := os.Create(filepath) - fileinfo := &fileInfo{filename: filename, size: int64(0)} + fileWithPath := filepath.Join(suite.s.runPath, filename) + file, err := os.Create(fileWithPath) + fileinfo := &fileInfo{fileWithPath: fileWithPath, size: int64(0)} assert.Nil(suite.T(), err) - info, err := os.Stat(filepath) + info, err := os.Stat(fileWithPath) assert.Nil(suite.T(), err) assert.Equal(suite.T(), int64(0), info.Size(), "Newly created file size not zero") @@ -229,14 +238,14 @@ func (suite *LauncherTestSuite) TestDeleteFile() { file.Write(data) file.Close() - info, err = os.Stat(filepath) + info, err = os.Stat(fileWithPath) assert.Nil(suite.T(), err) assert.Equal(suite.T(), int64(2*1024*1024), info.Size()) err = suite.s.deleteFile(fileinfo) assert.Nil(suite.T(), err) - _, err = os.Stat(filepath) + _, err = os.Stat(fileWithPath) assert.True(suite.T(), os.IsNotExist(err)) } @@ -281,8 +290,8 @@ func (suite *LauncherTestSuite) TestFileExceedsSingleFileLimit() { suite.s.fileSizeMax = oneMB filename := "sample_integration_123.log" - filepath := filepath.Join(suite.s.runPath, filename) - file, err := os.Create(filepath) + fileWithPath := filepath.Join(suite.s.runPath, filename) + file, err := os.Create(fileWithPath) assert.Nil(suite.T(), err) file.Write(make([]byte, oneMB)) @@ -308,7 +317,8 @@ func (suite *LauncherTestSuite) TestScanInitialFiles() { filename := "sample_integration_123.log" fileSize := int64(1 * 1024 * 1024) - file, err := os.Create(filepath.Join(suite.s.runPath, filename)) + fileWithPath := filepath.Join(suite.s.runPath, filename) + file, err := os.Create(fileWithPath) assert.Nil(suite.T(), err) data := make([]byte, fileSize) @@ -320,7 +330,7 @@ func (suite *LauncherTestSuite) TestScanInitialFiles() { actualFileInfo := suite.s.integrationToFile[fileID] assert.NotEmpty(suite.T(), suite.s.integrationToFile) - assert.Equal(suite.T(), actualFileInfo.filename, filename) + assert.Equal(suite.T(), actualFileInfo.fileWithPath, fileWithPath) assert.Equal(suite.T(), fileSize, actualFileInfo.size) assert.Equal(suite.T(), fileSize, suite.s.combinedUsageSize) } @@ -331,7 +341,8 @@ func (suite *LauncherTestSuite) TestCreateFileAfterScanInitialFile() { filename := "sample_integration_123.log" fileSize := int64(1 * 1024 * 1024) - file, err := os.Create(filepath.Join(suite.s.runPath, filename)) + fileWithPath := filepath.Join(suite.s.runPath, filename) + file, err := os.Create(fileWithPath) assert.Nil(suite.T(), err) data := make([]byte, fileSize) @@ -343,7 +354,7 @@ func (suite *LauncherTestSuite) TestCreateFileAfterScanInitialFile() { scannedFile := suite.s.integrationToFile[fileID] assert.NotEmpty(suite.T(), suite.s.integrationToFile) - assert.Equal(suite.T(), filename, scannedFile.filename) + assert.Equal(suite.T(), fileWithPath, scannedFile.fileWithPath) assert.Equal(suite.T(), fileSize, scannedFile.size) assert.Equal(suite.T(), fileSize, suite.s.combinedUsageSize) diff --git a/pkg/network/config/config.go b/pkg/network/config/config.go index 1b8ef55fb4df8..36c8557010e15 100644 --- a/pkg/network/config/config.go +++ b/pkg/network/config/config.go @@ -20,11 +20,10 @@ import ( ) const ( - spNS = "system_probe_config" - netNS = "network_config" - smNS = "service_monitoring_config" - evNS = "event_monitoring_config" - smjtNS = smNS + ".tls.java" + spNS = "system_probe_config" + netNS = "network_config" + smNS = "service_monitoring_config" + evNS = "event_monitoring_config" defaultUDPTimeoutSeconds = 30 defaultUDPStreamTimeoutSeconds = 120 @@ -110,10 +109,6 @@ type Config struct { // hooking the system-probe test binary. Defaults to true. GoTLSExcludeSelf bool - // EnableJavaTLSSupport specifies whether the tracer should monitor HTTPS - // traffic done through Java's TLS implementation - EnableJavaTLSSupport bool - // MaxTrackedHTTPConnections max number of http(s) flows that will be concurrently tracked. // value is currently Windows only MaxTrackedHTTPConnections int64 @@ -126,21 +121,6 @@ type Config struct { // Currently Windows only HTTPMaxRequestFragment int64 - // JavaAgentDebug will enable debug output of the injected USM agent - JavaAgentDebug bool - - // JavaAgentArgs arguments pass through injected USM agent - JavaAgentArgs string - - // JavaAgentAllowRegex (Higher priority) define a regex, if matching /proc/pid/cmdline the java agent will be injected - JavaAgentAllowRegex string - - // JavaAgentBlockRegex define a regex, if matching /proc/pid/cmdline the java agent will not be injected - JavaAgentBlockRegex string - - // JavaDir is the directory to load the java agent program from - JavaDir string - // UDPConnTimeout determines the length of traffic inactivity between two // (IP, port)-pairs before declaring a UDP connection as inactive. This is // set to /proc/sys/net/netfilter/nf_conntrack_udp_timeout on Linux by @@ -411,12 +391,6 @@ func New() *Config { EnableEbpfless: cfg.GetBool(join(netNS, "enable_ebpfless")), // Service Monitoring - EnableJavaTLSSupport: cfg.GetBool(join(smjtNS, "enabled")), - JavaAgentDebug: cfg.GetBool(join(smjtNS, "debug")), - JavaAgentArgs: cfg.GetString(join(smjtNS, "args")), - JavaAgentAllowRegex: cfg.GetString(join(smjtNS, "allow_regex")), - JavaAgentBlockRegex: cfg.GetString(join(smjtNS, "block_regex")), - JavaDir: cfg.GetString(join(smjtNS, "dir")), EnableGoTLSSupport: cfg.GetBool(join(smNS, "tls", "go", "enabled")), GoTLSExcludeSelf: cfg.GetBool(join(smNS, "tls", "go", "exclude_self")), EnableHTTPStatsByStatusCode: cfg.GetBool(join(smNS, "enable_http_stats_by_status_code")), diff --git a/pkg/network/config/config_test.go b/pkg/network/config/config_test.go index d732160b00b09..50c90c3947b1d 100644 --- a/pkg/network/config/config_test.go +++ b/pkg/network/config/config_test.go @@ -176,25 +176,6 @@ func TestEnableHTTPMonitoring(t *testing.T) { }) } -func TestEnableJavaTLSSupport(t *testing.T) { - t.Run("via YAML", func(t *testing.T) { - mockSystemProbe := mock.NewSystemProbe(t) - mockSystemProbe.SetWithoutSource("service_monitoring_config.tls.java.enabled", true) - cfg := New() - - require.True(t, cfg.EnableJavaTLSSupport) - }) - - t.Run("via ENV variable", func(t *testing.T) { - mock.NewSystemProbe(t) - t.Setenv("DD_SERVICE_MONITORING_CONFIG_TLS_JAVA_ENABLED", "true") - cfg := New() - - require.True(t, cfg.EnableJavaTLSSupport) - }) - -} - func TestEnableHTTP2Monitoring(t *testing.T) { t.Run("via YAML", func(t *testing.T) { mockSystemProbe := mock.NewSystemProbe(t) @@ -293,16 +274,6 @@ func TestEnableRedisMonitoring(t *testing.T) { }) } -func TestDefaultDisabledJavaTLSSupport(t *testing.T) { - mock.NewSystemProbe(t) - cfg := New() - - _, err := sysconfig.New("", "") - require.NoError(t, err) - - assert.False(t, cfg.EnableJavaTLSSupport) -} - func TestDefaultDisabledHTTP2Support(t *testing.T) { mock.NewSystemProbe(t) cfg := New() diff --git a/pkg/network/ebpf/c/prebuilt/usm.c b/pkg/network/ebpf/c/prebuilt/usm.c index d78dd93ab69bc..8b5e62ede108e 100644 --- a/pkg/network/ebpf/c/prebuilt/usm.c +++ b/pkg/network/ebpf/c/prebuilt/usm.c @@ -15,8 +15,6 @@ #include "protocols/postgres/decoding.h" #include "protocols/redis/decoding.h" #include "protocols/sockfd-probes.h" -#include "protocols/tls/java/erpc_dispatcher.h" -#include "protocols/tls/java/erpc_handlers.h" #include "protocols/tls/https.h" #include "protocols/tls/native-tls.h" #include "protocols/tls/tags-types.h" diff --git a/pkg/network/ebpf/c/protocols/tls/java/erpc_dispatcher.h b/pkg/network/ebpf/c/protocols/tls/java/erpc_dispatcher.h deleted file mode 100644 index b947b5468adc4..0000000000000 --- a/pkg/network/ebpf/c/protocols/tls/java/erpc_dispatcher.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef __ERPC_DISPATCHER_H -#define __ERPC_DISPATCHER_H - -#include "bpf_helpers.h" -#include "protocols/tls/java/types.h" -#include "protocols/tls/java/maps.h" - -#define USM_IOCTL_ID 0xda7ad09 - -static int __always_inline is_usm_erpc_request(struct pt_regs *ctx) { - u32 cmd = PT_REGS_PARM3(ctx); - return cmd == USM_IOCTL_ID; -}; - -/* - handle_erpc_request ioctl request format : - - struct { - u8 operation; // see erpc_message_type enum for supported operations - u8 data[]; // payload data - } -*/ - -static void __always_inline handle_erpc_request(struct pt_regs *ctx) { - #ifdef DEBUG - u64 pid_tgid = bpf_get_current_pid_tgid(); - u64 pid = pid_tgid >> 32; - #endif - - void *req = (void *)PT_REGS_PARM4(ctx); - - u8 op = 0; - if (0 != bpf_probe_read_user(&op, sizeof(op), req)){ - log_debug("[java_tls_handle_erpc_request] failed to parse opcode of java tls erpc request for: pid %llu", pid); - return; - } - - //for easier troubleshooting in case we get out of sync between java tracer's side of the erpc and systemprobe's side - #ifdef DEBUG - log_debug("[java_tls_handle_erpc_request] received %d op", op); - if (op >= MAX_MESSAGE_TYPE){ - log_debug("[java_tls_handle_erpc_request] got unsupported erpc request %x for: pid %llu",op, pid); - } - #endif - - bpf_tail_call_compat(ctx, &java_tls_erpc_handlers, op); -} - -SEC("kprobe/do_vfs_ioctl") -int BPF_BYPASSABLE_KPROBE(kprobe__do_vfs_ioctl) { - if (is_usm_erpc_request(ctx)) { - handle_erpc_request(ctx); - } - - return 0; -} - -#endif // __ERPC_DISPATCHER_H diff --git a/pkg/network/ebpf/c/protocols/tls/java/erpc_handlers.h b/pkg/network/ebpf/c/protocols/tls/java/erpc_handlers.h deleted file mode 100644 index 07570f6cbc0d5..0000000000000 --- a/pkg/network/ebpf/c/protocols/tls/java/erpc_handlers.h +++ /dev/null @@ -1,180 +0,0 @@ -#ifndef __ERPC_HANDLERS_H -#define __ERPC_HANDLERS_H - -#include "conn_tuple.h" -#include "protocols/tls/tags-types.h" -#include "protocols/tls/https.h" -#include "port_range.h" - -// macro to get the data pointer from the ctx, we skip the 1st byte as it is the operation byte read by the erpc dispatcher -#define GET_DATA_PTR(ctx) ((void *)(PT_REGS_PARM4(ctx) + 1)) - -/* - handle_sync_payload's pseudo format of *data that contains the http payload - - struct { - conn_tuple_t; - u32 payload_len; - u8 payload_buffer[payload_len]; - } -*/ - -SEC("kprobe/handle_sync_payload") -int kprobe_handle_sync_payload(struct pt_regs *ctx) { - // get connection tuple - conn_tuple_t connection = {0}; - const bool val = true; - u32 bytes_read = 0; - - //interactive pointer to read the data buffer - void* bufferPtr = GET_DATA_PTR(ctx); - - //read the connection tuple from the ioctl buffer - if (0 != bpf_probe_read_user(&connection, sizeof(conn_tuple_t), bufferPtr)){ - log_debug("[handle_sync_payload] failed to parse connection info"); - return 1; - } - normalize_tuple(&connection); - bufferPtr+=sizeof(conn_tuple_t); - - // read the actual length of the message (limited by HTTP_BUFFER_SIZE) - if (0 != bpf_probe_read_user(&bytes_read, sizeof(bytes_read), bufferPtr)){ -#ifdef DEBUG - u64 pid_tgid = bpf_get_current_pid_tgid(); - u64 pid = pid_tgid >> 32; - log_debug("[handle_sync_payload] failed reading message length location for pid %lld", pid); -#endif - return 1; - } - bufferPtr+=sizeof(bytes_read); - - // register the connection in our map - bpf_map_update_elem(&java_tls_connections, &connection, &val, BPF_ANY); - log_debug("[handle_sync_payload] handling tls request of size: %d for connection src addr: %llx; dst address %llx", - bytes_read, connection.saddr_l, connection.daddr_l); - tls_process(ctx, &connection, bufferPtr, bytes_read, JAVA_TLS); - return 0; -} - -/* - handle_close_connection gets only the connection information in form of conn_tuple_t struct from the close event of the socket -*/ -SEC("kprobe/handle_close_connection") -int kprobe_handle_close_connection(struct pt_regs *ctx) { - //interactive pointer to read the data buffer - void* bufferPtr = GET_DATA_PTR(ctx); - //read the connection tuple from the ioctl buffer - conn_tuple_t connection = {0}; - if (0 != bpf_probe_read_user(&connection, sizeof(conn_tuple_t), bufferPtr)){ - log_debug("[java_tls_handle_close] failed to parse connection info"); - return 1; - } - normalize_tuple(&connection); - - void *exists = bpf_map_lookup_elem(&java_tls_connections, &connection); - // if the connection exists in our map, finalize it and remove from the map - // otherwise just ignore - if (exists != NULL){ - // tls_finish can launch a tail call, thus cleanup should be done before. - bpf_map_delete_elem(&java_tls_connections, &connection); - tls_finish(ctx, &connection, false); - } - return 0; -} - -/* - handle_connection_by_peer gets connection information along the peer domain and port information - which helps to correlate later the plain payload with the relevant connection via the peer details -*/ -SEC("kprobe/handle_connection_by_peer") -int kprobe_handle_connection_by_peer(struct pt_regs *ctx) { - - connection_by_peer_key_t peer_key ={0}; - u64 pid_tgid = bpf_get_current_pid_tgid(); - peer_key.pid = pid_tgid >> 32; - - //interactive pointer to read the data buffer - void* bufferPtr = GET_DATA_PTR(ctx); - - //read the connection tuple from the ioctl buffer - conn_tuple_t connection = {0}; - if (0 != bpf_probe_read_user(&connection, sizeof(conn_tuple_t), bufferPtr)){ - log_debug("[handle_connection_by_peer] failed to parse connection info for pid: %d", peer_key.pid); - return 1; - } - normalize_tuple(&connection); - bufferPtr+=sizeof(conn_tuple_t); - - //read the peer tuple (domain string and port) - if (0 != bpf_probe_read_user(&peer_key.peer, sizeof(peer_t), bufferPtr)){ - log_debug("[handle_connection_by_peer] failed reading peer tuple information for pid %d", peer_key.pid); - return 1; - } - - // register the connection in conn_by_peer map - bpf_map_update_elem(&java_conn_tuple_by_peer, &peer_key, &connection, BPF_ANY); - - log_debug("[handle_connection_by_peer] created map entry for pid %d domain %s port: %d", - peer_key.pid, peer_key.peer.domain, peer_key.peer.port); - return 0; -} - -/* - handle_async_payload doesn't contain any transport layer information (connection), - buy instead send the actual payload in its plain form together with peer domain string and peer port. - - We try to locate the relevant connection info from the bpf map using peer information together with pid as a key -*/ -SEC("kprobe/handle_async_payload") -int kprobe_handle_async_payload(struct pt_regs *ctx) { - const bool val = true; - u32 bytes_read = 0; - - //interactive pointer to read the data buffer - void* bufferPtr = GET_DATA_PTR(ctx); - - connection_by_peer_key_t peer_key ={0}; - peer_key.pid = bpf_get_current_pid_tgid() >> 32; - - //read the peer tuple (domain string and port) - if (0 != bpf_probe_read_user(&peer_key.peer, sizeof(peer_t), bufferPtr)){ - log_debug("[handle_async_payload] failed allocating peer tuple struct on heap"); - return 1; - } - bufferPtr+=sizeof(peer_t); - log_debug("[handle_async_payload] pid: %d; peer domain: %s; peer port: %d", - peer_key.pid, - peer_key.peer.domain, - peer_key.peer.port); - - //get connection tuple - conn_tuple_t * actual_connection = bpf_map_lookup_elem(&java_conn_tuple_by_peer, &peer_key); - if (!actual_connection) { - log_debug("[handle_async_payload] couldn't correlate connection"); - return 1; - } - - // we need to copy the connection on the stack to be able to call bpf_map_update_elem on old kernels - conn_tuple_t conn_on_stack = *actual_connection; - log_debug("[handle_async_payload] found correlation conn src port: %d dst port: %d", - actual_connection->sport, - actual_connection->dport); - - // read the actual length of the message (limited to HTTP_BUFFER_SIZE bytes) - if (0 != bpf_probe_read_user(&bytes_read, sizeof(bytes_read), bufferPtr)){ - log_debug("[handle_async_payload] failed reading message length location for pid %d", peer_key.pid); - return 1; - } - bufferPtr+=sizeof(bytes_read); - - // register the connection in our map - bpf_map_update_elem(&java_tls_connections, &conn_on_stack, &val, BPF_ANY); - log_debug("[handle_async_payload] handling tls request of size: %d for connection src addr: %llx; dst address %llx", - bytes_read, - actual_connection->saddr_l, - actual_connection->daddr_l); - tls_process(ctx, actual_connection, bufferPtr, bytes_read, JAVA_TLS); - return 0; -} - -#endif // __ERPC_HANDLERS_H diff --git a/pkg/network/ebpf/c/protocols/tls/java/maps.h b/pkg/network/ebpf/c/protocols/tls/java/maps.h deleted file mode 100644 index 758139b581cff..0000000000000 --- a/pkg/network/ebpf/c/protocols/tls/java/maps.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef __JAVA_MAPS_H -#define __JAVA_MAPS_H - -#include "bpf_helpers.h" -#include "map-defs.h" -#include "conn_tuple.h" - -#include "protocols/tls/java/types.h" - -/* A set (map from a key to a const bool value, we care only if the key exists in the map, and not its value) to - mark if we've seen a specific java tls connection. - Map size is set to 1 as javaTLS is optional, this will be overwritten to MaxTrackedConnections - if javaTLS is enabled. */ -BPF_HASH_MAP(java_tls_connections, conn_tuple_t, bool, 1) - -/* map to correlate peer domain and port with the actual conn_tuple - Map size is set to 1 as javaTLS is optional, this will be overwritten to MaxTrackedConnections - if javaTLS is enabled. */ -BPF_HASH_MAP(java_conn_tuple_by_peer, connection_by_peer_key_t, conn_tuple_t, 1) - -/* - Map used to store the sub programs used by eRPC mechanism - This is done to avoid memory limitation when handling different operations sent via ioctl (eRPC) from our dd-java-agent -*/ -BPF_PROG_ARRAY(java_tls_erpc_handlers, MAX_MESSAGE_TYPE) - -#endif // __JAVA_MAPS_H diff --git a/pkg/network/ebpf/c/protocols/tls/java/types.h b/pkg/network/ebpf/c/protocols/tls/java/types.h deleted file mode 100644 index 2203f2a0a20a3..0000000000000 --- a/pkg/network/ebpf/c/protocols/tls/java/types.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef __JAVA_TLS_TYPES_H -#define __JAVA_TLS_TYPES_H - -#include "ktypes.h" - -// any change in this const is sensitive to stack limit of kprobe_handle_async_payload ebpf program, -// as it increases the size of structs defined below. -#define MAX_DOMAIN_NAME_LENGTH 48 - -enum erpc_message_type { - SYNCHRONOUS_PAYLOAD, - CLOSE_CONNECTION, - CONNECTION_BY_PEER, - ASYNC_PAYLOAD, - MAX_MESSAGE_TYPE, -}; - -typedef struct{ - __u16 port; - char domain[MAX_DOMAIN_NAME_LENGTH]; -} peer_t; - -typedef struct{ - __u32 pid; - peer_t peer; -} connection_by_peer_key_t; - - -#endif //__JAVA_TLS_TYPES_H diff --git a/pkg/network/ebpf/c/protocols/tls/tags-types.h b/pkg/network/ebpf/c/protocols/tls/tags-types.h index 913a806d626a1..8588558c24509 100644 --- a/pkg/network/ebpf/c/protocols/tls/tags-types.h +++ b/pkg/network/ebpf/c/protocols/tls/tags-types.h @@ -7,10 +7,9 @@ enum static_tags { LIBGNUTLS = (1<<0), LIBSSL = (1<<1), GO = (1<<2), - JAVA_TLS = (1<<3), - CONN_TLS = (1<<4), - ISTIO = (1<<5), - NODEJS = (1<<6), + CONN_TLS = (1<<3), + ISTIO = (1<<4), + NODEJS = (1<<5), }; #endif diff --git a/pkg/network/ebpf/c/runtime/usm.c b/pkg/network/ebpf/c/runtime/usm.c index 2912ea392f306..3cb3f1245a3ab 100644 --- a/pkg/network/ebpf/c/runtime/usm.c +++ b/pkg/network/ebpf/c/runtime/usm.c @@ -23,8 +23,6 @@ #include "protocols/postgres/decoding.h" #include "protocols/redis/decoding.h" #include "protocols/sockfd-probes.h" -#include "protocols/tls/java/erpc_dispatcher.h" -#include "protocols/tls/java/erpc_handlers.h" #include "protocols/tls/go-tls-types.h" #include "protocols/tls/go-tls-goid.h" #include "protocols/tls/go-tls-location.h" diff --git a/pkg/network/encoding/encoding_test.go b/pkg/network/encoding/encoding_test.go index 2e9806ea59297..ed50111ee7c66 100644 --- a/pkg/network/encoding/encoding_test.go +++ b/pkg/network/encoding/encoding_test.go @@ -38,7 +38,7 @@ type connTag = uint64 const ( tagGnuTLS connTag = 0x01 // network.ConnTagGnuTLS tagOpenSSL connTag = 0x02 // network.ConnTagOpenSSL - tagTLS connTag = 0x10 // network.ConnTagTLS + tagTLS connTag = 0x8 // network.ConnTagTLS ) func getBlobWriter(t *testing.T, assert *assert.Assertions, in *network.Connections, marshalerType string) *bytes.Buffer { diff --git a/pkg/network/java/testutil/JavaClientSimulator/Dockerfile b/pkg/network/java/testutil/JavaClientSimulator/Dockerfile deleted file mode 100644 index 295238d0e1884..0000000000000 --- a/pkg/network/java/testutil/JavaClientSimulator/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -# Base image with Java installed -FROM openjdk:11 -MAINTAINER val - -# Set the working directory inside the container -WORKDIR /app - -COPY target/JavaClientSimulator-1.0.jar app.jar -ENTRYPOINT ["sh", "-c", "java -jar app.jar -c \"$JAVA_TEST_CLIENT\" -u \"$JAVA_TARGET_URL\""] diff --git a/pkg/network/java/testutil/JavaClientSimulator/META-INF/MANIFEST.MF b/pkg/network/java/testutil/JavaClientSimulator/META-INF/MANIFEST.MF deleted file mode 100644 index 44f4d81f0c0dc..0000000000000 --- a/pkg/network/java/testutil/JavaClientSimulator/META-INF/MANIFEST.MF +++ /dev/null @@ -1,11 +0,0 @@ -Manifest-Version: 1.0 -Implementation-Title: dd-java-agent -Premain-Class: datadog.trace.bootstrap.AgentBootstrap -Implementation-Version: 1.13.0-SNAPSHOT -Agent-Class: datadog.trace.bootstrap.AgentBootstrap -Can-Redefine-Classes: true -Can-Retransform-Classes: true -Implementation-URL: https://github.com/datadog/dd-trace-java -Implementation-Vendor: Datadog -Main-Class: datadog.trace.bootstrap.AgentBootstrap - diff --git a/pkg/network/java/testutil/JavaClientSimulator/Readme.md b/pkg/network/java/testutil/JavaClientSimulator/Readme.md deleted file mode 100644 index d105d1a4bac66..0000000000000 --- a/pkg/network/java/testutil/JavaClientSimulator/Readme.md +++ /dev/null @@ -1,25 +0,0 @@ -JavaClientSimulator allows testing different java **https** frameworks. -You can run the attached docker or directly by executing the jar package.1 - -This test util was built using JDK 11 - -## Configuration - -you can control -- java client framework to use. Currently supported: ApacheHttp, OkHttp, URLConnection, HttpClient -- target url -- iterations - number of requests to send. By default infinitely -- timeout - interval between each iteration. By default 1 second - -## Standalone: - -Build: `mvn clean package` - -Run: `java -jar ./target/JavaClientSimulator-1.0.jar client= url=` - -## Docker - -Build: `docker build -t java-http-client .` - -Run: `docker run -e JAVA_TEST_CLIENT= -e JAVA_TARGET_URL= java-http-client` - diff --git a/pkg/network/java/testutil/JavaClientSimulator/dependency-reduced-pom.xml b/pkg/network/java/testutil/JavaClientSimulator/dependency-reduced-pom.xml deleted file mode 100644 index 1b733595f00eb..0000000000000 --- a/pkg/network/java/testutil/JavaClientSimulator/dependency-reduced-pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - 4.0.0 - JavaClients - JavaClientSimulator - client_example - 1.0 - - - Apache License Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - - - - - - maven-clean-plugin - 3.1.0 - - - maven-resources-plugin - 3.0.2 - - - maven-compiler-plugin - 3.8.0 - - 11 - 11 - - - - maven-surefire-plugin - 2.22.1 - - - maven-jar-plugin - 3.1.0 - - - - true - lib/ - Clients.Main - - - - - - maven-install-plugin - 2.5.2 - - - maven-deploy-plugin - 2.8.2 - - - maven-site-plugin - 3.7.1 - - - maven-shade-plugin - 3.4.1 - - - package - - shade - - - - - Clients.Main - - - - - - - - - - - confluent - https://packages.confluent.io/maven/ - - - - UTF-8 - - diff --git a/pkg/network/java/testutil/JavaClientSimulator/pom.xml b/pkg/network/java/testutil/JavaClientSimulator/pom.xml deleted file mode 100644 index fa70437f6a008..0000000000000 --- a/pkg/network/java/testutil/JavaClientSimulator/pom.xml +++ /dev/null @@ -1,127 +0,0 @@ - - - - 4.0.0 - - JavaClients - JavaClientSimulator - 1.0 - jar - - client_example - - - UTF-8 - - - - - - http://www.apache.org/licenses/LICENSE-2.0.txt - Apache License Version 2.0 - - - - - - confluent - https://packages.confluent.io/maven/ - - - - - - - - org.apache.httpcomponents - httpclient - 4.5.13 - - - commons-cli - commons-cli - 1.5.0 - - - com.squareup.okhttp3 - okhttp - 4.10.0 - - - - - - - - maven-clean-plugin - 3.1.0 - - - maven-resources-plugin - 3.0.2 - - - maven-compiler-plugin - - 11 - 11 - - 3.8.0 - - - maven-surefire-plugin - 2.22.1 - - - org.apache.maven.plugins - maven-jar-plugin - 3.1.0 - - - - true - lib/ - Clients.Main - - - - - - maven-install-plugin - 2.5.2 - - - maven-deploy-plugin - 2.8.2 - - - - maven-site-plugin - 3.7.1 - - - - org.apache.maven.plugins - maven-shade-plugin - 3.4.1 - - - - package - - shade - - - - - Clients.Main - - - - - - - - - diff --git a/pkg/network/java/testutil/JavaClientSimulator/src/main/java/Clients/ClientManager.java b/pkg/network/java/testutil/JavaClientSimulator/src/main/java/Clients/ClientManager.java deleted file mode 100644 index d2d704623504b..0000000000000 --- a/pkg/network/java/testutil/JavaClientSimulator/src/main/java/Clients/ClientManager.java +++ /dev/null @@ -1,106 +0,0 @@ -package Clients; - -import java.io.IOException; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; - -public class ClientManager { - - public enum ClientType{ - apache, - okhttp, - httpclient, - urlconnection, - unsupported, - } - - private static ClientType getClientType(String clientTypeArg) { - try { - return ClientType.valueOf(clientTypeArg.toLowerCase()); - } catch (IllegalArgumentException e) { - return ClientType.unsupported; - } - } - - public static void executeCallback(String clientTypeArg, int iterations, long sleepInterval, String url) throws InterruptedException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException, IOException { - - ClientType clientType = getClientType(clientTypeArg); - JavaClients clients = new JavaClients(); - clients.init(); - - System.out.println("Executing handler for " + clientType); - System.out.println("URL: " + url); - System.out.println("Iterations: " + iterations); - System.out.println("Interval: " + sleepInterval); - - Runnable callback; - - // Execute handler based on client type - switch (clientType) { - case apache: - - callback = () -> { - try { - clients.HttpApacheClient(url); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - break; - case okhttp: - callback = () -> { - try { - clients.OkHttpClient(url); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - break; - case httpclient: - callback = () -> { - try { - clients.HTTPClient(url); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - break; - case urlconnection: - callback = () -> { - try { - clients.HttpsURLConnection(url); - } catch (IOException e) { - throw new RuntimeException(e); - } - }; - break; - default: - throw new IllegalArgumentException("Unsupported callback type: " + clientType); - } - executeCallbackLogic(iterations, sleepInterval, callback); - clients.close(); - } - - private static void executeCallbackLogic(int iterations, long sleepInterval, Runnable callback) throws InterruptedException { - if (iterations == -1) { - // Infinite loop - while (true) { - callback.run(); - if (sleepInterval > 0) - { - Thread.sleep(sleepInterval); - } - } - } else { - // Fixed number of iterations - for (int i = 0; i < iterations; i++) { - callback.run(); - if (sleepInterval > 0) - { - Thread.sleep(sleepInterval); - } - } - } - } -} diff --git a/pkg/network/java/testutil/JavaClientSimulator/src/main/java/Clients/JavaClients.java b/pkg/network/java/testutil/JavaClientSimulator/src/main/java/Clients/JavaClients.java deleted file mode 100644 index f7c155d1b6cd6..0000000000000 --- a/pkg/network/java/testutil/JavaClientSimulator/src/main/java/Clients/JavaClients.java +++ /dev/null @@ -1,142 +0,0 @@ -package Clients; - -import okhttp3.Call; -import okhttp3.OkHttpClient; -import okhttp3.Protocol; -import okhttp3.Request; -import okhttp3.Response; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.io.Closeable; -import java.io.IOException; -import java.net.URI; -import java.net.URL; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.Arrays; - -public class JavaClients implements Closeable { - - // Create a custom TrustManager that accepts all certificates - // Create a TrustManager that accepts all certificates - static X509TrustManager trustManager = new X509TrustManager() { - @Override - public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { - // No validation needed, accepting all client certificates - } - - @Override - public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { - // No validation needed, accepting all server certificates - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - // Return an empty array to accept all issuers - return new X509Certificate[0]; - } - }; - private static final TrustManager[] trustAllCerts = new TrustManager[]{trustManager}; - - private static final String URL_SCHEME = "https://"; - - private CloseableHttpClient apacheClient; - private OkHttpClient okHttpClient; - private HttpClient httpClient; - - public void init() throws NoSuchAlgorithmException, KeyManagementException, KeyStoreException { - // Create a custom SSLContext to trust all certificates - SSLContext sslContext = SSLContext.getInstance("TLS"); - sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); - - //configure HttpsURLConnection to trust all certificates and ignore host validation - //URLConnection client will be recreated for each request - HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory()); - HttpsURLConnection.setDefaultHostnameVerifier(NoopHostnameVerifier.INSTANCE); - - //create apache client once and configure it to trust all certificates and ignore host validation - apacheClient = HttpClients.custom() - .setSSLContext(sslContext) - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .build(); - - //create http client once and configure it to trust all certificates and ignore host validation - httpClient = HttpClient.newBuilder().sslContext(sslContext).build(); - - //create okhttp client once and configure it to trust all certificates and ignore host validation - okHttpClient = new OkHttpClient.Builder() - .hostnameVerifier(NoopHostnameVerifier.INSTANCE) - .sslSocketFactory(sslContext.getSocketFactory(),(X509TrustManager) trustAllCerts[0]) - //by default okhttp is using http2.0 - .protocols(Arrays.asList(Protocol.HTTP_1_1)) - .build(); - } - - public void HttpsURLConnection(String url) throws IOException { - HttpsURLConnection urlConnection =(HttpsURLConnection) new URL(URL_SCHEME+url).openConnection(); - String response = urlConnection.getResponseMessage(); - int code = urlConnection.getResponseCode(); - System.out.println("Response code: " + code + " ; Message: " + response); - } - - public void OkHttpClient(String url) throws IOException { - Request request = new Request.Builder() - .url(URL_SCHEME+url) - .build(); - Call call = okHttpClient.newCall(request); - Response response = call.execute(); - System.out.println("Response: " + response); - } - - public void HttpApacheClient(String url) throws IOException { - HttpGet request = new HttpGet("https://"+url); - try { - CloseableHttpResponse response = apacheClient.execute(request); - System.out.println("Response: " + response); - } catch (IOException e) { - e.printStackTrace(); - } - finally { - // TODO: in the future we should support re-using the same connection for apache client, - // currently we are hitting the internal connection pool limit of the apacheclient, - // since we create a new request object for the same route, which in turn tries to use a new connection - // (the default connection limit of the apacheclient for the same route is 2 - request.releaseConnection(); - } - } - - public void HTTPClient(String url) throws IOException { - try { - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(URL_SCHEME+url)) - //by default HttpCLient is using http 2.0 - .version(HttpClient.Version.HTTP_1_1) - .build(); - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - System.out.println("Response " + response.toString()); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); - } - } - - @Override - public void close() throws IOException { - apacheClient.close(); - okHttpClient.dispatcher().executorService().shutdown(); - okHttpClient.connectionPool().evictAll(); - } -} diff --git a/pkg/network/java/testutil/JavaClientSimulator/src/main/java/Clients/Main.java b/pkg/network/java/testutil/JavaClientSimulator/src/main/java/Clients/Main.java deleted file mode 100644 index 2841b39052de9..0000000000000 --- a/pkg/network/java/testutil/JavaClientSimulator/src/main/java/Clients/Main.java +++ /dev/null @@ -1,72 +0,0 @@ -package Clients; -import org.apache.commons.cli.*; - -public class Main { - - //in milliseconds - private static final int DEFAULT_TIMEOUT = 1000; - - private static void printHelp(Options options){ - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("java -jar JavaClients client= url=\n", - "", - options, - "\nprovide the url WITHOUT the protocol scheme (always using https)"); - } - - public static void main(String[] args) throws Exception { - Options options = new Options(); - options.addRequiredOption("c", "client", true, - "Client type: apache ; okhttp ; urlconnection ; httpclient"); - options.addRequiredOption("u", "url", true, "Target URL"); - Option iterationOption = Option.builder("i") - .longOpt("iterations") - .hasArg() - .desc("Number of iterations. The default is infinitely") - .required(false) - .build(); - iterationOption.setType(Number.class); - options.addOption(iterationOption); - - Option timeoutOption = Option.builder("t") - .longOpt("timeout") - .hasArg() - .desc("Timeout between each call in ms. Default is 1 second, use 0 to send the requests without a timeout") - .required(false) - .build(); - timeoutOption.setType(Number.class); - options.addOption(timeoutOption); - - if (args.length == 0){ - printHelp(options); - return; - } - // Parse command-line arguments - CommandLineParser parser = new DefaultParser(); - try { - CommandLine cmd = parser.parse(options, args); - - // Get arguments - String clientTypeArg = cmd.getOptionValue("c"); - String url = cmd.getOptionValue("u"); - int iterationsValue = -1; - if (cmd.hasOption("i")) { - iterationsValue = ((Number)cmd.getParsedOptionValue("i")).intValue(); - } - - int interval = DEFAULT_TIMEOUT; - if (cmd.hasOption("t")) { - interval = ((Number)cmd.getParsedOptionValue("t")).intValue(); - } - - // Execute the appropriate handler based on client type - ClientManager.executeCallback(clientTypeArg, iterationsValue, interval, url); - } catch (ParseException e) { - System.err.println("Error parsing command-line arguments: " + e.getMessage()); - printHelp(options); - } catch (NumberFormatException e) { - System.err.println("Error parsing iterations argument: " + e.getMessage()); - printHelp(options); - } - } -} diff --git a/pkg/network/protocols/http/tls_counter.go b/pkg/network/protocols/http/tls_counter.go index b6fcccf69bec6..f0dd8534ebb8e 100644 --- a/pkg/network/protocols/http/tls_counter.go +++ b/pkg/network/protocols/http/tls_counter.go @@ -17,7 +17,6 @@ type TLSCounter struct { counterPlain *libtelemetry.Counter counterGnuTLS *libtelemetry.Counter counterOpenSSL *libtelemetry.Counter - counterJavaTLS *libtelemetry.Counter counterGoTLS *libtelemetry.Counter counterIstioTLS *libtelemetry.Counter counterNodeJSTLS *libtelemetry.Counter @@ -30,7 +29,6 @@ func NewTLSCounter(metricGroup *libtelemetry.MetricGroup, metricName string, tag counterPlain: metricGroup.NewCounter(metricName, append(tags, "encrypted:false", "tls_library:none")...), counterGnuTLS: metricGroup.NewCounter(metricName, append(tags, "encrypted:true", "tls_library:gnutls")...), counterOpenSSL: metricGroup.NewCounter(metricName, append(tags, "encrypted:true", "tls_library:openssl")...), - counterJavaTLS: metricGroup.NewCounter(metricName, append(tags, "encrypted:true", "tls_library:java")...), counterGoTLS: metricGroup.NewCounter(metricName, append(tags, "encrypted:true", "tls_library:go")...), counterIstioTLS: metricGroup.NewCounter(metricName, append(tags, "encrypted:true", "tls_library:istio")...), counterNodeJSTLS: metricGroup.NewCounter(metricName, append(tags, "encrypted:true", "tls_library:nodejs")...), diff --git a/pkg/network/protocols/http/tls_counter_linux.go b/pkg/network/protocols/http/tls_counter_linux.go index d217d692b4750..68b80b73e028c 100644 --- a/pkg/network/protocols/http/tls_counter_linux.go +++ b/pkg/network/protocols/http/tls_counter_linux.go @@ -14,8 +14,6 @@ func (t *TLSCounter) Add(tx Transaction) { t.counterGnuTLS.Add(1) case OpenSSL: t.counterOpenSSL.Add(1) - case Java: - t.counterJavaTLS.Add(1) case Go: t.counterGoTLS.Add(1) case Istio: diff --git a/pkg/network/protocols/http/types.go b/pkg/network/protocols/http/types.go index 13620a9bfcba8..3f0eb5c963937 100644 --- a/pkg/network/protocols/http/types.go +++ b/pkg/network/protocols/http/types.go @@ -31,7 +31,6 @@ const ( GnuTLS ConnTag = C.LIBGNUTLS OpenSSL ConnTag = C.LIBSSL Go ConnTag = C.GO - Java ConnTag = C.JAVA_TLS TLS ConnTag = C.CONN_TLS Istio ConnTag = C.ISTIO NodeJS ConnTag = C.NODEJS @@ -42,7 +41,6 @@ var ( GnuTLS: "tls.library:gnutls", OpenSSL: "tls.library:openssl", Go: "tls.library:go", - Java: "tls.library:java", TLS: "tls.connection:encrypted", Istio: "tls.library:istio", NodeJS: "tls.library:nodejs", diff --git a/pkg/network/protocols/http/types_linux.go b/pkg/network/protocols/http/types_linux.go index 0f219b6de1a9c..e4b3a97623457 100644 --- a/pkg/network/protocols/http/types_linux.go +++ b/pkg/network/protocols/http/types_linux.go @@ -49,10 +49,9 @@ const ( GnuTLS ConnTag = 0x1 OpenSSL ConnTag = 0x2 Go ConnTag = 0x4 - Java ConnTag = 0x8 - TLS ConnTag = 0x10 - Istio ConnTag = 0x20 - NodeJS ConnTag = 0x40 + TLS ConnTag = 0x8 + Istio ConnTag = 0x10 + NodeJS ConnTag = 0x20 ) var ( @@ -60,7 +59,6 @@ var ( GnuTLS: "tls.library:gnutls", OpenSSL: "tls.library:openssl", Go: "tls.library:go", - Java: "tls.library:java", TLS: "tls.connection:encrypted", Istio: "tls.library:istio", NodeJS: "tls.library:nodejs", diff --git a/pkg/network/protocols/kafka/statkeeper.go b/pkg/network/protocols/kafka/statkeeper.go index 855b002e5519e..fe2653d84726c 100644 --- a/pkg/network/protocols/kafka/statkeeper.go +++ b/pkg/network/protocols/kafka/statkeeper.go @@ -39,15 +39,24 @@ func NewStatkeeper(c *config.Config, telemetry *Telemetry) *StatKeeper { // Process processes the kafka transaction func (statKeeper *StatKeeper) Process(tx *EbpfTx) { - statKeeper.statsMutex.Lock() - defer statKeeper.statsMutex.Unlock() + latency := tx.RequestLatency() + // Produce requests with acks = 0 do not receive a response, and as a result, have no latency + if tx.APIKey() == FetchAPIKey && latency <= 0 { + statKeeper.telemetry.invalidLatency.Add(1) + return + } + // extractTopicName is an expensive operation but, it is also concurrent safe, so we can do it here + // without holding the lock. key := Key{ RequestAPIKey: tx.APIKey(), RequestVersion: tx.APIVersion(), TopicName: statKeeper.extractTopicName(&tx.Transaction), ConnectionKey: tx.ConnTuple(), } + + statKeeper.statsMutex.Lock() + defer statKeeper.statsMutex.Unlock() requestStats, ok := statKeeper.stats[key] if !ok { if len(statKeeper.stats) >= statKeeper.maxEntries { @@ -58,13 +67,6 @@ func (statKeeper *StatKeeper) Process(tx *EbpfTx) { statKeeper.stats[key] = requestStats } - latency := tx.RequestLatency() - // Produce requests with acks = 0 do not receive a response, and as a result, have no latency - if key.RequestAPIKey == FetchAPIKey && latency <= 0 { - statKeeper.telemetry.invalidLatency.Add(1) - return - } - requestStats.AddRequest(int32(tx.ErrorCode()), int(tx.RecordsCount()), uint64(tx.Transaction.Tags), latency) } diff --git a/pkg/network/protocols/postgres/model_linux.go b/pkg/network/protocols/postgres/model_linux.go index fb775ecdddfbc..8ee158de7b048 100644 --- a/pkg/network/protocols/postgres/model_linux.go +++ b/pkg/network/protocols/postgres/model_linux.go @@ -10,7 +10,6 @@ package postgres import ( "bytes" "fmt" - "regexp" "strings" "github.com/DataDog/go-sqllexer" @@ -26,6 +25,10 @@ const ( EmptyParameters = "EMPTY_PARAMETERS" ) +var ( + postgresDBMS = sqllexer.WithDBMS(sqllexer.DBMSPostgres) +) + // EventWrapper wraps an ebpf event and provides additional methods to extract information from it. // We use this wrapper to avoid recomputing the same values (operation and table name) multiple times. type EventWrapper struct { @@ -98,17 +101,10 @@ func (e *EventWrapper) extractParameters() string { return string(b[idxParam:]) } -var re = regexp.MustCompile(`(?i)if\s+exists`) - // extractTableName extracts the table name from the query. func (e *EventWrapper) extractTableName() string { - fragment := string(getFragment(&e.Tx)) - // Temp solution for the fact that ObfuscateSQLString does not support "IF EXISTS" or "if exists", so we remove - // it from the fragment if found. - fragment = re.ReplaceAllString(fragment, "") - // Normalize the query without obfuscating it. - _, statementMetadata, err := e.normalizer.Normalize(fragment, sqllexer.WithDBMS(sqllexer.DBMSPostgres)) + _, statementMetadata, err := e.normalizer.Normalize(string(getFragment(&e.Tx)), postgresDBMS) if err != nil { log.Debugf("unable to normalize due to: %s", err) return "UNKNOWN" diff --git a/pkg/network/protocols/tls/java/agent-usm.jar b/pkg/network/protocols/tls/java/agent-usm.jar deleted file mode 100644 index 8132059d52b0d..0000000000000 Binary files a/pkg/network/protocols/tls/java/agent-usm.jar and /dev/null differ diff --git a/pkg/network/protocols/tls/java/hotspot.go b/pkg/network/protocols/tls/java/hotspot.go deleted file mode 100644 index 383652ea4792d..0000000000000 --- a/pkg/network/protocols/tls/java/hotspot.go +++ /dev/null @@ -1,417 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build linux - -// Package java contains implementation for JavaTLS support. -package java - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "io/fs" - "net" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" - "syscall" - "time" - - "github.com/DataDog/datadog-agent/pkg/util/kernel" - "github.com/DataDog/datadog-agent/pkg/util/log" - "golang.org/x/sys/unix" -) - -// Hotspot java has a specific protocol, described here: -// -// o touch .attach_pid -// o kill -SIGQUIT -// o java process check if .attach_pid exit -// o then create a unix socket .java_pid -// o we can write command through the unix socket -// refers to the namespaced pid of the process. -// -// Public documentation https://openjdk.org/groups/hotspot/docs/Serviceability.html#battach -type Hotspot struct { - pid int - nsPid int - root string - cwd string // viewed by the process - conn *net.UnixConn - - socketPath string - uid int - gid int -} - -// NewHotspot create an object to connect to a JVM hotspot -// pid (host pid) and nsPid (within the namespace pid) -// -// NSPid field was introduced in kernel >= 4.1 -// So we can't support container on Centos 7 (kernel 3.10) -func NewHotspot(pid int, nsPid int) (*Hotspot, error) { - h := &Hotspot{ - pid: pid, - nsPid: nsPid, - } - // Centos 7 workaround to support host environment - if h.nsPid == 0 { - h.nsPid = pid - } - - var err error - procPath := fmt.Sprintf("%s/%d", kernel.ProcFSRoot(), pid) - h.root = procPath + "/root" - h.cwd, err = os.Readlink(procPath + "/cwd") - if err != nil { - return nil, err - } - h.socketPath = fmt.Sprintf("%s/.java_pid%d", h.tmpPath(), h.nsPid) - return h, nil -} - -func (h *Hotspot) tmpPath() string { - return fmt.Sprintf("%s/tmp", h.root) -} - -func (h *Hotspot) isSocketExists() bool { - mode, err := os.Stat(h.socketPath) - return err == nil && (mode.Mode()&fs.ModeSocket > 0) -} - -// getPathOwner return the uid/gid pointed by the path -func getPathOwner(path string) (uint32, uint32, error) { - mode, err := os.Stat(path) - if err != nil { - return 0, 0, err - } - stat, ok := mode.Sys().(*syscall.Stat_t) - if stat == nil || !ok { - return 0, 0, fmt.Errorf("stat cast issue on path %s %T", path, mode.Sys()) - } - return stat.Uid, stat.Gid, nil -} - -// findWritableDest looks for a writable destination for agent-usm.jar file. -// The default is to write this file into the working directory of the agent. -// If this is not possible then we try 'root/tmp', and finally fail. -func findWritableDest(cwd, root, agent string) (string, error) { - if unix.Access(cwd, unix.W_OK) == nil { - return filepath.Join(cwd, filepath.Base(agent)), nil - } - - log.Debugf("Current working directory %q is not writable", cwd) - - if unix.Access(filepath.Join(root, "tmp"), unix.W_OK) == nil { - dstPath := filepath.Join(root, "tmp", filepath.Base(agent)) - log.Debugf("Writing agent jar file to %q", dstPath) - return dstPath, nil - } - - return "", errors.New("unable to find writable destionation") -} - -// copyAgent copy the agent-usm.jar to a directory where the running java process can load it. -// the agent-usm.jar file must be readable from java process point of view -// copyAgent return : -// -// o dstPath is path to the copy of agent-usm.jar (from container perspective), this would be pass to the hotspot command -// o cleanup must be called to remove the created file -func (h *Hotspot) copyAgent(agent string, uid int, gid int) (string, func(), error) { - dstPath, err := findWritableDest(h.cwd, h.root, agent) - if err != nil { - return "", nil, err - } - - // path from the host point of view pointing to the process root namespace (/proc/pid/root/usr/...) - nsDstPath := h.root + dstPath - if dst, err := os.Stat(nsDstPath); err == nil { - // if the destination file already exist - // check if it's not the source agent file - if src, err := os.Stat(agent); err == nil { - s, oks := src.Sys().(*syscall.Stat_t) - d, okd := dst.Sys().(*syscall.Stat_t) - if s == nil || d == nil || !oks || !okd { - return "", nil, fmt.Errorf("stat cast issue on path %s %T %s %T", agent, src.Sys(), nsDstPath, dst.Sys()) - } - if s.Dev == d.Dev && s.Ino == d.Ino { - return "", func() {}, nil - } - } - } - - srcAgent, err := os.Open(agent) - if err != nil { - return "", nil, err - } - defer srcAgent.Close() - - dst, err := os.OpenFile(nsDstPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.FileMode(0444)) - if err != nil { - return "", nil, err - } - _, copyErr := io.Copy(dst, srcAgent) - dst.Close() // we are closing the file here as Chown will be called just after on the same path - if copyErr != nil { - return "", nil, err - } - if err := syscall.Chown(nsDstPath, uid, gid); err != nil { - os.Remove(nsDstPath) - return "", nil, err - } - - return dstPath, func() { - os.Remove(nsDstPath) - }, nil -} - -func (h *Hotspot) dialunix(raddr *net.UnixAddr, withCredential bool) (*net.UnixConn, error) { - // Hotspot reject connection credentials by checking uid/gid of the client calling connect() - // via getsockopt(SOL_SOCKET/SO_PEERCRED). - // but older hotspot JRE (1.8.0) accept only the same uid/gid and reject root - // - // For go, during the connect() syscall we don't want to fork() and stay on the same pthread - // to avoid side effect (pollution) of set effective uid/gid. - if withCredential { - runtime.LockOSThread() - syscall.ForkLock.Lock() - origeuid := syscall.Geteuid() - origegid := syscall.Getegid() - defer func() { - _ = syscall.Seteuid(origeuid) - _ = syscall.Setegid(origegid) - syscall.ForkLock.Unlock() - runtime.UnlockOSThread() - }() - - if err := syscall.Setegid(h.gid); err != nil { - return nil, err - } - if err := syscall.Seteuid(h.uid); err != nil { - return nil, err - } - } - return net.DialUnix("unix", nil, raddr) -} - -// connect to the previously created hotspot unix socket -// return close function must be call when finished -func (h *Hotspot) connect(withCredential bool) (func(), error) { - h.conn = nil - addr, err := net.ResolveUnixAddr("unix", h.socketPath) - if err != nil { - return nil, err - } - conn, err := h.dialunix(addr, withCredential) - if err != nil { - return nil, err - } - - if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil { - conn.Close() - return nil, err - } - h.conn = conn - return func() { - if h.conn != nil { - h.conn.Close() - } - }, nil -} - -// parseResponse parse the response from the hotspot command -// JVM will return a command error code and some command have a specific return code -// the response will contain the full message -func (h *Hotspot) parseResponse(buf []byte) (returnCommand int, returnCode int, response string, err error) { - line := 0 - scanner := bufio.NewScanner(bytes.NewReader(buf)) - for scanner.Scan() { - s := string(scanner.Bytes()) - switch line { - case 0: - returnCommand, err = strconv.Atoi(s) - if err != nil { - return 0, 0, "", fmt.Errorf("parsing hotspot response failed %d %s", line, s) - } - case 1: - if strings.HasPrefix(s, "return code: ") { - returnCode, err = strconv.Atoi(s[len("return code: "):]) - if err != nil { - return 0, 0, "", fmt.Errorf("parsing hotspot response failed %d %s", line, s) - } - } - default: - break - } - line++ - } - return returnCommand, returnCode, string(buf), nil -} - -// command: tailingNull is necessary here to flush command -// -// otherwise the JVM is blocked and waiting for more bytes -// This applies only for some command like : 'load agent.so true' -func (h *Hotspot) command(cmd string, tailingNull bool) error { - if err := h.commandWriteRead(cmd, tailingNull); err != nil { - // if we receive EPIPE (write) or ECONNRESET (read) from the kernel may from old hotspot JVM - // let's retry with credentials, see dialunix() for more info - if !errors.Is(err, syscall.EPIPE) && !errors.Is(err, syscall.ECONNRESET) { - return err - } - log.Debugf("java attach hotspot pid %d/%d old hotspot JVM detected, requires credentials", h.pid, h.nsPid) - _ = h.conn.Close() - // we don't need to propagate the cleanConn() callback as it's doing the same thing. - if _, err := h.connect(true); err != nil { - return err - } - - if err := h.commandWriteRead(cmd, tailingNull); err != nil { - return err - } - } - return nil -} - -// commandWriteRead: tailingNull is necessary here to flush command -// -// otherwise the JVM is blocked and waiting for more bytes -// This applies only for some command like : 'load agent.so true' -func (h *Hotspot) commandWriteRead(cmd string, tailingNull bool) error { - if _, err := h.conn.Write([]byte{'1', 0}); err != nil { // Protocol version - return err - } - - // We split by space for at most 4 words, since our longest command is "load instrument false " which is 4 words - for _, c := range strings.SplitN(cmd, " ", 4) { - cmd := append([]byte(c), 0) - if _, err := h.conn.Write(cmd); err != nil { - return err - } - } - if tailingNull { - if _, err := h.conn.Write([]byte{0}); err != nil { - return err - } - } - - buf := make([]byte, 8192) - if _, err := h.conn.Read(buf); err != nil { - return err - } - returnCommand, returnCode, responseText, err := h.parseResponse(buf) - if err != nil { - return err - } - - if returnCommand != 0 { - return fmt.Errorf("command sent to hotspot JVM %q return %d and return code %d, response text:\n%s", cmd, returnCommand, returnCode, responseText) - } - return nil -} - -// attachJVMProtocol use this (short) protocol : -// -// o create a file .attach_pid%d -// o send a SIGQUIT signal -// o wait for socket file to be created by the java process -func (h *Hotspot) attachJVMProtocol(uid int, gid int) error { - attachPIDPath := func(root string) string { - return fmt.Sprintf("%s/.attach_pid%d", root, h.nsPid) - } - - attachPath := attachPIDPath(h.root + h.cwd) - hook, err := os.OpenFile(attachPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) - hookUID := uint32(0) - var ownerErr error - if err == nil { - hook.Close() - // we don't check Chown() return error here as it can fail on some filesystems (read below) - _ = syscall.Chown(attachPath, uid, gid) - hookUID, _, ownerErr = getPathOwner(attachPath) - } - if err != nil || ownerErr != nil || hookUID != uint32(uid) { - // We are trying an alternative attach path (in /tmp) - // o if we can't create one in the cwd of the java process (probably read only filesystem) - // o the filesystem changed the owner (id mapped mounts, like nfs force_uid, ...) - - if ownerErr == nil { - os.Remove(attachPath) - } - - attachPath = attachPIDPath(h.tmpPath()) - hook, err = os.OpenFile(attachPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0660) - if err != nil { - return err - } - hook.Close() - } - defer os.Remove(attachPath) - - process, _ := os.FindProcess(h.pid) // os.FindProcess() will never fail on linux - if err := process.Signal(syscall.SIGQUIT); err != nil { - return fmt.Errorf("process %d/%d SIGQUIT failed : %s", h.pid, h.nsPid, err) - } - - end := time.Now().Add(6 * time.Second) - for end.After(time.Now()) { - time.Sleep(200 * time.Millisecond) - if h.isSocketExists() { - return nil - } - } - return fmt.Errorf("the java process %d/%d didn't create a socket file", h.pid, h.nsPid) -} - -// Attach an agent to the hotspot, uid/gid must be accessible read-only by the targeted hotspot -func (h *Hotspot) Attach(agentPath string, args string, uid int, gid int) error { - if !h.isSocketExists() { - // ask JVM to create a socket to communicate - if err := h.attachJVMProtocol(uid, gid); err != nil { - return err - } - } - - // copy the agent in the cwd of the process and change his owner/group - dstAgentPath, agentCleanup, err := h.copyAgent(agentPath, uid, gid) - if err != nil { - return err - } - defer agentCleanup() - - h.uid = uid - h.gid = gid - // connect and ask to load the agent .jar or .so - cleanConn, err := h.connect(false) - if err != nil { - return err - } - defer cleanConn() - - var loadCommand string - isJar := strings.HasSuffix(agentPath, ".jar") - if isJar { // agent is a .jar - loadCommand = fmt.Sprintf("load instrument false %s", dstAgentPath) - if args != "" { - loadCommand += "=" + args - } - } else { - loadCommand = fmt.Sprintf("load %s true", dstAgentPath) - if args != "" { - loadCommand += " " + args - } - } - - if err := h.command(loadCommand, !isJar); err != nil { - log.Debugf("java attach hotspot pid %d/%d command '%s' failed isJar=%v : %s", h.pid, h.nsPid, loadCommand, isJar, err) - return err - } - return nil -} diff --git a/pkg/network/protocols/tls/java/jattach.go b/pkg/network/protocols/tls/java/jattach.go deleted file mode 100644 index 8630a7572a207..0000000000000 --- a/pkg/network/protocols/tls/java/jattach.go +++ /dev/null @@ -1,72 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build linux - -package java - -import ( - "time" - - // needs NsPid from fork - "github.com/DataDog/gopsutil/process" - - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -// minJavaAgeToAttachMS is the minimum age of a java process to be able to attach it -// else the java process would crash if he receives the SIGQUIT too early ("Signal Dispatch" thread is not ready) -// In other words that the only reliable safety thing we could check to assume a java process started (System.main execution) -// Looking a proc/pid/status.Thread numbers is not reliable as it depend on numbers of cores and JRE version/implementation -// -// The issue is described here https://bugs.openjdk.org/browse/JDK-8186709 see Kevin Walls comment -// if java received a SIGQUIT and the JVM is not started yet, java will print 'quit (core dumped)' -// SIGQUIT is sent as part of the hotspot protocol handshake -const minJavaAgeToAttachMS = 10000 - -func injectAttach(pid int, agent, args string, nsPid, fsUID, fsGID int) error { - h, err := NewHotspot(pid, nsPid) - if err != nil { - return err - } - - return h.Attach(agent, args, fsUID, fsGID) -} - -// InjectAgent injects the given agent into the given java process -func InjectAgent(pid int, agent, args string) error { - proc, err := process.NewProcess(int32(pid)) - if err != nil { - return err - } - uids, err := proc.Uids() - if err != nil { - return err - } - gids, err := proc.Gids() - if err != nil { - return err - } - // we return the process uid and gid from the filesystem point of view - // as attach file need to be created with uid/gid accessible from the java hotspot - // index 3 here point to the 4th columns of /proc/pid/status Uid/Gid => filesystem uid/gid - fsUID, fsGID := int(uids[3]), int(gids[3]) - - ctime, _ := proc.CreateTime() - ageMs := time.Now().UnixMilli() - ctime - if ageMs < minJavaAgeToAttachMS { - log.Debugf("java attach pid %d will be delayed by %d ms", pid, minJavaAgeToAttachMS-ageMs) - // wait and inject the agent asynchronously - go func() { - time.Sleep(time.Duration(minJavaAgeToAttachMS-ageMs) * time.Millisecond) - if err := injectAttach(pid, agent, args, int(proc.NsPid), fsUID, fsGID); err != nil { - log.Errorf("java attach pid %d failed %s", pid, err) - } - }() - return nil - } - - return injectAttach(pid, agent, args, int(proc.NsPid), fsUID, fsGID) -} diff --git a/pkg/network/protocols/tls/java/jattach_test.go b/pkg/network/protocols/tls/java/jattach_test.go deleted file mode 100644 index 14bdcff71b09f..0000000000000 --- a/pkg/network/protocols/tls/java/jattach_test.go +++ /dev/null @@ -1,145 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build linux_bpf - -package java - -import ( - "os" - "path/filepath" - "syscall" - "testing" - "time" - - "github.com/cihub/seelog" - "github.com/stretchr/testify/require" - - javatestutil "github.com/DataDog/datadog-agent/pkg/network/protocols/tls/java/testutil" - "github.com/DataDog/datadog-agent/pkg/network/testutil" - "github.com/DataDog/datadog-agent/pkg/util/kernel" - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -func TestMain(m *testing.M) { - logLevel := os.Getenv("DD_LOG_LEVEL") - if logLevel == "" { - logLevel = "warn" - } - log.SetupLogger(seelog.Default, logLevel) - os.Exit(m.Run()) -} - -func testInject(t *testing.T, prefix string) { - go func() { - o, err := testutil.RunCommand(prefix + "java -cp testdata Wait JustWait") - if err != nil { - t.Logf("%v\n", err) - } - t.Log(o) - }() - - var pids []int - var err error - require.Eventually(t, func() bool { - pids, err = javatestutil.FindProcessByCommandLine("java", "JustWait") - return len(pids) == 1 - }, time.Second*5, time.Millisecond*100) - require.NoError(t, err) - - t.Cleanup(func() { - process, err := os.FindProcess(pids[0]) - if err != nil { - return - } - _ = process.Signal(syscall.SIGKILL) - _, _ = process.Wait() - }) - - tfile, err := os.CreateTemp("", "TestAgentLoaded.agentmain.*") - require.NoError(t, err) - require.NoError(t, tfile.Close()) - require.NoError(t, os.Remove(tfile.Name())) - // equivalent to jattach load instrument false testdata/TestAgentLoaded.jar= - require.NoError(t, InjectAgent(pids[0], filepath.Join("testdata", "TestAgentLoaded.jar"), "testfile="+tfile.Name())) - require.Eventually(t, func() bool { - _, err = os.Stat(tfile.Name()) - return err == nil - }, time.Second*15, time.Millisecond*100) -} - -// We test injection on a java hotspot running -// -// o on the host -// o in the container, _simulated_ by running java in his own PID namespace -func runTestInject(t *testing.T) { - currKernelVersion, err := kernel.HostVersion() - require.NoError(t, err) - if currKernelVersion < kernel.VersionCode(4, 14, 0) { - t.Skip("Java TLS injection tests can run only on USM supported machines.") - } - - javaVersion, err := testutil.RunCommand("java -version") - require.NoErrorf(t, err, "java is not installed") - t.Logf("java version %v", javaVersion) - - t.Run("host", func(t *testing.T) { - testInject(t, "") - }) - - t.Run("PID namespace", func(t *testing.T) { - p := "unshare -p --fork " - _, err = testutil.RunCommand(p + "id") - if err != nil { - t.Skipf("unshare not supported on this platform %s", err) - } - - // running the target process in a new PID namespace - // and testing if the test/platform give enough permission to do that - testInject(t, p) - }) -} - -func TestInject(t *testing.T) { - t.Skip("JavaTLS tests are currently disabled") - runTestInject(t) -} - -func TestInjectInReadOnlyFS(t *testing.T) { - t.Skip("JavaTLS tests are currently disabled") - curDir, err := os.Getwd() - require.NoError(t, err) - rodir := filepath.Join(os.TempDir(), "rodir") - - err = os.Mkdir(rodir, 0766) - require.NoError(t, err) - defer func() { - err = os.RemoveAll(rodir) - if err != nil { - t.Log(err) - } - }() - - err = syscall.Mount(curDir, rodir, "auto", syscall.MS_BIND|syscall.MS_RDONLY, "") - require.NoError(t, err) - err = syscall.Mount("none", rodir, "", syscall.MS_RDONLY|syscall.MS_REMOUNT|syscall.MS_BIND, "") - require.NoError(t, err) - defer func() { - // without the sleep the unmount fails with a 'resource busy' error - time.Sleep(1 * time.Second) - err = syscall.Unmount(rodir, 0) - if err != nil { - t.Log(err) - } - }() - - err = os.Chdir(rodir) - require.NoError(t, err) - - runTestInject(t) - - err = os.Chdir(curDir) - require.NoError(t, err) -} diff --git a/pkg/network/protocols/tls/java/testdata/.gitignore b/pkg/network/protocols/tls/java/testdata/.gitignore deleted file mode 100644 index 24a22407e6e3c..0000000000000 --- a/pkg/network/protocols/tls/java/testdata/.gitignore +++ /dev/null @@ -1 +0,0 @@ -TestAgentLoaded.agentmain.* diff --git a/pkg/network/protocols/tls/java/testdata/META-INF/MANIFEST.MF b/pkg/network/protocols/tls/java/testdata/META-INF/MANIFEST.MF deleted file mode 100644 index eca2e74bb62bb..0000000000000 --- a/pkg/network/protocols/tls/java/testdata/META-INF/MANIFEST.MF +++ /dev/null @@ -1,6 +0,0 @@ -Manifest-Version: 1.0 -Can-Redefine-Classes: true -Can-Retransform-Classes: true -Agent-Class: TestAgentLoaded -Premain-Class: TestAgentLoaded -Created-By: 1.2.3.4 (Private Build) diff --git a/pkg/network/protocols/tls/java/testdata/TestAgentLoaded.class b/pkg/network/protocols/tls/java/testdata/TestAgentLoaded.class deleted file mode 100644 index c47486dfa6ca7..0000000000000 Binary files a/pkg/network/protocols/tls/java/testdata/TestAgentLoaded.class and /dev/null differ diff --git a/pkg/network/protocols/tls/java/testdata/TestAgentLoaded.jar b/pkg/network/protocols/tls/java/testdata/TestAgentLoaded.jar deleted file mode 100644 index 9093502416b34..0000000000000 Binary files a/pkg/network/protocols/tls/java/testdata/TestAgentLoaded.jar and /dev/null differ diff --git a/pkg/network/protocols/tls/java/testdata/TestAgentLoaded.java b/pkg/network/protocols/tls/java/testdata/TestAgentLoaded.java deleted file mode 100644 index 27b84d8b3dd2a..0000000000000 --- a/pkg/network/protocols/tls/java/testdata/TestAgentLoaded.java +++ /dev/null @@ -1,34 +0,0 @@ -/* -Need to be compiled with java7 - -javac TestAgentLoaded.java -jar cvmf META-INF/MANIFEST.MF TestAgentLoaded.jar TestAgentLoaded.class - */ - -import java.lang.instrument.Instrumentation; -import java.io.FileOutputStream; - -public class TestAgentLoaded { - - public static void agentmain(String agentArgs, Instrumentation inst) { - try { - // parsing the argument like agent-usm.jar - if (agentArgs != ""){ - //split arguments by comma character - String[] args = agentArgs.split(","); - for (String arg : args){ - //we only parse the arguments of the form "arg=value" (e.g: dd.debug.enabled=true) - String[] keyValTuple = arg.split("="); - if ((keyValTuple.length == 2) && (keyValTuple[0].equals("testfile"))) { - System.out.println("touch file "+keyValTuple[1]); - new FileOutputStream(keyValTuple[1]).close(); - } - } - } - } catch (Exception ex) { - System.out.println(ex); - } finally { - System.out.println("loading TestAgentLoaded.agentmain("+agentArgs+")"); - } - } -} diff --git a/pkg/network/protocols/tls/java/testdata/Wait.class b/pkg/network/protocols/tls/java/testdata/Wait.class deleted file mode 100644 index 7235400ed0475..0000000000000 Binary files a/pkg/network/protocols/tls/java/testdata/Wait.class and /dev/null differ diff --git a/pkg/network/protocols/tls/java/testdata/Wait.java b/pkg/network/protocols/tls/java/testdata/Wait.java deleted file mode 100644 index 61b3c2b894dc9..0000000000000 --- a/pkg/network/protocols/tls/java/testdata/Wait.java +++ /dev/null @@ -1,41 +0,0 @@ -// Need to be compiled with java7 - -import java.lang.Thread; -import java.lang.management.ManagementFactory; - -class Wait { - private static String getProcessId(final String fallback) { - // Note: may fail in some JVM implementations - // therefore fallback has to be provided - - // something like '@', at least in SUN / Oracle JVMs - final String jvmName = ManagementFactory.getRuntimeMXBean().getName(); - final int index = jvmName.indexOf('@'); - - if (index < 1) { - // part before '@' empty (index = 0) / '@' not found (index = -1) - return fallback; - } - - try { - return Long.toString(Long.parseLong(jvmName.substring(0, index))); - } catch (NumberFormatException e) { - // ignore - } - return fallback; - } - - public static void main(String[] args) { - String progName = args[0]; - String pid = getProcessId(""); - System.out.println(progName + " pid "+ pid); - try { - for(;;) { - Thread.sleep(1000); - } - } catch (Exception ex) { - System.out.println(ex); - } - System.out.println(progName + " pid "+pid+" ended"); - } -} diff --git a/pkg/network/protocols/tls/java/testdata/Wget$1.class b/pkg/network/protocols/tls/java/testdata/Wget$1.class deleted file mode 100644 index d586e3e758256..0000000000000 Binary files a/pkg/network/protocols/tls/java/testdata/Wget$1.class and /dev/null differ diff --git a/pkg/network/protocols/tls/java/testdata/Wget$2.class b/pkg/network/protocols/tls/java/testdata/Wget$2.class deleted file mode 100644 index 4e424dc0efc57..0000000000000 Binary files a/pkg/network/protocols/tls/java/testdata/Wget$2.class and /dev/null differ diff --git a/pkg/network/protocols/tls/java/testdata/Wget$InvalidCertificateHostVerifier.class b/pkg/network/protocols/tls/java/testdata/Wget$InvalidCertificateHostVerifier.class deleted file mode 100644 index c748c2835541b..0000000000000 Binary files a/pkg/network/protocols/tls/java/testdata/Wget$InvalidCertificateHostVerifier.class and /dev/null differ diff --git a/pkg/network/protocols/tls/java/testdata/Wget.class b/pkg/network/protocols/tls/java/testdata/Wget.class deleted file mode 100644 index 2bbd3ec21d053..0000000000000 Binary files a/pkg/network/protocols/tls/java/testdata/Wget.class and /dev/null differ diff --git a/pkg/network/protocols/tls/java/testdata/Wget.java b/pkg/network/protocols/tls/java/testdata/Wget.java deleted file mode 100644 index 4a66711cd6dc0..0000000000000 --- a/pkg/network/protocols/tls/java/testdata/Wget.java +++ /dev/null @@ -1,102 +0,0 @@ -// Need to be compiled with java7 - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.URL; -import javax.net.ssl.HttpsURLConnection; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLSession; -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.security.cert.X509Certificate; -import java.security.cert.CertificateException; -import java.security.NoSuchAlgorithmException; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; - -public class Wget { - - public static void main(String[] args) { - URL url; - boolean dumpContent = false; - if (args.length == 0) { - System.out.println("Wget "); - System.exit(1); - } - - try { - System.out.println("waiting 11 seconds"); - // sleep 11 seconds before doing the request, as the process need to be injected - Thread.sleep(11000); - } catch (InterruptedException intException) { - intException.printStackTrace(); - System.exit(1); - } - System.out.println("finished waiting"); - - try { - TrustManager[] trustAllCerts = new TrustManager[] { - new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - @Override - public void checkClientTrusted(X509Certificate[] arg0, String arg1) - throws CertificateException {} - - @Override - public void checkServerTrusted(X509Certificate[] arg0, String arg1) - throws CertificateException {} - } - }; - - KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); - trustStore.load(null, null); - - SSLContext sc=null; - try { - sc = SSLContext.getInstance("TLSv1.3"); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } - try { - sc.init(null, trustAllCerts, new java.security.SecureRandom()); - } catch (KeyManagementException e) { - e.printStackTrace(); - } - HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); - - - url = new URL(args[0]); - - HttpsURLConnection connection = (HttpsURLConnection) url.openConnection(); - connection.setRequestMethod("GET"); - connection.setConnectTimeout(15 * 1000); - connection.setReadTimeout(15 * 1000); - - // skip certificate validation - connection.setHostnameVerifier(new HostnameVerifier() { - public boolean verify(String s, SSLSession sslSession) { - return true; - } - }); - System.out.println("Response code = " + connection.getResponseCode()); - - BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream())); - String input; - while ((input = br.readLine()) != null) { - if (dumpContent) { - System.out.println(input); - } - } - connection.disconnect(); - } catch (Exception e) { - e.printStackTrace(); - System.exit(1); - } - } -} diff --git a/pkg/network/protocols/tls/java/testdata/docker-compose.yml b/pkg/network/protocols/tls/java/testdata/docker-compose.yml deleted file mode 100644 index 24b6066e510c6..0000000000000 --- a/pkg/network/protocols/tls/java/testdata/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '3' -name: java -services: - java: - image: ${IMAGE_VERSION} - entrypoint: java -cp /v ${ENTRYCLASS} - extra_hosts: - - ${EXTRA_HOSTS} - # setting the limits here as new os confuse (abort()) old glibc with default big numbers - # https://datadoghq.atlassian.net/browse/USMO-295 - # numbers here are default from /etc/systemd/system.conf - ulimits: - nofile: - soft: 1024 - hard: 524288 - volumes: - - ${TESTDIR}:/v:z diff --git a/pkg/network/protocols/tls/java/testutil/inject.go b/pkg/network/protocols/tls/java/testutil/inject.go deleted file mode 100644 index a9593308435c6..0000000000000 --- a/pkg/network/protocols/tls/java/testutil/inject.go +++ /dev/null @@ -1,152 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build linux && test - -// Package testutil provides utilities for testing the java tracer. -package testutil - -import ( - "io" - "regexp" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/cihub/seelog" - "github.com/stretchr/testify/require" - - "github.com/shirou/gopsutil/v3/process" - - "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" - "github.com/DataDog/datadog-agent/pkg/util/kernel" - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -// RunJavaVersion run class under java version -func RunJavaVersion(t testing.TB, version, class string, testdir string, waitForParam ...*regexp.Regexp) error { - t.Helper() - var waitFor *regexp.Regexp - if len(waitForParam) == 0 { - // test if injection happen - waitFor = regexp.MustCompile(`loading TestAgentLoaded\.agentmain.*`) - } else { - waitFor = waitForParam[0] - } - - dir, _ := testutil.CurDir() - addr := "172.17.0.1" // for some reason docker network inspect bridge --format='{{(index .IPAM.Config 0).Gateway}}' is not reliable and doesn't report Gateway ip sometime - env := []string{ - "IMAGE_VERSION=" + version, - "ENTRYCLASS=" + class, - "EXTRA_HOSTS=host.docker.internal:" + addr, - "TESTDIR=" + testdir, - } - - return protocolsUtils.RunDockerServer(t, version, dir+"/../testdata/docker-compose.yml", env, waitFor, protocolsUtils.DefaultTimeout, 3) -} - -// FindProcessByCommandLine gets a proc name and part of its command line, and returns all PIDs matched for those conditions. -func FindProcessByCommandLine(procName, command string) ([]int, error) { - res := make([]int, 0) - fn := func(pid int) error { - proc, err := process.NewProcess(int32(pid)) - if err != nil { - return nil // ignore process that didn't exist anymore - } - - name, err := proc.Name() - if err == nil && name == procName { - cmdline, err := proc.Cmdline() - if err == nil && strings.Contains(cmdline, command) { - res = append(res, pid) - } - } - return nil - } - - if err := kernel.WithAllProcs(kernel.ProcFSRoot(), fn); err != nil { - return nil, err - } - return res, nil -} - -// RunJavaVersionAndWaitForRejection is running a java version program, waiting for it to successfully load, and then -// checking the java TLS program didn't attach to it (rejected the injection). The last part is done using log scanner -// we're registering a new log scanner and looking for a specific log (java pid (\d+) attachment rejected). -func RunJavaVersionAndWaitForRejection(t testing.TB, version, class string, testdir string, waitForCondition *regexp.Regexp) { - t.Helper() - - dir, _ := testutil.CurDir() - addr := "172.17.0.1" // for some reason docker network inspect bridge --format='{{(index .IPAM.Config 0).Gateway}}' is not reliable and doesn't report Gateway ip sometime - env := []string{ - "IMAGE_VERSION=" + version, - "ENTRYCLASS=" + class, - "EXTRA_HOSTS=host.docker.internal:" + addr, - "TESTDIR=" + testdir, - } - - l := javaInjectionRejectionLogger{ - t: t, - lock: sync.RWMutex{}, - rejectedPIDs: make(map[int32]struct{}), - } - configureLoggerForTest(t, &l) - - require.NoError(t, protocolsUtils.RunDockerServer(t, version, dir+"/../testdata/docker-compose.yml", env, waitForCondition, time.Second*15, 3)) - pids, err := FindProcessByCommandLine("java", class) - require.NoError(t, err) - require.Lenf(t, pids, 1, "found more process (%d) than expected (1)", len(pids)) - require.Eventuallyf(t, func() bool { - return l.HasRejectedPID(int32(pids[0])) - }, time.Second*30, time.Millisecond*100, "pid %d was not rejected", pids[0]) -} - -func configureLoggerForTest(t testing.TB, w io.Writer) func() { - logger, err := seelog.LoggerFromWriterWithMinLevel(w, seelog.TraceLvl) - if err != nil { - t.Fatalf("unable to configure logger, err: %v", err) - } - seelog.ReplaceLogger(logger) //nolint:errcheck - log.SetupLogger(logger, "debug") - return log.Flush -} - -type javaInjectionRejectionLogger struct { - t testing.TB - lock sync.RWMutex - rejectedPIDs map[int32]struct{} -} - -var ( - rejectionRegex = regexp.MustCompile(`java pid (\d+) attachment rejected`) -) - -// Write implements the io.Writer interface. -func (l *javaInjectionRejectionLogger) Write(p []byte) (n int, err error) { - res := rejectionRegex.FindAllSubmatch(p, -1) - // We expect to have 1 group (len(res) == 1) and the match (res[0]) should have 2 entries, the first is the full string - // and the second (res[0][1]) is the group (the PID). - if len(res) == 1 && len(res[0]) == 2 { - i, err := strconv.Atoi(string(res[0][1])) - if err == nil { - l.lock.Lock() - l.rejectedPIDs[int32(i)] = struct{}{} - l.lock.Unlock() - } - } - return len(p), nil -} - -// HasRejectedPID returns true if the given pid was rejected by java tls. -func (l *javaInjectionRejectionLogger) HasRejectedPID(pid int32) bool { - l.lock.RLock() - defer l.lock.RUnlock() - _, ok := l.rejectedPIDs[pid] - return ok -} diff --git a/pkg/network/tags_linux.go b/pkg/network/tags_linux.go index 53f350ea8e96e..4f81692cfc31a 100644 --- a/pkg/network/tags_linux.go +++ b/pkg/network/tags_linux.go @@ -18,8 +18,6 @@ const ( ConnTagOpenSSL = http.OpenSSL // ConnTagGo is the tag for GO TLS connections ConnTagGo = http.Go - // ConnTagJava is the tag for Java TLS connections - ConnTagJava = http.Java // ConnTagTLS is the tag for TLS connections in general ConnTagTLS = http.TLS // ConnTagIstio is the tag for Istio TLS connections @@ -40,5 +38,5 @@ func GetStaticTags(staticTags uint64) (tags []string) { // IsTLSTag return if the tag is a TLS tag func IsTLSTag(staticTags uint64) bool { - return staticTags&(ConnTagGnuTLS|ConnTagOpenSSL|ConnTagGo|ConnTagJava|ConnTagTLS|ConnTagIstio|ConnTagNodeJS) > 0 + return staticTags&(ConnTagGnuTLS|ConnTagOpenSSL|ConnTagGo|ConnTagTLS|ConnTagIstio|ConnTagNodeJS) > 0 } diff --git a/pkg/network/usm/config/config.go b/pkg/network/usm/config/config.go index 00a4cc9764c5b..361b23e9325c5 100644 --- a/pkg/network/usm/config/config.go +++ b/pkg/network/usm/config/config.go @@ -78,5 +78,5 @@ func IsUSMSupportedAndEnabled(config *config.Config) bool { // NeedProcessMonitor returns true if the process monitor is needed for the given configuration func NeedProcessMonitor(config *config.Config) bool { - return config.EnableNativeTLSMonitoring || config.EnableGoTLSSupport || config.EnableJavaTLSSupport || config.EnableIstioMonitoring || config.EnableNodeJSMonitoring + return config.EnableNativeTLSMonitoring || config.EnableGoTLSSupport || config.EnableIstioMonitoring || config.EnableNodeJSMonitoring } diff --git a/pkg/network/usm/ebpf_javatls.go b/pkg/network/usm/ebpf_javatls.go deleted file mode 100644 index b2e3f65ac08f6..0000000000000 --- a/pkg/network/usm/ebpf_javatls.go +++ /dev/null @@ -1,329 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2022-present Datadog, Inc. - -//go:build linux_bpf - -package usm - -import ( - "bytes" - "fmt" - "io" - "math/rand" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "time" - - "github.com/cilium/ebpf" - - manager "github.com/DataDog/ebpf-manager" - - "github.com/DataDog/datadog-agent/pkg/network/config" - "github.com/DataDog/datadog-agent/pkg/network/protocols" - "github.com/DataDog/datadog-agent/pkg/network/protocols/tls/java" - "github.com/DataDog/datadog-agent/pkg/network/usm/buildmode" - usmconfig "github.com/DataDog/datadog-agent/pkg/network/usm/config" - "github.com/DataDog/datadog-agent/pkg/process/monitor" - "github.com/DataDog/datadog-agent/pkg/util/kernel" - "github.com/DataDog/datadog-agent/pkg/util/log" -) - -const ( - agentUSMJar = "agent-usm.jar" - javaTLSConnectionsMap = "java_tls_connections" - javaDomainsToConnectionsMap = "java_conn_tuple_by_peer" - eRPCHandlersMap = "java_tls_erpc_handlers" - - doVfsIoctlKprobeName = "kprobe__do_vfs_ioctl" - handleSyncPayloadKprobeName = "kprobe_handle_sync_payload" - handleCloseConnectionKprobeName = "kprobe_handle_close_connection" - handleConnectionByPeerKprobeName = "kprobe_handle_connection_by_peer" - handleAsyncPayloadKprobeName = "kprobe_handle_async_payload" -) - -const ( - // syncPayload is the key to the program that handles the SYNCHRONOUS_PAYLOAD eRPC operation - syncPayload uint32 = iota - // closeConnection is the key to the program that handles the CLOSE_CONNECTION eRPC operation - closeConnection - // connectionByPeer is the key to the program that handles the CONNECTION_BY_PEER eRPC operation - connectionByPeer - // asyncPayload is the key to the program that handles the ASYNC_PAYLOAD eRPC operation - asyncPayload -) - -var ( - javaProcessName = []byte("java") -) - -type javaTLSProgram struct { - cfg *config.Config - processMonitor *monitor.ProcessMonitor - cleanupExec func() - - // tracerJarPath path to the USM agent TLS tracer. - tracerJarPath string - - // tracerArguments default arguments passed to the injected agent-usm.jar - tracerArguments string - - // injectionAllowRegex is matched against /proc/pid/cmdline, to determine if we should attach to the process. - injectionAllowRegex *regexp.Regexp - // injectionAllowRegex is matched against /proc/pid/cmdline, to determine if we should deny attachment to the process. - injectionBlockRegex *regexp.Regexp - - procRoot string -} - -var javaTLSSpec = &protocols.ProtocolSpec{ - Factory: newJavaTLSProgram, - Maps: []*manager.Map{ - { - Name: javaTLSConnectionsMap, - }, - { - Name: javaDomainsToConnectionsMap, - }, - { - Name: eRPCHandlersMap, - }, - }, - Probes: []*manager.Probe{ - { - ProbeIdentificationPair: manager.ProbeIdentificationPair{ - EBPFFuncName: doVfsIoctlKprobeName, - UID: probeUID, - }, - }, - }, - TailCalls: []manager.TailCallRoute{ - { - ProgArrayName: eRPCHandlersMap, - Key: syncPayload, - ProbeIdentificationPair: manager.ProbeIdentificationPair{ - EBPFFuncName: handleSyncPayloadKprobeName, - }, - }, - { - ProgArrayName: eRPCHandlersMap, - Key: closeConnection, - ProbeIdentificationPair: manager.ProbeIdentificationPair{ - EBPFFuncName: handleCloseConnectionKprobeName, - }, - }, - { - ProgArrayName: eRPCHandlersMap, - Key: connectionByPeer, - ProbeIdentificationPair: manager.ProbeIdentificationPair{ - EBPFFuncName: handleConnectionByPeerKprobeName, - }, - }, - { - ProgArrayName: eRPCHandlersMap, - Key: asyncPayload, - ProbeIdentificationPair: manager.ProbeIdentificationPair{ - EBPFFuncName: handleAsyncPayloadKprobeName, - }, - }, - }, -} - -func newJavaTLSProgram(c *config.Config) (protocols.Protocol, error) { - if !c.EnableJavaTLSSupport || !usmconfig.TLSSupported(c) { - return nil, nil - } - - javaUSMAgentJarPath := filepath.Join(c.JavaDir, agentUSMJar) - // We tried switching os.Open to os.Stat, but it seems it does not guarantee we'll be able to copy the file. - f, err := os.Open(javaUSMAgentJarPath) - if err != nil { - return nil, fmt.Errorf("java TLS can't access java tracer payload %s : %s", javaUSMAgentJarPath, err) - } - // If we managed to open the file, then we close it, as we just needed to check if the file exists. - _ = f.Close() - - return &javaTLSProgram{ - cfg: c, - processMonitor: monitor.GetProcessMonitor(), - tracerArguments: buildTracerArguments(c), - tracerJarPath: javaUSMAgentJarPath, - injectionAllowRegex: buildRegex(c.JavaAgentAllowRegex, "allow"), - injectionBlockRegex: buildRegex(c.JavaAgentBlockRegex, "block"), - procRoot: kernel.ProcFSRoot(), - }, nil -} - -// Name return the program's name. -func (p *javaTLSProgram) Name() string { - return "Java TLS" -} - -// ConfigureOptions changes map attributes to the given options. -func (p *javaTLSProgram) ConfigureOptions(_ *manager.Manager, options *manager.Options) { - options.MapSpecEditors[javaTLSConnectionsMap] = manager.MapSpecEditor{ - MaxEntries: p.cfg.MaxUSMConcurrentRequests, - EditorFlag: manager.EditMaxEntries, - } - options.MapSpecEditors[javaDomainsToConnectionsMap] = manager.MapSpecEditor{ - MaxEntries: p.cfg.MaxUSMConcurrentRequests, - EditorFlag: manager.EditMaxEntries, - } - options.ActivatedProbes = append(options.ActivatedProbes, - &manager.ProbeSelector{ - ProbeIdentificationPair: manager.ProbeIdentificationPair{ - EBPFFuncName: doVfsIoctlKprobeName, - UID: probeUID, - }, - }, - ) -} - -// isJavaProcess checks if the given PID comm's name is java. -// The method is much faster and efficient that using process.NewProcess(pid).Name(). -func (p *javaTLSProgram) isJavaProcess(pid uint32) bool { - filePath := filepath.Join(p.procRoot, strconv.Itoa(int(pid)), "comm") - content, err := os.ReadFile(filePath) - if err != nil { - // Waiting a bit, as we might get the event of process creation before the directory was created. - for i := 0; i < 3; i++ { - time.Sleep(10 * time.Millisecond) - // reading again. - content, err = os.ReadFile(filePath) - if err == nil { - break - } - } - } - - if err != nil { - // short living process can hit here, or slow start of another process. - return false - } - return bytes.Equal(bytes.TrimSpace(content), javaProcessName) -} - -// isAttachmentAllowed will return true if the pid can be attached -// The filter is based on the process command line matching injectionAllowRegex and injectionBlockRegex regex -// injectionAllowRegex has a higher priority -// -// # In case of only one regex (allow or block) is set, the regex will be evaluated as exclusive filter -// / match | not match -// allowRegex only true | false -// blockRegex only false | true -func (p *javaTLSProgram) isAttachmentAllowed(pid uint32) bool { - allowIsSet := p.injectionAllowRegex != nil - blockIsSet := p.injectionBlockRegex != nil - // filter is disabled (default configuration) - if !allowIsSet && !blockIsSet { - return true - } - - procCmdline := fmt.Sprintf("%s/%d/cmdline", p.procRoot, pid) - cmd, err := os.ReadFile(procCmdline) - if err != nil { - log.Debugf("injection filter can't open commandline %q : %s", procCmdline, err) - return false - } - fullCmdline := strings.ReplaceAll(string(cmd), "\000", " ") // /proc/pid/cmdline format : arguments are separated by '\0' - - // Allow to have a higher priority - if allowIsSet && p.injectionAllowRegex.MatchString(fullCmdline) { - return true - } - if blockIsSet && p.injectionBlockRegex.MatchString(fullCmdline) { - return false - } - - // if only one regex is set, allow regex if not match should not attach - if allowIsSet != blockIsSet { // allow xor block - if allowIsSet { - return false - } - } - return true -} - -func (p *javaTLSProgram) newJavaProcess(pid uint32) { - if !p.isJavaProcess(pid) { - return - } - if !p.isAttachmentAllowed(pid) { - log.Debugf("java pid %d attachment rejected", pid) - return - } - - if err := java.InjectAgent(int(pid), p.tracerJarPath, p.tracerArguments); err != nil { - log.Error(err) - } -} - -// PreStart subscribes to the exec events to inject the java agent. -func (p *javaTLSProgram) PreStart(*manager.Manager) error { - p.cleanupExec = p.processMonitor.SubscribeExec(p.newJavaProcess) - return nil -} - -// PostStart is a no-op. -func (p *javaTLSProgram) PostStart(*manager.Manager) error { - return nil -} - -// Stop unsubscribes from the exec events. -func (p *javaTLSProgram) Stop(*manager.Manager) { - if p.cleanupExec != nil { - p.cleanupExec() - } - - if p.processMonitor != nil { - p.processMonitor.Stop() - } -} - -// DumpMaps is a no-op. -func (p *javaTLSProgram) DumpMaps(io.Writer, string, *ebpf.Map) {} - -// GetStats is a no-op. -func (p *javaTLSProgram) GetStats() *protocols.ProtocolStats { - return nil -} - -// buildRegex is similar to regexp.MustCompile, but without panic. -func buildRegex(re, reType string) *regexp.Regexp { - if re == "" { - return nil - } - res, err := regexp.Compile(re) - if err != nil { - log.Errorf("%s regex can't be compiled %s", reType, err) - return nil - } - - return res -} - -// buildTracerArguments returns the command line arguments we'll pass to the injected tracer. -func buildTracerArguments(c *config.Config) string { - // Randomizing the seed to ensure we get a truly random number. - rand.Seed(int64(os.Getpid()) + time.Now().UnixMicro()) - - allArgs := []string{ - c.JavaAgentArgs, - // authID is used here as an identifier, simple proof of authenticity - // between the injected java process and the ebpf ioctl that receive the payload. - fmt.Sprintf("dd.usm.authID=%d", rand.Int63()), - } - if c.JavaAgentDebug { - allArgs = append(allArgs, "dd.trace.debug=true") - } - return strings.Join(allArgs, ",") -} - -// IsBuildModeSupported returns always true, as java tls module is supported by all modes. -func (*javaTLSProgram) IsBuildModeSupported(buildmode.Type) bool { - return true -} diff --git a/pkg/network/usm/ebpf_javatls_test.go b/pkg/network/usm/ebpf_javatls_test.go deleted file mode 100644 index 947bab3e3597e..0000000000000 --- a/pkg/network/usm/ebpf_javatls_test.go +++ /dev/null @@ -1,198 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build linux_bpf - -package usm - -import ( - "os" - "path/filepath" - "regexp" - "testing" - - "github.com/stretchr/testify/require" - - networkconfig "github.com/DataDog/datadog-agent/pkg/network/config" - "github.com/DataDog/datadog-agent/pkg/network/protocols/http/testutil" - javatestutil "github.com/DataDog/datadog-agent/pkg/network/protocols/tls/java/testutil" - nettestutil "github.com/DataDog/datadog-agent/pkg/network/testutil" - usmconfig "github.com/DataDog/datadog-agent/pkg/network/usm/config" -) - -func createJavaTempFile(t *testing.T, dir string) string { - tempfile, err := os.CreateTemp(dir, "TestAgentLoaded.agentmain.*") - require.NoError(t, err) - tempfile.Close() - os.Remove(tempfile.Name()) - t.Cleanup(func() { os.Remove(tempfile.Name()) }) - - return tempfile.Name() -} - -func TestJavaInjection(t *testing.T) { - t.Skip("JavaTLS tests are currently disabled") - cfg := networkconfig.New() - cfg.EnableJavaTLSSupport = true - if !usmconfig.TLSSupported(cfg) { - t.Skip("Java injection tests are not supported on this machine") - } - - defaultCfg := cfg - - dir, _ := testutil.CurDir() - testdataDir := filepath.Join(dir, "../protocols/tls/java/testdata") - // create a fake agent-usm.jar based on TestAgentLoaded.jar by forcing cfg.JavaDir - fakeAgentDir, err := os.MkdirTemp("", "fake.agent-usm.jar.") - require.NoError(t, err) - defer os.RemoveAll(fakeAgentDir) - _, err = nettestutil.RunCommand("install -m444 " + filepath.Join(testdataDir, "TestAgentLoaded.jar") + " " + filepath.Join(fakeAgentDir, "agent-usm.jar")) - require.NoError(t, err) - _, err = nettestutil.RunCommand("install -m444 " + filepath.Join(testdataDir, "Wait.class") + " " + filepath.Join(fakeAgentDir, "Wait.class")) - require.NoError(t, err) - - commonTearDown := func(_ *testing.T, ctx map[string]interface{}) { - cfg.JavaAgentArgs = ctx["JavaAgentArgs"].(string) - - testfile := ctx["testfile"].(string) - _, err := os.Stat(testfile) - if err == nil { - os.Remove(testfile) - } - } - - commonValidation := func(t *testing.T, ctx map[string]interface{}) { - testfile := ctx["testfile"].(string) - _, err := os.Stat(testfile) - require.NoError(t, err) - } - - tests := []struct { - name string - context map[string]interface{} - preTracerSetup func(t *testing.T, ctx map[string]interface{}) - postTracerSetup func(t *testing.T, ctx map[string]interface{}) - validation func(t *testing.T, ctx map[string]interface{}) - teardown func(t *testing.T, ctx map[string]interface{}) - }{ - { - // Test the java hotspot injection is working - name: "java_hotspot_injection_8u151", - context: make(map[string]interface{}), - preTracerSetup: func(t *testing.T, ctx map[string]interface{}) { - cfg.JavaDir = fakeAgentDir - ctx["JavaAgentArgs"] = cfg.JavaAgentArgs - ctx["testfile"] = createJavaTempFile(t, fakeAgentDir) - cfg.JavaAgentArgs += ",testfile=/v/" + filepath.Base(ctx["testfile"].(string)) - }, - postTracerSetup: func(t *testing.T, _ map[string]interface{}) { - // if RunJavaVersion failing to start it's probably because the java process has not been injected - require.NoError(t, javatestutil.RunJavaVersion(t, "openjdk:8u151-jre", "Wait JustWait", fakeAgentDir), "Failed running Java version") - }, - validation: commonValidation, - teardown: commonTearDown, - }, - { - // Test the java hotspot injection is working - name: "java_hotspot_injection_21_allow_only", - context: make(map[string]interface{}), - preTracerSetup: func(t *testing.T, ctx map[string]interface{}) { - cfg.JavaDir = fakeAgentDir - ctx["JavaAgentArgs"] = cfg.JavaAgentArgs - ctx["testfile"] = createJavaTempFile(t, fakeAgentDir) - cfg.JavaAgentArgs += ",testfile=/v/" + filepath.Base(ctx["testfile"].(string)) - - // testing allow/block list, as Allow list have higher priority - // this test will pass normally - cfg.JavaAgentAllowRegex = ".*JustWait.*" - cfg.JavaAgentBlockRegex = "" - }, - postTracerSetup: func(t *testing.T, _ map[string]interface{}) { - // if RunJavaVersion failing to start it's probably because the java process has not been injected - require.NoError(t, javatestutil.RunJavaVersion(t, "openjdk:21-oraclelinux8", "Wait JustWait", fakeAgentDir), "Failed running Java version") - javatestutil.RunJavaVersionAndWaitForRejection(t, "openjdk:21-oraclelinux8", "Wait AnotherWait", fakeAgentDir, regexp.MustCompile(`AnotherWait pid.*`)) - }, - validation: commonValidation, - teardown: commonTearDown, - }, - { - // Test the java hotspot injection is working - name: "java_hotspot_injection_21_block_only", - context: make(map[string]interface{}), - preTracerSetup: func(t *testing.T, ctx map[string]interface{}) { - ctx["JavaAgentArgs"] = cfg.JavaAgentArgs - ctx["testfile"] = createJavaTempFile(t, fakeAgentDir) - cfg.JavaAgentArgs += ",testfile=/v/" + filepath.Base(ctx["testfile"].(string)) - - // block the agent attachment - cfg.JavaAgentAllowRegex = "" - cfg.JavaAgentBlockRegex = ".*JustWait.*" - }, - postTracerSetup: func(t *testing.T, _ map[string]interface{}) { - // if RunJavaVersion failing to start it's probably because the java process has not been injected - require.NoError(t, javatestutil.RunJavaVersion(t, "openjdk:21-oraclelinux8", "Wait AnotherWait", fakeAgentDir), "Failed running Java version") - javatestutil.RunJavaVersionAndWaitForRejection(t, "openjdk:21-oraclelinux8", "Wait JustWait", fakeAgentDir, regexp.MustCompile(`JustWait pid.*`)) - }, - validation: commonValidation, - teardown: commonTearDown, - }, - { - name: "java_hotspot_injection_21_allowblock", - context: make(map[string]interface{}), - preTracerSetup: func(t *testing.T, ctx map[string]interface{}) { - ctx["JavaAgentArgs"] = cfg.JavaAgentArgs - ctx["testfile"] = createJavaTempFile(t, fakeAgentDir) - cfg.JavaAgentArgs += ",testfile=/v/" + filepath.Base(ctx["testfile"].(string)) - - // block the agent attachment - cfg.JavaAgentAllowRegex = ".*JustWait.*" - cfg.JavaAgentBlockRegex = ".*AnotherWait.*" - }, - postTracerSetup: func(t *testing.T, _ map[string]interface{}) { - require.NoError(t, javatestutil.RunJavaVersion(t, "openjdk:21-oraclelinux8", "Wait JustWait", fakeAgentDir), "Failed running Java version") - javatestutil.RunJavaVersionAndWaitForRejection(t, "openjdk:21-oraclelinux8", "Wait AnotherWait", fakeAgentDir, regexp.MustCompile(`AnotherWait pid.*`)) - }, - validation: commonValidation, - teardown: commonTearDown, - }, - { - name: "java_hotspot_injection_21_allow_higher_priority", - context: make(map[string]interface{}), - preTracerSetup: func(t *testing.T, ctx map[string]interface{}) { - ctx["JavaAgentArgs"] = cfg.JavaAgentArgs - ctx["testfile"] = createJavaTempFile(t, fakeAgentDir) - cfg.JavaAgentArgs += ",testfile=/v/" + filepath.Base(ctx["testfile"].(string)) - - // allow has a higher priority - cfg.JavaAgentAllowRegex = ".*JustWait.*" - cfg.JavaAgentBlockRegex = ".*JustWait.*" - }, - postTracerSetup: func(t *testing.T, _ map[string]interface{}) { - require.NoError(t, javatestutil.RunJavaVersion(t, "openjdk:21-oraclelinux8", "Wait JustWait", fakeAgentDir), "Failed running Java version") - }, - validation: commonValidation, - teardown: commonTearDown, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Cleanup(func() { - tt.teardown(t, tt.context) - }) - cfg = defaultCfg - tt.preTracerSetup(t, tt.context) - javaTLSProg, err := newJavaTLSProgram(cfg) - require.NoError(t, err) - require.NoError(t, javaTLSProg.PreStart(nil)) - t.Cleanup(func() { - javaTLSProg.Stop(nil) - }) - require.NoError(t, javaTLSProg.(*javaTLSProgram).processMonitor.Initialize(true)) - - tt.postTracerSetup(t, tt.context) - tt.validation(t, tt.context) - }) - } -} diff --git a/pkg/network/usm/ebpf_main.go b/pkg/network/usm/ebpf_main.go index 63137d318ff91..cc1bec4f89bf4 100644 --- a/pkg/network/usm/ebpf_main.go +++ b/pkg/network/usm/ebpf_main.go @@ -51,7 +51,6 @@ var ( kafka.Spec, postgres.Spec, redis.Spec, - javaTLSSpec, // opensslSpec is unique, as we're modifying its factory during runtime to allow getting more parameters in the // factory. opensslSpec, diff --git a/pkg/network/usm/monitor_testutil.go b/pkg/network/usm/monitor_testutil.go index 450081f3fbd78..bbf01267af343 100644 --- a/pkg/network/usm/monitor_testutil.go +++ b/pkg/network/usm/monitor_testutil.go @@ -75,7 +75,7 @@ func (p *protocolMock) Stop(mgr *manager.Manager) { func (p *protocolMock) DumpMaps(io.Writer, string, *ebpf.Map) {} func (p *protocolMock) GetStats() *protocols.ProtocolStats { return nil } -// IsBuildModeSupported returns always true, as java tls module is supported by all modes. +// IsBuildModeSupported returns always true, as the mock is supported by all modes. func (*protocolMock) IsBuildModeSupported(buildmode.Type) bool { return true } // patchProtocolMock updates the map of known protocols to replace the mock diff --git a/pkg/network/usm/monitor_tls_test.go b/pkg/network/usm/monitor_tls_test.go index c706b5fe874d4..d4c012a0747f5 100644 --- a/pkg/network/usm/monitor_tls_test.go +++ b/pkg/network/usm/monitor_tls_test.go @@ -39,9 +39,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/network/protocols/http2" protocolsUtils "github.com/DataDog/datadog-agent/pkg/network/protocols/testutil" gotlstestutil "github.com/DataDog/datadog-agent/pkg/network/protocols/tls/gotls/testutil" - javatestutil "github.com/DataDog/datadog-agent/pkg/network/protocols/tls/java/testutil" "github.com/DataDog/datadog-agent/pkg/network/protocols/tls/nodejs" - nettestutil "github.com/DataDog/datadog-agent/pkg/network/testutil" usmconfig "github.com/DataDog/datadog-agent/pkg/network/usm/config" usmtestutil "github.com/DataDog/datadog-agent/pkg/network/usm/testutil" "github.com/DataDog/datadog-agent/pkg/network/usm/utils" @@ -455,98 +453,6 @@ func isRequestIncluded(allStats map[http.Key]*http.RequestStats, req *nethttp.Re return false } -func (s *tlsSuite) TestJavaInjection() { - t := s.T() - t.Skip("JavaTLS tests are currently disabled") - - cfg := config.New() - cfg.EnableHTTPMonitoring = true - cfg.EnableJavaTLSSupport = true - defaultCfg := cfg - - dir, _ := testutil.CurDir() - testdataDir := filepath.Join(dir, "../protocols/tls/java/testdata") - legacyJavaDir := cfg.JavaDir - // create a fake agent-usm.jar based on TestAgentLoaded.jar by forcing cfg.JavaDir - fakeAgentDir, err := os.MkdirTemp("", "fake.agent-usm.jar.") - require.NoError(t, err) - defer os.RemoveAll(fakeAgentDir) - _, err = nettestutil.RunCommand("install -m444 " + filepath.Join(testdataDir, "TestAgentLoaded.jar") + " " + filepath.Join(fakeAgentDir, "agent-usm.jar")) - require.NoError(t, err) - - tests := []struct { - name string - context testContext - preTracerSetup func(t *testing.T) - postTracerSetup func(t *testing.T) - validation func(t *testing.T, monitor *Monitor) - teardown func(t *testing.T) - }{ - { - // Test the java jdk client https request is working - name: "java_jdk_client_httpbin_docker_withTLSClassification_java15", - preTracerSetup: func(t *testing.T) { - cfg.JavaDir = legacyJavaDir - cfg.ProtocolClassificationEnabled = true - cfg.CollectTCPv4Conns = true - cfg.CollectTCPv6Conns = true - - serverDoneFn := testutil.HTTPServer(t, "0.0.0.0:5443", testutil.Options{ - EnableTLS: true, - }) - t.Cleanup(serverDoneFn) - }, - postTracerSetup: func(t *testing.T) { - require.NoError(t, javatestutil.RunJavaVersion(t, "openjdk:15-oraclelinux8", "Wget https://host.docker.internal:5443/200/anything/java-tls-request", "./", regexp.MustCompile("Response code = .*")), "Failed running Java version") - }, - validation: func(t *testing.T, monitor *Monitor) { - // Iterate through active connections until we find connection created above - require.Eventually(t, func() bool { - stats := getHTTPLikeProtocolStats(monitor, protocols.HTTP) - if stats == nil { - return false - } - for key, stats := range stats { - if key.Path.Content.Get() == "/200/anything/java-tls-request" { - t.Log("path content found") - - req, exists := stats.Data[200] - if !exists { - t.Logf("wrong response, not 200 : %#+v", key) - continue - } - - if req.StaticTags != network.ConnTagJava { - t.Logf("tag not java : %#+v", key) - continue - } - return true - } - } - - return false - }, 4*time.Second, 100*time.Millisecond, "couldn't find http connection matching: https://host.docker.internal:5443/200/anything/java-tls-request") - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.teardown != nil { - t.Cleanup(func() { - tt.teardown(t) - }) - } - cfg = defaultCfg - if tt.preTracerSetup != nil { - tt.preTracerSetup(t) - } - usmMonitor := setupUSMTLSMonitor(t, cfg) - tt.postTracerSetup(t) - tt.validation(t, usmMonitor) - }) - } -} - func TestHTTPGoTLSAttachProbes(t *testing.T) { t.Skip("skipping GoTLS tests while we investigate their flakiness") diff --git a/pkg/network/usm/nodejs.go b/pkg/network/usm/nodejs.go index d4e44523fc444..4b5996830c2b4 100644 --- a/pkg/network/usm/nodejs.go +++ b/pkg/network/usm/nodejs.go @@ -125,7 +125,6 @@ func newNodeJSMonitor(c *config.Config, mgr *manager.Manager) (*nodeJSMonitor, e EbpfConfig: &c.Config, ExcludeTargets: uprobes.ExcludeSelf | uprobes.ExcludeInternal | uprobes.ExcludeBuildkit | uprobes.ExcludeContainerdTmp, EnablePeriodicScanNewProcesses: true, - ProcessMonitorEventStream: c.EnableUSMEventStream, } attacher, err := uprobes.NewUprobeAttacher(nodeJsAttacherName, attachCfg, mgr, uprobes.NopOnAttachCallback, &uprobes.NativeBinaryInspector{}) diff --git a/pkg/networkdevice/metadata/payload.go b/pkg/networkdevice/metadata/payload.go index 8e5efc545770e..54b2f45eb0996 100644 --- a/pkg/networkdevice/metadata/payload.go +++ b/pkg/networkdevice/metadata/payload.go @@ -6,8 +6,6 @@ // Package metadata defines types for describing data about a device. package metadata -import "github.com/DataDog/datadog-agent/pkg/snmp/gosnmplib" - // PayloadMetadataBatchSize is the number of resources per event payload // Resources are devices, interfaces, etc const PayloadMetadataBatchSize = 100 @@ -77,8 +75,10 @@ type DeviceMetadata struct { // DeviceOID device scan oid data type DeviceOID struct { - *gosnmplib.PDU DeviceID string `json:"device_id"` + OID string `json:"oid"` + Type string `json:"type"` + Value string `json:"value"` } // InterfaceMetadata contains interface metadata diff --git a/pkg/networkdevice/metadata/payload_utils.go b/pkg/networkdevice/metadata/payload_utils.go index ca6bf632aafb2..5ae8f5de786c6 100644 --- a/pkg/networkdevice/metadata/payload_utils.go +++ b/pkg/networkdevice/metadata/payload_utils.go @@ -94,6 +94,8 @@ func DeviceOIDFromPDU(deviceID string, snmpPDU *gosnmp.SnmpPDU) (*DeviceOID, err } return &DeviceOID{ DeviceID: deviceID, - PDU: pdu, + OID: pdu.OID, + Type: pdu.Type.String(), + Value: pdu.Value, }, nil } diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod index 14e8631fd7867..487bf4de5b631 100644 --- a/pkg/obfuscate/go.mod +++ b/pkg/obfuscate/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( github.com/DataDog/datadog-go/v5 v5.5.0 - github.com/DataDog/go-sqllexer v0.0.15 + github.com/DataDog/go-sqllexer v0.0.16 github.com/outcaste-io/ristretto v0.2.1 github.com/stretchr/testify v1.9.0 go.uber.org/atomic v1.10.0 diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum index 3e672458d1eed..e58cf0c1e6e5c 100644 --- a/pkg/obfuscate/go.sum +++ b/pkg/obfuscate/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= -github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.16 h1:RoSUMS6MECyB3gTUIdydzXwK5NhEhv6GMJkS7ptsgRA= +github.com/DataDog/go-sqllexer v0.0.16/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= diff --git a/pkg/process/checks/container.go b/pkg/process/checks/container.go index 590cb8f9ac359..8d9c8330ebd47 100644 --- a/pkg/process/checks/container.go +++ b/pkg/process/checks/container.go @@ -56,7 +56,7 @@ func (c *ContainerCheck) Init(syscfg *SysProbeConfig, info *HostInfo, _ bool) er c.containerProvider = proccontainers.GetSharedContainerProvider(c.wmeta) c.hostInfo = info - var tu *net.RemoteSysProbeUtil + var tu net.SysProbeUtil var err error if syscfg.NetworkTracerModuleEnabled { // Calling the remote tracer will cause it to initialize and check connectivity diff --git a/pkg/process/checks/net.go b/pkg/process/checks/net.go index 13b33d87f9dca..816ae56a42134 100644 --- a/pkg/process/checks/net.go +++ b/pkg/process/checks/net.go @@ -505,7 +505,7 @@ func convertAndEnrichWithServiceCtx(tags []string, tagOffsets []uint32, serviceC } // fetches network_id from the current netNS or from the system probe if necessary, where the root netNS is used -func retryGetNetworkID(sysProbeUtil *net.RemoteSysProbeUtil) (string, error) { +func retryGetNetworkID(sysProbeUtil net.SysProbeUtil) (string, error) { networkID, err := cloudproviders.GetNetworkID(context.TODO()) if err != nil && sysProbeUtil != nil { log.Infof("no network ID detected. retrying via system-probe: %s", err) diff --git a/pkg/process/checks/process.go b/pkg/process/checks/process.go index 5d905a135702c..05a7ac5c72d28 100644 --- a/pkg/process/checks/process.go +++ b/pkg/process/checks/process.go @@ -136,7 +136,7 @@ func (p *ProcessCheck) Init(syscfg *SysProbeConfig, info *HostInfo, oneShot bool p.notInitializedLogLimit = log.NewLogLimit(1, time.Minute*10) - var tu *net.RemoteSysProbeUtil + var tu net.SysProbeUtil var err error if syscfg.NetworkTracerModuleEnabled { // Calling the remote tracer will cause it to initialize and check connectivity @@ -657,7 +657,7 @@ func skipProcess( return false } -func (p *ProcessCheck) getRemoteSysProbeUtil() *net.RemoteSysProbeUtil { +func (p *ProcessCheck) getRemoteSysProbeUtil() net.SysProbeUtil { if !p.sysProbeConfig.ProcessModuleEnabled { return nil } diff --git a/pkg/process/checks/process_rt.go b/pkg/process/checks/process_rt.go index 9fd866a735ce6..33bf93f06a793 100644 --- a/pkg/process/checks/process_rt.go +++ b/pkg/process/checks/process_rt.go @@ -160,7 +160,7 @@ func calculateRate(cur, prev uint64, before time.Time) float32 { } // mergeStatWithSysprobeStats takes a process by PID map and fill the stats from system probe into the processes in the map -func mergeStatWithSysprobeStats(pids []int32, stats map[int32]*procutil.Stats, pu *net.RemoteSysProbeUtil) { +func mergeStatWithSysprobeStats(pids []int32, stats map[int32]*procutil.Stats, pu net.SysProbeUtil) { pStats, err := pu.GetProcStats(pids) if err == nil { for pid, stats := range stats { diff --git a/pkg/process/net/check.go b/pkg/process/net/check.go index 6ff1c50dc259c..748c3dacd52cf 100644 --- a/pkg/process/net/check.go +++ b/pkg/process/net/check.go @@ -19,6 +19,7 @@ import ( ebpfcheck "github.com/DataDog/datadog-agent/pkg/collector/corechecks/ebpf/probe/ebpfcheck/model" oomkill "github.com/DataDog/datadog-agent/pkg/collector/corechecks/ebpf/probe/oomkill/model" tcpqueuelength "github.com/DataDog/datadog-agent/pkg/collector/corechecks/ebpf/probe/tcpqueuelength/model" + gpu "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu/model" ) const ( @@ -67,6 +68,13 @@ func (r *RemoteSysProbeUtil) GetCheck(module sysconfigtypes.ModuleName) (interfa return nil, err } return stats, nil + } else if module == sysconfig.GPUMonitoringModule { + var stats gpu.GPUStats + err = json.Unmarshal(body, &stats) + if err != nil { + return nil, err + } + return stats, nil } return nil, fmt.Errorf("invalid check name: %s", module) diff --git a/pkg/process/net/common.go b/pkg/process/net/common.go index 252361f6d8983..8af3c4e7e65bb 100644 --- a/pkg/process/net/common.go +++ b/pkg/process/net/common.go @@ -65,8 +65,11 @@ type RemoteSysProbeUtil struct { tracerouteClient http.Client } +// ensure that GetRemoteSystemProbeUtil implements SysProbeUtilGetter +var _ SysProbeUtilGetter = GetRemoteSystemProbeUtil + // GetRemoteSystemProbeUtil returns a ready to use RemoteSysProbeUtil. It is backed by a shared singleton. -func GetRemoteSystemProbeUtil(path string) (*RemoteSysProbeUtil, error) { +func GetRemoteSystemProbeUtil(path string) (SysProbeUtil, error) { err := CheckPath(path) if err != nil { return nil, fmt.Errorf("error setting up remote system probe util, %v", err) diff --git a/pkg/process/net/common_unsupported.go b/pkg/process/net/common_unsupported.go index af7eaef555926..0d0a502236ad5 100644 --- a/pkg/process/net/common_unsupported.go +++ b/pkg/process/net/common_unsupported.go @@ -9,12 +9,18 @@ package net import ( + "time" + model "github.com/DataDog/agent-payload/v5/process" + sysconfigtypes "github.com/DataDog/datadog-agent/cmd/system-probe/config/types" + discoverymodel "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" "github.com/DataDog/datadog-agent/pkg/languagedetection/languagemodels" + nppayload "github.com/DataDog/datadog-agent/pkg/networkpath/payload" ) var _ SysProbeUtil = &RemoteSysProbeUtil{} +var _ SysProbeUtilGetter = GetRemoteSystemProbeUtil // RemoteSysProbeUtil is not supported type RemoteSysProbeUtil struct{} @@ -29,7 +35,7 @@ func CheckPath(_ string) error { // GetRemoteSystemProbeUtil is not supported // //nolint:revive // TODO(PROC) Fix revive linter -func GetRemoteSystemProbeUtil(_ string) (*RemoteSysProbeUtil, error) { +func GetRemoteSystemProbeUtil(_ string) (SysProbeUtil, error) { return &RemoteSysProbeUtil{}, ErrNotImplemented } @@ -76,3 +82,19 @@ func (r *RemoteSysProbeUtil) GetPprof(_ string) ([]byte, error) { // GetTelemetry is not supported func (r *RemoteSysProbeUtil) GetTelemetry() ([]byte, error) { return nil, ErrNotImplemented } + +func (r *RemoteSysProbeUtil) GetDiscoveryServices() (*discoverymodel.ServicesResponse, error) { + return nil, ErrNotImplemented +} + +func (r *RemoteSysProbeUtil) GetCheck(module sysconfigtypes.ModuleName) (interface{}, error) { + return nil, ErrNotImplemented +} + +func (r *RemoteSysProbeUtil) GetPing(clientID string, host string, count int, interval time.Duration, timeout time.Duration) ([]byte, error) { + return nil, ErrNotImplemented +} + +func (r *RemoteSysProbeUtil) GetTraceroute(clientID string, host string, port uint16, protocol nppayload.Protocol, maxTTL uint8, timeout time.Duration) ([]byte, error) { + return nil, ErrNotImplemented +} diff --git a/pkg/process/net/mocks/sys_probe_util.go b/pkg/process/net/mocks/sys_probe_util.go index c4a1bc3492645..1e623aac6cc23 100644 --- a/pkg/process/net/mocks/sys_probe_util.go +++ b/pkg/process/net/mocks/sys_probe_util.go @@ -3,9 +3,18 @@ package mocks import ( + languagemodels "github.com/DataDog/datadog-agent/pkg/languagedetection/languagemodels" mock "github.com/stretchr/testify/mock" + model "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" + + payload "github.com/DataDog/datadog-agent/pkg/networkpath/payload" + process "github.com/DataDog/agent-payload/v5/process" + + time "time" + + types "github.com/DataDog/datadog-agent/cmd/system-probe/config/types" ) // SysProbeUtil is an autogenerated mock type for the SysProbeUtil type @@ -13,6 +22,130 @@ type SysProbeUtil struct { mock.Mock } +type SysProbeUtil_Expecter struct { + mock *mock.Mock +} + +func (_m *SysProbeUtil) EXPECT() *SysProbeUtil_Expecter { + return &SysProbeUtil_Expecter{mock: &_m.Mock} +} + +// DetectLanguage provides a mock function with given fields: pids +func (_m *SysProbeUtil) DetectLanguage(pids []int32) ([]languagemodels.Language, error) { + ret := _m.Called(pids) + + if len(ret) == 0 { + panic("no return value specified for DetectLanguage") + } + + var r0 []languagemodels.Language + var r1 error + if rf, ok := ret.Get(0).(func([]int32) ([]languagemodels.Language, error)); ok { + return rf(pids) + } + if rf, ok := ret.Get(0).(func([]int32) []languagemodels.Language); ok { + r0 = rf(pids) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]languagemodels.Language) + } + } + + if rf, ok := ret.Get(1).(func([]int32) error); ok { + r1 = rf(pids) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SysProbeUtil_DetectLanguage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DetectLanguage' +type SysProbeUtil_DetectLanguage_Call struct { + *mock.Call +} + +// DetectLanguage is a helper method to define mock.On call +// - pids []int32 +func (_e *SysProbeUtil_Expecter) DetectLanguage(pids interface{}) *SysProbeUtil_DetectLanguage_Call { + return &SysProbeUtil_DetectLanguage_Call{Call: _e.mock.On("DetectLanguage", pids)} +} + +func (_c *SysProbeUtil_DetectLanguage_Call) Run(run func(pids []int32)) *SysProbeUtil_DetectLanguage_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].([]int32)) + }) + return _c +} + +func (_c *SysProbeUtil_DetectLanguage_Call) Return(_a0 []languagemodels.Language, _a1 error) *SysProbeUtil_DetectLanguage_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_DetectLanguage_Call) RunAndReturn(run func([]int32) ([]languagemodels.Language, error)) *SysProbeUtil_DetectLanguage_Call { + _c.Call.Return(run) + return _c +} + +// GetCheck provides a mock function with given fields: module +func (_m *SysProbeUtil) GetCheck(module types.ModuleName) (interface{}, error) { + ret := _m.Called(module) + + if len(ret) == 0 { + panic("no return value specified for GetCheck") + } + + var r0 interface{} + var r1 error + if rf, ok := ret.Get(0).(func(types.ModuleName) (interface{}, error)); ok { + return rf(module) + } + if rf, ok := ret.Get(0).(func(types.ModuleName) interface{}); ok { + r0 = rf(module) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(interface{}) + } + } + + if rf, ok := ret.Get(1).(func(types.ModuleName) error); ok { + r1 = rf(module) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SysProbeUtil_GetCheck_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetCheck' +type SysProbeUtil_GetCheck_Call struct { + *mock.Call +} + +// GetCheck is a helper method to define mock.On call +// - module types.ModuleName +func (_e *SysProbeUtil_Expecter) GetCheck(module interface{}) *SysProbeUtil_GetCheck_Call { + return &SysProbeUtil_GetCheck_Call{Call: _e.mock.On("GetCheck", module)} +} + +func (_c *SysProbeUtil_GetCheck_Call) Run(run func(module types.ModuleName)) *SysProbeUtil_GetCheck_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(types.ModuleName)) + }) + return _c +} + +func (_c *SysProbeUtil_GetCheck_Call) Return(_a0 interface{}, _a1 error) *SysProbeUtil_GetCheck_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetCheck_Call) RunAndReturn(run func(types.ModuleName) (interface{}, error)) *SysProbeUtil_GetCheck_Call { + _c.Call.Return(run) + return _c +} + // GetConnections provides a mock function with given fields: clientID func (_m *SysProbeUtil) GetConnections(clientID string) (*process.Connections, error) { ret := _m.Called(clientID) @@ -43,6 +176,91 @@ func (_m *SysProbeUtil) GetConnections(clientID string) (*process.Connections, e return r0, r1 } +// SysProbeUtil_GetConnections_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetConnections' +type SysProbeUtil_GetConnections_Call struct { + *mock.Call +} + +// GetConnections is a helper method to define mock.On call +// - clientID string +func (_e *SysProbeUtil_Expecter) GetConnections(clientID interface{}) *SysProbeUtil_GetConnections_Call { + return &SysProbeUtil_GetConnections_Call{Call: _e.mock.On("GetConnections", clientID)} +} + +func (_c *SysProbeUtil_GetConnections_Call) Run(run func(clientID string)) *SysProbeUtil_GetConnections_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *SysProbeUtil_GetConnections_Call) Return(_a0 *process.Connections, _a1 error) *SysProbeUtil_GetConnections_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetConnections_Call) RunAndReturn(run func(string) (*process.Connections, error)) *SysProbeUtil_GetConnections_Call { + _c.Call.Return(run) + return _c +} + +// GetDiscoveryServices provides a mock function with given fields: +func (_m *SysProbeUtil) GetDiscoveryServices() (*model.ServicesResponse, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetDiscoveryServices") + } + + var r0 *model.ServicesResponse + var r1 error + if rf, ok := ret.Get(0).(func() (*model.ServicesResponse, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() *model.ServicesResponse); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.ServicesResponse) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SysProbeUtil_GetDiscoveryServices_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetDiscoveryServices' +type SysProbeUtil_GetDiscoveryServices_Call struct { + *mock.Call +} + +// GetDiscoveryServices is a helper method to define mock.On call +func (_e *SysProbeUtil_Expecter) GetDiscoveryServices() *SysProbeUtil_GetDiscoveryServices_Call { + return &SysProbeUtil_GetDiscoveryServices_Call{Call: _e.mock.On("GetDiscoveryServices")} +} + +func (_c *SysProbeUtil_GetDiscoveryServices_Call) Run(run func()) *SysProbeUtil_GetDiscoveryServices_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *SysProbeUtil_GetDiscoveryServices_Call) Return(_a0 *model.ServicesResponse, _a1 error) *SysProbeUtil_GetDiscoveryServices_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetDiscoveryServices_Call) RunAndReturn(run func() (*model.ServicesResponse, error)) *SysProbeUtil_GetDiscoveryServices_Call { + _c.Call.Return(run) + return _c +} + // GetNetworkID provides a mock function with given fields: func (_m *SysProbeUtil) GetNetworkID() (string, error) { ret := _m.Called() @@ -71,6 +289,153 @@ func (_m *SysProbeUtil) GetNetworkID() (string, error) { return r0, r1 } +// SysProbeUtil_GetNetworkID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetNetworkID' +type SysProbeUtil_GetNetworkID_Call struct { + *mock.Call +} + +// GetNetworkID is a helper method to define mock.On call +func (_e *SysProbeUtil_Expecter) GetNetworkID() *SysProbeUtil_GetNetworkID_Call { + return &SysProbeUtil_GetNetworkID_Call{Call: _e.mock.On("GetNetworkID")} +} + +func (_c *SysProbeUtil_GetNetworkID_Call) Run(run func()) *SysProbeUtil_GetNetworkID_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *SysProbeUtil_GetNetworkID_Call) Return(_a0 string, _a1 error) *SysProbeUtil_GetNetworkID_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetNetworkID_Call) RunAndReturn(run func() (string, error)) *SysProbeUtil_GetNetworkID_Call { + _c.Call.Return(run) + return _c +} + +// GetPing provides a mock function with given fields: clientID, host, count, interval, timeout +func (_m *SysProbeUtil) GetPing(clientID string, host string, count int, interval time.Duration, timeout time.Duration) ([]byte, error) { + ret := _m.Called(clientID, host, count, interval, timeout) + + if len(ret) == 0 { + panic("no return value specified for GetPing") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(string, string, int, time.Duration, time.Duration) ([]byte, error)); ok { + return rf(clientID, host, count, interval, timeout) + } + if rf, ok := ret.Get(0).(func(string, string, int, time.Duration, time.Duration) []byte); ok { + r0 = rf(clientID, host, count, interval, timeout) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(string, string, int, time.Duration, time.Duration) error); ok { + r1 = rf(clientID, host, count, interval, timeout) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SysProbeUtil_GetPing_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetPing' +type SysProbeUtil_GetPing_Call struct { + *mock.Call +} + +// GetPing is a helper method to define mock.On call +// - clientID string +// - host string +// - count int +// - interval time.Duration +// - timeout time.Duration +func (_e *SysProbeUtil_Expecter) GetPing(clientID interface{}, host interface{}, count interface{}, interval interface{}, timeout interface{}) *SysProbeUtil_GetPing_Call { + return &SysProbeUtil_GetPing_Call{Call: _e.mock.On("GetPing", clientID, host, count, interval, timeout)} +} + +func (_c *SysProbeUtil_GetPing_Call) Run(run func(clientID string, host string, count int, interval time.Duration, timeout time.Duration)) *SysProbeUtil_GetPing_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(int), args[3].(time.Duration), args[4].(time.Duration)) + }) + return _c +} + +func (_c *SysProbeUtil_GetPing_Call) Return(_a0 []byte, _a1 error) *SysProbeUtil_GetPing_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetPing_Call) RunAndReturn(run func(string, string, int, time.Duration, time.Duration) ([]byte, error)) *SysProbeUtil_GetPing_Call { + _c.Call.Return(run) + return _c +} + +// GetPprof provides a mock function with given fields: path +func (_m *SysProbeUtil) GetPprof(path string) ([]byte, error) { + ret := _m.Called(path) + + if len(ret) == 0 { + panic("no return value specified for GetPprof") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]byte, error)); ok { + return rf(path) + } + if rf, ok := ret.Get(0).(func(string) []byte); ok { + r0 = rf(path) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(path) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SysProbeUtil_GetPprof_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetPprof' +type SysProbeUtil_GetPprof_Call struct { + *mock.Call +} + +// GetPprof is a helper method to define mock.On call +// - path string +func (_e *SysProbeUtil_Expecter) GetPprof(path interface{}) *SysProbeUtil_GetPprof_Call { + return &SysProbeUtil_GetPprof_Call{Call: _e.mock.On("GetPprof", path)} +} + +func (_c *SysProbeUtil_GetPprof_Call) Run(run func(path string)) *SysProbeUtil_GetPprof_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *SysProbeUtil_GetPprof_Call) Return(_a0 []byte, _a1 error) *SysProbeUtil_GetPprof_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetPprof_Call) RunAndReturn(run func(string) ([]byte, error)) *SysProbeUtil_GetPprof_Call { + _c.Call.Return(run) + return _c +} + // GetProcStats provides a mock function with given fields: pids func (_m *SysProbeUtil) GetProcStats(pids []int32) (*process.ProcStatsWithPermByPID, error) { ret := _m.Called(pids) @@ -101,6 +466,34 @@ func (_m *SysProbeUtil) GetProcStats(pids []int32) (*process.ProcStatsWithPermBy return r0, r1 } +// SysProbeUtil_GetProcStats_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetProcStats' +type SysProbeUtil_GetProcStats_Call struct { + *mock.Call +} + +// GetProcStats is a helper method to define mock.On call +// - pids []int32 +func (_e *SysProbeUtil_Expecter) GetProcStats(pids interface{}) *SysProbeUtil_GetProcStats_Call { + return &SysProbeUtil_GetProcStats_Call{Call: _e.mock.On("GetProcStats", pids)} +} + +func (_c *SysProbeUtil_GetProcStats_Call) Run(run func(pids []int32)) *SysProbeUtil_GetProcStats_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].([]int32)) + }) + return _c +} + +func (_c *SysProbeUtil_GetProcStats_Call) Return(_a0 *process.ProcStatsWithPermByPID, _a1 error) *SysProbeUtil_GetProcStats_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetProcStats_Call) RunAndReturn(run func([]int32) (*process.ProcStatsWithPermByPID, error)) *SysProbeUtil_GetProcStats_Call { + _c.Call.Return(run) + return _c +} + // GetStats provides a mock function with given fields: func (_m *SysProbeUtil) GetStats() (map[string]interface{}, error) { ret := _m.Called() @@ -131,6 +524,33 @@ func (_m *SysProbeUtil) GetStats() (map[string]interface{}, error) { return r0, r1 } +// SysProbeUtil_GetStats_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetStats' +type SysProbeUtil_GetStats_Call struct { + *mock.Call +} + +// GetStats is a helper method to define mock.On call +func (_e *SysProbeUtil_Expecter) GetStats() *SysProbeUtil_GetStats_Call { + return &SysProbeUtil_GetStats_Call{Call: _e.mock.On("GetStats")} +} + +func (_c *SysProbeUtil_GetStats_Call) Run(run func()) *SysProbeUtil_GetStats_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *SysProbeUtil_GetStats_Call) Return(_a0 map[string]interface{}, _a1 error) *SysProbeUtil_GetStats_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetStats_Call) RunAndReturn(run func() (map[string]interface{}, error)) *SysProbeUtil_GetStats_Call { + _c.Call.Return(run) + return _c +} + // GetTelemetry provides a mock function with given fields: func (_m *SysProbeUtil) GetTelemetry() ([]byte, error) { ret := _m.Called() @@ -161,6 +581,96 @@ func (_m *SysProbeUtil) GetTelemetry() ([]byte, error) { return r0, r1 } +// SysProbeUtil_GetTelemetry_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetTelemetry' +type SysProbeUtil_GetTelemetry_Call struct { + *mock.Call +} + +// GetTelemetry is a helper method to define mock.On call +func (_e *SysProbeUtil_Expecter) GetTelemetry() *SysProbeUtil_GetTelemetry_Call { + return &SysProbeUtil_GetTelemetry_Call{Call: _e.mock.On("GetTelemetry")} +} + +func (_c *SysProbeUtil_GetTelemetry_Call) Run(run func()) *SysProbeUtil_GetTelemetry_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *SysProbeUtil_GetTelemetry_Call) Return(_a0 []byte, _a1 error) *SysProbeUtil_GetTelemetry_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetTelemetry_Call) RunAndReturn(run func() ([]byte, error)) *SysProbeUtil_GetTelemetry_Call { + _c.Call.Return(run) + return _c +} + +// GetTraceroute provides a mock function with given fields: clientID, host, port, protocol, maxTTL, timeout +func (_m *SysProbeUtil) GetTraceroute(clientID string, host string, port uint16, protocol payload.Protocol, maxTTL uint8, timeout time.Duration) ([]byte, error) { + ret := _m.Called(clientID, host, port, protocol, maxTTL, timeout) + + if len(ret) == 0 { + panic("no return value specified for GetTraceroute") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(string, string, uint16, payload.Protocol, uint8, time.Duration) ([]byte, error)); ok { + return rf(clientID, host, port, protocol, maxTTL, timeout) + } + if rf, ok := ret.Get(0).(func(string, string, uint16, payload.Protocol, uint8, time.Duration) []byte); ok { + r0 = rf(clientID, host, port, protocol, maxTTL, timeout) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(string, string, uint16, payload.Protocol, uint8, time.Duration) error); ok { + r1 = rf(clientID, host, port, protocol, maxTTL, timeout) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// SysProbeUtil_GetTraceroute_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetTraceroute' +type SysProbeUtil_GetTraceroute_Call struct { + *mock.Call +} + +// GetTraceroute is a helper method to define mock.On call +// - clientID string +// - host string +// - port uint16 +// - protocol payload.Protocol +// - maxTTL uint8 +// - timeout time.Duration +func (_e *SysProbeUtil_Expecter) GetTraceroute(clientID interface{}, host interface{}, port interface{}, protocol interface{}, maxTTL interface{}, timeout interface{}) *SysProbeUtil_GetTraceroute_Call { + return &SysProbeUtil_GetTraceroute_Call{Call: _e.mock.On("GetTraceroute", clientID, host, port, protocol, maxTTL, timeout)} +} + +func (_c *SysProbeUtil_GetTraceroute_Call) Run(run func(clientID string, host string, port uint16, protocol payload.Protocol, maxTTL uint8, timeout time.Duration)) *SysProbeUtil_GetTraceroute_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string), args[1].(string), args[2].(uint16), args[3].(payload.Protocol), args[4].(uint8), args[5].(time.Duration)) + }) + return _c +} + +func (_c *SysProbeUtil_GetTraceroute_Call) Return(_a0 []byte, _a1 error) *SysProbeUtil_GetTraceroute_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *SysProbeUtil_GetTraceroute_Call) RunAndReturn(run func(string, string, uint16, payload.Protocol, uint8, time.Duration) ([]byte, error)) *SysProbeUtil_GetTraceroute_Call { + _c.Call.Return(run) + return _c +} + // Register provides a mock function with given fields: clientID func (_m *SysProbeUtil) Register(clientID string) error { ret := _m.Called(clientID) @@ -179,6 +689,34 @@ func (_m *SysProbeUtil) Register(clientID string) error { return r0 } +// SysProbeUtil_Register_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Register' +type SysProbeUtil_Register_Call struct { + *mock.Call +} + +// Register is a helper method to define mock.On call +// - clientID string +func (_e *SysProbeUtil_Expecter) Register(clientID interface{}) *SysProbeUtil_Register_Call { + return &SysProbeUtil_Register_Call{Call: _e.mock.On("Register", clientID)} +} + +func (_c *SysProbeUtil_Register_Call) Run(run func(clientID string)) *SysProbeUtil_Register_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *SysProbeUtil_Register_Call) Return(_a0 error) *SysProbeUtil_Register_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *SysProbeUtil_Register_Call) RunAndReturn(run func(string) error) *SysProbeUtil_Register_Call { + _c.Call.Return(run) + return _c +} + // NewSysProbeUtil creates a new instance of SysProbeUtil. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewSysProbeUtil(t interface { diff --git a/pkg/process/net/shared.go b/pkg/process/net/shared.go index b54f7ab7087e7..4ebc5bd45a294 100644 --- a/pkg/process/net/shared.go +++ b/pkg/process/net/shared.go @@ -5,7 +5,20 @@ package net -import model "github.com/DataDog/agent-payload/v5/process" +import ( + "time" + + model "github.com/DataDog/agent-payload/v5/process" + + sysconfigtypes "github.com/DataDog/datadog-agent/cmd/system-probe/config/types" + discoverymodel "github.com/DataDog/datadog-agent/pkg/collector/corechecks/servicediscovery/model" + "github.com/DataDog/datadog-agent/pkg/languagedetection/languagemodels" + nppayload "github.com/DataDog/datadog-agent/pkg/networkpath/payload" +) + +// SysProbeUtilGetter is a function that returns a SysProbeUtil for the given path +// The standard implementation is GetRemoteSysProbeUtil +type SysProbeUtilGetter func(string) (SysProbeUtil, error) // SysProbeUtil fetches info from the SysProbe running remotely type SysProbeUtil interface { @@ -15,4 +28,10 @@ type SysProbeUtil interface { Register(clientID string) error GetNetworkID() (string, error) GetTelemetry() ([]byte, error) + DetectLanguage(pids []int32) ([]languagemodels.Language, error) + GetPprof(path string) ([]byte, error) + GetDiscoveryServices() (*discoverymodel.ServicesResponse, error) + GetCheck(module sysconfigtypes.ModuleName) (interface{}, error) + GetPing(clientID string, host string, count int, interval time.Duration, timeout time.Duration) ([]byte, error) + GetTraceroute(clientID string, host string, port uint16, protocol nppayload.Protocol, maxTTL uint8, timeout time.Duration) ([]byte, error) } diff --git a/pkg/security/ebpf/c/include/helpers/activity_dump.h b/pkg/security/ebpf/c/include/helpers/activity_dump.h index be6ff4a927f0f..82ed6834e10fb 100644 --- a/pkg/security/ebpf/c/include/helpers/activity_dump.h +++ b/pkg/security/ebpf/c/include/helpers/activity_dump.h @@ -100,10 +100,11 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, containe return 0; } - if (cgroup->cgroup_flags != 0 && ((cgroup->cgroup_flags & 0b111) != CGROUP_MANAGER_SYSTEMD)) { - copy_container_id(container_id, evt->container.container_id); + if ((cgroup->cgroup_flags & 0b111) == CGROUP_MANAGER_SYSTEMD) { + return 0; } + copy_container_id(container_id, evt->container.container_id); evt->container.cgroup_context = *cgroup; evt->cookie = cookie; evt->config = config; diff --git a/pkg/security/ebpf/c/include/maps.h b/pkg/security/ebpf/c/include/maps.h index 4dea16af729fd..2d567c8e2c0c5 100644 --- a/pkg/security/ebpf/c/include/maps.h +++ b/pkg/security/ebpf/c/include/maps.h @@ -98,6 +98,5 @@ BPF_PROG_ARRAY(dentry_resolver_kprobe_or_fentry_progs, 6) BPF_PROG_ARRAY(dentry_resolver_tracepoint_progs, 3) BPF_PROG_ARRAY(classifier_router, 100) BPF_PROG_ARRAY(sys_exit_progs, 64) -BPF_PROG_ARRAY(raw_packet_filters, 10) #endif diff --git a/pkg/security/module/cws.go b/pkg/security/module/cws.go index 1ec45284e524d..e27a7c80d8da3 100644 --- a/pkg/security/module/cws.go +++ b/pkg/security/module/cws.go @@ -10,10 +10,12 @@ import ( "context" "errors" "fmt" + "runtime" "sync" "time" "github.com/DataDog/datadog-go/v5/statsd" + "go.uber.org/atomic" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/eventmonitor" @@ -30,6 +32,12 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/seclog" "github.com/DataDog/datadog-agent/pkg/security/serializers" "github.com/DataDog/datadog-agent/pkg/security/telemetry" + "github.com/DataDog/datadog-agent/pkg/security/utils" +) + +const ( + maxSelftestRetry = 3 + selftestDelay = 5 * time.Second ) // CWSConsumer represents the system-probe module for the runtime security agent @@ -50,6 +58,7 @@ type CWSConsumer struct { grpcServer *GRPCServer ruleEngine *rulesmodule.RuleEngine selfTester *selftests.SelfTester + selfTestRetry *atomic.Int32 reloader ReloaderInterface crtelemetry *telemetry.ContainersRunningTelemetry } @@ -90,6 +99,7 @@ func NewCWSConsumer(evm *eventmonitor.EventMonitor, cfg *config.RuntimeSecurityC sendStatsChan: make(chan chan bool, 1), grpcServer: NewGRPCServer(family, address), selfTester: selfTester, + selfTestRetry: atomic.NewInt32(0), reloader: NewReloader(), crtelemetry: crtelemetry, } @@ -170,11 +180,22 @@ func (c *CWSConsumer) Start() error { // we can now wait for self test events cb := func(success []eval.RuleID, fails []eval.RuleID, testEvents map[eval.RuleID]*serializers.EventSerializer) { + seclog.Debugf("self-test results : success : %v, failed : %v, retry %d/%d", success, fails, c.selfTestRetry.Load()+1, maxSelftestRetry) + + if len(fails) > 0 && c.selfTestRetry.Load() < maxSelftestRetry { + c.selfTestRetry.Inc() + + time.Sleep(selftestDelay) + + if _, err := c.RunSelfTest(false); err != nil { + seclog.Errorf("self-test error: %s", err) + } + return + } + if c.config.SelfTestSendReport { c.reportSelfTest(success, fails, testEvents) } - - seclog.Debugf("self-test results : success : %v, failed : %v", success, fails) } if c.selfTester != nil { go c.selfTester.WaitForResult(cb) @@ -226,6 +247,9 @@ func (c *CWSConsumer) reportSelfTest(success []eval.RuleID, fails []eval.RuleID, tags := []string{ fmt.Sprintf("success:%d", len(success)), fmt.Sprintf("fails:%d", len(fails)), + fmt.Sprintf("os:%s", runtime.GOOS), + fmt.Sprintf("arch:%s", utils.RuntimeArch()), + fmt.Sprintf("origin:%s", c.probe.Origin()), } if err := c.statsdClient.Count(metrics.MetricSelfTest, 1, tags, 1.0); err != nil { seclog.Errorf("failed to send self_test metric: %s", err) diff --git a/pkg/security/module/grpc.go b/pkg/security/module/grpc.go index 7320713cc9fe6..c2b7f1e6c8be3 100644 --- a/pkg/security/module/grpc.go +++ b/pkg/security/module/grpc.go @@ -19,18 +19,19 @@ import ( // GRPCServer defines a gRPC server type GRPCServer struct { - server *grpc.Server - netListener net.Listener - wg sync.WaitGroup - family string - address string + server *grpc.Server + wg sync.WaitGroup + family string + address string } // NewGRPCServer returns a new gRPC server func NewGRPCServer(family string, address string) *GRPCServer { // force socket cleanup of previous socket not cleanup if family == "unix" { - _ = os.Remove(address) + if err := os.Remove(address); err != nil && !os.IsNotExist(err) { + seclog.Errorf("error removing the previous runtime security socket: %v", err) + } } return &GRPCServer{ @@ -53,8 +54,6 @@ func (g *GRPCServer) Start() error { } } - g.netListener = ln - g.wg.Add(1) go func() { defer g.wg.Done() @@ -73,10 +72,9 @@ func (g *GRPCServer) Stop() { g.server.Stop() } - if g.netListener != nil { - g.netListener.Close() - if g.family == "unix" { - _ = os.Remove(g.address) + if g.family == "unix" { + if err := os.Remove(g.address); err != nil && !os.IsNotExist(err) { + seclog.Errorf("error removing the runtime security socket: %v", err) } } diff --git a/pkg/security/probe/constantfetch/btfhub/constants.json b/pkg/security/probe/constantfetch/btfhub/constants.json index f656a0ec99242..4e70d557da035 100644 --- a/pkg/security/probe/constantfetch/btfhub/constants.json +++ b/pkg/security/probe/constantfetch/btfhub/constants.json @@ -18281,6 +18281,13 @@ "uname_release": "4.14.35-2047.540.4.1.el7uek.aarch64", "cindex": 89 }, + { + "distrib": "ol", + "version": "7", + "arch": "arm64", + "uname_release": "4.14.35-2047.540.4.2.el7uek.aarch64", + "cindex": 89 + }, { "distrib": "ol", "version": "7", @@ -20871,6 +20878,13 @@ "uname_release": "4.1.12-124.89.4.el7uek.x86_64", "cindex": 94 }, + { + "distrib": "ol", + "version": "7", + "arch": "x86_64", + "uname_release": "4.1.12-124.90.3.1.el7uek.x86_64", + "cindex": 94 + }, { "distrib": "ol", "version": "7", @@ -23734,6 +23748,13 @@ "uname_release": "4.14.35-2047.540.4.1.el7uek.x86_64", "cindex": 96 }, + { + "distrib": "ol", + "version": "7", + "arch": "x86_64", + "uname_release": "4.14.35-2047.540.4.2.el7uek.x86_64", + "cindex": 96 + }, { "distrib": "ol", "version": "7", diff --git a/pkg/security/probe/field_handlers_ebpf.go b/pkg/security/probe/field_handlers_ebpf.go index 6e391707cb57d..0f3d8d66b1d05 100644 --- a/pkg/security/probe/field_handlers_ebpf.go +++ b/pkg/security/probe/field_handlers_ebpf.go @@ -423,21 +423,27 @@ func (fh *EBPFFieldHandlers) ResolveAsync(ev *model.Event) bool { return ev.Async } +func (fh *EBPFFieldHandlers) resolveSBOMFields(ev *model.Event, f *model.FileEvent) { + // Force the resolution of file path to be able to map to a package provided file + if fh.ResolveFilePath(ev, f) == "" { + return + } + + if fh.resolvers.SBOMResolver == nil { + return + } + + if pkg := fh.resolvers.SBOMResolver.ResolvePackage(string(ev.ContainerContext.ContainerID), f); pkg != nil { + f.PkgName = pkg.Name + f.PkgVersion = pkg.Version + f.PkgSrcVersion = pkg.SrcVersion + } +} + // ResolvePackageName resolves the name of the package providing this file func (fh *EBPFFieldHandlers) ResolvePackageName(ev *model.Event, f *model.FileEvent) string { if f.PkgName == "" { - // Force the resolution of file path to be able to map to a package provided file - if fh.ResolveFilePath(ev, f) == "" { - return "" - } - - if fh.resolvers.SBOMResolver == nil { - return "" - } - - if pkg := fh.resolvers.SBOMResolver.ResolvePackage(string(ev.ContainerContext.ContainerID), f); pkg != nil { - f.PkgName = pkg.Name - } + fh.resolveSBOMFields(ev, f) } return f.PkgName } @@ -445,18 +451,7 @@ func (fh *EBPFFieldHandlers) ResolvePackageName(ev *model.Event, f *model.FileEv // ResolvePackageVersion resolves the version of the package providing this file func (fh *EBPFFieldHandlers) ResolvePackageVersion(ev *model.Event, f *model.FileEvent) string { if f.PkgVersion == "" { - // Force the resolution of file path to be able to map to a package provided file - if fh.ResolveFilePath(ev, f) == "" { - return "" - } - - if fh.resolvers.SBOMResolver == nil { - return "" - } - - if pkg := fh.resolvers.SBOMResolver.ResolvePackage(string(ev.ContainerContext.ContainerID), f); pkg != nil { - f.PkgVersion = pkg.Version - } + fh.resolveSBOMFields(ev, f) } return f.PkgVersion } @@ -464,18 +459,7 @@ func (fh *EBPFFieldHandlers) ResolvePackageVersion(ev *model.Event, f *model.Fil // ResolvePackageSourceVersion resolves the version of the source package of the package providing this file func (fh *EBPFFieldHandlers) ResolvePackageSourceVersion(ev *model.Event, f *model.FileEvent) string { if f.PkgSrcVersion == "" { - // Force the resolution of file path to be able to map to a package provided file - if fh.ResolveFilePath(ev, f) == "" { - return "" - } - - if fh.resolvers.SBOMResolver == nil { - return "" - } - - if pkg := fh.resolvers.SBOMResolver.ResolvePackage(string(ev.ContainerContext.ContainerID), f); pkg != nil { - f.PkgSrcVersion = pkg.SrcVersion - } + fh.resolveSBOMFields(ev, f) } return f.PkgSrcVersion } diff --git a/pkg/security/probe/kfilters/approvers.go b/pkg/security/probe/kfilters/approvers.go index 1314a54e958bb..7910f06771d78 100644 --- a/pkg/security/probe/kfilters/approvers.go +++ b/pkg/security/probe/kfilters/approvers.go @@ -162,7 +162,7 @@ func fimKFiltersGetter(eventType model.EventType, fields []eval.Field) kfiltersG fieldHandled = append(fieldHandled, handled...) } - kfs, handled, err := getProcessKFilters(model.FileOpenEventType, approvers) + kfs, handled, err := getProcessKFilters(eventType, approvers) if err != nil { return nil, nil, err } diff --git a/pkg/security/probe/kfilters/process.go b/pkg/security/probe/kfilters/process.go index 14d538765d882..84d65b3db9d3a 100644 --- a/pkg/security/probe/kfilters/process.go +++ b/pkg/security/probe/kfilters/process.go @@ -62,9 +62,10 @@ func getProcessKFilters(eventType model.EventType, approvers rules.Approvers) ([ switch value.Type { case eval.ScalarValueType: kfilters = append(kfilters, &eventMaskEntry{ - tableName: auidApproversTable, - tableKey: ebpf.Uint32MapItem(value.Value.(int)), - eventMask: uint64(1 << (eventType - 1)), + approverType: AUIDApproverType, + tableName: auidApproversTable, + tableKey: ebpf.Uint32MapItem(value.Value.(int)), + eventMask: uint64(1 << (eventType - 1)), }) case eval.RangeValueType: min, max := value.Value.(rules.RangeFilterValue).Min, value.Value.(rules.RangeFilterValue).Max diff --git a/pkg/security/probe/probe.go b/pkg/security/probe/probe.go index d22c41504a49a..18c91108643e7 100644 --- a/pkg/security/probe/probe.go +++ b/pkg/security/probe/probe.go @@ -437,6 +437,11 @@ func (p *Probe) IsNetworkEnabled() bool { return p.Config.Probe.NetworkEnabled } +// IsNetworkRawPacketEnabled returns whether network raw packet is enabled +func (p *Probe) IsNetworkRawPacketEnabled() bool { + return p.IsNetworkEnabled() && p.Config.Probe.NetworkRawPacketEnabled +} + // IsActivityDumpEnabled returns whether activity dump is enabled func (p *Probe) IsActivityDumpEnabled() bool { return p.Config.RuntimeSecurity.ActivityDumpEnabled diff --git a/pkg/security/probe/probe_ebpf.go b/pkg/security/probe/probe_ebpf.go index b8c965d8619c5..7ea35bdc91802 100644 --- a/pkg/security/probe/probe_ebpf.go +++ b/pkg/security/probe/probe_ebpf.go @@ -395,7 +395,7 @@ func (p *EBPFProbe) Setup() error { p.profileManagers.Start(p.ctx, &p.wg) - if p.config.Probe.NetworkRawPacketEnabled { + if p.probe.IsNetworkRawPacketEnabled() { if err := p.setupRawPacketProgs(); err != nil { return err } diff --git a/pkg/security/probe/probe_others.go b/pkg/security/probe/probe_others.go index 6112ab5b188c5..c3f168ab03414 100644 --- a/pkg/security/probe/probe_others.go +++ b/pkg/security/probe/probe_others.go @@ -90,6 +90,11 @@ func (p *Probe) IsNetworkEnabled() bool { return p.Config.Probe.NetworkEnabled } +// IsNetworkRawPacketEnabled returns whether network raw packet is enabled +func (p *Probe) IsNetworkRawPacketEnabled() bool { + return p.IsNetworkEnabled() && p.Config.Probe.NetworkRawPacketEnabled +} + // IsActivityDumpEnabled returns whether activity dump is enabled func (p *Probe) IsActivityDumpEnabled() bool { return p.Config.RuntimeSecurity.ActivityDumpEnabled diff --git a/pkg/security/probe/selftests/chmod.go b/pkg/security/probe/selftests/chmod.go index 0f98c08e9b86a..aaf9d3b7e0dd7 100644 --- a/pkg/security/probe/selftests/chmod.go +++ b/pkg/security/probe/selftests/chmod.go @@ -31,6 +31,7 @@ func (o *ChmodSelfTest) GetRuleDefinition() *rules.RuleDefinition { return &rules.RuleDefinition{ ID: o.ruleID, Expression: fmt.Sprintf(`chmod.file.path == "%s"`, o.filename), + Silent: true, } } diff --git a/pkg/security/probe/selftests/chown.go b/pkg/security/probe/selftests/chown.go index 558857de69a01..edf775fe4c2c2 100644 --- a/pkg/security/probe/selftests/chown.go +++ b/pkg/security/probe/selftests/chown.go @@ -32,6 +32,7 @@ func (o *ChownSelfTest) GetRuleDefinition() *rules.RuleDefinition { return &rules.RuleDefinition{ ID: o.ruleID, Expression: fmt.Sprintf(`chown.file.path == "%s"`, o.filename), + Silent: true, } } diff --git a/pkg/security/probe/selftests/create_file_windows.go b/pkg/security/probe/selftests/create_file_windows.go index eb713a5c926ff..eebc878f61b24 100644 --- a/pkg/security/probe/selftests/create_file_windows.go +++ b/pkg/security/probe/selftests/create_file_windows.go @@ -38,6 +38,7 @@ func (o *WindowsCreateFileSelfTest) GetRuleDefinition() *rules.RuleDefinition { return &rules.RuleDefinition{ ID: o.ruleID, Expression: fmt.Sprintf(`create.file.name == "%s" && create.file.device_path =~ "%s" && process.pid == %d`, basename, filepath.ToSlash(devicePath), os.Getpid()), + Silent: true, } } diff --git a/pkg/security/probe/selftests/ebpfless.go b/pkg/security/probe/selftests/ebpfless.go index 578143231e490..cd5b692fb5070 100644 --- a/pkg/security/probe/selftests/ebpfless.go +++ b/pkg/security/probe/selftests/ebpfless.go @@ -31,6 +31,7 @@ func (o *EBPFLessSelfTest) GetRuleDefinition() *rules.RuleDefinition { ID: o.ruleID, Expression: `exec.file.path != "" && process.parent.pid == 0 && process.ppid == 0`, Every: time.Duration(math.MaxInt64), + Silent: true, } } diff --git a/pkg/security/probe/selftests/open.go b/pkg/security/probe/selftests/open.go index 272ec1e1c3b55..e5ddf92d76dfa 100644 --- a/pkg/security/probe/selftests/open.go +++ b/pkg/security/probe/selftests/open.go @@ -31,6 +31,7 @@ func (o *OpenSelfTest) GetRuleDefinition() *rules.RuleDefinition { return &rules.RuleDefinition{ ID: o.ruleID, Expression: fmt.Sprintf(`open.file.path == "%s" && open.flags & O_CREAT > 0`, o.filename), + Silent: true, } } diff --git a/pkg/security/probe/selftests/open_registry_key_windows.go b/pkg/security/probe/selftests/open_registry_key_windows.go index 7b87fa69c483a..f405e0ace0e0f 100644 --- a/pkg/security/probe/selftests/open_registry_key_windows.go +++ b/pkg/security/probe/selftests/open_registry_key_windows.go @@ -34,6 +34,7 @@ func (o *WindowsOpenRegistryKeyTest) GetRuleDefinition() *rules.RuleDefinition { return &rules.RuleDefinition{ ID: o.ruleID, Expression: fmt.Sprintf(`open.registry.key_name == "%s" && process.pid == %d`, filepath.Base(o.keyPath), os.Getpid()), + Silent: true, } } diff --git a/pkg/security/resolvers/process/resolver_ebpf.go b/pkg/security/resolvers/process/resolver_ebpf.go index 8b5e87c94c656..e35c3cb725633 100644 --- a/pkg/security/resolvers/process/resolver_ebpf.go +++ b/pkg/security/resolvers/process/resolver_ebpf.go @@ -352,9 +352,6 @@ func (p *EBPFResolver) enrichEventFromProc(entry *model.ProcessCacheEntry, proc entry.FileEvent.MountOrigin = model.MountOriginProcfs entry.FileEvent.MountSource = model.MountSourceSnapshot - var id containerutils.CGroupID - id, entry.Process.ContainerID = containerutils.GetCGroupContext(containerID, containerFlags) - entry.Process.CGroup.CGroupID = id entry.Process.CGroup.CGroupFlags = containerFlags var fileStats unix.Statx_t @@ -372,6 +369,21 @@ func (p *EBPFResolver) enrichEventFromProc(entry *model.ProcessCacheEntry, proc } } + if cgroupFileContent, err := os.ReadFile(taskPath); err == nil { + lines := strings.Split(string(cgroupFileContent), "\n") + for _, line := range lines { + parts := strings.SplitN(line, ":", 3) + + // Skip potentially malformed lines + if len(parts) != 3 { + continue + } + + entry.Process.CGroup.CGroupID = containerutils.CGroupID(parts[2]) + break + } + } + if entry.FileEvent.IsFileless() { entry.FileEvent.Filesystem = model.TmpFS } else { diff --git a/pkg/security/rules/engine.go b/pkg/security/rules/engine.go index 16a0d00297974..00ad1233946d1 100644 --- a/pkg/security/rules/engine.go +++ b/pkg/security/rules/engine.go @@ -484,7 +484,12 @@ func (e *RuleEngine) getEventTypeEnabled() map[eval.EventType]bool { if e.probe.IsNetworkEnabled() { if eventTypes, exists := categories[model.NetworkCategory]; exists { for _, eventType := range eventTypes { - enabled[eventType] = true + switch eventType { + case model.RawPacketEventType.String(): + enabled[eventType] = e.probe.IsNetworkRawPacketEnabled() + default: + enabled[eventType] = true + } } } } diff --git a/pkg/security/secl/rules/ruleset.go b/pkg/security/secl/rules/ruleset.go index 981109f7ad02e..061e9d230c5a5 100644 --- a/pkg/security/secl/rules/ruleset.go +++ b/pkg/security/secl/rules/ruleset.go @@ -340,7 +340,7 @@ func (rs *RuleSet) AddRule(parsingContext *ast.ParsingContext, pRule *PolicyRule // ignore event types not supported if _, exists := rs.opts.EventTypeEnabled["*"]; !exists { - if _, exists := rs.opts.EventTypeEnabled[eventType]; !exists { + if enabled, exists := rs.opts.EventTypeEnabled[eventType]; !exists || !enabled { return nil, &ErrRuleLoad{Rule: pRule, Err: ErrEventTypeNotEnabled} } } diff --git a/pkg/security/security_profile/dump/activity_dump.go b/pkg/security/security_profile/dump/activity_dump.go index 0f345ddf77d31..b9b0f34e00e71 100644 --- a/pkg/security/security_profile/dump/activity_dump.go +++ b/pkg/security/security_profile/dump/activity_dump.go @@ -615,7 +615,8 @@ func (ad *ActivityDump) ResolveTags() error { // resolveTags thread unsafe version ot ResolveTags func (ad *ActivityDump) resolveTags() error { - if len(ad.Tags) >= 10 || len(ad.Metadata.ContainerID) == 0 { + selector := ad.GetWorkloadSelector() + if selector != nil { return nil } diff --git a/pkg/security/security_profile/dump/load_controller.go b/pkg/security/security_profile/dump/load_controller.go index fcce3b6de6a51..3d0148f168232 100644 --- a/pkg/security/security_profile/dump/load_controller.go +++ b/pkg/security/security_profile/dump/load_controller.go @@ -87,6 +87,7 @@ func (lc *ActivityDumpLoadController) NextPartialDump(ad *ActivityDump) *Activit newDump.Metadata.ContainerID = ad.Metadata.ContainerID newDump.Metadata.DifferentiateArgs = ad.Metadata.DifferentiateArgs newDump.Tags = ad.Tags + newDump.selector = ad.selector // copy storage requests for _, reqList := range ad.StorageRequests { @@ -101,7 +102,7 @@ func (lc *ActivityDumpLoadController) NextPartialDump(ad *ActivityDump) *Activit } // compute the duration it took to reach the dump size threshold - timeToThreshold := ad.End.Sub(ad.Start) + timeToThreshold := time.Since(ad.Start) // set new load parameters newDump.SetTimeout(ad.LoadConfig.Timeout - timeToThreshold) diff --git a/pkg/security/security_profile/dump/manager.go b/pkg/security/security_profile/dump/manager.go index 10af2f9d787f7..a0b8c490b965e 100644 --- a/pkg/security/security_profile/dump/manager.go +++ b/pkg/security/security_profile/dump/manager.go @@ -852,6 +852,9 @@ func (adm *ActivityDumpManager) triggerLoadController() { // handle overweight dumps for _, ad := range dumps { + // restart a new dump for the same workload + newDump := adm.loadController.NextPartialDump(ad) + // stop the dump but do not release the cgroup ad.Finalize(false) seclog.Infof("tracing paused for [%s]", ad.GetSelectorStr()) @@ -867,9 +870,6 @@ func (adm *ActivityDumpManager) triggerLoadController() { adm.emptyDropped.Inc() } - // restart a new dump for the same workload - newDump := adm.loadController.NextPartialDump(ad) - adm.Lock() if err := adm.insertActivityDump(newDump); err != nil { seclog.Errorf("couldn't resume tracing [%s]: %v", newDump.GetSelectorStr(), err) diff --git a/pkg/security/tests/fake_tags_resolver.go b/pkg/security/tests/fake_tags_resolver.go index 45d1d9c548450..9c97502c8d266 100644 --- a/pkg/security/tests/fake_tags_resolver.go +++ b/pkg/security/tests/fake_tags_resolver.go @@ -40,7 +40,6 @@ func (fr *FakeResolver) Stop() error { func (fr *FakeResolver) Resolve(containerID string) []string { fakeTags := []string{ "image_tag:latest", - "container_id:" + containerID, } fr.Lock() defer fr.Unlock() diff --git a/pkg/security/tests/filters_test.go b/pkg/security/tests/filters_test.go index 8c4116980a7fe..78841d2305f50 100644 --- a/pkg/security/tests/filters_test.go +++ b/pkg/security/tests/filters_test.go @@ -341,7 +341,7 @@ func TestFilterOpenGrandParentDiscarder(t *testing.T) { testFilterOpenParentDiscarder(t, "grandparent", "parent") } -func runAUIDTest(t *testing.T, test *testModule, goSyscallTester, auidOK, auidKO string) { +func runAUIDTest(t *testing.T, test *testModule, goSyscallTester string, eventType model.EventType, field eval.Field, path string, auidOK, auidKO string) { var cmdWrapper *dockerCmdWrapper cmdWrapper, err := test.StartADocker() if err != nil { @@ -352,16 +352,20 @@ func runAUIDTest(t *testing.T, test *testModule, goSyscallTester, auidOK, auidKO // reset stats test.statsdClient.Flush() - if err := waitForOpenProbeEvent(test, func() error { + if err := waitForProbeEvent(test, func() error { args := []string{ - "-login-uid-open-test", - "-login-uid-open-path", "/tmp/test-auid", - "-login-uid-open-uid", auidOK, + "-login-uid-test", + "-login-uid-event-type", eventType.String(), + "-login-uid-path", "/tmp/test-auid", + "-login-uid-value", auidOK, } cmd := cmdWrapper.Command(goSyscallTester, args, []string{}) return cmd.Run() - }, "/tmp/test-auid"); err != nil { + }, eventType, eventKeyValueFilter{ + key: field, + value: path, + }); err != nil { t.Fatal(err) } @@ -372,7 +376,7 @@ func runAUIDTest(t *testing.T, test *testModule, goSyscallTester, auidOK, auidKO return fmt.Errorf("expected metrics not found: %+v", test.statsdClient.GetByPrefix(metrics.MetricEventApproved)) } - if count := test.statsdClient.Get(metrics.MetricEventApproved + ":event_type:open"); count == 0 { + if count := test.statsdClient.Get(metrics.MetricEventApproved + ":event_type:" + eventType.String()); count == 0 { return fmt.Errorf("expected metrics not found: %+v", test.statsdClient.GetByPrefix(metrics.MetricEventApproved)) } @@ -380,16 +384,20 @@ func runAUIDTest(t *testing.T, test *testModule, goSyscallTester, auidOK, auidKO }, retry.Delay(1*time.Second), retry.Attempts(5), retry.DelayType(retry.FixedDelay)) assert.NoError(t, err) - if err := waitForOpenProbeEvent(test, func() error { + if err := waitForProbeEvent(test, func() error { args := []string{ - "-login-uid-open-test", - "-login-uid-open-path", "/tmp/test-auid", - "-login-uid-open-uid", auidKO, + "-login-uid-test", + "-login-uid-event-type", eventType.String(), + "-login-uid-path", "/tmp/test-auid", + "-login-uid-value", auidKO, } cmd := cmdWrapper.Command(goSyscallTester, args, []string{}) return cmd.Run() - }, "/tmp/test-auid"); err == nil { + }, eventType, eventKeyValueFilter{ + key: field, + value: path, + }); err == nil { t.Fatal("shouldn't get an event") } } @@ -432,15 +440,15 @@ func TestFilterOpenAUIDEqualApprover(t *testing.T) { } t.Run("equal-fixed-value", func(t *testing.T) { - runAUIDTest(t, test, goSyscallTester, "1005", "6000") + runAUIDTest(t, test, goSyscallTester, model.FileOpenEventType, "open.file.path", "/tmp/test-auid", "1005", "6000") }) t.Run("equal-zero", func(t *testing.T) { - runAUIDTest(t, test, goSyscallTester, "0", "6000") + runAUIDTest(t, test, goSyscallTester, model.FileOpenEventType, "open.file.path", "/tmp/test-auid", "0", "6000") }) t.Run("equal-unset", func(t *testing.T) { - runAUIDTest(t, test, goSyscallTester, "-1", "6000") + runAUIDTest(t, test, goSyscallTester, model.FileOpenEventType, "open.file.path", "/tmp/test-auid", "-1", "6000") }) } @@ -473,7 +481,7 @@ func TestFilterOpenAUIDLesserApprover(t *testing.T) { t.Fatal(err) } - runAUIDTest(t, test, goSyscallTester, "450", "605") + runAUIDTest(t, test, goSyscallTester, model.FileOpenEventType, "open.file.path", "/tmp/test-auid", "450", "605") } func TestFilterOpenAUIDGreaterApprover(t *testing.T) { @@ -505,7 +513,7 @@ func TestFilterOpenAUIDGreaterApprover(t *testing.T) { t.Fatal(err) } - runAUIDTest(t, test, goSyscallTester, "1500", "605") + runAUIDTest(t, test, goSyscallTester, model.FileOpenEventType, "open.file.path", "/tmp/test-auid", "1500", "605") } func TestFilterOpenAUIDNotEqualUnsetApprover(t *testing.T) { @@ -537,7 +545,41 @@ func TestFilterOpenAUIDNotEqualUnsetApprover(t *testing.T) { t.Fatal(err) } - runAUIDTest(t, test, goSyscallTester, "6000", "-1") + runAUIDTest(t, test, goSyscallTester, model.FileOpenEventType, "open.file.path", "/tmp/test-auid", "6000", "-1") +} + +func TestFilterUnlinkAUIDEqualApprover(t *testing.T) { + SkipIfNotAvailable(t) + + // skip test that are about to be run on docker (to avoid trying spawning docker in docker) + if testEnvironment == DockerEnvironment { + t.Skip("Skip test spawning docker containers on docker") + } + if _, err := whichNonFatal("docker"); err != nil { + t.Skip("Skip test where docker is unavailable") + } + + ruleDefs := []*rules.RuleDefinition{ + { + ID: "test_equal_1", + Expression: `unlink.file.path =~ "/tmp/test-auid" && process.auid == 1009`, + }, + } + + test, err := newTestModule(t, nil, ruleDefs) + if err != nil { + t.Fatal(err) + } + defer test.Close() + + goSyscallTester, err := loadSyscallTester(t, test, "syscall_go_tester") + if err != nil { + t.Fatal(err) + } + + t.Run("equal-fixed-value", func(t *testing.T) { + runAUIDTest(t, test, goSyscallTester, model.FileUnlinkEventType, "unlink.file.path", "/tmp/test-auid", "1009", "6000") + }) } func TestFilterDiscarderMask(t *testing.T) { diff --git a/pkg/security/tests/login_uid_test.go b/pkg/security/tests/login_uid_test.go index 5e1e65c9d784f..31d548788e349 100644 --- a/pkg/security/tests/login_uid_test.go +++ b/pkg/security/tests/login_uid_test.go @@ -55,12 +55,13 @@ func TestLoginUID(t *testing.T) { } defer dockerInstance.stop() - t.Run("login-uid-open-test", func(t *testing.T) { + t.Run("open", func(t *testing.T) { test.WaitSignal(t, func() error { args := []string{ - "-login-uid-open-test", - "-login-uid-open-path", "/tmp/test-auid", - "-login-uid-open-uid", "1005", + "-login-uid-test", + "-login-uid-event-type", "open", + "-login-uid-path", "/tmp/test-auid", + "-login-uid-value", "1005", } cmd := dockerInstance.Command(goSyscallTester, args, []string{}) @@ -72,11 +73,13 @@ func TestLoginUID(t *testing.T) { }) }) - t.Run("login-uid-exec-test", func(t *testing.T) { + t.Run("exec", func(t *testing.T) { test.WaitSignal(t, func() error { args := []string{ - "-login-uid-exec-test", - "-login-uid-exec-path", goSyscallTester, + "-login-uid-test", + "-login-uid-event-type", "exec", + "-login-uid-path", goSyscallTester, + "-login-uid-value", "1005", } cmd := dockerInstance.Command(goSyscallTester, args, []string{}) diff --git a/pkg/security/tests/syscall_tester/go/syscall_go_tester.go b/pkg/security/tests/syscall_tester/go/syscall_go_tester.go index c8c72ff67831b..ef5833f48e406 100644 --- a/pkg/security/tests/syscall_tester/go/syscall_go_tester.go +++ b/pkg/security/tests/syscall_tester/go/syscall_go_tester.go @@ -42,11 +42,10 @@ var ( userSessionExecutable string userSessionOpenPath string syscallDriftTest bool - loginUIDOpenTest bool - loginUIDOpenPath string - loginUIDOpenUID int - loginUIDExecTest bool - loginUIDExecPath string + loginUIDTest bool + loginUIDPath string + loginUIDEventType string + loginUIDValue int ) //go:embed ebpf_probe.o @@ -232,35 +231,39 @@ func setSelfLoginUID(uid int) error { return nil } -func RunLoginUIDOpenTest() error { - if loginUIDOpenUID != -1 { - if err := setSelfLoginUID(loginUIDOpenUID); err != nil { +func RunLoginUIDTest() error { + if loginUIDValue != -1 { + if err := setSelfLoginUID(loginUIDValue); err != nil { return err } } - // open test file to trigger an event - f, err := os.OpenFile(loginUIDOpenPath, os.O_RDWR|os.O_CREATE, 0755) - if err != nil { - return fmt.Errorf("couldn't create test-auid file: %v", err) - } - defer os.Remove(loginUIDOpenPath) - - if err = f.Close(); err != nil { - return fmt.Errorf("couldn't close test file: %v", err) - } - return nil -} - -func RunLoginUIDExecTest() error { - if err := setSelfLoginUID(1005); err != nil { - return err - } + switch loginUIDEventType { + case "open": + // open test file to trigger an event + f, err := os.OpenFile(loginUIDPath, os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return fmt.Errorf("couldn't create test-auid file: %v", err) + } + defer os.Remove(loginUIDPath) - // exec ls to trigger an execution with auid = 1005 - cmd := exec.Command(loginUIDExecPath) - if err := cmd.Run(); err != nil { - return fmt.Errorf("'%s' execution returned an error: %v", loginUIDExecPath, err) + if err = f.Close(); err != nil { + return fmt.Errorf("couldn't close test file: %v", err) + } + case "exec": + cmd := exec.Command(loginUIDPath) + if err := cmd.Run(); err != nil { + return fmt.Errorf("'%s' execution returned an error: %v", loginUIDPath, err) + } + case "unlink": + f, err := os.OpenFile(loginUIDPath, os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return fmt.Errorf("couldn't create test-auid file: %v", err) + } + f.Close() + os.Remove(loginUIDPath) + default: + panic("unknown event type") } return nil } @@ -277,11 +280,10 @@ func main() { flag.BoolVar(&cleanupIMDSTest, "cleanup-imds-test", false, "when set, removes the dummy interface of the IMDS test") flag.BoolVar(&runIMDSTest, "run-imds-test", false, "when set, binds an IMDS server locally and sends a query to it") flag.BoolVar(&syscallDriftTest, "syscall-drift-test", false, "when set, runs the syscall drift test") - flag.BoolVar(&loginUIDOpenTest, "login-uid-open-test", false, "when set, runs the login_uid open test") - flag.StringVar(&loginUIDOpenPath, "login-uid-open-path", "", "file used for the login_uid open test") - flag.IntVar(&loginUIDOpenUID, "login-uid-open-uid", 0, "uid used for the login_uid open test") - flag.BoolVar(&loginUIDExecTest, "login-uid-exec-test", false, "when set, runs the login_uid exec test") - flag.StringVar(&loginUIDExecPath, "login-uid-exec-path", "", "path to the executable to run during the login_uid exec test") + flag.BoolVar(&loginUIDTest, "login-uid-test", false, "when set, runs the login_uid open test") + flag.StringVar(&loginUIDPath, "login-uid-path", "", "file used for the login_uid open test") + flag.StringVar(&loginUIDEventType, "login-uid-event-type", "", "event type used for the login_uid open test") + flag.IntVar(&loginUIDValue, "login-uid-value", 0, "uid used for the login_uid open test") flag.Parse() @@ -337,14 +339,8 @@ func main() { } } - if loginUIDOpenTest { - if err := RunLoginUIDOpenTest(); err != nil { - panic(err) - } - } - - if loginUIDExecTest { - if err := RunLoginUIDExecTest(); err != nil { + if loginUIDTest { + if err := RunLoginUIDTest(); err != nil { panic(err) } } diff --git a/pkg/security/utils/cgroup.go b/pkg/security/utils/cgroup.go index 2059e587333e9..2e4214e5290c6 100644 --- a/pkg/security/utils/cgroup.go +++ b/pkg/security/utils/cgroup.go @@ -89,9 +89,9 @@ func GetProcContainerContext(tgid, pid uint32) (containerutils.ContainerID, cont } for _, cgroup := range cgroups { - if containerID, runtime := cgroup.GetContainerContext(); containerID != "" { - return containerID, runtime, nil - } + containerID, runtime := cgroup.GetContainerContext() + return containerID, runtime, nil } + return "", 0, nil } diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index 695a1bb5029de..958e3c3243fbc 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -41,8 +41,8 @@ require ( go.opentelemetry.io/collector/pdata v1.11.0 go.opentelemetry.io/collector/processor v0.104.0 go.opentelemetry.io/collector/semconv v0.104.0 - go.opentelemetry.io/otel v1.30.0 - go.opentelemetry.io/otel/metric v1.30.0 + go.opentelemetry.io/otel v1.31.0 + go.opentelemetry.io/otel/metric v1.31.0 go.uber.org/atomic v1.11.0 golang.org/x/sys v0.26.0 golang.org/x/time v0.7.0 @@ -53,7 +53,7 @@ require ( ) require ( - github.com/DataDog/go-sqllexer v0.0.15 // indirect + github.com/DataDog/go-sqllexer v0.0.16 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -98,7 +98,7 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.21.0 // indirect diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index 154e9b2a918db..40a1eafcd4fc9 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -2,8 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= -github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.16 h1:RoSUMS6MECyB3gTUIdydzXwK5NhEhv6GMJkS7ptsgRA= +github.com/DataDog/go-sqllexer v0.0.16/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 h1:fKv05WFWHCXQmUTehW1eEZvXJP65Qv00W4V01B1EqSA= @@ -266,8 +266,8 @@ go.opentelemetry.io/contrib/config v0.7.0 h1:b1rK5tGTuhhPirJiMxOcyQfZs76j2VapY6O go.opentelemetry.io/contrib/config v0.7.0/go.mod h1:8tdiFd8N5etOi3XzBmAoMxplEzI3TcL8dU5rM5/xcOQ= go.opentelemetry.io/contrib/propagators/b3 v1.27.0 h1:IjgxbomVrV9za6bRi8fWCNXENs0co37SZedQilP2hm0= go.opentelemetry.io/contrib/propagators/b3 v1.27.0/go.mod h1:Dv9obQz25lCisDvvs4dy28UPh974CxkahRDUPsY7y9E= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/bridge/opencensus v1.27.0 h1:ao9aGGHd+G4YfjBpGs6vbkvt5hoC67STlJA9fCnOAcs= go.opentelemetry.io/otel/bridge/opencensus v1.27.0/go.mod h1:uRvWtAAXzyVOST0WMPX5JHGBaAvBws+2F8PcC5gMnTk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 h1:bFgvUr3/O4PHj3VQcFEuYKvRZJX1SJDQ+11JXuSB3/w= @@ -286,14 +286,14 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0 h1:/jlt1Y8gXWiHG9 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.27.0/go.mod h1:bmToOGOBZ4hA9ghphIc1PAf66VA8KOtsuy3+ScStG20= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 8631edd2eba78..5446366e3543a 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -13,7 +13,7 @@ require ( go.opentelemetry.io/collector/component v0.104.0 go.opentelemetry.io/collector/pdata v1.11.0 go.opentelemetry.io/collector/semconv v0.104.0 - go.opentelemetry.io/otel/metric v1.30.0 + go.opentelemetry.io/otel/metric v1.31.0 google.golang.org/protobuf v1.34.2 ) @@ -26,7 +26,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/pointer v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/util/scrubber v0.56.0-rc.3 // indirect - github.com/DataDog/go-sqllexer v0.0.15 // indirect + github.com/DataDog/go-sqllexer v0.0.16 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/sketches-go v1.4.2 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect @@ -69,11 +69,11 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.104.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index be59ca525ee43..a201b6217a5a0 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -1,7 +1,7 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI6LDrKU= github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= -github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.16 h1:RoSUMS6MECyB3gTUIdydzXwK5NhEhv6GMJkS7ptsgRA= +github.com/DataDog/go-sqllexer v0.0.16/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 h1:fKv05WFWHCXQmUTehW1eEZvXJP65Qv00W4V01B1EqSA= @@ -168,18 +168,18 @@ go.opentelemetry.io/collector/processor v0.104.0 h1:KSvMDu4DWmK1/k2z2rOzMtTvAa00 go.opentelemetry.io/collector/processor v0.104.0/go.mod h1:qU2/xCCYdvVORkN6aq0H/WUWkvo505VGYg2eOwPvaTg= go.opentelemetry.io/collector/semconv v0.104.0 h1:dUvajnh+AYJLEW/XOPk0T0BlwltSdi3vrjO7nSOos3k= go.opentelemetry.io/collector/semconv v0.104.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/prometheus v0.49.0 h1:Er5I1g/YhfYv9Affk9nJLfH/+qCCVVg1f2R9AbJfqDQ= go.opentelemetry.io/otel/exporters/prometheus v0.49.0/go.mod h1:KfQ1wpjf3zsHjzP149P4LyAwWRupc6c7t1ZJ9eXpKQM= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= diff --git a/pkg/util/containerd/containerd_util.go b/pkg/util/containerd/containerd_util.go index 9b80304c894b4..88fe4ecd91820 100644 --- a/pkg/util/containerd/containerd_util.go +++ b/pkg/util/containerd/containerd_util.go @@ -13,12 +13,13 @@ import ( "encoding/json" "errors" "fmt" + "os" "strings" "time" + "github.com/hashicorp/go-multierror" "github.com/opencontainers/image-spec/identity" - "github.com/DataDog/datadog-agent/pkg/config/env" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" dderrors "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -453,15 +454,33 @@ func (c *ContainerdUtil) getMounts(ctx context.Context, expiration time.Duration return nil, nil, fmt.Errorf("No snapshots returned for image: %s", imageID) } - // Transforming mounts in case we're running in a container - if env.IsContainerized() { - for i := range mounts { - mounts[i].Source = strings.ReplaceAll(mounts[i].Source, "/var/lib", "/host/var/lib") - for j := range mounts[i].Options { - mounts[i].Options[j] = strings.ReplaceAll(mounts[i].Options[j], "/var/lib", "/host/var/lib") + for i := range mounts { + mounts[i].Source = sanitizePath(mounts[i].Source) + + var errs error + for j, opt := range mounts[i].Options { + for _, prefix := range []string{"upperdir=", "lowerdir=", "workdir="} { + if strings.HasPrefix(opt, prefix) { + trimmedOpt := strings.TrimPrefix(opt, prefix) + dirs := strings.Split(trimmedOpt, ":") + for n, dir := range dirs { + dirs[n] = sanitizePath(dir) + if _, err := os.Stat(dirs[n]); err != nil { + errs = multierror.Append(errs, fmt.Errorf("unreachable folder %s for overlayfs mount: %w", dir, err)) + } + } + mounts[i].Options[j] = prefix + strings.Join(dirs, ":") + } } + + log.Debugf("Sanitized overlayfs mount options to %s", strings.Join(mounts[i].Options, ",")) + } + + if errs != nil { + log.Warnf("Unreachable path detected in mounts for image %s: %s", imageID, errs.Error()) } } + return mounts, func(ctx context.Context) error { ctx = namespaces.WithNamespace(ctx, namespace) if err := cleanSnapshot(ctx); err != nil { @@ -474,6 +493,14 @@ func (c *ContainerdUtil) getMounts(ctx context.Context, expiration time.Duration }, nil } +func sanitizePath(path string) string { + if index := strings.Index(path, "/var/lib"); index != -1 { + return "/host" + path[index:] + } + + return path +} + // Mounts returns the mounts for an image func (c *ContainerdUtil) Mounts(ctx context.Context, expiration time.Duration, namespace string, img containerd.Image) ([]mount.Mount, error) { mounts, clean, err := c.getMounts(ctx, expiration, namespace, img) diff --git a/pkg/util/crashreport/crashreport.go b/pkg/util/crashreport/crashreport.go index 0bab9ca167051..8c04cdbb6c11c 100644 --- a/pkg/util/crashreport/crashreport.go +++ b/pkg/util/crashreport/crashreport.go @@ -115,6 +115,13 @@ func (wcr *WinCrashReporter) CheckForCrash() (*probe.WinCrashStatus, error) { if !ok { return nil, fmt.Errorf("Raw data has incorrect type") } + + // Crash dump processing is not done yet, nothing to send at the moment. Try later. + if crash.StatusCode == probe.WinCrashStatusCodeBusy { + log.Infof("Crash dump processing is busy") + return nil, nil + } + /* * originally did this with a sync.once. The problem is the check is run prior to the * system probe being successfully started. This is OK; we just need to detect the BSOD @@ -124,7 +131,7 @@ func (wcr *WinCrashReporter) CheckForCrash() (*probe.WinCrashStatus, error) { * we don't need to run any more */ wcr.hasRunOnce = true - if !crash.Success { + if crash.StatusCode == probe.WinCrashStatusCodeFailed { return nil, fmt.Errorf("Error getting crash data %s", crash.ErrString) } diff --git a/pkg/util/trivy/trivy.go b/pkg/util/trivy/trivy.go index 385068d0518aa..a4fdb87e79c57 100644 --- a/pkg/util/trivy/trivy.go +++ b/pkg/util/trivy/trivy.go @@ -318,6 +318,7 @@ func (c *Collector) ScanContainerdImageFromSnapshotter(ctx context.Context, imgM if err != nil { return nil, fmt.Errorf("unable to get mounts for image %s, err: %w", imgMeta.ID, err) } + layers := extractLayersFromOverlayFSMounts(mounts) if len(layers) == 0 { return nil, fmt.Errorf("unable to extract layers from overlayfs mounts %+v for image %s", mounts, imgMeta.ID) diff --git a/pyproject.toml b/pyproject.toml index 526757ec84200..09227ca7e8729 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ exclude = [ "docs", "google-marketplace", "omnibus", - "pkg-config", "releasenotes", "vendor", "venv", diff --git a/release.json b/release.json index 3b1075d639a2f..3e09fada9d482 100644 --- a/release.json +++ b/release.json @@ -7,7 +7,7 @@ }, "nightly": { "INTEGRATIONS_CORE_VERSION": "master", - "OMNIBUS_SOFTWARE_VERSION": "375618d70253293d71b13f9385260aa3dedd7125", + "OMNIBUS_SOFTWARE_VERSION": "cf7fc9c3ab792ebf97dd7c2ce0be39a6e197f6f5", "OMNIBUS_RUBY_VERSION": "bc5e0fb395c4b106967f97f8738d23705e5bc933", "JMXFETCH_VERSION": "0.49.3", "JMXFETCH_HASH": "258085a94d529a6bdf914db36dd50faf6fde2cebc44b1f54a60eb209a5d8917c", @@ -26,7 +26,7 @@ }, "nightly-a7": { "INTEGRATIONS_CORE_VERSION": "master", - "OMNIBUS_SOFTWARE_VERSION": "375618d70253293d71b13f9385260aa3dedd7125", + "OMNIBUS_SOFTWARE_VERSION": "cf7fc9c3ab792ebf97dd7c2ce0be39a6e197f6f5", "OMNIBUS_RUBY_VERSION": "bc5e0fb395c4b106967f97f8738d23705e5bc933", "JMXFETCH_VERSION": "0.49.3", "JMXFETCH_HASH": "258085a94d529a6bdf914db36dd50faf6fde2cebc44b1f54a60eb209a5d8917c", diff --git a/releasenotes/notes/flare-provider-timeout-cli-1961b537734bcc84.yaml b/releasenotes/notes/flare-provider-timeout-cli-1961b537734bcc84.yaml new file mode 100644 index 0000000000000..4e3958aa262d7 --- /dev/null +++ b/releasenotes/notes/flare-provider-timeout-cli-1961b537734bcc84.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + The ``flare`` subcommand includes a ``--provider-timeout`` option to set a timeout for each file collection (default is 10s), useful for unblocking slow flare creation. diff --git a/releasenotes/notes/windowscrashreportasync-1b2c77f9ebeafdd5.yaml b/releasenotes/notes/windowscrashreportasync-1b2c77f9ebeafdd5.yaml new file mode 100644 index 0000000000000..16552ffd1708b --- /dev/null +++ b/releasenotes/notes/windowscrashreportasync-1b2c77f9ebeafdd5.yaml @@ -0,0 +1,14 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + On Windows, the endpoint /windows_crash_detection/check has been modified to report crashes in + an asynchronous manner, to allow processing of large crash dumps without blocking or timing out. + The first check will return a busy status and continue to do so until the processing is completed. + diff --git a/rtloader/CMakeLists.txt b/rtloader/CMakeLists.txt index 1ad9855378039..9398d863bc3ee 100644 --- a/rtloader/CMakeLists.txt +++ b/rtloader/CMakeLists.txt @@ -24,17 +24,10 @@ if(ARCH_I386) endif() ## Config options -option(DISABLE_PYTHON2 "Do not build Python2 support") -option(DISABLE_PYTHON3 "Do not build Python3 support") option(BUILD_DEMO "Build the demo app" ON) ## Add Build Targets -if (NOT DISABLE_PYTHON2) - add_subdirectory(two) -endif() -if (NOT DISABLE_PYTHON3) - add_subdirectory(three) -endif() +add_subdirectory(three) add_subdirectory(rtloader) add_subdirectory(test) if (BUILD_DEMO) diff --git a/rtloader/README.md b/rtloader/README.md index 9b9b033f1a530..13b2ce442d5d4 100644 --- a/rtloader/README.md +++ b/rtloader/README.md @@ -15,14 +15,13 @@ RtLoader will `dlopen` the proper backend libraries accordingly. ### libdatadog-agent-rtloader RtLoader exposes its C89-compatible API through `include/datadog_agent_rtloader.h`. By -using the `make2` and `make3` functions, the corresponding Python backend will +using the `make3` function, the corresponding Python backend will be loaded at runtime. Under the hood the library provides `RtLoader`, a C++ interface that must be implemented by any supported backend, see `include/rtloader.h` for details. ### Two and Three -`libdatadog-agent-three` and `libdatadog-agent-two` libraries provide Python support -for extending and embedding by linking different versions of the CPython library. +`libdatadog-agent-three` library provides Python3 support. Python2 isn't supported anymore. ### Common @@ -33,7 +32,6 @@ Most of the code used to extend the embedded interpreter is there. ## Requirements * C/C++ compiler -* Python 2.7.x development packages * Python 3.12.x development packages * Cmake version 3.12 or above * Go compiler with `cgo` capabilities to run the tests diff --git a/rtloader/common/builtins/_util.c b/rtloader/common/builtins/_util.c index c552c441624e1..7dd43a1e0d099 100644 --- a/rtloader/common/builtins/_util.c +++ b/rtloader/common/builtins/_util.c @@ -34,7 +34,6 @@ static PyMethodDef methods[] = { { NULL, NULL } // guards }; -#ifdef DATADOG_AGENT_THREE static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, _UTIL_MODULE_NAME, NULL, -1, methods }; PyMODINIT_FUNC PyInit__util(void) @@ -43,16 +42,6 @@ PyMODINIT_FUNC PyInit__util(void) addSubprocessException(m); return m; } -#elif defined(DATADOG_AGENT_TWO) -// in Python2 keep the object alive for the program lifetime -static PyObject *module; - -void Py2_init__util() -{ - module = Py_InitModule(_UTIL_MODULE_NAME, methods); - addSubprocessException(module); -} -#endif void _set_get_subprocess_output_cb(cb_get_subprocess_output_t cb) { diff --git a/rtloader/common/builtins/_util.h b/rtloader/common/builtins/_util.h index 6bed75db34bff..f29712f087057 100644 --- a/rtloader/common/builtins/_util.h +++ b/rtloader/common/builtins/_util.h @@ -46,21 +46,13 @@ // The keyword-only arguments separator ($) for PyArg_ParseTupleAndKeywords() // has been introduced in Python 3.3 // https://docs.python.org/3/c-api/arg.html#other-objects -#ifdef DATADOG_AGENT_THREE -# define PY_ARG_PARSE_TUPLE_KEYWORD_ONLY "$" -#elif defined(DATADOG_AGENT_TWO) -# define PY_ARG_PARSE_TUPLE_KEYWORD_ONLY "" -#endif +#define PY_ARG_PARSE_TUPLE_KEYWORD_ONLY "$" #ifdef __cplusplus extern "C" { #endif -#ifdef DATADOG_AGENT_THREE PyMODINIT_FUNC PyInit__util(void); -#elif defined(DATADOG_AGENT_TWO) -void Py2_init__util(); -#endif void _set_get_subprocess_output_cb(cb_get_subprocess_output_t); #ifdef __cplusplus diff --git a/rtloader/common/builtins/aggregator.c b/rtloader/common/builtins/aggregator.c index 3ee1b6b56678e..15343248b3709 100644 --- a/rtloader/common/builtins/aggregator.c +++ b/rtloader/common/builtins/aggregator.c @@ -48,7 +48,6 @@ static void add_constants(PyObject *m) PyModule_AddIntConstant(m, "HISTORATE", DATADOG_AGENT_RTLOADER_HISTORATE); } -#ifdef DATADOG_AGENT_THREE static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, AGGREGATOR_MODULE_NAME, NULL, -1, methods }; PyMODINIT_FUNC PyInit_aggregator(void) @@ -57,16 +56,6 @@ PyMODINIT_FUNC PyInit_aggregator(void) add_constants(m); return m; } -#elif defined(DATADOG_AGENT_TWO) -// module object storage -static PyObject *module; - -void Py2_init_aggregator() -{ - module = Py_InitModule(AGGREGATOR_MODULE_NAME, methods); - add_constants(module); -} -#endif void _set_submit_metric_cb(cb_submit_metric_t cb) { diff --git a/rtloader/common/builtins/aggregator.h b/rtloader/common/builtins/aggregator.h index 6097239ee676d..99b444fd23872 100644 --- a/rtloader/common/builtins/aggregator.h +++ b/rtloader/common/builtins/aggregator.h @@ -74,12 +74,8 @@ extern "C" { #endif -#ifdef DATADOG_AGENT_THREE // PyMODINIT_FUNC macro already specifies extern "C", nesting these is legal PyMODINIT_FUNC PyInit_aggregator(void); -#elif defined(DATADOG_AGENT_TWO) -void Py2_init_aggregator(); -#endif void _set_submit_metric_cb(cb_submit_metric_t cb); void _set_submit_service_check_cb(cb_submit_service_check_t cb); diff --git a/rtloader/common/builtins/containers.c b/rtloader/common/builtins/containers.c index a182703da8a48..bc528fd13e2a7 100644 --- a/rtloader/common/builtins/containers.c +++ b/rtloader/common/builtins/containers.c @@ -18,22 +18,12 @@ static PyMethodDef methods[] = { { NULL, NULL } // guards }; -#ifdef DATADOG_AGENT_THREE static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, CONTAINERS_MODULE_NAME, NULL, -1, methods }; PyMODINIT_FUNC PyInit_containers(void) { return PyModule_Create(&module_def); } -#elif defined(DATADOG_AGENT_TWO) -// in Python2 keep the object alive for the program lifetime -static PyObject *module; - -void Py2_init_containers() -{ - module = Py_InitModule(CONTAINERS_MODULE_NAME, methods); -} -#endif void _set_is_excluded_cb(cb_is_excluded_t cb) { diff --git a/rtloader/common/builtins/containers.h b/rtloader/common/builtins/containers.h index 09a129a4e91d3..fadc5442c67ff 100644 --- a/rtloader/common/builtins/containers.h +++ b/rtloader/common/builtins/containers.h @@ -50,10 +50,6 @@ PyMODINIT_FUNC PyInit_containers(void); extern "C" { #endif -#ifdef DATADOG_AGENT_TWO -void Py2_init_containers(); -#endif - void _set_is_excluded_cb(cb_is_excluded_t); #ifdef __cplusplus diff --git a/rtloader/common/builtins/datadog_agent.c b/rtloader/common/builtins/datadog_agent.c index 2e606f8238279..e20b8f8ff30c8 100644 --- a/rtloader/common/builtins/datadog_agent.c +++ b/rtloader/common/builtins/datadog_agent.c @@ -70,22 +70,12 @@ static PyMethodDef methods[] = { { NULL, NULL } // guards }; -#ifdef DATADOG_AGENT_THREE static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, DATADOG_AGENT_MODULE_NAME, NULL, -1, methods }; PyMODINIT_FUNC PyInit_datadog_agent(void) { return PyModule_Create(&module_def); } -#elif defined(DATADOG_AGENT_TWO) -// in Python2 keep the object alive for the program lifetime -static PyObject *module; - -void Py2_init_datadog_agent() -{ - module = Py_InitModule(DATADOG_AGENT_MODULE_NAME, methods); -} -#endif void _set_get_version_cb(cb_get_version_t cb) { diff --git a/rtloader/common/builtins/datadog_agent.h b/rtloader/common/builtins/datadog_agent.h index 055878d6490ba..20a7b97391a6c 100644 --- a/rtloader/common/builtins/datadog_agent.h +++ b/rtloader/common/builtins/datadog_agent.h @@ -145,11 +145,7 @@ extern "C" { #endif -#ifdef DATADOG_AGENT_THREE PyMODINIT_FUNC PyInit_datadog_agent(void); -#elif defined(DATADOG_AGENT_TWO) -void Py2_init_datadog_agent(); -#endif void _set_get_clustername_cb(cb_get_clustername_t); void _set_get_config_cb(cb_get_config_t); diff --git a/rtloader/common/builtins/kubeutil.c b/rtloader/common/builtins/kubeutil.c index 804deaeeb6c1c..57f8ec65c14d0 100644 --- a/rtloader/common/builtins/kubeutil.c +++ b/rtloader/common/builtins/kubeutil.c @@ -19,22 +19,12 @@ static PyMethodDef methods[] = { { NULL, NULL } // guards }; -#ifdef DATADOG_AGENT_THREE static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, KUBEUTIL_MODULE_NAME, NULL, -1, methods }; PyMODINIT_FUNC PyInit_kubeutil(void) { return PyModule_Create(&module_def); } -#elif defined(DATADOG_AGENT_TWO) -// in Python2 keep the object alive for the program lifetime -static PyObject *module; - -void Py2_init_kubeutil() -{ - module = Py_InitModule(KUBEUTIL_MODULE_NAME, methods); -} -#endif void _set_get_connection_info_cb(cb_get_connection_info_t cb) { diff --git a/rtloader/common/builtins/kubeutil.h b/rtloader/common/builtins/kubeutil.h index db9c95ac863c9..d5eb20ef8b1e1 100644 --- a/rtloader/common/builtins/kubeutil.h +++ b/rtloader/common/builtins/kubeutil.h @@ -45,12 +45,8 @@ extern "C" { #endif -#ifdef DATADOG_AGENT_THREE // PyMODINIT_FUNC macro already specifies extern "C", nesting these is legal PyMODINIT_FUNC PyInit_kubeutil(void); -#elif defined(DATADOG_AGENT_TWO) -void Py2_init_kubeutil(); -#endif void _set_get_connection_info_cb(cb_get_connection_info_t); diff --git a/rtloader/common/builtins/tagger.c b/rtloader/common/builtins/tagger.c index 662174d0be966..4c35738e7c3a2 100644 --- a/rtloader/common/builtins/tagger.c +++ b/rtloader/common/builtins/tagger.c @@ -172,7 +172,6 @@ static void add_constants(PyObject *m) PyModule_AddIntConstant(m, "HIGH", DATADOG_AGENT_RTLOADER_TAGGER_HIGH); } -#ifdef DATADOG_AGENT_THREE static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, TAGGER_MODULE_NAME, NULL, -1, methods }; PyMODINIT_FUNC PyInit_tagger(void) @@ -181,13 +180,3 @@ PyMODINIT_FUNC PyInit_tagger(void) add_constants(module); return module; } -#elif defined(DATADOG_AGENT_TWO) -// in Python2 keep the object alive for the program lifetime -static PyObject *module; - -void Py2_init_tagger() -{ - module = Py_InitModule(TAGGER_MODULE_NAME, methods); - add_constants(module); -} -#endif diff --git a/rtloader/common/builtins/tagger.h b/rtloader/common/builtins/tagger.h index fc8230de8bb73..7d30c1a219ed7 100644 --- a/rtloader/common/builtins/tagger.h +++ b/rtloader/common/builtins/tagger.h @@ -46,18 +46,12 @@ #define TAGGER_MODULE_NAME "tagger" -#ifdef DATADOG_AGENT_THREE PyMODINIT_FUNC PyInit_tagger(void); -#endif #ifdef __cplusplus extern "C" { #endif -#ifdef DATADOG_AGENT_TWO -void Py2_init_tagger(); -#endif - void _set_tags_cb(cb_tags_t); #ifdef __cplusplus diff --git a/rtloader/common/builtins/util.c b/rtloader/common/builtins/util.c index 38df33b86f428..c184e681af3ae 100644 --- a/rtloader/common/builtins/util.c +++ b/rtloader/common/builtins/util.c @@ -19,22 +19,12 @@ static PyMethodDef methods[] = { { NULL, NULL } // guards }; -#ifdef DATADOG_AGENT_THREE static struct PyModuleDef module_def = { PyModuleDef_HEAD_INIT, UTIL_MODULE_NAME, NULL, -1, methods }; PyMODINIT_FUNC PyInit_util(void) { return PyModule_Create(&module_def); } -#elif defined(DATADOG_AGENT_TWO) -// in Python2 keep the object alive for the program lifetime -static PyObject *module; - -void Py2_init_util() -{ - module = Py_InitModule(UTIL_MODULE_NAME, methods); -} -#endif /*! \fn PyObject *headers(PyObject *self, PyObject *args, PyObject *kwargs) \brief This function provides a standard set of HTTP headers the caller might want to diff --git a/rtloader/common/builtins/util.h b/rtloader/common/builtins/util.h index 40073cc5c1400..86f5ed7cec1a1 100644 --- a/rtloader/common/builtins/util.h +++ b/rtloader/common/builtins/util.h @@ -21,8 +21,6 @@ extern "C" { #endif -#ifdef DATADOG_AGENT_THREE - /*! \fn void PyInit_util() \brief Initializes the util builtin python module. @@ -30,16 +28,6 @@ extern "C" { array in 'util.c' and registered into python. This function is python3 only. */ PyMODINIT_FUNC PyInit_util(void); -#elif defined(DATADOG_AGENT_TWO) - -/*! \fn void Py2_init_util() - \brief Initializes the util builtin python module. - - The 'util' python builtin is created with the methods from the PyMethodDef - array in 'util.c' and registered into python. This function is python2 only. -*/ -void Py2_init_util(); -#endif #ifdef __cplusplus } diff --git a/rtloader/common/stringutils.c b/rtloader/common/stringutils.c index 1e6315635c9a5..68a5d7820568c 100644 --- a/rtloader/common/stringutils.c +++ b/rtloader/common/stringutils.c @@ -31,21 +31,6 @@ char *as_string(PyObject *object) char *retval = NULL; -// DATADOG_AGENT_THREE implementation is the default -#ifdef DATADOG_AGENT_TWO - if (!PyString_Check(object) && !PyUnicode_Check(object)) { - return NULL; - } - - char *tmp = PyString_AsString(object); - if (tmp == NULL) { - // PyString_AsString might raise an error when python can't encode a - // unicode string to byte - PyErr_Clear(); - return NULL; - } - retval = strdupe(tmp); -#else PyObject *temp_bytes = NULL; if (PyBytes_Check(object)) { @@ -67,7 +52,6 @@ char *as_string(PyObject *object) retval = strdupe(PyBytes_AS_STRING(temp_bytes)); Py_XDECREF(temp_bytes); -#endif return retval; } diff --git a/rtloader/common/stringutils.h b/rtloader/common/stringutils.h index b0a70c02d102c..1e19c8dd7e297 100644 --- a/rtloader/common/stringutils.h +++ b/rtloader/common/stringutils.h @@ -73,11 +73,7 @@ char *as_string(PyObject *); PyObject *from_yaml(const char *); char *as_yaml(PyObject *); -#ifdef DATADOG_AGENT_THREE -# define PyStringFromCString(x) PyUnicode_FromString(x) -#elif defined(DATADOG_AGENT_TWO) -# define PyStringFromCString(x) PyString_FromString(x) -#endif +#define PyStringFromCString(x) PyUnicode_FromString(x) #ifdef __cplusplus } diff --git a/rtloader/demo/main.c b/rtloader/demo/main.c index 2db57cee12677..c9b8f09c3791b 100644 --- a/rtloader/demo/main.c +++ b/rtloader/demo/main.c @@ -78,16 +78,8 @@ int main(int argc, char *argv[]) } char *init_error = NULL; - // Embed Python2 - if (strcmp(argv[1], "2") == 0) { - rtloader = make2(python_home, "", &init_error); - if (!rtloader) { - printf("Unable to init Python2: %s\n", init_error); - return 1; - } - } // Embed Python3 - else if (strcmp(argv[1], "3") == 0) { + if (strcmp(argv[1], "3") == 0) { rtloader = make3(python_home, "", &init_error); if (!rtloader) { printf("Unable to init Python3: %s\n", init_error); diff --git a/rtloader/include/datadog_agent_rtloader.h b/rtloader/include/datadog_agent_rtloader.h index f61f1ffec9b48..dc5a906373418 100644 --- a/rtloader/include/datadog_agent_rtloader.h +++ b/rtloader/include/datadog_agent_rtloader.h @@ -28,17 +28,7 @@ typedef struct rtloader_s rtloader_t; struct rtloader_pyobject_s; typedef struct rtloader_pyobject_s rtloader_pyobject_t; -// FACTORIES -/*! \fn rtloader_t *make2(const char *python_home, const char *python_exe, char **error) - \brief Factory function to load the python2 backend DLL and create its relevant RtLoader - instance. - \param python_home A C-string with the path to the PYTHONHOME for said DLL. - \param python_exe A C-string with the path to the python interpreter. - \param error A C-string pointer output parameter to return error messages. - \return A rtloader_t * pointer to the RtLoader instance. - \sa rtloader_t -*/ -DATADOG_AGENT_RTLOADER_API rtloader_t *make2(const char *python_home, const char *python_exe, char **error); +// FACTORY /*! \fn rtloader_t *make3(const char *python_home, const char *python_exe, char **error) \brief Factory function to load the python3 backend DLL and create its relevant RtLoader instance. diff --git a/rtloader/rtloader/api.cpp b/rtloader/rtloader/api.cpp index 1026c566fe7ca..da99a969f82e1 100644 --- a/rtloader/rtloader/api.cpp +++ b/rtloader/rtloader/api.cpp @@ -33,16 +33,12 @@ #include "rtloader_mem.h" #if __linux__ -# define DATADOG_AGENT_TWO "libdatadog-agent-two.so" # define DATADOG_AGENT_THREE "libdatadog-agent-three.so" #elif __APPLE__ -# define DATADOG_AGENT_TWO "libdatadog-agent-two.dylib" # define DATADOG_AGENT_THREE "libdatadog-agent-three.dylib" #elif __FreeBSD__ -# define DATADOG_AGENT_TWO "libdatadog-agent-two.so" # define DATADOG_AGENT_THREE "libdatadog-agent-three.so" #elif _WIN32 -# define DATADOG_AGENT_TWO "libdatadog-agent-two.dll" # define DATADOG_AGENT_THREE "libdatadog-agent-three.dll" #else # error Platform not supported @@ -69,7 +65,7 @@ static void *rtloader_backend = NULL; \return A create_t * function pointer that will allow us to create the relevant python backend. In case of failure NULL is returned and the error string is set on the output parameter. - \sa create_t, make2, make3 + \sa create_t, make3 This function is windows only. Required by the backend "makers". */ @@ -102,21 +98,6 @@ create_t *loadAndCreate(const char *dll, const char *python_home, char **error) return create; } -rtloader_t *make2(const char *python_home, const char *python_exe, char **error) -{ - - if (rtloader_backend != NULL) { - *error = strdupe("RtLoader already initialized!"); - return NULL; - } - - create_t *create = loadAndCreate(DATADOG_AGENT_TWO, python_home, error); - if (!create) { - return NULL; - } - return AS_TYPE(rtloader_t, create(python_home, python_exe, _get_memory_tracker_cb())); -} - rtloader_t *make3(const char *python_home, const char *python_exe, char **error) { if (rtloader_backend != NULL) { @@ -152,37 +133,6 @@ void destroy(rtloader_t *rtloader) } #else -rtloader_t *make2(const char *python_home, const char *python_exe, char **error) -{ - if (rtloader_backend != NULL) { - std::string err_msg = "RtLoader already initialized!"; - *error = strdupe(err_msg.c_str()); - return NULL; - } - // load library - rtloader_backend = dlopen(DATADOG_AGENT_TWO, RTLD_LAZY | RTLD_GLOBAL); - if (!rtloader_backend) { - std::ostringstream err_msg; - err_msg << "Unable to open two library: " << dlerror(); - *error = strdupe(err_msg.str().c_str()); - return NULL; - } - - // reset dl errors - dlerror(); - - // dlsym class factory - create_t *create = (create_t *)dlsym(rtloader_backend, "create"); - const char *dlsym_error = dlerror(); - if (dlsym_error) { - std::ostringstream err_msg; - err_msg << "Unable to open two factory: " << dlsym_error; - *error = strdupe(err_msg.str().c_str()); - return NULL; - } - - return AS_TYPE(rtloader_t, create(python_home, python_exe, _get_memory_tracker_cb())); -} rtloader_t *make3(const char *python_home, const char *python_exe, char **error) { diff --git a/rtloader/test/CMakeLists.txt b/rtloader/test/CMakeLists.txt index 04b667825c3f3..6f4f509ca0a45 100644 --- a/rtloader/test/CMakeLists.txt +++ b/rtloader/test/CMakeLists.txt @@ -14,9 +14,9 @@ set(PKGS ) if (WIN32) - set(LIBS_PATH \"${PROJECT_BINARY_DIR}/rtloader/\;${PROJECT_BINARY_DIR}/two/\;${PROJECT_BINARY_DIR}/three/\") + set(LIBS_PATH \"${PROJECT_BINARY_DIR}/rtloader/\;${PROJECT_BINARY_DIR}/three/\") else() - set(LIBS_PATH "${PROJECT_BINARY_DIR}/rtloader/:${PROJECT_BINARY_DIR}/two/:${PROJECT_BINARY_DIR}/three/") + set(LIBS_PATH "${PROJECT_BINARY_DIR}/rtloader/:${PROJECT_BINARY_DIR}/three/") endif() set (CGO_CFLAGS \"-I${CMAKE_SOURCE_DIR}/include -I${CMAKE_SOURCE_DIR}/common -Wno-deprecated-declarations\") @@ -27,22 +27,11 @@ else() set (CGO_LDFLAGS -L${PROJECT_BINARY_DIR}/rtloader -ldatadog-agent-rtloader -ldl) endif() -if (NOT DISABLE_PYTHON2) - set (CGO_LDFLAGS \"-L${PROJECT_BINARY_DIR}/two/ ${CGO_LDFLAGS}\") - add_custom_command( - OUTPUT testPy2 - COMMAND ${CMAKE_COMMAND} -E env CGO_CFLAGS=${CGO_CFLAGS} CGO_LDFLAGS=${CGO_LDFLAGS} DYLD_LIBRARY_PATH=${LIBS_PATH} LD_LIBRARY_PATH=${LIBS_PATH} go test -mod=mod -tags "two" -count=1 -p=1 ${PKGS} - ) - list(APPEND TARGETS "testPy2") -endif() - -if (NOT DISABLE_PYTHON3) - set (CGO_LDFLAGS \"-L${PROJECT_BINARY_DIR}/three/ ${CGO_LDFLAGS}\") - add_custom_command( - OUTPUT testPy3 - COMMAND ${CMAKE_COMMAND} -E env CGO_CFLAGS=${CGO_CFLAGS} CGO_LDFLAGS=${CGO_LDFLAGS} DYLD_LIBRARY_PATH=${LIBS_PATH} LD_LIBRARY_PATH=${LIBS_PATH} go test -mod=mod -tags "three" -count=1 -p=1 ${PKGS} - ) - list(APPEND TARGETS "testPy3") -endif() +set (CGO_LDFLAGS \"-L${PROJECT_BINARY_DIR}/three/ ${CGO_LDFLAGS}\") +add_custom_command( + OUTPUT testPy3 + COMMAND ${CMAKE_COMMAND} -E env CGO_CFLAGS=${CGO_CFLAGS} CGO_LDFLAGS=${CGO_LDFLAGS} DYLD_LIBRARY_PATH=${LIBS_PATH} LD_LIBRARY_PATH=${LIBS_PATH} go test -mod=mod -tags "three" -count=1 -p=1 ${PKGS} +) +list(APPEND TARGETS "testPy3") add_custom_target(run DEPENDS ${TARGETS}) diff --git a/rtloader/test/common/cgo_free_two.go b/rtloader/test/common/cgo_free_two.go deleted file mode 100644 index 5b5b65dc1c8e5..0000000000000 --- a/rtloader/test/common/cgo_free_two.go +++ /dev/null @@ -1,19 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build two - -package testcommon - -/* -#cgo !windows LDFLAGS: -L../../two/ -ldatadog-agent-two -#cgo windows LDFLAGS: -L../../two/ -ldatadog-agent-two.dll -#include "cgo_free.h" - -void c_callCgoFree(void *ptr) { - cgo_free(ptr); -} -*/ -import "C" diff --git a/rtloader/test/common/common_two.go b/rtloader/test/common/common_two.go deleted file mode 100644 index 16515cab089d6..0000000000000 --- a/rtloader/test/common/common_two.go +++ /dev/null @@ -1,27 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build two - -package testcommon - -// #include -// -import "C" - -import "unsafe" - -// UsingTwo states whether we're using Two as backend -const UsingTwo bool = true - -// GetRtLoader returns a RtLoader instance using Two -func GetRtLoader() *C.rtloader_t { - var err *C.char = nil - - executablePath := C.CString("/folder/mock_python_interpeter_bin_path") - defer C.free(unsafe.Pointer(executablePath)) - - return C.make2(nil, executablePath, &err) -} diff --git a/rtloader/test/init/init_two.go b/rtloader/test/init/init_two.go deleted file mode 100644 index 7626d458c727a..0000000000000 --- a/rtloader/test/init/init_two.go +++ /dev/null @@ -1,12 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build two - -package testinit - -const ( - initAllocations = 2 -) diff --git a/rtloader/three/three.h b/rtloader/three/three.h index 1f2ae429308eb..829d8d6d88275 100644 --- a/rtloader/three/three.h +++ b/rtloader/three/three.h @@ -6,13 +6,6 @@ #ifndef DATADOG_AGENT_RTLOADER_THREE_H #define DATADOG_AGENT_RTLOADER_THREE_H -// Some preprocessor sanity for builds (2+3 common sources) -#ifndef DATADOG_AGENT_THREE -# error Build requires defining DATADOG_AGENT_THREE -#elif defined(DATADOG_AGENT_TWO) && defined(DATADOG_AGENT_THREE) -# error "DATADOG_AGENT_TWO and DATADOG_AGENT_THREE are mutually exclusive - define only one of the two." -#endif - #include #include #include diff --git a/rtloader/two/CMakeLists.txt b/rtloader/two/CMakeLists.txt deleted file mode 100644 index 8b4fc0f97445f..0000000000000 --- a/rtloader/two/CMakeLists.txt +++ /dev/null @@ -1,82 +0,0 @@ -cmake_minimum_required(VERSION 3.12) -find_package (Python2 COMPONENTS Interpreter Development) - -if(Python2_VERSION_MINOR LESS "7") - message( - FATAL_ERROR - "Python2 version found is too old: found ${Python2_EXECUTABLE} (version \"${Python2_VERSION}\"), minimum required version is 2.7" - ) -endif() - -project(datadog-agent-two VERSION 0.1.0 DESCRIPTION "CPython backend for the Datadog Agent") - -if(WIN32) - if(MSVC) - - # explicitly set the compiler flags to use the static C runtime (/MT(d) instead of the DLL - # c runtime (/MD(d) so that we don't have to worry about redistributing the CRT). - foreach(flag_var - CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE - CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO) - if(${flag_var} MATCHES "/MD") - string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") - endif(${flag_var} MATCHES "/MD") - endforeach(flag_var) - else() # assume gnuC on windows - string(REPLACE "\\" "\\\\" Python2_STDLIB ${Python2_STDLIB}) - if(ARCH_I386) - set(CMAKE_C_FLAGS "-D_hypot=hypot ") - set(CMAKE_CXX_FLAGS "-D_hypot=hypot ") - else() - set(CMAKE_C_FLAGS "-D_hypot=hypot -DMS_WIN64") - set(CMAKE_CXX_FLAGS "-D_hypot=hypot -DMS_WIN64") - endif() - endif() -endif() - -include(GNUInstallDirs) -configure_file(constants.h.in constants.h) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) -add_library(datadog-agent-two SHARED - two.cpp - ../common/cgo_free.c - ../common/stringutils.c - ../common/log.c - ../common/builtins/aggregator.c - ../common/builtins/datadog_agent.c - ../common/builtins/util.c - ../common/builtins/_util.c - ../common/builtins/tagger.c - ../common/builtins/kubeutil.c - ../common/builtins/containers.c -) -add_compile_definitions(DATADOG_AGENT_TWO) -target_include_directories(datadog-agent-two PRIVATE .) -target_include_directories(datadog-agent-two PUBLIC - ${CMAKE_SOURCE_DIR}/include - ${CMAKE_SOURCE_DIR}/common - ${CMAKE_SOURCE_DIR}/common/builtins - ${Python2_INCLUDE_DIRS} -) -if(WIN32) - if(ARCH_I386) - set_target_properties(datadog-agent-two PROPERTIES COMPILE_FLAGS "-m32" LINK_FLAGS "-m32 -static") - else() - set_target_properties(datadog-agent-two PROPERTIES LINK_FLAGS "-static") - endif() -elseif(APPLE) - set_target_properties(datadog-agent-two PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE) -endif() - -target_link_libraries(datadog-agent-two ${Python2_LIBRARIES} datadog-agent-rtloader) - -if(WIN32) - install(TARGETS datadog-agent-two - RUNTIME DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) -else() - target_compile_options(datadog-agent-two PRIVATE "-Wno-deprecated-register") - install(TARGETS datadog-agent-two - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) -endif() diff --git a/rtloader/two/constants.h.in b/rtloader/two/constants.h.in deleted file mode 100644 index 78413d0d29b74..0000000000000 --- a/rtloader/two/constants.h.in +++ /dev/null @@ -1,5 +0,0 @@ -#ifdef _WIN32 -static const char *_defaultPythonHome = "@Python2_STDLIB@\\.."; -#else -static const char *_defaultPythonHome = "@Python2_STDLIB@/../.."; -#endif diff --git a/rtloader/two/two.cpp b/rtloader/two/two.cpp deleted file mode 100644 index 4901adb5723b7..0000000000000 --- a/rtloader/two/two.cpp +++ /dev/null @@ -1,1120 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog -// (https://www.datadoghq.com/). -// Copyright 2019-present Datadog, Inc. -#include "two.h" - -#include "constants.h" - -#include "_util.h" -#include "aggregator.h" -#include "cgo_free.h" -#include "containers.h" -#include "datadog_agent.h" -#include "kubeutil.h" -#include "rtloader_mem.h" -#include "rtloader_types.h" -#include "stringutils.h" -#include "tagger.h" -#include "util.h" - -#include -#include -#include - -extern "C" DATADOG_AGENT_RTLOADER_API RtLoader *create(const char *python_home, const char *python_exe, - cb_memory_tracker_t memtrack_cb) -{ - return new Two(python_home, python_exe, memtrack_cb); -} - -extern "C" DATADOG_AGENT_RTLOADER_API void destroy(RtLoader *p) -{ - delete p; -} - -Two::Two(const char *python_home, const char *python_exe, cb_memory_tracker_t memtrack_cb) - : RtLoader(memtrack_cb) - , _pythonHome(NULL) - , _pythonExe(NULL) - , _baseClass(NULL) - , _pythonPaths() -{ - initPythonHome(python_home); - - // If not empty, set our Python interpreter path - if (python_exe && strlen(python_exe) > 0) { - initPythonExe(python_exe); - } -} - -Two::~Two() -{ - // For more information on why Py_Finalize() isn't called here please - // refer to the header file or the doxygen documentation. - PyEval_RestoreThread(_threadState); - Py_XDECREF(_baseClass); -} - -void Two::initPythonHome(const char *pythonHome) -{ - // Py_SetPythonHome stores a pointer to the string we pass to it, so we must keep it in memory - char *oldPythonHome = _pythonHome; - - if (pythonHome == NULL || strlen(pythonHome) == 0) { - _pythonHome = strdupe(_defaultPythonHome); - } else { - _pythonHome = strdupe(pythonHome); - } - - Py_SetPythonHome(_pythonHome); - if (oldPythonHome) { - _free(oldPythonHome); - } -} - -void Two::initPythonExe(const char *python_exe) -{ - // Py_SetProgramName stores a pointer to the string we pass to it, so we must keep it in memory - char *oldPythonExe = _pythonExe; - - _pythonExe = strdupe(python_exe); - - Py_SetProgramName(_pythonExe); - - if (oldPythonExe) { - _free(oldPythonExe); - } -} - -bool Two::init() -{ - - Py_Initialize(); - - if (!Py_IsInitialized()) { - return false; - } - - // In recent versions of Python3 this is called from Py_Initialize already, - // for Python2 it has to be explicit. - PyEval_InitThreads(); - - // Set PYTHONPATH - if (!_pythonPaths.empty()) { - char pathchr[] = "path"; - PyObject *path = PySys_GetObject(pathchr); // borrowed - if (path == NULL) { - // sys.path doesn't exist, which should never happen. - // No exception is set on the interpreter, so no need to handle any. - setError("could not access sys.path"); - goto done; - } - for (PyPaths::iterator pit = _pythonPaths.begin(); pit != _pythonPaths.end(); ++pit) { - PyObject *p = PyString_FromString(pit->c_str()); - if (p == NULL) { - setError("could not set pythonPath: " + _fetchPythonError()); - goto done; - } - int retval = PyList_Append(path, p); - Py_XDECREF(p); - if (retval == -1) { - setError("could not append path to pythonPath: " + _fetchPythonError()); - goto done; - } - } - } - - // init custom builtins - if (init_stringutils() != EXIT_SUCCESS) { - setError("error initializing string utils: " + _fetchPythonError()); - goto done; - } - Py2_init_aggregator(); - Py2_init_datadog_agent(); - Py2_init_util(); - Py2_init__util(); - Py2_init_tagger(); - Py2_init_kubeutil(); - Py2_init_containers(); - - // import the base class - _baseClass = _importFrom("datadog_checks.checks", "AgentCheck"); - if (_baseClass == NULL) { - setError("could not import base class: " + std::string(getError())); - } - -done: - // save thread state and release the GIL - _threadState = PyEval_SaveThread(); - - return _baseClass != NULL; -} - -py_info_t *Two::getPyInfo() -{ - PyObject *sys = NULL; - PyObject *path = NULL; - PyObject *str_path = NULL; - - py_info_t *info = (py_info_t *)_malloc(sizeof(*info)); - if (!info) { - setError("could not allocate a py_info_t struct"); - return NULL; - } - - info->version = Py_GetVersion(); - info->path = NULL; - - sys = PyImport_ImportModule("sys"); - if (!sys) { - setError("could not import module 'sys': " + _fetchPythonError()); - goto done; - } - - path = PyObject_GetAttrString(sys, "path"); - if (!path) { - setError("could not get 'sys.path': " + _fetchPythonError()); - goto done; - } - - str_path = PyObject_Repr(path); - if (!str_path) { - setError("could not compute a string representation of 'sys.path': " + _fetchPythonError()); - goto done; - } - - info->path = as_string(str_path); - -done: - Py_XDECREF(sys); - Py_XDECREF(path); - Py_XDECREF(str_path); - return info; -} -/** - * freePyInfo() - */ -void Two::freePyInfo(py_info_t *info) -{ - info->version = NULL; - if (info->path) { - _free(info->path); - } - _free(info); - return; -} - -bool Two::runSimpleString(const char *code) const -{ - return PyRun_SimpleString(code) == 0; -} - -bool Two::addPythonPath(const char *path) -{ - if (std::find(_pythonPaths.begin(), _pythonPaths.end(), path) == _pythonPaths.end()) { - _pythonPaths.push_back(path); - return true; - } - return false; -} - -rtloader_gilstate_t Two::GILEnsure() -{ - PyGILState_STATE state = PyGILState_Ensure(); - if (state == PyGILState_LOCKED) { - return DATADOG_AGENT_RTLOADER_GIL_LOCKED; - } - return DATADOG_AGENT_RTLOADER_GIL_UNLOCKED; -} - -void Two::GILRelease(rtloader_gilstate_t state) -{ - if (state == DATADOG_AGENT_RTLOADER_GIL_LOCKED) { - PyGILState_Release(PyGILState_LOCKED); - } else { - PyGILState_Release(PyGILState_UNLOCKED); - } -} - -bool Two::getClass(const char *module, RtLoaderPyObject *&pyModule, RtLoaderPyObject *&pyClass) -{ - PyObject *obj_module = NULL; - PyObject *obj_class = NULL; - - obj_module = PyImport_ImportModule(module); - if (obj_module == NULL) { - std::ostringstream err; - err << "unable to import module '" << module << "': " + _fetchPythonError(); - setError(err.str()); - return false; - } - - obj_class = _findSubclassOf(_baseClass, obj_module); - if (obj_class == NULL) { - // `_findSubclassOf` does not set the interpreter's error flag, but leaves an error on rtloader - std::ostringstream err; - err << "unable to find a subclass of the base check in module '" << module << "': " << getError(); - setError(err.str()); - Py_XDECREF(obj_module); - return false; - } - - pyModule = reinterpret_cast(obj_module); - pyClass = reinterpret_cast(obj_class); - return true; -} - -bool Two::getCheck(RtLoaderPyObject *py_class, const char *init_config_str, const char *instance_str, - const char *check_id_str, const char *check_name, const char *agent_config_str, - RtLoaderPyObject *&check) -{ - - PyObject *klass = reinterpret_cast(py_class); - PyObject *agent_config = NULL; - PyObject *init_config = NULL; - PyObject *instance = NULL; - PyObject *instances = NULL; - PyObject *py_check = NULL; - PyObject *args = NULL; - PyObject *kwargs = NULL; - PyObject *check_id = NULL; - PyObject *name = NULL; - - char load_config[] = "load_config"; - char format[] = "(s)"; // use parentheses to force Tuple creation - - // call `AgentCheck.load_config(init_config)` - init_config = PyObject_CallMethod(klass, load_config, format, init_config_str); - if (init_config == NULL) { - setError("error parsing init_config: " + _fetchPythonError()); - goto done; - } - // replace an empty init_config by a empty dict - if (init_config == Py_None) { - Py_XDECREF(init_config); - init_config = PyDict_New(); - if (init_config == NULL) { - setError("error 'init_config' can't be initialized to an empty dict: " + _fetchPythonError()); - goto done; - } - } else if (!PyDict_Check(init_config)) { - setError("error 'init_config' is not a dict"); - goto done; - } - - // call `AgentCheck.load_config(instance)` - instance = PyObject_CallMethod(klass, load_config, format, instance_str); - if (instance == NULL) { - setError("error parsing instance: " + _fetchPythonError()); - goto done; - } else if (!PyDict_Check(instance)) { - setError("error instance is not a dict"); - Py_XDECREF(instance); // we still own the reference to instance, so we need to decref it here - goto done; - } - - instances = PyTuple_New(1); - if (instances == NULL) { - setError("could not create tuple for instances: " + _fetchPythonError()); - Py_XDECREF(instance); // we still own the reference to instance, so we need to decref it here - goto done; - } - // As stated in the Python C-API documentation - // https://github.com/python/cpython/blob/2.7/Doc/c-api/intro.rst#reference-count-details, PyTuple_SetItem takes - // over ownership of the given item (instance in this case). This means that we should NOT DECREF it - if (PyTuple_SetItem(instances, 0, instance) != 0) { - setError("could not set instance item on instances: " + _fetchPythonError()); - goto done; - } - - // create `args` and `kwargs` to invoke `AgentCheck` constructor - args = PyTuple_New(0); - if (args == NULL) { - setError("error 'args' can't be initialized to an empty tuple: " + _fetchPythonError()); - goto done; - } - kwargs = PyDict_New(); - if (kwargs == NULL) { - setError("error 'kwargs' can't be initialized to an empty dict: " + _fetchPythonError()); - goto done; - } - name = PyString_FromString(check_name); - if (name == NULL) { - setError("error 'name' can't be initialized: " + _fetchPythonError()); - goto done; - } - if (PyDict_SetItemString(kwargs, "name", name) == -1) { - setError("error 'name' key can't be set: " + _fetchPythonError()); - goto done; - } - if (PyDict_SetItemString(kwargs, "init_config", init_config) == -1) { - setError("error 'init_config' key can't be set: " + _fetchPythonError()); - goto done; - } - if (PyDict_SetItemString(kwargs, "instances", instances) == -1) { - setError("error 'instances' key can't be set: " + _fetchPythonError()); - goto done; - } - - if (agent_config_str != NULL) { - agent_config = PyObject_CallMethod(klass, load_config, format, agent_config_str); - if (agent_config == NULL) { - setError("error parsing agent_config: " + _fetchPythonError()); - goto done; - } else if (!PyDict_Check(agent_config)) { - setError("error agent_config is not a dict"); - goto done; - } - - if (PyDict_SetItemString(kwargs, "agentConfig", agent_config) == -1) { - setError("error 'agentConfig' key can't be set: " + _fetchPythonError()); - goto done; - } - } - - // call `AgentCheck` constructor - py_check = PyObject_Call(klass, args, kwargs); - if (py_check == NULL) { - setError(_fetchPythonError()); - goto done; - } - - if (check_id_str != NULL && strlen(check_id_str) != 0) { - check_id = PyString_FromString(check_id_str); - if (check_id == NULL) { - std::ostringstream err; - err << "error could not set check_id: " << check_id_str; - setError(err.str()); - Py_XDECREF(py_check); - py_check = NULL; - goto done; - } - - if (PyObject_SetAttrString(py_check, "check_id", check_id) != 0) { - setError("error could not set 'check_id' attr: " + _fetchPythonError()); - Py_XDECREF(py_check); - py_check = NULL; - goto done; - } - } - -done: - // We purposefully avoid calling Py_XDECREF on instance because we lost ownership earlier by - // calling PyTuple_SetItem. More details are available in the comment above this PyTuple_SetItem - // call - Py_XDECREF(name); - Py_XDECREF(check_id); - Py_XDECREF(init_config); - Py_XDECREF(instances); - Py_XDECREF(agent_config); - Py_XDECREF(args); - Py_XDECREF(kwargs); - - if (py_check == NULL) { - return false; - } - - check = reinterpret_cast(py_check); - return true; -} - -char *Two::runCheck(RtLoaderPyObject *check) -{ - if (check == NULL) { - return NULL; - } - - PyObject *py_check = reinterpret_cast(check); - - // result will be eventually returned as a copy and the corresponding Python - // string decref'ed, caller will be responsible for memory deallocation. - char *ret = NULL; - char *ret_copy = NULL; - char run[] = "run"; - PyObject *result = NULL; - - result = PyObject_CallMethod(py_check, run, NULL); - if (result == NULL) { - setError("error invoking 'run' method: " + _fetchPythonError()); - goto done; - } - - // `ret` points to the Python string internal storage and will be eventually - // deallocated along with the corresponding Python object. - ret = PyString_AsString(result); - if (ret == NULL) { - setError("error converting 'run' result to string: " + _fetchPythonError()); - goto done; - } - - ret_copy = strdupe(ret); - -done: - Py_XDECREF(result); - return ret_copy; -} - -void Two::cancelCheck(RtLoaderPyObject *check) -{ - if (check == NULL) { - return; - } - - PyObject *py_check = reinterpret_cast(check); - - char cancel[] = "cancel"; - PyObject *result = NULL; - - result = PyObject_CallMethod(py_check, cancel, NULL); - // at least None should be returned - if (result == NULL) { - setError("error invoking 'cancel' method: " + _fetchPythonError()); - } - Py_XDECREF(result); -} - -char **Two::getCheckWarnings(RtLoaderPyObject *check) -{ - if (check == NULL) { - return NULL; - } - - PyObject *py_check = reinterpret_cast(check); - char **warnings = NULL; - - char func_name[] = "get_warnings"; - Py_ssize_t numWarnings; - - PyObject *warns_list = PyObject_CallMethod(py_check, func_name, NULL); - if (warns_list == NULL) { - setError("error invoking 'get_warnings' method: " + _fetchPythonError()); - goto done; - } - - numWarnings = PyList_Size(warns_list); - // docs are not clear but `PyList_Size` can actually fail and in case it would - // return -1, see https://github.com/python/cpython/blob/2.7/Objects/listobject.c#L170 - if (numWarnings == -1) { - setError("error computing 'len(warnings)': " + _fetchPythonError()); - goto done; - } - - warnings = (char **)_malloc(sizeof(*warnings) * (numWarnings + 1)); - if (!warnings) { - setError("could not allocate memory to store warnings"); - goto done; - } - warnings[numWarnings] = NULL; - - for (Py_ssize_t idx = 0; idx < numWarnings; idx++) { - PyObject *warn = PyList_GetItem(warns_list, idx); // borrowed ref - if (warn == NULL) { - setError("there was an error browsing 'warnings' list: " + _fetchPythonError()); - - for (int jdx = 0; jdx < idx; jdx++) { - _free(warnings[jdx]); - } - _free(warnings); - - warnings = NULL; - goto done; - } - warnings[idx] = as_string(warn); - } - -done: - Py_XDECREF(warns_list); - return warnings; -} - -char *Two::getCheckDiagnoses(RtLoaderPyObject *check) -{ - if (check == NULL) { - return NULL; - } - - PyObject *py_check = reinterpret_cast(check); - - // result will be eventually returned as a copy and the corresponding Python - // string decref'ed, caller will be responsible for memory deallocation. - char *ret = NULL; - char *ret_copy = NULL; - char func_name[] = "get_diagnoses"; - PyObject *result = NULL; - - result = PyObject_CallMethod(py_check, func_name, NULL); - if (result == NULL) { - ret = _createInternalErrorDiagnoses(_fetchPythonError().c_str()); - goto done; - } - - // `ret` points to the Python string internal storage and will be eventually - // deallocated along with the corresponding Python object. - ret = PyString_AsString(result); - if (ret == NULL) { - std::string errorMsg = std::string("error converting 'get_diagnoses' result to string: ") + _fetchPythonError(); - ret = _createInternalErrorDiagnoses(errorMsg.c_str()); - goto done; - } - - ret_copy = strdupe(ret); - -done: - Py_XDECREF(result); - return ret_copy; -} - -// return new reference -PyObject *Two::_importFrom(const char *module, const char *name) -{ - PyObject *obj_module = NULL; - PyObject *obj_symbol = NULL; - - obj_module = PyImport_ImportModule(module); - if (obj_module == NULL) { - setError(_fetchPythonError()); - goto error; - } - - obj_symbol = PyObject_GetAttrString(obj_module, name); - if (obj_symbol == NULL) { - setError(_fetchPythonError()); - goto error; - } - - Py_XDECREF(obj_module); - return obj_symbol; - -error: - Py_XDECREF(obj_module); - Py_XDECREF(obj_symbol); - return NULL; -} - -PyObject *Two::_findSubclassOf(PyObject *base, PyObject *module) -{ - // baseClass is not a Class type - if (base == NULL || !PyType_Check(base)) { - setError("base class is not of type 'Class'"); - return NULL; - } - - // module is not a Module object - if (module == NULL || !PyModule_Check(module)) { - setError("module is not of type 'Module'"); - return NULL; - } - - PyObject *dir = PyObject_Dir(module); - if (dir == NULL) { - PyErr_Clear(); - setError("there was an error calling dir() on module object"); - return NULL; - } - - PyObject *klass = NULL; - for (int i = 0; i < PyList_GET_SIZE(dir); i++) { - // Reset `klass` at every iteration so its state is always clean when we - // continue the loop or return early. Reset at first iteration is useless - // but it keeps the code readable. - Py_XDECREF(klass); - klass = NULL; - - // get the symbol in current list item - PyObject *symbol = PyList_GetItem(dir, i); - if (symbol == NULL) { - // This should never happen as it means we're out of bounds - PyErr_Clear(); - setError("there was an error browsing dir() output"); - goto done; - } - - // get symbol name - char *symbol_name = PyString_AsString(symbol); - if (symbol_name == NULL) { - // PyString_AsString returns NULL if `symbol` is not a string object - // and raises TypeError. Let's clear the error and keep going. - PyErr_Clear(); - continue; - } - - // get symbol instance. It's a new ref but in case of success we don't - // DecRef since we return it and the caller will be owner - klass = PyObject_GetAttrString(module, symbol_name); - if (klass == NULL) { - PyErr_Clear(); - continue; - } - - // not a class, ignore - if (!PyType_Check(klass)) { - continue; - } - - // this is an unrelated class, ignore - if (!PyType_IsSubtype((PyTypeObject *)klass, (PyTypeObject *)base)) { - continue; - } - - // check whether `klass` is actually `base` itself - int retval = PyObject_RichCompareBool(klass, base, Py_EQ); - if (retval == 1) { - // `klass` is indeed `base`, continue - continue; - } else if (retval == -1) { - // an error occurred calling __eq__, clear and continue - PyErr_Clear(); - continue; - } - - // does `klass` have subclasses? - char func_name[] = "__subclasses__"; - PyObject *children = PyObject_CallMethod(klass, func_name, NULL); - if (children == NULL) { - PyErr_Clear(); - continue; - } - - // how many? - int children_count = PyList_GET_SIZE(children); - Py_XDECREF(children); - - // Agent integrations are supposed to have no subclasses - if (children_count > 0) { - continue; - } - - // got it, return the check class - goto done; - } - - // we couldn't find any good subclass, set an error and reset - // `klass` state for one last time before moving to `done`. - setError("cannot find a subclass"); - Py_XDECREF(klass); - klass = NULL; - -done: - Py_XDECREF(dir); - return klass; -} - -std::string Two::_fetchPythonError() -{ - std::string ret_val = ""; - - if (PyErr_Occurred() == NULL) { - return ret_val; - } - - PyObject *ptype = NULL; - PyObject *pvalue = NULL; - PyObject *ptraceback = NULL; - PyObject *format_exception = NULL; - PyObject *traceback = NULL; - PyObject *fmt_exc = NULL; - - // Fetch error and make sure exception values are normalized, as per python C - // API docs. - // PyErr_Fetch returns void, no need to check its return value - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - // PyErr_NormalizeException returns void, no need to check its return value - PyErr_NormalizeException(&ptype, &pvalue, &ptraceback); - - // There's a traceback, try to format it nicely - if (ptraceback != NULL) { - traceback = PyImport_ImportModule("traceback"); - if (traceback != NULL) { - char fname[] = "format_exception"; - format_exception = PyObject_GetAttrString(traceback, fname); - if (format_exception != NULL) { - fmt_exc = PyObject_CallFunctionObjArgs(format_exception, ptype, pvalue, ptraceback, NULL); - if (fmt_exc != NULL) { - Py_ssize_t len = PyList_Size(fmt_exc); - // docs are not clear but `PyList_Size` can actually fail and in case it would - // return -1, see https://github.com/python/cpython/blob/2.7/Objects/listobject.c#L170 - if (len == -1) { - // don't fetch the actual error or the caller might think it was the root cause, - // while it's not. Setting `ret_val` empty will make the function return "unknown error". - // PyErr_Clear() will be called before returning. - ret_val = ""; - goto done; - } - - // "format_exception" returns a list of strings (one per line) - for (int i = 0; i < len; i++) { - PyObject *s = PyList_GetItem(fmt_exc, i); // borrowed ref - if (s == NULL || !PyString_Check(s)) { - // unlikely to happen but same as above, do not propagate this error upstream - // to avoid confusing the caller. PyErr_Clear() will be called before returning. - ret_val = ""; - goto done; - } - // traceback.format_exception returns a list of strings, each ending in a *newline* - // and some containing internal newlines. No need to add any CRLF/newlines. - // PyString_AsString returns a char* to a temporary object, no need to strdup it because - // of the implicit conversion to std::string, which already makes a copy. - ret_val += PyString_AsString(s); - } - } - } - } else { - // If we reach this point, there was an error while formatting the - // exception - ret_val = "can't format exception"; - } - } - // we sometimes do not get a traceback but an error in pvalue - else if (pvalue != NULL) { - PyObject *pvalue_obj = PyObject_Str(pvalue); - if (pvalue_obj != NULL) { - // we know pvalue_obj is a string (we just casted it), no need to PyString_Check() - // PyString_AsString returns a char* to a temporary object, no need to strdup it because - // of the implicit conversion to std::string, which already makes a copy. - ret_val = PyString_AsString(pvalue_obj); - Py_XDECREF(pvalue_obj); - } - } else if (ptype != NULL) { - PyObject *ptype_obj = PyObject_Str(ptype); - if (ptype_obj != NULL) { - // we know ptype_obj is a string (we just casted it), no need to PyString_Check() - // PyString_AsString returns a char* to a temporary object, no need to strdup it because - // of the implicit conversion to std::string, which already makes a copy. - ret_val = PyString_AsString(ptype_obj); - Py_XDECREF(ptype_obj); - } - } - -done: - if (ret_val == "") { - ret_val = "unknown error"; - } - - // something might've gone wrong while fetching the error, ensure - // the error state is clean before returning - PyErr_Clear(); - - Py_XDECREF(traceback); - Py_XDECREF(format_exception); - Py_XDECREF(fmt_exc); - Py_XDECREF(ptype); - Py_XDECREF(pvalue); - Py_XDECREF(ptraceback); - return ret_val; -} - -bool Two::getAttrString(RtLoaderPyObject *obj, const char *attributeName, char *&value) const -{ - if (obj == NULL) { - return false; - } - - bool res = false; - PyObject *py_attr = NULL; - PyObject *py_obj = reinterpret_cast(obj); - - py_attr = PyObject_GetAttrString(py_obj, attributeName); - if (py_attr != NULL && PyString_Check(py_attr)) { - value = as_string(py_attr); - if (value == NULL) { - // as_string clears the error, so we can't fetch it here - setError("error converting attribute " + std::string(attributeName) + " to string"); - } else { - res = true; - } - } else if (py_attr != NULL && !PyString_Check(py_attr)) { - setError("error attribute " + std::string(attributeName) + " has a different type than string"); - PyErr_Clear(); - } else { - PyErr_Clear(); - } - - Py_XDECREF(py_attr); - return res; -} - -void Two::decref(RtLoaderPyObject *obj) -{ - Py_XDECREF(reinterpret_cast(obj)); -} - -void Two::incref(RtLoaderPyObject *obj) -{ - Py_XINCREF(reinterpret_cast(obj)); -} - -void Two::setModuleAttrString(char *module, char *attr, char *value) -{ - PyObject *py_module = PyImport_ImportModule(module); - if (!py_module) { - setError("error importing python '" + std::string(module) + "' module: " + _fetchPythonError()); - return; - } - - PyObject *py_value = PyStringFromCString(value); - if (PyObject_SetAttrString(py_module, attr, py_value) != 0) { - setError("error setting the '" + std::string(module) + "." + std::string(attr) - + "' attribute: " + _fetchPythonError()); - } - - Py_XDECREF(py_module); - Py_XDECREF(py_value); -} - -void Two::setSubmitMetricCb(cb_submit_metric_t cb) -{ - _set_submit_metric_cb(cb); -} - -void Two::setSubmitServiceCheckCb(cb_submit_service_check_t cb) -{ - _set_submit_service_check_cb(cb); -} - -void Two::setSubmitEventCb(cb_submit_event_t cb) -{ - _set_submit_event_cb(cb); -} - -void Two::setSubmitHistogramBucketCb(cb_submit_histogram_bucket_t cb) -{ - _set_submit_histogram_bucket_cb(cb); -} - -void Two::setSubmitEventPlatformEventCb(cb_submit_event_platform_event_t cb) -{ - _set_submit_event_platform_event_cb(cb); -} - -void Two::setGetVersionCb(cb_get_version_t cb) -{ - _set_get_version_cb(cb); -} - -void Two::setGetConfigCb(cb_get_config_t cb) -{ - _set_get_config_cb(cb); -} - -void Two::setHeadersCb(cb_headers_t cb) -{ - _set_headers_cb(cb); -} - -void Two::setGetHostnameCb(cb_get_hostname_t cb) -{ - _set_get_hostname_cb(cb); -} - -void Two::setGetHostTagsCb(cb_get_host_tags_t cb) -{ - _set_get_host_tags_cb(cb); -} - -void Two::setGetClusternameCb(cb_get_clustername_t cb) -{ - _set_get_clustername_cb(cb); -} - -void Two::setGetTracemallocEnabledCb(cb_tracemalloc_enabled_t cb) -{ - _set_tracemalloc_enabled_cb(cb); -} - -void Two::setLogCb(cb_log_t cb) -{ - _set_log_cb(cb); -} - -void Two::setSendLogCb(cb_send_log_t cb) -{ - _set_send_log_cb(cb); -} - -void Two::setSetCheckMetadataCb(cb_set_check_metadata_t cb) -{ - _set_set_check_metadata_cb(cb); -} - -void Two::setSetExternalTagsCb(cb_set_external_tags_t cb) -{ - _set_set_external_tags_cb(cb); -} - -void Two::setSubprocessOutputCb(cb_get_subprocess_output_t cb) -{ - _set_get_subprocess_output_cb(cb); -} - -void Two::setCGOFreeCb(cb_cgo_free_t cb) -{ - _set_cgo_free_cb(cb); -} - -void Two::setTagsCb(cb_tags_t cb) -{ - _set_tags_cb(cb); -} - -void Two::setGetConnectionInfoCb(cb_get_connection_info_t cb) -{ - _set_get_connection_info_cb(cb); -} - -void Two::setIsExcludedCb(cb_is_excluded_t cb) -{ - _set_is_excluded_cb(cb); -} - -void Two::setWritePersistentCacheCb(cb_write_persistent_cache_t cb) -{ - _set_write_persistent_cache_cb(cb); -} - -void Two::setReadPersistentCacheCb(cb_read_persistent_cache_t cb) -{ - _set_read_persistent_cache_cb(cb); -} - -void Two::setObfuscateSqlCb(cb_obfuscate_sql_t cb) -{ - _set_obfuscate_sql_cb(cb); -} - -void Two::setObfuscateSqlExecPlanCb(cb_obfuscate_sql_exec_plan_t cb) -{ - _set_obfuscate_sql_exec_plan_cb(cb); -} - -void Two::setGetProcessStartTimeCb(cb_get_process_start_time_t cb) -{ - _set_get_process_start_time_cb(cb); -} - -void Two::setObfuscateMongoDBStringCb(cb_obfuscate_mongodb_string_t cb) -{ - _set_obfuscate_mongodb_string_cb(cb); -} - -void Two::setEmitAgentTelemetryCb(cb_emit_agent_telemetry_t cb) -{ - _set_emit_agent_telemetry_cb(cb); -} - -// Python Helpers - -// get_integration_list return a list of every datadog's wheels installed. -char *Two::getIntegrationList() -{ - PyObject *pyPackages = NULL; - PyObject *pkgLister = NULL; - PyObject *args = NULL; - PyObject *packages = NULL; - char *wheels = NULL; - - rtloader_gilstate_t state = GILEnsure(); - - pyPackages = PyImport_ImportModule("datadog_checks.base.utils.agent.packages"); - if (pyPackages == NULL) { - setError("could not import datadog_checks.base.utils.agent.packages: " + _fetchPythonError()); - goto done; - } - - pkgLister = PyObject_GetAttrString(pyPackages, "get_datadog_wheels"); - if (pkgLister == NULL) { - setError("could not fetch get_datadog_wheels attr: " + _fetchPythonError()); - goto done; - } - - args = PyTuple_New(0); - if (args == NULL) { - setError("could not initialize args to empty tuple: " + _fetchPythonError()); - goto done; - } - - packages = PyObject_Call(pkgLister, args, NULL); - if (packages == NULL) { - setError("error fetching wheels list: " + _fetchPythonError()); - goto done; - } - - if (PyList_Check(packages) == 0) { - setError("'get_datadog_wheels' did not return a list"); - goto done; - } - - wheels = as_yaml(packages); - if (wheels == NULL) { - setError("'packages' could not be serialized to yaml: " + _fetchPythonError()); - goto done; - } - -done: - Py_XDECREF(pyPackages); - Py_XDECREF(pkgLister); - Py_XDECREF(args); - Py_XDECREF(packages); - GILRelease(state); - - return wheels; -} - -// getInterpreterMemoryUsage return a dict with the python interpreters memory -// usage snapshot. The returned dict must be freed by calling.... -char *Two::getInterpreterMemoryUsage() -{ - PyObject *pyMemory = NULL; - PyObject *memSummary = NULL; - PyObject *args = NULL; - PyObject *summary = NULL; - char *memUsage = NULL; - - rtloader_gilstate_t state = GILEnsure(); - - pyMemory = PyImport_ImportModule(_PY_MEM_MODULE); - if (pyMemory == NULL) { - setError("could not import " _PY_MEM_MODULE ": " + _fetchPythonError()); - goto done; - } - - memSummary = PyObject_GetAttrString(pyMemory, _PY_MEM_SUMMARY_FUNC); - if (memSummary == NULL) { - setError("could not fetch " _PY_MEM_SUMMARY_FUNC ": " + _fetchPythonError()); - goto done; - } - - // an empty tuple *must* be used when no arguments are used - args = PyTuple_New(0); - if (args == NULL) { - setError("could not initialize args to empty tuple: " + _fetchPythonError()); - goto done; - } - - summary = PyObject_Call(memSummary, args, NULL); - if (summary == NULL) { - setError("error fetching interpreter memory usage: " + _fetchPythonError()); - goto done; - } - - if (PyDict_Check(summary) == 0) { - setError("'" _PY_MEM_SUMMARY_FUNC "' did not return a dictionary"); - goto done; - } - - memUsage = as_yaml(summary); - if (memUsage == NULL) { - setError("'packages' could not be serialized to yaml: " + _fetchPythonError()); - goto done; - } - -done: - Py_XDECREF(summary); - Py_XDECREF(args); - Py_XDECREF(memSummary); - Py_XDECREF(pyMemory); - GILRelease(state); - - return memUsage; -} diff --git a/rtloader/two/two.h b/rtloader/two/two.h deleted file mode 100644 index 88137ad16e4f7..0000000000000 --- a/rtloader/two/two.h +++ /dev/null @@ -1,194 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog -// (https://www.datadoghq.com/). -// Copyright 2019-present Datadog, Inc. -#ifndef DATADOG_AGENT_RTLOADER_TWO_H -#define DATADOG_AGENT_RTLOADER_TWO_H - -// Some preprocessor sanity for builds (2+3 common sources) -#ifndef DATADOG_AGENT_TWO -# error Build requires defining DATADOG_AGENT_TWO -#elif defined(DATADOG_AGENT_TWO) && defined(DATADOG_AGENT_THREE) -# error "DATADOG_AGENT_TWO and DATADOG_AGENT_THREE are mutually exclusive - define only one of the two." -#endif - -#include -#include -#include - -#include -#include - -class Two : public RtLoader -{ -public: - //! Constructor. - /*! - \param python_home A C-string with the path to the python home for the - python interpreter. - \param python_exe A C-string with the path to the python interpreter. - - Basic constructor, initializes the _error string to an empty string and - errorFlag to false and set the supplied PYTHONHOME. - */ - Two(const char *python_home, const char *python_exe, cb_memory_tracker_t memtrack_cb); - - //! Destructor. - /*! - Destroys the Two instance, including relevant python teardown calls. - - We do not call Py_Finalize() since we won't be calling it from the same - thread where we called Py_Initialize(), this is a product of the go runtime - switch threads constantly. It's not really an issue here as we destroy this - class instance just before exiting the agent. - Calling Py_Finalize from a different thread cause the "threading" - package to raise an exception: "Exception KeyError: KeyError(,) in (Py_None); - } - - // Python Helpers - char *getIntegrationList(); - char *getInterpreterMemoryUsage(); - - // aggregator API - void setSubmitMetricCb(cb_submit_metric_t); - void setSubmitServiceCheckCb(cb_submit_service_check_t); - void setSubmitEventCb(cb_submit_event_t); - void setSubmitHistogramBucketCb(cb_submit_histogram_bucket_t); - void setSubmitEventPlatformEventCb(cb_submit_event_platform_event_t); - - // datadog_agent API - void setGetVersionCb(cb_get_version_t); - void setGetConfigCb(cb_get_config_t); - void setHeadersCb(cb_headers_t); - void setGetHostnameCb(cb_get_hostname_t); - void setGetHostTagsCb(cb_get_host_tags_t); - void setGetClusternameCb(cb_get_clustername_t); - void setGetTracemallocEnabledCb(cb_tracemalloc_enabled_t); - void setLogCb(cb_log_t); - void setSendLogCb(cb_send_log_t); - void setSetCheckMetadataCb(cb_set_check_metadata_t); - void setSetExternalTagsCb(cb_set_external_tags_t); - void setWritePersistentCacheCb(cb_write_persistent_cache_t); - void setReadPersistentCacheCb(cb_read_persistent_cache_t); - void setObfuscateSqlCb(cb_obfuscate_sql_t); - void setObfuscateSqlExecPlanCb(cb_obfuscate_sql_exec_plan_t); - void setGetProcessStartTimeCb(cb_get_process_start_time_t); - void setObfuscateMongoDBStringCb(cb_obfuscate_mongodb_string_t); - void setEmitAgentTelemetryCb(cb_emit_agent_telemetry_t); - - // _util API - virtual void setSubprocessOutputCb(cb_get_subprocess_output_t); - - // CGO API - void setCGOFreeCb(cb_cgo_free_t); - - // tagger - void setTagsCb(cb_tags_t); - - // kubeutil - void setGetConnectionInfoCb(cb_get_connection_info_t); - - // containers - void setIsExcludedCb(cb_is_excluded_t); - -private: - //! initPythonHome member. - /*! - \brief This member function sets the Python home for the underlying python2.7 interpreter. - \param pythonHome A C-string to the target python home for the python runtime. - */ - void initPythonHome(const char *pythonHome = NULL); - - //! initPythonExe member. - /*! - \brief This member function sets the path to the underlying python2 interpreter. - \param python_exe A C-string to the target python executable. - */ - void initPythonExe(const char *python_exe = NULL); - - //! _importFrom member. - /*! - \brief This member function imports a Python object by name from the specified - module. - \param module A C-string representation of the Python module we wish to import from. - \param name A C-string representation of the target Python object we wish to import. - \return A PyObject * pointer to the imported Python object, or NULL in case of error. - - This function returns a new reference to the underlying PyObject. In case of error, - NULL is returned with clean interpreter error flag. - */ - PyObject *_importFrom(const char *module, const char *name); - - //! _findSubclassOf member. - /*! - \brief This member function attemts to find a subclass of the provided base class in - the specified Python module. - \param base A PyObject * pointer to the Python base class we wish to search for. - \param moduleName A PyObject * pointer to the module we wish to find a derived class - in. - \return A PyObject * pointer to the found subclass Python object, or NULL in case of error. - - This function returns a new reference to the underlying PyObject. In case of error, - NULL is returned with clean interpreter error flag. - */ - PyObject *_findSubclassOf(PyObject *base, PyObject *moduleName); - - //! _fetchPythonError member. - /*! - \brief This member function retrieves the error set on the python interpreter. - \return A string describing the python error/exception set on the underlying python - interpreter. - */ - std::string _fetchPythonError(); - - /*! PyPaths type prototype - \typedef PyPaths defines a vector of strings. - */ - typedef std::vector PyPaths; - - char *_pythonHome; /*!< string with the PYTHONHOME for the underlying interpreter */ - char *_pythonExe; /*!< string with the path to the executable of the underlying interpreter */ - PyObject *_baseClass; /*!< PyObject * pointer to the base Agent check class */ - PyPaths _pythonPaths; /*!< string vector containing paths in the PYTHONPATH */ - PyThreadState *_threadState; /*!< PyThreadState * pointer to the saved Python interpreter thread state */ -}; - -#endif diff --git a/tasks/gotest.py b/tasks/gotest.py index cf0a5efd540b7..abdbdfb86a8d1 100644 --- a/tasks/gotest.py +++ b/tasks/gotest.py @@ -32,7 +32,7 @@ from tasks.libs.common.datadog_api import create_count, send_metrics from tasks.libs.common.git import get_modified_files from tasks.libs.common.junit_upload_core import enrich_junitxml, produce_junit_tar -from tasks.libs.common.utils import clean_nested_paths, get_build_flags, gitlab_section +from tasks.libs.common.utils import clean_nested_paths, get_build_flags, gitlab_section, running_in_ci from tasks.libs.releasing.json import _get_release_json_value from tasks.modules import DEFAULT_MODULES, GoModule, get_module_by_path from tasks.test_core import ModuleTestResult, process_input_args, process_module_results, test_core @@ -215,7 +215,10 @@ def process_test_result(test_results, junit_tar: str, flavor: AgentFlavor, test_ print(color_message("All tests passed", "green")) return True - if test_washer: + if test_washer or running_in_ci(): + if not test_washer: + print("Test washer is always enabled in the CI, enforcing it") + tw = TestWasher() should_succeed = tw.process_module_results(test_results) if should_succeed: diff --git a/tasks/kmt.py b/tasks/kmt.py index 436ce486e662a..fb5ada35894a8 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -1023,13 +1023,6 @@ def kmt_sysprobe_prepare( variables=variables, ) - if pkg.endswith("java"): - nw.build( - inputs=[os.path.join(pkg, "agent-usm.jar")], - outputs=[os.path.join(target_path, "agent-usm.jar")], - rule="copyfiles", - ) - # handle testutils and testdata separately since they are # shared across packages target_pkgs = build_target_packages([], build_tags) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index d91b2649d27ab..6809892feac7e 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -121,7 +121,7 @@ def env_filter(item): "HOSTNAME", "HOST_IP", "INFOPATH", - "INSTALL_SCRIPT_API_KEY", + "INSTALL_SCRIPT_API_KEY_ORG2", "INTEGRATION_WHEELS_CACHE_BUCKET", "IRBRC", "KITCHEN_INFRASTRUCTURE_FLAKES_RETRY", @@ -141,7 +141,6 @@ def env_filter(item): "PROCESS_S3_BUCKET", "PWD", "PROMPT", - "PYTHON_RUNTIMES", "RESTORE_CACHE_ATTEMPTS", "RUSTC_SHA256", "SIGN", @@ -234,9 +233,7 @@ def should_retry_bundle_install(res): def send_build_metrics(ctx, overall_duration): # We only want to generate those metrics from the CI src_dir = os.environ.get('CI_PROJECT_DIR') - aws_cmd = "aws" if sys.platform == 'win32': - aws_cmd = "aws.cmd" if src_dir is None: src_dir = os.environ.get("REPO_ROOT", os.getcwd()) @@ -317,10 +314,16 @@ def send_build_metrics(ctx, overall_duration): 'type': 0, } ) - dd_api_key = ctx.run( - f'{aws_cmd} ssm get-parameter --region us-east-1 --name {os.environ["API_KEY_ORG2"]} --with-decryption --query "Parameter.Value" --out text', - hide=True, - ).stdout.strip() + if sys.platform == 'win32': + dd_api_key = ctx.run( + f'aws.cmd ssm get-parameter --region us-east-1 --name {os.environ["API_KEY_ORG2"]} --with-decryption --query "Parameter.Value" --out text', + hide=True, + ).stdout.strip() + else: + dd_api_key = ctx.run( + f'vault kv get -field=token kv/k8s/gitlab-runner/datadog-agent/{os.environ["AGENT_API_KEY_ORG2"]}', + hide=True, + ).stdout.strip() headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'DD-API-KEY': dd_api_key} r = requests.post("https://api.datadoghq.com/api/v2/series", json={'series': series}, headers=headers) if r.ok: @@ -332,13 +335,15 @@ def send_build_metrics(ctx, overall_duration): def send_cache_miss_event(ctx, pipeline_id, job_name, job_id): if sys.platform == 'win32': - aws_cmd = "aws.cmd" + dd_api_key = ctx.run( + f'aws.cmd ssm get-parameter --region us-east-1 --name {os.environ["API_KEY_ORG2"]} --with-decryption --query "Parameter.Value" --out text', + hide=True, + ).stdout.strip() else: - aws_cmd = "aws" - dd_api_key = ctx.run( - f'{aws_cmd} ssm get-parameter --region us-east-1 --name {os.environ["API_KEY_ORG2"]} --with-decryption --query "Parameter.Value" --out text', - hide=True, - ).stdout.strip() + dd_api_key = ctx.run( + f'vault kv get -field=token kv/k8s/gitlab-runner/datadog-agent/{os.environ["AGENT_API_KEY_ORG2"]}', + hide=True, + ).stdout.strip() headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'DD-API-KEY': dd_api_key} payload = { 'title': 'omnibus cache miss', diff --git a/tasks/libs/common/utils.py b/tasks/libs/common/utils.py index 9b8138176a624..3b3afbc80c027 100644 --- a/tasks/libs/common/utils.py +++ b/tasks/libs/common/utils.py @@ -337,6 +337,11 @@ def get_build_flags( if os.getenv('DD_CXX'): env['CXX'] = os.getenv('DD_CXX') + if sys.platform == 'linux': + # Enable lazy binding, which seems to be overridden when loading containerd + # Required to fix go-nvml compilation (see https://github.com/NVIDIA/go-nvml/issues/18) + extldflags += "-Wl,-z,lazy " + if extldflags: ldflags += f"'-extldflags={extldflags}' " @@ -669,7 +674,6 @@ def file_match(word): 'internal', 'omnibus', 'pkg', - 'pkg-config', 'rtloader', 'tasks', 'test', diff --git a/tasks/linter.py b/tasks/linter.py index 02e7a52c428d0..082c3065cc666 100644 --- a/tasks/linter.py +++ b/tasks/linter.py @@ -667,6 +667,7 @@ def job_change_path(ctx, job_files=None): 'new-e2e-package-signing-debian-a7-x86_64', 'new-e2e-package-signing-suse-a7-x86_64', 'new-e2e_windows_powershell_module_test', + 'new-e2e-eks-cleanup-on-failure', 'trigger-flakes-finder', } diff --git a/tasks/pipeline.py b/tasks/pipeline.py index 1bfad63fb27ae..9a0ff2b3821d3 100644 --- a/tasks/pipeline.py +++ b/tasks/pipeline.py @@ -889,11 +889,11 @@ def trigger_external(ctx, owner_branch_name: str, no_verify=False): owner_branch_name = owner_branch_name.lower() assert ( - owner_branch_name.count('/') == 1 + owner_branch_name.count('/') >= 1 ), f'owner_branch_name should be "/" but is {owner_branch_name}' assert "'" not in owner_branch_name - owner, branch = owner_branch_name.split('/') + owner, branch = owner_branch_name.split('/', 1) no_verify_flag = ' --no-verify' if no_verify else '' # Can checkout diff --git a/tasks/release.py b/tasks/release.py index 52dbc4fde4be9..57ed76ba1671f 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -1043,7 +1043,7 @@ def create_qa_cards(ctx, tag): """ from tasks.libs.releasing.qa import get_labels, setup_ddqa - version = _create_version_from_match(RC_VERSION_RE.match(tag)) + version = _create_version_from_match(VERSION_RE.match(tag)) if not version.rc: print(f"{tag} is not a release candidate, skipping") return diff --git a/tasks/security_agent.py b/tasks/security_agent.py index 1a75915295a02..2dc906d6d318f 100644 --- a/tasks/security_agent.py +++ b/tasks/security_agent.py @@ -159,7 +159,7 @@ def build_dev_image(ctx, image=None, push=False, base_image="datadog/agent:lates ctx.run(f"touch {docker_context}/agent") core_agent_dest = "/dev/null" - copy_ebpf_and_related_files(ctx, docker_context, copy_usm_jar=False) + copy_ebpf_and_related_files(ctx, docker_context) with ctx.cd(docker_context): # --pull in the build will force docker to grab the latest base image @@ -610,8 +610,6 @@ def docker_functional_tests( cmd += '-v ./pkg/security/tests:/tests {image_tag} sleep 3600' args = { - "GOPATH": get_gopath(ctx), - "REPO_PATH": REPO_PATH, "container_name": container_name, "caps": ' '.join(f"--cap-add {cap}" for cap in capabilities), "image_tag": image_tag, diff --git a/tasks/system_probe.py b/tasks/system_probe.py index 6b8da8f057bee..88cadc19603b8 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -68,7 +68,6 @@ } CWS_PREBUILT_MINIMUM_KERNEL_VERSION = (5, 8, 0) EMBEDDED_SHARE_DIR = os.path.join("/opt", "datadog-agent", "embedded", "share", "system-probe", "ebpf") -EMBEDDED_SHARE_JAVA_DIR = os.path.join("/opt", "datadog-agent", "embedded", "share", "system-probe", "java") is_windows = sys.platform == "win32" is_macos = sys.platform == "darwin" @@ -969,9 +968,6 @@ def kitchen_prepare(ctx, kernel_release=None, ci=False, packages=""): if os.path.isdir(extra_path): shutil.copytree(extra_path, os.path.join(target_path, extra)) - if pkg.endswith("java"): - shutil.copy(os.path.join(pkg, "agent-usm.jar"), os.path.join(target_path, "agent-usm.jar")) - for gobin in [ "external_unix_proxy_server", "fmapper", @@ -1440,10 +1436,6 @@ def build_object_files( sudo = "" if is_root() else "sudo" ctx.run(f"{sudo} mkdir -p {EMBEDDED_SHARE_DIR}") - java_dir = os.path.join("pkg", "network", "protocols", "tls", "java") - ctx.run(f"{sudo} mkdir -p {EMBEDDED_SHARE_JAVA_DIR}") - ctx.run(f"{sudo} install -m644 -oroot -groot {java_dir}/agent-usm.jar {EMBEDDED_SHARE_JAVA_DIR}/agent-usm.jar") - if ctx.run("command -v rsync >/dev/null 2>&1", warn=True, hide=True).ok: rsync_filter = "--filter='+ */' --filter='+ *.o' --filter='+ *.c' --filter='- *'" ctx.run( @@ -1902,8 +1894,6 @@ def _test_docker_image_list(): for component in docker_compose["services"]: images.add(docker_compose["services"][component]["image"]) - # Special use-case in javatls - images.remove("${IMAGE_VERSION}") # Temporary: GoTLS monitoring inside containers tests are flaky in the CI, so at the meantime, the tests are # disabled, so we can skip downloading a redundant image. images.remove("public.ecr.aws/b1o7r7e0/usm-team/go-httpbin:https") @@ -2001,7 +1991,7 @@ def save_build_outputs(ctx, destfile): ctx.run(f"sha256sum {outfile} >> {absdest}.sum") -def copy_ebpf_and_related_files(ctx: Context, target: Path | str, arch: Arch | None = None, copy_usm_jar: bool = True): +def copy_ebpf_and_related_files(ctx: Context, target: Path | str, arch: Arch | None = None): if arch is None: arch = Arch.local() @@ -2014,6 +2004,3 @@ def copy_ebpf_and_related_files(ctx: Context, target: Path | str, arch: Arch | N ctx.run(f"chmod 0444 {target}/*.o {target}/*.c {target}/co-re/*.o") ctx.run(f"cp /opt/datadog-agent/embedded/bin/clang-bpf {target}") ctx.run(f"cp /opt/datadog-agent/embedded/bin/llc-bpf {target}") - - if copy_usm_jar: - ctx.run(f"cp pkg/network/protocols/tls/java/agent-usm.jar {target}") diff --git a/tasks/testwasher.py b/tasks/testwasher.py index 93e74d997ee50..96eac587e403b 100644 --- a/tasks/testwasher.py +++ b/tasks/testwasher.py @@ -2,10 +2,11 @@ import copy import json +import os from collections import defaultdict import yaml -from invoke import task +from invoke import Exit, task from tasks.libs.ciproviders.gitlab_api import ( resolve_gitlab_ci_configuration, @@ -128,13 +129,21 @@ def process_module_results(self, module_results: list[ModuleTestResult]): @task -def generate_flake_finder_pipeline(ctx, n=3): +def generate_flake_finder_pipeline(ctx, n=3, generate_config=False): """ Generate a child pipeline where jobs marked with SHOULD_RUN_IN_FLAKES_FINDER are run n times """ - - # Read gitlab config - config = resolve_gitlab_ci_configuration(ctx, ".gitlab-ci.yml") + if generate_config: + # Read gitlab config + config = resolve_gitlab_ci_configuration(ctx, ".gitlab-ci.yml") + else: + # Read gitlab config, which is computed and stored in compute_gitlab_ci_config job + if not os.path.exists("artifacts/after.gitlab-ci.yml"): + raise Exit( + "The configuration is not stored as artifact. Please ensure you ran the compute_gitlab_ci_config job, or set generate_config to True" + ) + with open("artifacts/after.gitlab-ci.yml") as f: + config = yaml.safe_load(f)[".gitlab-ci.yml"] # Lets keep only variables and jobs with flake finder variable kept_job = {} diff --git a/tasks/unit_tests/linter_tests.py b/tasks/unit_tests/linter_tests.py index 715b93e802db8..7178e9010f891 100644 --- a/tasks/unit_tests/linter_tests.py +++ b/tasks/unit_tests/linter_tests.py @@ -51,7 +51,7 @@ def test_with_wrapper_no_env(self): def test_with_wrapper_with_env(self): with open(self.test_file, "w") as f: f.write( - "DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $APP_KEY_ORG2) || exit $?; export DD_APP_KEY" + "DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_APP_KEY_ORG2 token) || exit $?; export DD_APP_KEY" ) matched = linter.list_get_parameter_calls(self.test_file) self.assertListEqual([], matched) diff --git a/tasks/unit_tests/omnibus_tests.py b/tasks/unit_tests/omnibus_tests.py index 1aeaf35e411f7..dba84635d9503 100644 --- a/tasks/unit_tests/omnibus_tests.py +++ b/tasks/unit_tests/omnibus_tests.py @@ -42,6 +42,7 @@ def _run_calls_to_string(mock_calls): 'RELEASE_VERSION_7': 'nightly', 'S3_OMNIBUS_CACHE_BUCKET': 'omnibus-cache', 'API_KEY_ORG2': 'api-key', + 'AGENT_API_KEY_ORG2': 'agent-api-key', }, clear=True, ) @@ -60,6 +61,7 @@ def _set_up_default_command_mocks(self): (r'go mod .*', Result()), (r'grep .*', Result()), (r'aws ssm .*', Result()), + (r'vault kv get .*', Result()), ] for pattern, result in patterns: self.mock_ctx.set_result_for('run', re.compile(pattern), result) diff --git a/tasks/winbuildscripts/unittests.bat b/tasks/winbuildscripts/unittests.bat index ab7eb626ed44a..9673769c4c329 100644 --- a/tasks/winbuildscripts/unittests.bat +++ b/tasks/winbuildscripts/unittests.bat @@ -15,7 +15,10 @@ xcopy /e/s/h/q c:\mnt\*.* call %TEST_ROOT%\datadog-agent\tasks\winbuildscripts\extract-modcache.bat %TEST_ROOT%\datadog-agent modcache call %TEST_ROOT%\datadog-agent\tasks\winbuildscripts\extract-modcache.bat %TEST_ROOT%\datadog-agent modcache_tools -Powershell -C "%TEST_ROOT%\datadog-agent\tasks\winbuildscripts\unittests.ps1" || exit /b 2 +Powershell -File "%TEST_ROOT%\datadog-agent\tasks\winbuildscripts\unittests.ps1" +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) goto :EOF diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index 64840df96df3a..6406e7c61bb8f 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -59,7 +59,7 @@ require ( // `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version // Example: github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB // => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB - github.com/DataDog/test-infra-definitions v0.0.0-20241010155348-7e55b9e3279a + github.com/DataDog/test-infra-definitions v0.0.0-20241015101741-f694c4dc33e4 github.com/aws/aws-sdk-go-v2 v1.32.0 github.com/aws/aws-sdk-go-v2/config v1.27.40 github.com/aws/aws-sdk-go-v2/service/ec2 v1.164.2 @@ -75,7 +75,7 @@ require ( github.com/pkg/sftp v1.13.6 github.com/pulumi/pulumi-aws/sdk/v6 v6.54.2 github.com/pulumi/pulumi-awsx/sdk/v2 v2.14.0 - github.com/pulumi/pulumi-eks/sdk/v2 v2.7.8 + github.com/pulumi/pulumi-eks/sdk/v2 v2.7.8 // indirect github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.17.1 github.com/pulumi/pulumi/sdk/v3 v3.133.0 github.com/samber/lo v1.47.0 @@ -94,7 +94,7 @@ require ( ) require ( - dario.cat/mergo v1.0.0 // indirect + dario.cat/mergo v1.0.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/BurntSushi/toml v1.2.1 // indirect github.com/DataDog/datadog-agent/comp/netflow/payload v0.56.0-rc.3 // indirect @@ -252,11 +252,11 @@ require ( github.com/zclconf/go-cty v1.14.4 // indirect github.com/zorkian/go-datadog-api v2.30.0+incompatible go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/otel/sdk v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/atomic v1.11.0 // indirect golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index f4bb21313e204..28321f6b27f51 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -1,6 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -16,8 +16,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a h1:m9REhmyaWD5YJ0P53ygRHxKKo+KM+nw+zz0hEdKztMo= github.com/DataDog/mmh3 v0.0.0-20200805151601-30884ca2197a/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= -github.com/DataDog/test-infra-definitions v0.0.0-20241010155348-7e55b9e3279a h1:pUK99AnBBTVtAX5T+haaLftyjf0BIkFHTJHmAyE+fZI= -github.com/DataDog/test-infra-definitions v0.0.0-20241010155348-7e55b9e3279a/go.mod h1:spBV6LBmaH17vb/oe6n2kVfsuEUR+eQGsoaCGJclHF0= +github.com/DataDog/test-infra-definitions v0.0.0-20241015101741-f694c4dc33e4 h1:9P0Ecgmk+so7agQVftk6ojEs7qlquhGakJ1BMh3jZ7A= +github.com/DataDog/test-infra-definitions v0.0.0-20241015101741-f694c4dc33e4/go.mod h1:0YN66dG8119K5c1QDt11MUwYxov6Yd0JX9+kmj45yQE= github.com/DataDog/viper v1.13.5 h1:SZMcyMknYQN2jRY/40A16gUXexlNJOI8sDs1cWZnI64= github.com/DataDog/viper v1.13.5/go.mod h1:wDdUVJ2SHaMaPrCZrlRCObwkubsX8j5sme3LaR/SGTc= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= @@ -656,20 +656,20 @@ github.com/zorkian/go-datadog-api v2.30.0+incompatible/go.mod h1:PkXwHX9CUQa/FpB go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 h1:j9+03ymgYhPKmeXGk5Zu+cIZOlVzd9Zv7QIiyItjFBU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0/go.mod h1:Y5+XiUG4Emn1hTfciPzGPJaSI+RpDts6BnCIir0SLqk= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/eks.go b/test/new-e2e/pkg/environments/aws/kubernetes/eks.go index 6e3f5e6070577..873acf174a044 100644 --- a/test/new-e2e/pkg/environments/aws/kubernetes/eks.go +++ b/test/new-e2e/pkg/environments/aws/kubernetes/eks.go @@ -10,30 +10,19 @@ import ( "context" "fmt" - "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/common/utils" - "github.com/DataDog/test-infra-definitions/components" "github.com/DataDog/test-infra-definitions/components/datadog/agent/helm" dogstatsdstandalone "github.com/DataDog/test-infra-definitions/components/datadog/dogstatsd-standalone" fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" - kubeComp "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/DataDog/test-infra-definitions/resources/aws" - localEks "github.com/DataDog/test-infra-definitions/resources/aws/eks" + "github.com/DataDog/test-infra-definitions/scenarios/aws/eks" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" - "github.com/pulumi/pulumi-aws/sdk/v6/go/aws/ec2" - awsEks "github.com/pulumi/pulumi-aws/sdk/v6/go/aws/eks" - awsIam "github.com/pulumi/pulumi-aws/sdk/v6/go/aws/iam" - "github.com/pulumi/pulumi-eks/sdk/v2/go/eks" - "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" - appsv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1" - corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" - metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -79,296 +68,64 @@ func EKSRunFunc(ctx *pulumi.Context, env *environments.Kubernetes, params *Provi } } - clusterComp, err := components.NewComponent(&awsEnv, awsEnv.Namer.ResourceName("eks"), func(comp *kubeComp.Cluster) error { - // Create Cluster SG - clusterSG, err := ec2.NewSecurityGroup(ctx, awsEnv.Namer.ResourceName("eks-sg"), &ec2.SecurityGroupArgs{ - NamePrefix: awsEnv.CommonNamer().DisplayName(255, pulumi.String("eks-sg")), - Description: pulumi.StringPtr("EKS Cluster sg for stack: " + ctx.Stack()), - Ingress: ec2.SecurityGroupIngressArray{ - ec2.SecurityGroupIngressArgs{ - SecurityGroups: pulumi.ToStringArray(awsEnv.EKSAllowedInboundSecurityGroups()), - PrefixListIds: pulumi.ToStringArray(awsEnv.EKSAllowedInboundPrefixLists()), - ToPort: pulumi.Int(22), - FromPort: pulumi.Int(22), - Protocol: pulumi.String("tcp"), - }, - ec2.SecurityGroupIngressArgs{ - SecurityGroups: pulumi.ToStringArray(awsEnv.EKSAllowedInboundSecurityGroups()), - PrefixListIds: pulumi.ToStringArray(awsEnv.EKSAllowedInboundPrefixLists()), - ToPort: pulumi.Int(443), - FromPort: pulumi.Int(443), - Protocol: pulumi.String("tcp"), - }, - }, - VpcId: pulumi.StringPtr(awsEnv.DefaultVPCID()), - }, awsEnv.WithProviders(config.ProviderAWS)) - if err != nil { - return err + cluster, err := eks.NewCluster(awsEnv, params.name, params.eksOptions...) + if err != nil { + return err + } + + if err := cluster.Export(ctx, &env.KubernetesCluster.ClusterOutput); err != nil { + return err + } + + var fakeIntake *fakeintakeComp.Fakeintake + if params.fakeintakeOptions != nil { + fakeIntakeOptions := []fakeintake.Option{ + fakeintake.WithCPU(1024), + fakeintake.WithMemory(6144), + } + if awsEnv.GetCommonEnvironment().InfraShouldDeployFakeintakeWithLB() { + fakeIntakeOptions = append(fakeIntakeOptions, fakeintake.WithLoadBalancer()) } - // Cluster role - clusterRole, err := localEks.GetClusterRole(awsEnv, "eks-cluster-role") - if err != nil { + if fakeIntake, err = fakeintake.NewECSFargateInstance(awsEnv, "ecs", fakeIntakeOptions...); err != nil { return err } - - // IAM Node role - linuxNodeRole, err := localEks.GetNodeRole(awsEnv, "eks-linux-node-role") - if err != nil { + if err := fakeIntake.Export(awsEnv.Ctx(), &env.FakeIntake.FakeintakeOutput); err != nil { return err } + } else { + env.FakeIntake = nil + } - windowsNodeRole, err := localEks.GetNodeRole(awsEnv, "eks-windows-node-role") + // Deploy the agent + dependsOnSetup := utils.PulumiDependsOn(cluster) + if params.agentOptions != nil { + params.agentOptions = append(params.agentOptions, kubernetesagentparams.WithPulumiResourceOptions(dependsOnSetup), kubernetesagentparams.WithFakeintake(fakeIntake)) + kubernetesAgent, err := helm.NewKubernetesAgent(&awsEnv, "eks", cluster.KubeProvider, params.agentOptions...) if err != nil { return err } - - // Fargate Configuration - var fargateProfile pulumi.Input - if fargateNamespace := awsEnv.EKSFargateNamespace(); fargateNamespace != "" { - fargateProfile = pulumi.Any( - eks.FargateProfile{ - Selectors: []awsEks.FargateProfileSelector{ - { - Namespace: fargateNamespace, - }, - }, - }, - ) - } - - // Create an EKS cluster with the default configuration. - cluster, err := eks.NewCluster(ctx, awsEnv.Namer.ResourceName("eks"), &eks.ClusterArgs{ - Name: awsEnv.CommonNamer().DisplayName(100), - Version: pulumi.StringPtr(awsEnv.KubernetesVersion()), - EndpointPrivateAccess: pulumi.BoolPtr(true), - EndpointPublicAccess: pulumi.BoolPtr(false), - Fargate: fargateProfile, - ClusterSecurityGroup: clusterSG, - NodeAssociatePublicIpAddress: pulumi.BoolRef(false), - PrivateSubnetIds: awsEnv.RandomSubnets(), - VpcId: pulumi.StringPtr(awsEnv.DefaultVPCID()), - SkipDefaultNodeGroup: pulumi.BoolRef(true), - // The content of the aws-auth map is the merge of `InstanceRoles` and `RoleMappings`. - // For managed node groups, we push the value in `InstanceRoles`. - // For unmanaged node groups, we push the value in `RoleMappings` - RoleMappings: eks.RoleMappingArray{ - eks.RoleMappingArgs{ - Groups: pulumi.ToStringArray([]string{"system:bootstrappers", "system:nodes", "eks:kube-proxy-windows"}), - Username: pulumi.String("system:node:{{EC2PrivateDNSName}}"), - RoleArn: windowsNodeRole.Arn, - }, - }, - InstanceRoles: awsIam.RoleArray{ - linuxNodeRole, - }, - ServiceRole: clusterRole, - ProviderCredentialOpts: &eks.KubeconfigOptionsArgs{ - ProfileName: pulumi.String(awsEnv.Profile()), - }, - }, pulumi.Timeouts(&pulumi.CustomTimeouts{ - Create: "30m", - Update: "30m", - Delete: "30m", - }), awsEnv.WithProviders(config.ProviderEKS, config.ProviderAWS)) + err = kubernetesAgent.Export(ctx, &env.Agent.KubernetesAgentOutput) if err != nil { return err } + } else { + env.Agent = nil + } - if awsEnv.InitOnly() { - return nil - } - - kubeConfig, err := cluster.GetKubeconfig(ctx, &eks.ClusterGetKubeconfigArgs{ - ProfileName: pulumi.String(awsEnv.Profile()), - }) - - if err != nil { + // Deploy standalone dogstatsd + if params.deployDogstatsd { + if _, err := dogstatsdstandalone.K8sAppDefinition(&awsEnv, cluster.KubeProvider, "dogstatsd-standalone", fakeIntake, true, ""); err != nil { return err } + } - // Building Kubernetes provider - eksKubeProvider, err := kubernetes.NewProvider(awsEnv.Ctx(), awsEnv.Namer.ResourceName("k8s-provider"), &kubernetes.ProviderArgs{ - Kubeconfig: kubeConfig, - EnableServerSideApply: pulumi.BoolPtr(true), - DeleteUnreachable: pulumi.BoolPtr(true), - }, awsEnv.WithProviders(config.ProviderAWS)) + // Deploy workloads + for _, appFunc := range params.workloadAppFuncs { + _, err := appFunc(&awsEnv, cluster.KubeProvider) if err != nil { return err } - - // Filling Kubernetes component from EKS cluster - comp.ClusterName = cluster.EksCluster.Name() - comp.KubeConfig = kubeConfig - comp.KubeProvider = eksKubeProvider - - // Create configuration for POD subnets if any - workloadDeps := make([]pulumi.Resource, 0) - if podSubnets := awsEnv.EKSPODSubnets(); len(podSubnets) > 0 { - eniConfigs, err := localEks.NewENIConfigs(awsEnv, podSubnets, awsEnv.DefaultSecurityGroups(), pulumi.Provider(eksKubeProvider)) - if err != nil { - return err - } - - // Setting AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG is mandatory for EKS CNI to work with ENIConfig CRD - dsPatch, err := appsv1.NewDaemonSetPatch(awsEnv.Ctx(), awsEnv.Namer.ResourceName("eks-custom-network"), &appsv1.DaemonSetPatchArgs{ - Metadata: metav1.ObjectMetaPatchArgs{ - Namespace: pulumi.String("kube-system"), - Name: pulumi.String("aws-node"), - Annotations: pulumi.StringMap{ - "pulumi.com/patchForce": pulumi.String("true"), - }, - }, - Spec: appsv1.DaemonSetSpecPatchArgs{ - Template: corev1.PodTemplateSpecPatchArgs{ - Spec: corev1.PodSpecPatchArgs{ - Containers: corev1.ContainerPatchArray{ - corev1.ContainerPatchArgs{ - Name: pulumi.StringPtr("aws-node"), - Env: corev1.EnvVarPatchArray{ - corev1.EnvVarPatchArgs{ - Name: pulumi.String("AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG"), - Value: pulumi.String("true"), - }, - corev1.EnvVarPatchArgs{ - Name: pulumi.String("ENI_CONFIG_LABEL_DEF"), - Value: pulumi.String("topology.kubernetes.io/zone"), - }, - corev1.EnvVarPatchArgs{ - Name: pulumi.String("ENABLE_PREFIX_DELEGATION"), - Value: pulumi.String("true"), - }, - corev1.EnvVarPatchArgs{ - Name: pulumi.String("WARM_IP_TARGET"), - Value: pulumi.String("1"), - }, - corev1.EnvVarPatchArgs{ - Name: pulumi.String("MINIMUM_IP_TARGET"), - Value: pulumi.String("1"), - }, - }, - }, - }, - }, - }, - }, - }, pulumi.Provider(eksKubeProvider), utils.PulumiDependsOn(eniConfigs)) - if err != nil { - return err - } - - workloadDeps = append(workloadDeps, eniConfigs, dsPatch) - } - - // Create managed node groups - if params.eksLinuxNodeGroup { - ng, err := localEks.NewLinuxNodeGroup(awsEnv, cluster, linuxNodeRole) - if err != nil { - return err - } - workloadDeps = append(workloadDeps, ng) - } - - if params.eksLinuxARMNodeGroup { - ng, err := localEks.NewLinuxARMNodeGroup(awsEnv, cluster, linuxNodeRole) - if err != nil { - return err - } - workloadDeps = append(workloadDeps, ng) - } - - if params.eksBottlerocketNodeGroup { - ng, err := localEks.NewBottlerocketNodeGroup(awsEnv, cluster, linuxNodeRole) - if err != nil { - return err - } - workloadDeps = append(workloadDeps, ng) - } - - // Create unmanaged node groups - if params.eksWindowsNodeGroup { - _, err := localEks.NewWindowsNodeGroup(awsEnv, cluster, windowsNodeRole) - if err != nil { - return err - } - } - - // Applying necessary Windows configuration if Windows nodes - // Custom networking is not available for Windows nodes, using normal subnets IPs - if params.eksWindowsNodeGroup { - _, err := corev1.NewConfigMapPatch(awsEnv.Ctx(), awsEnv.Namer.ResourceName("eks-cni-cm"), &corev1.ConfigMapPatchArgs{ - Metadata: metav1.ObjectMetaPatchArgs{ - Namespace: pulumi.String("kube-system"), - Name: pulumi.String("amazon-vpc-cni"), - Annotations: pulumi.StringMap{ - "pulumi.com/patchForce": pulumi.String("true"), - }, - }, - Data: pulumi.StringMap{ - "enable-windows-ipam": pulumi.String("true"), - }, - }, pulumi.Provider(eksKubeProvider)) - if err != nil { - return err - } - } - - var fakeIntake *fakeintakeComp.Fakeintake - if params.fakeintakeOptions != nil { - fakeIntakeOptions := []fakeintake.Option{ - fakeintake.WithCPU(1024), - fakeintake.WithMemory(6144), - } - if awsEnv.GetCommonEnvironment().InfraShouldDeployFakeintakeWithLB() { - fakeIntakeOptions = append(fakeIntakeOptions, fakeintake.WithLoadBalancer()) - } - - if fakeIntake, err = fakeintake.NewECSFargateInstance(awsEnv, "ecs", fakeIntakeOptions...); err != nil { - return err - } - if err := fakeIntake.Export(awsEnv.Ctx(), &env.FakeIntake.FakeintakeOutput); err != nil { - return err - } - } else { - env.FakeIntake = nil - } - - // Deploy the agent - dependsOnSetup := utils.PulumiDependsOn(workloadDeps...) - if params.agentOptions != nil { - params.agentOptions = append(params.agentOptions, kubernetesagentparams.WithPulumiResourceOptions(dependsOnSetup), kubernetesagentparams.WithFakeintake(fakeIntake)) - kubernetesAgent, err := helm.NewKubernetesAgent(&awsEnv, "eks", eksKubeProvider, params.agentOptions...) - if err != nil { - return err - } - err = kubernetesAgent.Export(ctx, &env.Agent.KubernetesAgentOutput) - if err != nil { - return err - } - } else { - env.Agent = nil - } - - // Deploy standalone dogstatsd - if params.deployDogstatsd { - if _, err := dogstatsdstandalone.K8sAppDefinition(&awsEnv, eksKubeProvider, "dogstatsd-standalone", fakeIntake, true, ""); err != nil { - return err - } - } - - // Deploy workloads - for _, appFunc := range params.workloadAppFuncs { - _, err := appFunc(&awsEnv, eksKubeProvider) - if err != nil { - return err - } - } - - return nil - }) - if err != nil { - return err } - - return clusterComp.Export(ctx, &env.KubernetesCluster.ClusterOutput) + return nil } diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/params.go b/test/new-e2e/pkg/environments/aws/kubernetes/params.go index 1084e6f41bc32..abc939d523bfb 100644 --- a/test/new-e2e/pkg/environments/aws/kubernetes/params.go +++ b/test/new-e2e/pkg/environments/aws/kubernetes/params.go @@ -17,6 +17,7 @@ import ( kubeComp "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/DataDog/test-infra-definitions/resources/aws" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/DataDog/test-infra-definitions/scenarios/aws/eks" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" @@ -28,6 +29,7 @@ type ProvisionerParams struct { vmOptions []ec2.VMOption agentOptions []kubernetesagentparams.Option fakeintakeOptions []fakeintake.Option + eksOptions []eks.Option extraConfigParams runner.ConfigMap workloadAppFuncs []WorkloadAppFunc @@ -45,6 +47,7 @@ func newProvisionerParams() *ProvisionerParams { vmOptions: []ec2.VMOption{}, agentOptions: []kubernetesagentparams.Option{}, fakeintakeOptions: []fakeintake.Option{}, + eksOptions: []eks.Option{}, extraConfigParams: runner.ConfigMap{}, workloadAppFuncs: []WorkloadAppFunc{}, @@ -101,34 +104,10 @@ func WithFakeIntakeOptions(opts ...fakeintake.Option) ProvisionerOption { } } -// WithEKSLinuxNodeGroup enable Linux node group -func WithEKSLinuxNodeGroup() ProvisionerOption { +// WithEKSOptions adds options to the EKS cluster +func WithEKSOptions(opts ...eks.Option) ProvisionerOption { return func(params *ProvisionerParams) error { - params.eksLinuxNodeGroup = true - return nil - } -} - -// WithEKSLinuxARMNodeGroup enable ARM node group -func WithEKSLinuxARMNodeGroup() ProvisionerOption { - return func(params *ProvisionerParams) error { - params.eksLinuxARMNodeGroup = true - return nil - } -} - -// WithEKSBottlerocketNodeGroup enable AWS Bottle rocket node group -func WithEKSBottlerocketNodeGroup() ProvisionerOption { - return func(params *ProvisionerParams) error { - params.eksBottlerocketNodeGroup = true - return nil - } -} - -// WithEKSWindowsNodeGroup enable Windows node group -func WithEKSWindowsNodeGroup() ProvisionerOption { - return func(params *ProvisionerParams) error { - params.eksWindowsNodeGroup = true + params.eksOptions = opts return nil } } diff --git a/test/new-e2e/system-probe/test-runner/files/all_tests.json b/test/new-e2e/system-probe/test-runner/files/all_tests.json deleted file mode 100644 index d19097180afab..0000000000000 --- a/test/new-e2e/system-probe/test-runner/files/all_tests.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "*": { - "exclude": false - } -} diff --git a/test/new-e2e/system-probe/test-runner/files/cws_docker.json b/test/new-e2e/system-probe/test-runner/files/cws_docker.json new file mode 100644 index 0000000000000..c445dbc9b193c --- /dev/null +++ b/test/new-e2e/system-probe/test-runner/files/cws_docker.json @@ -0,0 +1,8 @@ +{ + "filters": { + "*": { + "exclude": false + } + }, + "testcontainer": "ghcr.io/datadog/apps-cws-centos7:main" +} diff --git a/test/new-e2e/system-probe/test-runner/files/cws_host.json b/test/new-e2e/system-probe/test-runner/files/cws_host.json new file mode 100644 index 0000000000000..03ac2f0fc2aeb --- /dev/null +++ b/test/new-e2e/system-probe/test-runner/files/cws_host.json @@ -0,0 +1,7 @@ +{ + "filters": { + "*": { + "exclude": false + } + } +} diff --git a/test/new-e2e/system-probe/test-runner/files/no_usm.json b/test/new-e2e/system-probe/test-runner/files/no_usm.json index f700502621fd3..60dcef477454a 100644 --- a/test/new-e2e/system-probe/test-runner/files/no_usm.json +++ b/test/new-e2e/system-probe/test-runner/files/no_usm.json @@ -1,8 +1,10 @@ { - "pkg/network/usm": { - "exclude": true - }, - "*": { - "exclude": false + "filters": { + "pkg/network/usm": { + "exclude": true + }, + "*": { + "exclude": false + } } } diff --git a/test/new-e2e/system-probe/test-runner/files/only_usm.json b/test/new-e2e/system-probe/test-runner/files/only_usm.json index 42ee8200ecf42..d63d56fbd9617 100644 --- a/test/new-e2e/system-probe/test-runner/files/only_usm.json +++ b/test/new-e2e/system-probe/test-runner/files/only_usm.json @@ -1,5 +1,7 @@ { - "pkg/network/usm": { - "exclude": false + "filters": { + "pkg/network/usm": { + "exclude": false + } } } diff --git a/test/new-e2e/system-probe/test-runner/main.go b/test/new-e2e/system-probe/test-runner/main.go index a7b21754bd50e..4d9c993406ce1 100644 --- a/test/new-e2e/system-probe/test-runner/main.go +++ b/test/new-e2e/system-probe/test-runner/main.go @@ -49,6 +49,12 @@ type testConfig struct { testingTools string extraParams string extraEnv string + inContainerImage string +} + +type userProvidedConfig struct { + PackagesRunConfig map[string]packageRunConfiguration `json:"filters"` + InContainerImage string `json:"testcontainer"` } const ciVisibility = "/ci-visibility" @@ -119,7 +125,7 @@ func glob(dir, filePattern string, filterFn func(path string) bool) ([]string, e return matches, nil } -func buildCommandArgs(pkg string, xmlpath string, jsonpath string, file string, testConfig *testConfig) []string { +func buildCommandArgs(pkg string, xmlpath string, jsonpath string, testArgs []string, testConfig *testConfig) []string { verbosity := "testname" if testConfig.verbose { verbosity = "standard-verbose" @@ -132,8 +138,15 @@ func buildCommandArgs(pkg string, xmlpath string, jsonpath string, file string, fmt.Sprintf("--rerun-fails=%d", testConfig.retryCount), "--rerun-fails-max-failures=100", "--raw-command", "--", - filepath.Join(testConfig.testingTools, "go/bin/test2json"), "-t", "-p", pkg, file, "-test.v", fmt.Sprintf("-test.count=%d", testConfig.runCount), "-test.timeout=" + getTimeout(pkg).String(), + filepath.Join(testConfig.testingTools, "go/bin/test2json"), "-t", "-p", pkg, } + args = append(args, testArgs...) + args = append( + args, + "-test.v", + fmt.Sprintf("-test.count=%d", testConfig.runCount), + "-test.timeout="+getTimeout(pkg).String(), + ) if testConfig.extraParams != "" { args = append(args, strings.Split(testConfig.extraParams, " ")...) @@ -213,6 +226,20 @@ func testPass(testConfig *testConfig, props map[string]string) error { } } + buildDir, err := getEBPFBuildDir() + if err != nil { + return fmt.Errorf("getEBPFBuildDir: %w", err) + } + bpfDir := filepath.Join(testConfig.testDirRoot, buildDir) + + var testContainer *testContainer + if testConfig.inContainerImage != "" { + testContainer = newTestContainer(testConfig.inContainerImage, bpfDir) + if err := testContainer.start(); err != nil { + return fmt.Errorf("error creating test container: %w", err) + } + } + for _, testsuite := range testsuites { pkg, err := filepath.Rel(testConfig.testDirRoot, filepath.Dir(testsuite)) if err != nil { @@ -221,17 +248,19 @@ func testPass(testConfig *testConfig, props map[string]string) error { junitfilePrefix := strings.ReplaceAll(pkg, "/", "-") xmlpath := filepath.Join(xmlDir, fmt.Sprintf("%s.xml", junitfilePrefix)) jsonpath := filepath.Join(jsonDir, fmt.Sprintf("%s.json", junitfilePrefix)) - args := buildCommandArgs(pkg, xmlpath, jsonpath, testsuite, testConfig) + + testsuiteArgs := []string{testsuite} + if testContainer != nil { + testsuiteArgs = testContainer.buildDockerExecArgs(testsuite, "--env", "docker") + } + + args := buildCommandArgs(pkg, xmlpath, jsonpath, testsuiteArgs, testConfig) cmd := exec.Command(filepath.Join(testConfig.testingTools, "go/bin/gotestsum"), args...) - buildDir, err := getEBPFBuildDir() - if err != nil { - return fmt.Errorf("getEBPFBuildDir: %w", err) - } baseEnv = append( baseEnv, - "DD_SYSTEM_PROBE_BPF_DIR="+filepath.Join(testConfig.testDirRoot, buildDir), + "DD_SYSTEM_PROBE_BPF_DIR="+bpfDir, "DD_SERVICE_MONITORING_CONFIG_TLS_JAVA_DIR="+filepath.Join(testConfig.testDirRoot, "pkg/network/protocols/tls/java"), ) if testConfig.extraEnv != "" { @@ -284,7 +313,7 @@ func buildTestConfiguration() (*testConfig, error) { flag.Parse() - breakdown := make(map[string]packageRunConfiguration) + var userConfig userProvidedConfig if *packageRunConfigPtr != "" { configData, err := os.ReadFile(*packageRunConfigPtr) if err != nil { @@ -294,7 +323,7 @@ func buildTestConfiguration() (*testConfig, error) { dec := json.NewDecoder(bytes.NewReader(configData)) dec.DisallowUnknownFields() - if err := dec.Decode(&breakdown); err != nil { + if err := dec.Decode(&userConfig); err != nil { return nil, err } } @@ -314,11 +343,12 @@ func buildTestConfiguration() (*testConfig, error) { runCount: *runCount, verbose: *verbose, retryCount: *retryPtr, - packagesRunConfig: breakdown, + packagesRunConfig: userConfig.PackagesRunConfig, testDirRoot: root, testingTools: tools, extraParams: *extraParams, extraEnv: *extraEnv, + inContainerImage: userConfig.InContainerImage, }, nil } diff --git a/test/new-e2e/system-probe/test-runner/testcontainer.go b/test/new-e2e/system-probe/test-runner/testcontainer.go new file mode 100644 index 0000000000000..1901432a296c0 --- /dev/null +++ b/test/new-e2e/system-probe/test-runner/testcontainer.go @@ -0,0 +1,95 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build linux + +package main + +import ( + "fmt" + "os" + "os/exec" +) + +const containerName = "kmt-test-container" + +type testContainer struct { + image string + bpfDir string +} + +func newTestContainer(image, bpfDir string) *testContainer { + return &testContainer{ + image: image, + bpfDir: bpfDir, + } +} + +func (ctc *testContainer) runDockerCmd(args []string) error { + cmd := exec.Command("docker", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func (ctc *testContainer) start() error { + args := []string{ + "run", + "--name", containerName, + "--privileged", + "--detach", + } + + var capabilities = []string{"SYS_ADMIN", "SYS_RESOURCE", "SYS_PTRACE", "NET_ADMIN", "IPC_LOCK", "ALL"} + for _, cap := range capabilities { + args = append(args, "--cap-add", cap) + } + + var mounts = []string{ + "/dev:/dev", + "/proc:/host/proc", + "/etc:/host/etc", + "/sys:/host/sys", + "/etc/os-release:/host/etc/os-release", + "/usr/lib/os-release:/host/usr/lib/os-release", + "/etc/passwd:/etc/passwd", + "/etc/group:/etc/group", + "/opt/datadog-agent/embedded/:/opt/datadog-agent/embedded/", + "/opt/kmt-ramfs:/opt/kmt-ramfs", + fmt.Sprintf("%s:/opt/bpf", ctc.bpfDir), + } + for _, mount := range mounts { + args = append(args, "-v", mount) + } + + var envs = []string{ + "HOST_PROC=/host/proc", + "HOST_ETC=/host/etc", + "HOST_SYS=/host/sys", + "DD_SYSTEM_PROBE_BPF_DIR=/opt/bpf", + } + for _, env := range envs { + args = append(args, "-e", env) + } + + // create container + args = append(args, ctc.image) // image tag + args = append(args, "sleep", "infinity") + if err := ctc.runDockerCmd(args); err != nil { + return fmt.Errorf("run docker: %s", err) + } + + // mount debugfs + args = []string{"exec", containerName, "mount", "-t", "debugfs", "none", "/sys/kernel/debug"} + if err := ctc.runDockerCmd(args); err != nil { + return fmt.Errorf("run docker: %w", err) + } + + return nil +} + +func (ctc *testContainer) buildDockerExecArgs(args ...string) []string { + return append([]string{"docker", "exec", containerName}, args...) +} diff --git a/test/new-e2e/tests/agent-subcommands/check/check_common_test.go b/test/new-e2e/tests/agent-subcommands/check/check_common_test.go index 59db2a49022c2..b117050474fa0 100644 --- a/test/new-e2e/tests/agent-subcommands/check/check_common_test.go +++ b/test/new-e2e/tests/agent-subcommands/check/check_common_test.go @@ -7,19 +7,26 @@ package check import ( + _ "embed" "fmt" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) type baseCheckSuite struct { e2e.BaseSuite[environments.Host] } +//go:embed fixtures/hello.yaml +var customCheckYaml []byte + +//go:embed fixtures/hello.py +var customCheckPython []byte + func (v *baseCheckSuite) TestCheckDisk() { check := v.Env().Agent.Client.Check(agentclient.WithArgs([]string{"disk"})) @@ -43,8 +50,7 @@ func (v *baseCheckSuite) TestCustomCheck() { func (v *baseCheckSuite) TestCheckRate() { check := v.Env().Agent.Client.Check(agentclient.WithArgs([]string{"hello", "--check-rate", "--json"})) - data := parseCheckOutput([]byte(check)) - require.NotNil(v.T(), data) + data := parseCheckOutput(v.T(), []byte(check)) metrics := data[0].Aggregator.Metrics @@ -59,8 +65,7 @@ func (v *baseCheckSuite) TestCheckTimes() { times := 10 check := v.Env().Agent.Client.Check(agentclient.WithArgs([]string{"hello", "--check-times", fmt.Sprint(times), "--json"})) - data := parseCheckOutput([]byte(check)) - require.NotNil(v.T(), data) + data := parseCheckOutput(v.T(), []byte(check)) metrics := data[0].Aggregator.Metrics diff --git a/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go b/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go index 92e80c401a72e..253ecc8075cc3 100644 --- a/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go @@ -7,7 +7,6 @@ package check import ( - _ "embed" "testing" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" @@ -22,12 +21,6 @@ type linuxCheckSuite struct { baseCheckSuite } -//go:embed fixtures/hello.yaml -var customCheckYaml []byte - -//go:embed fixtures/hello.py -var customCheckPython []byte - func TestLinuxCheckSuite(t *testing.T) { t.Parallel() e2e.Run(t, &linuxCheckSuite{}, e2e.WithProvisioner(awshost.ProvisionerNoFakeIntake(awshost.WithAgentOptions( diff --git a/test/new-e2e/tests/agent-subcommands/check/check_win_test.go b/test/new-e2e/tests/agent-subcommands/check/check_win_test.go index 123a2442cc13a..383663c37f0e6 100644 --- a/test/new-e2e/tests/agent-subcommands/check/check_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/check/check_win_test.go @@ -22,7 +22,6 @@ type windowsCheckSuite struct { } func TestWindowsCheckSuite(t *testing.T) { - t.Skip("not working because of the following error: unable to import module 'hello': source code string cannot contain null bytes") t.Parallel() e2e.Run(t, &windowsCheckSuite{}, e2e.WithProvisioner( awshost.ProvisionerNoFakeIntake( @@ -30,5 +29,7 @@ func TestWindowsCheckSuite(t *testing.T) { awshost.WithAgentOptions( agentparams.WithFile("C:/ProgramData/Datadog/conf.d/hello.d/conf.yaml", string(customCheckYaml), true), agentparams.WithFile("C:/ProgramData/Datadog/checks.d/hello.py", string(customCheckPython), true), - )))) + ), + ), + )) } diff --git a/test/new-e2e/tests/agent-subcommands/check/parse.go b/test/new-e2e/tests/agent-subcommands/check/parse.go index 36b665c36c471..dac78070b1deb 100644 --- a/test/new-e2e/tests/agent-subcommands/check/parse.go +++ b/test/new-e2e/tests/agent-subcommands/check/parse.go @@ -6,7 +6,11 @@ package check import ( + "bytes" "encoding/json" + "testing" + + "github.com/stretchr/testify/require" ) type root struct { @@ -27,11 +31,17 @@ type metric struct { Type string `json:"type"` } -func parseCheckOutput(check []byte) []root { +func parseCheckOutput(t *testing.T, check []byte) []root { + // On Windows a warning is printed when running the check command with the wrong user + // This warning is not part of the JSON output and needs to be ignored when parsing + startIdx := bytes.IndexAny(check, "[{") + require.NotEqual(t, -1, startIdx, "Failed to find start of JSON output in check output: %v", string(check)) + + check = check[startIdx:] + var data []root - if err := json.Unmarshal([]byte(check), &data); err != nil { - return nil - } + err := json.Unmarshal([]byte(check), &data) + require.NoErrorf(t, err, "Failed to unmarshal check output: %v", string(check)) return data } diff --git a/test/new-e2e/tests/containers/eks_test.go b/test/new-e2e/tests/containers/eks_test.go index ab63cfba09bdf..6562eff6abb80 100644 --- a/test/new-e2e/tests/containers/eks_test.go +++ b/test/new-e2e/tests/containers/eks_test.go @@ -75,7 +75,7 @@ func (suite *eksSuite) SetupSuite() { suite.Fakeintake = fakeintake.Client() kubeCluster := &components.KubernetesCluster{} - kubeSerialized, err := json.Marshal(stackOutput.Outputs["dd-Cluster-aws-eks"].Value) + kubeSerialized, err := json.Marshal(stackOutput.Outputs["dd-Cluster-eks"].Value) suite.Require().NoError(err) suite.Require().NoError(kubeCluster.Import(kubeSerialized, &kubeCluster)) suite.Require().NoError(kubeCluster.Init(suite)) diff --git a/test/new-e2e/tests/gpu/gpu_test.go b/test/new-e2e/tests/gpu/gpu_test.go new file mode 100644 index 0000000000000..56a49051c8e46 --- /dev/null +++ b/test/new-e2e/tests/gpu/gpu_test.go @@ -0,0 +1,102 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +package gpu + +import ( + "encoding/json" + "flag" + "fmt" + "testing" + + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/DataDog/test-infra-definitions/components/os" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" +) + +var devMode = flag.Bool("devmode", false, "enable dev mode") + +type gpuSuite struct { + e2e.BaseSuite[environments.Host] +} + +const defaultGpuCheckConfig = ` +init_config: + min_collection_interval: 5 + +instances: + - {} +` + +const defaultSysprobeConfig = ` +gpu_monitoring: + enabled: true +` + +const vectorAddDockerImg = "nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda10.2" +const gpuEnabledAMI = "ami-0f71e237bb2ba34be" // Ubuntu 22.04 with GPU drivers + +// TestGPUSuite runs tests for the VM interface to ensure its implementation is correct. +func TestGPUSuite(t *testing.T) { + provisioner := awshost.Provisioner( + awshost.WithEC2InstanceOptions( + ec2.WithInstanceType("g4dn.xlarge"), + ec2.WithAMI(gpuEnabledAMI, os.Ubuntu2204, os.AMD64Arch), + ), + awshost.WithAgentOptions( + agentparams.WithIntegration("gpu.d", defaultGpuCheckConfig), + agentparams.WithSystemProbeConfig(defaultSysprobeConfig), + ), + awshost.WithDocker(), + ) + + suiteParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioner)} + if *devMode { + suiteParams = append(suiteParams, e2e.WithDevMode()) + } + + e2e.Run(t, &gpuSuite{}, suiteParams...) +} + +func (v *gpuSuite) SetupSuite() { + v.BaseSuite.SetupSuite() + + v.Env().RemoteHost.MustExecute(fmt.Sprintf("docker pull %s", vectorAddDockerImg)) +} + +// TODO: Extract this to common package? service_discovery uses it too +type checkStatus struct { + CheckID string `json:"CheckID"` + CheckName string `json:"CheckName"` + CheckConfigSource string `json:"CheckConfigSource"` + ExecutionTimes []int `json:"ExecutionTimes"` + LastError string `json:"LastError"` +} + +type runnerStats struct { + Checks map[string]checkStatus `json:"Checks"` +} + +type collectorStatus struct { + RunnerStats runnerStats `json:"runnerStats"` +} + +func (v *gpuSuite) TestGPUCheckIsEnabled() { + statusOutput := v.Env().Agent.Client.Status(agentclient.WithArgs([]string{"collector", "--json"})) + + var status collectorStatus + err := json.Unmarshal([]byte(statusOutput.Content), &status) + v.Require().NoError(err, "failed to unmarshal agent status") + v.Require().Contains(status.RunnerStats.Checks, "gpu") + + gpuCheckStatus := status.RunnerStats.Checks["gpu"] + v.Require().Equal(gpuCheckStatus.LastError, "") +} diff --git a/test/new-e2e/tests/installer/unix/package_apm_inject_test.go b/test/new-e2e/tests/installer/unix/package_apm_inject_test.go index 389da3c392f3e..926a6210e00ef 100644 --- a/test/new-e2e/tests/installer/unix/package_apm_inject_test.go +++ b/test/new-e2e/tests/installer/unix/package_apm_inject_test.go @@ -54,6 +54,10 @@ func (s *packageApmInjectSuite) TestInstall() { state.AssertFileExists("/usr/bin/dd-host-install", 0755, "root", "root") state.AssertFileExists("/usr/bin/dd-container-install", 0755, "root", "root") state.AssertDirExists("/etc/datadog-agent/inject", 0755, "root", "root") + if s.os == e2eos.Ubuntu2204 || s.os == e2eos.Debian12 { + state.AssertDirExists("/etc/apparmor.d/abstractions/base.d", 0755, "root", "root") + state.AssertFileExists("/etc/apparmor.d/abstractions/base.d/datadog", 0644, "root", "root") + } s.assertLDPreloadInstrumented(injectOCIPath) s.assertSocketPath() s.assertDockerdInstrumented(injectOCIPath) @@ -78,6 +82,7 @@ func (s *packageApmInjectSuite) TestUninstall() { state := s.host.State() state.AssertPathDoesNotExist("/usr/bin/dd-host-install") state.AssertPathDoesNotExist("/usr/bin/dd-container-install") + state.AssertPathDoesNotExist("/etc/apparmor.d/abstractions/base.d/datadog") } func (s *packageApmInjectSuite) TestDockerAdditionalFields() { @@ -421,6 +426,23 @@ func (s *packageApmInjectSuite) TestInstallWithUmask() { s.TestInstall() } +func (s *packageApmInjectSuite) TestAppArmor() { + if s.os != e2eos.Ubuntu2204 && s.os != e2eos.Debian12 { + s.T().Skip("AppArmor not installed by default") + } + assert.Contains(s.T(), s.Env().RemoteHost.MustExecute("sudo aa-enabled"), "Yes") + s.RunInstallScript( + "DD_APM_INSTRUMENTATION_ENABLED=host", + "DD_APM_INSTRUMENTATION_LIBRARIES=python", + envForceInstall("datadog-agent"), + ) + defer s.Purge() + s.assertAppArmorProfile() + assert.Contains(s.T(), s.Env().RemoteHost.MustExecute("sudo aa-enabled"), "Yes") + res := s.Env().RemoteHost.MustExecute("sudo DD_APM_INSTRUMENTATION_DEBUG=true /usr/sbin/dhclient 2>&1") + assert.Contains(s.T(), res, "not injecting; on deny list") +} + func (s *packageApmInjectSuite) assertTraceReceived(traceID uint64) { found := assert.Eventually(s.T(), func() bool { tracePayloads, err := s.Env().FakeIntake.Client().GetTraces() @@ -495,6 +517,13 @@ func (s *packageApmInjectSuite) assertDockerdNotInstrumented() { assert.Equal(s.T(), runtimeConfig, "") } +func (s *packageApmInjectSuite) assertAppArmorProfile() { + content, err := s.host.ReadFile("/etc/apparmor.d/abstractions/base.d/datadog") + assert.NoError(s.T(), err) + assert.Equal(s.T(), string(content), "/opt/datadog-packages/** rix,\n/proc/@{pid}/** rix,") + assert.Contains(s.T(), s.Env().RemoteHost.MustExecute("sudo aa-enabled"), "Yes") +} + func (s *packageApmInjectSuite) purgeInjectorDebInstall() { s.Env().RemoteHost.MustExecute("sudo rm -f /opt/datadog-packages/run/environment") s.Env().RemoteHost.MustExecute("sudo rm -f /etc/datadog-agent/datadog.yaml") diff --git a/test/new-e2e/tests/npm/eks_1host_test.go b/test/new-e2e/tests/npm/eks_1host_test.go index e882d3387304c..12f86c5fd37e2 100644 --- a/test/new-e2e/tests/npm/eks_1host_test.go +++ b/test/new-e2e/tests/npm/eks_1host_test.go @@ -17,6 +17,7 @@ import ( kubeComp "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/DataDog/test-infra-definitions/resources/aws" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/DataDog/test-infra-definitions/scenarios/aws/eks" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" @@ -73,7 +74,7 @@ func eksHttpbinEnvProvisioner(opts ...envkube.ProvisionerOption) e2e.PulumiEnvRu provisionerOpts := []envkube.ProvisionerOption{ envkube.WithAwsEnv(&awsEnv), - envkube.WithEKSLinuxNodeGroup(), + envkube.WithEKSOptions(eks.WithLinuxNodeGroup()), envkube.WithAgentOptions(kubernetesagentparams.WithHelmValues(systemProbeConfigNPMHelmValues)), envkube.WithWorkloadApp(npmToolsWorkload), } diff --git a/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go b/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go index a94e4ad83de88..8e9fd649ed124 100644 --- a/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go +++ b/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" + "github.com/DataDog/test-infra-definitions/scenarios/aws/eks" "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" @@ -33,7 +34,7 @@ datadog: containerCollectUsingFiles: false ` t.Parallel() - e2e.Run(t, &iaEKSTestSuite{}, e2e.WithProvisioner(awskubernetes.EKSProvisioner(awskubernetes.WithEKSLinuxNodeGroup(), awskubernetes.WithAgentOptions(kubernetesagentparams.WithoutDualShipping(), kubernetesagentparams.WithHelmValues(values), kubernetesagentparams.WithOTelAgent(), kubernetesagentparams.WithOTelConfig(iaConfig))))) + e2e.Run(t, &iaEKSTestSuite{}, e2e.WithProvisioner(awskubernetes.EKSProvisioner(awskubernetes.WithEKSOptions(eks.WithLinuxNodeGroup()), awskubernetes.WithAgentOptions(kubernetesagentparams.WithoutDualShipping(), kubernetesagentparams.WithHelmValues(values), kubernetesagentparams.WithOTelAgent(), kubernetesagentparams.WithOTelConfig(iaConfig))))) } var eksParams = utils.IAParams{ diff --git a/test/otel/go.mod b/test/otel/go.mod index 933ce7f09d138..76543d05d11fe 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -177,7 +177,7 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.26.0 // indirect github.com/DataDog/datadog-go/v5 v5.5.0 // indirect github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 // indirect - github.com/DataDog/go-sqllexer v0.0.15 // indirect + github.com/DataDog/go-sqllexer v0.0.16 // indirect github.com/DataDog/go-tuf v1.1.0-0.5.2 // indirect github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 // indirect github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.20.0 // indirect @@ -288,12 +288,12 @@ require ( go.opentelemetry.io/collector/pdata v1.11.0 // indirect go.opentelemetry.io/collector/semconv v0.104.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect go.opentelemetry.io/otel/exporters/prometheus v0.49.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect go.opentelemetry.io/otel/sdk v1.27.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.27.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.18.0 // indirect go.uber.org/fx v1.22.2 // indirect diff --git a/test/otel/go.sum b/test/otel/go.sum index 249e64737379b..71a0a814b244b 100644 --- a/test/otel/go.sum +++ b/test/otel/go.sum @@ -8,8 +8,8 @@ github.com/DataDog/datadog-go/v5 v5.5.0 h1:G5KHeB8pWBNXT4Jtw0zAkhdxEAWSpWH00geHI github.com/DataDog/datadog-go/v5 v5.5.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= -github.com/DataDog/go-sqllexer v0.0.15 h1:rUUu52dP8EQhJLnUw0MIAxZp0BQx2fOTuMztr3vtHUU= -github.com/DataDog/go-sqllexer v0.0.15/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= +github.com/DataDog/go-sqllexer v0.0.16 h1:RoSUMS6MECyB3gTUIdydzXwK5NhEhv6GMJkS7ptsgRA= +github.com/DataDog/go-sqllexer v0.0.16/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc= github.com/DataDog/go-tuf v1.1.0-0.5.2 h1:4CagiIekonLSfL8GMHRHcHudo1fQnxELS9g4tiAupQ4= github.com/DataDog/go-tuf v1.1.0-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= @@ -433,18 +433,18 @@ go.opentelemetry.io/collector/semconv v0.104.0 h1:dUvajnh+AYJLEW/XOPk0T0BlwltSdi go.opentelemetry.io/collector/semconv v0.104.0/go.mod h1:yMVUCNoQPZVq/IPfrHrnntZTWsLf5YGZ7qwKulIl5hw= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= go.opentelemetry.io/otel/exporters/prometheus v0.49.0 h1:Er5I1g/YhfYv9Affk9nJLfH/+qCCVVg1f2R9AbJfqDQ= go.opentelemetry.io/otel/exporters/prometheus v0.49.0/go.mod h1:KfQ1wpjf3zsHjzP149P4LyAwWRupc6c7t1ZJ9eXpKQM= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2NemcCrOL8gI= go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= diff --git a/tools/ci/docker-login.ps1 b/tools/ci/docker-login.ps1 index 840b6b786492d..39f03f4fb7632 100644 --- a/tools/ci/docker-login.ps1 +++ b/tools/ci/docker-login.ps1 @@ -9,12 +9,14 @@ If ($lastExitCode -ne "0") { $tmpfile = [System.IO.Path]::GetTempFileName() & "C:\mnt\tools\ci\fetch_secret.ps1" -parameterName "$Env:DOCKER_REGISTRY_LOGIN" -tempFile "$tmpfile" If ($lastExitCode -ne "0") { - throw "Previous command returned $lastExitCode" + Write-Host "Previous command returned $lastExitCode" + exit "$lastExitCode" } $DOCKER_REGISTRY_LOGIN = $(cat "$tmpfile") & "C:\mnt\tools\ci\fetch_secret.ps1" -parameterName "$Env:DOCKER_REGISTRY_PWD" -tempFile "$tmpfile" If ($lastExitCode -ne "0") { - throw "Previous command returned $lastExitCode" + Write-Host "Previous command returned $lastExitCode" + exit "$lastExitCode" } $DOCKER_REGISTRY_PWD = $(cat "$tmpfile") Remove-Item "$tmpfile" diff --git a/tools/ci/junit_upload.sh b/tools/ci/junit_upload.sh index 27690cced9747..97312066045e1 100755 --- a/tools/ci/junit_upload.sh +++ b/tools/ci/junit_upload.sh @@ -6,7 +6,7 @@ if [[ -n "$1" ]]; then junit_files="$1" fi -DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$API_KEY_ORG2")" +DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_API_KEY_ORG2" token)" export DATADOG_API_KEY error=0 for file in $junit_files; do diff --git a/tools/ebpf/Dockerfiles/Dockerfile-process-agent-dev b/tools/ebpf/Dockerfiles/Dockerfile-process-agent-dev index 3cef80b912fb4..0f964fbd783c4 100644 --- a/tools/ebpf/Dockerfiles/Dockerfile-process-agent-dev +++ b/tools/ebpf/Dockerfiles/Dockerfile-process-agent-dev @@ -23,4 +23,3 @@ COPY agent $CORE_AGENT_DEST COPY *.o /opt/datadog-agent/embedded/share/system-probe/ebpf/ COPY co-re/*.o /opt/datadog-agent/embedded/share/system-probe/ebpf/co-re/ COPY *.c /opt/datadog-agent/embedded/share/system-probe/ebpf/runtime/ -COPY agent-usm.jar /opt/datadog-agent/embedded/share/system-probe/java/ diff --git a/tools/windows/DatadogAgentInstallScript/Install-Datadog.ps1 b/tools/windows/DatadogAgentInstallScript/Install-Datadog.ps1 index ef89be5eb5c5e..023a11938a148 100644 --- a/tools/windows/DatadogAgentInstallScript/Install-Datadog.ps1 +++ b/tools/windows/DatadogAgentInstallScript/Install-Datadog.ps1 @@ -6,6 +6,17 @@ $SCRIPT_VERSION = "1.0.0" $GENERAL_ERROR_CODE = 1 +# Set some defaults if not provided +$ddInstallerUrl = $env:DD_INSTALLER_URL +if (-Not $ddInstallerUrl) { + $ddInstallerUrl = "https://s3.amazonaws.com/dd-agent-mstesting/datadog-installer-x86_64.exe" +} + +$ddRemoteUpdates = $env:DD_REMOTE_UPDATES +if (-Not $ddRemoteUpdates) { + $ddRemoteUpdates = "false" +} + # ExitCodeException can be used to report failures from executables that set $LASTEXITCODE class ExitCodeException : Exception { [string] $LastExitCode @@ -15,11 +26,18 @@ class ExitCodeException : Exception { } } -function Update-ConfigFile($regex, $replacement) { - $configFile = Join-Path (Get-ItemPropertyValue -Path "HKLM:\\SOFTWARE\\Datadog\\Datadog Agent" -Name "ConfigRoot") "datadog.yaml" - if (-Not $configFile) { - $configFile = "C:\\ProgramData\\Datadog\\datadog.yaml" +function Get-DatadogConfigPath() { + if ( + (Test-Path "HKLM:\\SOFTWARE\\Datadog\\Datadog Agent") -and + ($null -ne (Get-Item -Path "HKLM:\\SOFTWARE\\Datadog\\Datadog Agent").GetValue("ConfigRoot")) + ) { + return (Join-Path (Get-ItemPropertyValue -Path "HKLM:\\SOFTWARE\\Datadog\\Datadog Agent" -Name "ConfigRoot") "datadog.yaml") } + return "C:\\ProgramData\\Datadog\\datadog.yaml" +} + +function Update-DatadogConfigFile($regex, $replacement) { + $configFile = Get-DatadogConfigPath if (-Not (Test-Path $configFile)) { throw "datadog.yaml doesn't exist" } @@ -44,8 +62,15 @@ function Send-Telemetry($payload) { "DD-Api-Key" = $env:DD_API_KEY "Content-Type" = "application/json" } - $result = Invoke-WebRequest -Uri $telemetryUrl -Method POST -Body $payload -Headers $requestHeaders - Write-Host "Sending telemetry: $($result.StatusCode)" + try { + $result = Invoke-WebRequest -Uri $telemetryUrl -Method POST -Body $payload -Headers $requestHeaders + Write-Host "Sending telemetry: $($result.StatusCode)" + } catch { + # Don't propagate errors when sending telemetry, because our error handling code will also + # try to send telemetry. + # It's enough to just print a message since there's no further error handling to be done. + Write-Host "Error sending telemetry" + } } function Show-Error($errorMessage, $errorCode) { @@ -125,15 +150,36 @@ function Start-ProcessWithOutput { return $process.ExitCode } -# Set some defaults if not provided -$ddInstallerUrl = $env:DD_INSTALLER_URL -if (-Not $ddInstallerUrl) { - $ddInstallerUrl = "https://s3.amazonaws.com/dd-agent-mstesting/datadog-installer-x86_64.exe" +function Test-DatadogAgentPresence() { + # Rudimentary check for the Agent presence, the `datadogagent` service should exist, and so should the `InstallPath` key in the registry. + # We check that particular key since we use it later in the script to restart the service. + return ( + ((Get-Service "datadogagent" -ea silent | Measure-Object).Count -eq 1) -and + (Test-Path "HKLM:\\SOFTWARE\\Datadog\\Datadog Agent") -and + ($null -ne (Get-Item -Path "HKLM:\\SOFTWARE\\Datadog\\Datadog Agent").GetValue("InstallPath")) + ) } -$ddRemoteUpdates = $env:DD_REMOTE_UPDATES -if (-Not $ddRemoteUpdates) { - $ddRemoteUpdates = "false" +function Update-DatadogAgentConfig() { + if ($env:DD_API_KEY) { + Write-Host "Writing DD_API_KEY" + Update-DatadogConfigFile "^[ #]*api_key:.*" "api_key: $env:DD_API_KEY" + } + + if ($env:DD_SITE) { + Write-Host "Writing DD_SITE" + Update-DatadogConfigFile "^[ #]*site:.*" "site: $env:DD_SITE" + } + + if ($env:DD_URL) { + Write-Host "Writing DD_URL" + Update-DatadogConfigFile "^[ #]*dd_url:.*" "dd_url: $env:DD_URL" + } + + if ($ddRemoteUpdates) { + Write-Host "Writing DD_REMOTE_UPDATES" + Update-DatadogConfigFile "^[ #]*remote_updates:.*" "remote_updates: $($ddRemoteUpdates.ToLower())" + } } try { @@ -145,11 +191,7 @@ try { $myWindowsID = [System.Security.Principal.WindowsIdentity]::GetCurrent() $myWindowsPrincipal = new-object System.Security.Principal.WindowsPrincipal($myWindowsID) $adminRole = [System.Security.Principal.WindowsBuiltInRole]::Administrator - if ($myWindowsPrincipal.IsInRole($adminRole)) { - # We are running "as Administrator" - $Host.UI.RawUI.WindowTitle = $myInvocation.MyCommand.Definition + "(Elevated)" - } - else { + if (-not $myWindowsPrincipal.IsInRole($adminRole)) { # We are not running "as Administrator" $newProcess = new-object System.Diagnostics.ProcessStartInfo "PowerShell"; $newProcess.Arguments = $myInvocation.MyCommand.Definition; @@ -159,6 +201,26 @@ try { return $proc.ExitCode } + # First thing to do is to stop the services if they are started + if (Test-DatadogAgentPresence) { + Write-Host "Stopping Datadog Agent services" + & ((Get-ItemProperty "HKLM:\\SOFTWARE\\Datadog\\Datadog Agent").InstallPath + "bin\\agent.exe") stop-service + } + + if ((Get-Service "Datadog Installer" -ea silent | Measure-Object).Count -eq 1) { + Write-Host "Stopping Datadog Installer service" + Stop-Service "Datadog Installer" + } + + $configUpdated = $False + # Write the config before-hand if it exists, that way if the Agent/Installer services start + # once installed, they will have a valid configuration. + # This allows the MSI to emit some telemetry as well. + if (Test-Path (Get-DatadogConfigPath)) { + Update-DatadogAgentConfig + $configUpdated = $True + } + # Powershell does not enable TLS 1.2 by default, & we want it enabled for faster downloads Write-Host "Forcing web requests to TLS v1.2" [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor [System.Net.SecurityProtocolType]::Tls12 @@ -177,33 +239,17 @@ try { Write-Host "Starting bootstrap process" $result = Start-ProcessWithOutput -Path $installer -ArgumentList "bootstrap" if ($result -ne 0) { - # bootstrap only fails if it fails to install to install the Datadog Installer, so it's possible the Agent was not installed + # bootstrap only fails if it fails to install to install the Datadog Installer, so it's possible the Agent was not installed throw [ExitCodeException]::new("Bootstrap failed", $result) } Write-Host "Bootstrap execution done" - if (-Not (Test-Path "HKLM:\\SOFTWARE\\Datadog\\Datadog Agent")) { + if (-Not (Test-DatadogAgentPresence)) { throw "Agent is not installed" } - if ($env:DD_API_KEY) { - Write-Host "Writing DD_API_KEY" - Update-ConfigFile "^[ #]*api_key:.*" "api_key: $env:DD_API_KEY" - } - - if ($env:DD_SITE) { - Write-Host "Writing DD_SITE" - Update-ConfigFile "^[ #]*site:.*" "site: $env:DD_SITE" - } - - if ($env:DD_URL) { - Write-Host "Writing DD_URL" - Update-ConfigFile "^[ #]*dd_url:.*" "dd_url: $env:DD_URL" - } - - if ($ddRemoteUpdates) { - Write-Host "Writing DD_REMOTE_UPDATES" - Update-ConfigFile "^[ #]*remote_updates:.*" "remote_updates: $($ddRemoteUpdates.ToLower())" + if (-Not ($configUpdated)) { + Update-DatadogAgentConfig } Send-Telemetry @" @@ -223,7 +269,12 @@ try { } } "@ - Start-Service "Datadog Installer" + # The datadog.yaml configuration was potentially modified so restart the services + Write-Host "Starting Datadog Installer service" + Restart-Service "Datadog Installer" + # This command handles restarting the dependent services as well + Write-Host "Starting Datadog Agent services" + & ((Get-ItemProperty "HKLM:\\SOFTWARE\\Datadog\\Datadog Agent").InstallPath + "bin\\agent.exe") restart-service } catch [ExitCodeException] { Show-Error $_.Exception.Message $_.Exception.LastExitCode @@ -233,6 +284,8 @@ catch { } finally { Write-Host "Cleaning up..." - Remove-Item -Force -EA SilentlyContinue $installer + if ($installer -and (Test-Path $installer)) { + Remove-Item -Force -EA SilentlyContinue $installer + } } Write-Host "Datadog Install Script finished!" diff --git a/tools/windows/DatadogAgentInstaller/InstallerCustomActions/CustomAction.cs b/tools/windows/DatadogAgentInstaller/InstallerCustomActions/CustomAction.cs index 69c42d537056c..fb6db05527a77 100644 --- a/tools/windows/DatadogAgentInstaller/InstallerCustomActions/CustomAction.cs +++ b/tools/windows/DatadogAgentInstaller/InstallerCustomActions/CustomAction.cs @@ -48,12 +48,6 @@ public static ActionResult ReadConfig(Session session) return Datadog.CustomActions.ConfigCustomActions.ReadConfig(session); } - [CustomAction] - public static ActionResult WriteConfig(Session session) - { - return Datadog.CustomActions.ConfigCustomActions.WriteConfig(session); - } - [CustomAction] public static ActionResult ProcessDdAgentUserCredentials(Session session) { diff --git a/tools/windows/DatadogAgentInstaller/WixSetup/Datadog Installer/DatadogInstallerCustomActions.cs b/tools/windows/DatadogAgentInstaller/WixSetup/Datadog Installer/DatadogInstallerCustomActions.cs index a8f7f86bab33f..79ebb0d9d3378 100644 --- a/tools/windows/DatadogAgentInstaller/WixSetup/Datadog Installer/DatadogInstallerCustomActions.cs +++ b/tools/windows/DatadogAgentInstaller/WixSetup/Datadog Installer/DatadogInstallerCustomActions.cs @@ -7,7 +7,6 @@ public class DatadogInstallerCustomActions { public ManagedAction RunAsAdmin { get; } public ManagedAction ReadConfig { get; } - public ManagedAction WriteConfig { get; } public ManagedAction ReadInstallState { get; } public ManagedAction WriteInstallState { get; } public ManagedAction RollbackWriteInstallState { get; } @@ -72,24 +71,6 @@ public DatadogInstallerCustomActions() } .SetProperties("APPLICATIONDATADIRECTORY=[APPLICATIONDATADIRECTORY]"); - WriteConfig = new CustomAction( - new Id(nameof(WriteConfig)), - CustomActions.WriteConfig, - Return.check, - When.Before, - Step.InstallServices, - Conditions.FirstInstall | Conditions.Upgrading | Conditions.Maintenance - ) - { - Execute = Execute.deferred, - Impersonate = false - } - .SetProperties( - "APPLICATIONDATADIRECTORY=[APPLICATIONDATADIRECTORY]," + - "APIKEY=[APIKEY], " + - "SITE=[SITE]") - .HideTarget(true); - OpenMsiLog = new CustomAction( new Id(nameof(OpenMsiLog)), CustomActions.OpenMsiLog