diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8c82bee16308e..24c15c4a3b6cd 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -126,6 +126,7 @@ /.gitlab/deploy_dca/ @DataDog/container-integrations @DataDog/agent-delivery /.gitlab/deploy_packages/ @DataDog/agent-delivery +/.gitlab/deploy_packages/oci.yml @DataDog/agent-delivery @DataDog/fleet /.gitlab/deploy_packages/windows.yml @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/deploy_packages/winget.yml @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/deploy_packages/cluster_agent_cloudfoundry.yml @DataDog/platform-integrations @DataDog/agent-devx-infra @@ -148,7 +149,6 @@ /.gitlab/powershell_script_deploy @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/choco_build/choco_build.yml @DataDog/agent-delivery @DataDog/windows-agent -/.gitlab/choco_deploy/choco_deploy.yml @DataDog/agent-delivery @DataDog/windows-agent /.gitlab/integration_test/windows.yml @DataDog/agent-devx-infra @DataDog/windows-agent @@ -176,6 +176,7 @@ /.gitlab/package_build/ @DataDog/agent-delivery /.gitlab/package_build/windows.yml @DataDog/agent-delivery @DataDog/windows-agent +/.gitlab/package_build/installer.yml @DataDog/agent-delivery @DataDog/fleet /.gitlab/packaging/ @DataDog/agent-delivery /.gitlab/benchmarks/benchmarks.yml @DataDog/agent-apm @@ -271,6 +272,7 @@ /go.mod # do not notify anyone /go.sum # do not notify anyone + /Makefile.trace @DataDog/agent-apm /omnibus/ @DataDog/agent-delivery @@ -647,3 +649,16 @@ /internal/third_party/client-go @DataDog/container-platform /internal/third_party/kubernetes @DataDog/container-integrations /internal/third_party/golang/ @DataDog/container-integrations + +# With the introduction of go.work, dependencies bump modify go.mod and go.sum in a lot of file. +# Which bring a lot of team in the review each time. To make it smoother we no longer consider go.mod and go.sum owned by teams. +# Each team can individually decide to update CODEOWNERS to be requested for a review on each modification of their go.mod/sum +/**/go.mod # do not notify anyone +/**/go.sum # do not notify anyone + +# Add here modules that need explicit review from the team owning them +/internal/tools/**/go.mod @DataDog/agent-devx-loops +/internal/tools/**/go.sum @DataDog/agent-devx-loops + +/pkg/util/scrubber/go.mod @DataDog/agent-shared-components +/pkg/util/scrubber/go.sum @DataDog/agent-shared-components diff --git a/.github/workflows/add_milestone.yml b/.github/workflows/add_milestone.yml index 33885369f0e0c..6bcd28e967c2e 100644 --- a/.github/workflows/add_milestone.yml +++ b/.github/workflows/add_milestone.yml @@ -22,7 +22,7 @@ jobs: GH_REPO: ${{ github.repository }} steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false diff --git a/.github/workflows/buildimages-update.yml b/.github/workflows/buildimages-update.yml index 1138e8557d011..a480105bf17e8 100644 --- a/.github/workflows/buildimages-update.yml +++ b/.github/workflows/buildimages-update.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: # credentials are needed to create the PR at the end of the workflow persist-credentials: true @@ -53,7 +53,7 @@ jobs: fi - name: Checkout branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 if: ${{ steps.branch_fetch.outputs.RESULT == 'true' }} with: ref: ${{ inputs.branch }} diff --git a/.github/workflows/chase_release_managers.yml b/.github/workflows/chase_release_managers.yml index 436217659ae48..475068736efea 100644 --- a/.github/workflows/chase_release_managers.yml +++ b/.github/workflows/chase_release_managers.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} persist-credentials: false diff --git a/.github/workflows/code_review_complexity.yml b/.github/workflows/code_review_complexity.yml index 8c26254f6c626..3f36b387e0f34 100644 --- a/.github/workflows/code_review_complexity.yml +++ b/.github/workflows/code_review_complexity.yml @@ -21,7 +21,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Setup python diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index bb6d39efa5e6d..bc52cb502bdaf 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,7 +23,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 persist-credentials: false diff --git a/.github/workflows/create_rc_pr.yml b/.github/workflows/create_rc_pr.yml index 2a685c3cc85df..66fbf9462a168 100644 --- a/.github/workflows/create_rc_pr.yml +++ b/.github/workflows/create_rc_pr.yml @@ -21,7 +21,7 @@ jobs: warning: ${{ steps.warning.outputs.value }} steps: - name: Checkout repository - if: ${{ env.IS_AGENT6_RELEASE == 'false' }} + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 with: sparse-checkout: 'tasks' @@ -69,7 +69,7 @@ jobs: fail-fast: false steps: - name: Checkout the main branch - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: true diff --git a/.github/workflows/create_release_schedule.yml b/.github/workflows/create_release_schedule.yml index f8dc006160e3c..4b749ba3bdc97 100644 --- a/.github/workflows/create_release_schedule.yml +++ b/.github/workflows/create_release_schedule.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} persist-credentials: false diff --git a/.github/workflows/cws-btfhub-sync.yml b/.github/workflows/cws-btfhub-sync.yml index 795953b22a354..88f198f5b114d 100644 --- a/.github/workflows/cws-btfhub-sync.yml +++ b/.github/workflows/cws-btfhub-sync.yml @@ -20,7 +20,7 @@ permissions: {} jobs: generate: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -52,13 +52,13 @@ jobs: df -h - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.base_branch || 'main' }} persist-credentials: false - name: Checkout btfhub-archive repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/btfhub-archive path: dev/dist/archive @@ -104,13 +104,13 @@ jobs: combine: needs: generate - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: contents: write pull-requests: write steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ inputs.base_branch || 'main' }} diff --git a/.github/workflows/datadog-static-analysis.yml b/.github/workflows/datadog-static-analysis.yml index 3063c7c0db456..a77f4ba632c1d 100644 --- a/.github/workflows/datadog-static-analysis.yml +++ b/.github/workflows/datadog-static-analysis.yml @@ -11,7 +11,7 @@ jobs: name: Datadog Static Analyzer steps: - name: Checkout - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Check code meets quality and security standards diff --git a/.github/workflows/docs-dev.yml b/.github/workflows/docs-dev.yml index 1eab6460db4b6..59e803598f6cd 100644 --- a/.github/workflows/docs-dev.yml +++ b/.github/workflows/docs-dev.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false # Fetch all history for applying timestamps to every page diff --git a/.github/workflows/external-contributor.yml b/.github/workflows/external-contributor.yml index f0beb805c2a50..03cf46fd03511 100644 --- a/.github/workflows/external-contributor.yml +++ b/.github/workflows/external-contributor.yml @@ -17,7 +17,7 @@ jobs: if: github.event.pull_request.head.repo.full_name != github.repository steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: main fetch-depth: 0 diff --git a/.github/workflows/go-update-commenter.yml b/.github/workflows/go-update-commenter.yml index 1028110bc4fda..c74a404a09c4c 100644 --- a/.github/workflows/go-update-commenter.yml +++ b/.github/workflows/go-update-commenter.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: # get the Go version of the target branch - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.base_ref }} persist-credentials: false @@ -26,7 +26,7 @@ jobs: echo version="$(cat .go-version)" >> $GITHUB_OUTPUT # get the Go version of the PR branch - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Get current Go version diff --git a/.github/workflows/go_mod_tidy.yml b/.github/workflows/go_mod_tidy.yml index e48d806c9dce9..4cc1a55690e6f 100644 --- a/.github/workflows/go_mod_tidy.yml +++ b/.github/workflows/go_mod_tidy.yml @@ -17,7 +17,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} - name: Checkout PR diff --git a/.github/workflows/gohai.yml b/.github/workflows/gohai.yml index 851c2f9d0ccd1..7ab3b5e49d2fc 100644 --- a/.github/workflows/gohai.yml +++ b/.github/workflows/gohai.yml @@ -34,7 +34,7 @@ jobs: go-file: [.go-version, pkg/gohai/go.mod] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 diff --git a/.github/workflows/label-analysis.yml b/.github/workflows/label-analysis.yml index ec53d5a695fed..7c4345e2ec323 100644 --- a/.github/workflows/label-analysis.yml +++ b/.github/workflows/label-analysis.yml @@ -23,7 +23,7 @@ jobs: pull-requests: write steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Setup python @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 persist-credentials: false @@ -125,7 +125,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.head_ref }} - name: Setup python diff --git a/.github/workflows/markdown-lint-check.yml b/.github/workflows/markdown-lint-check.yml index 5f440614390ae..b5fe87f21f0f8 100644 --- a/.github/workflows/markdown-lint-check.yml +++ b/.github/workflows/markdown-lint-check.yml @@ -9,7 +9,7 @@ jobs: markdown-link-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - uses: gaurav-nelson/github-action-markdown-link-check@d53a906aa6b22b8979d33bc86170567e619495ec # v1.0.15 diff --git a/.github/workflows/report-merged-pr.yml b/.github/workflows/report-merged-pr.yml index feefb3c5446bd..879b5b7e13330 100644 --- a/.github/workflows/report-merged-pr.yml +++ b/.github/workflows/report-merged-pr.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false diff --git a/.github/workflows/serverless-benchmarks.yml b/.github/workflows/serverless-benchmarks.yml index 8ad83b34d36d1..a3a4ecbf609e5 100644 --- a/.github/workflows/serverless-benchmarks.yml +++ b/.github/workflows/serverless-benchmarks.yml @@ -24,7 +24,7 @@ jobs: sha: ${{ steps.prepare.outputs.sha }} steps: - name: Checkout ${{ github.base_ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.base_ref }} persist-credentials: false @@ -63,7 +63,7 @@ jobs: steps: - name: Checkout ${{ github.ref }} - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.sha }} persist-credentials: false diff --git a/.github/workflows/serverless-binary-size.yml b/.github/workflows/serverless-binary-size.yml index 7be692d81d51a..fda6772398b9e 100644 --- a/.github/workflows/serverless-binary-size.yml +++ b/.github/workflows/serverless-binary-size.yml @@ -19,7 +19,7 @@ jobs: pull-requests: write # Add comment to PR steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: go/src/github.com/DataDog/datadog-agent persist-credentials: false @@ -36,7 +36,7 @@ jobs: fi - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/datadog-lambda-extension path: go/src/github.com/DataDog/datadog-lambda-extension diff --git a/.github/workflows/serverless-integration.yml b/.github/workflows/serverless-integration.yml index ace5e88fbda98..f11209b420ef0 100644 --- a/.github/workflows/serverless-integration.yml +++ b/.github/workflows/serverless-integration.yml @@ -26,13 +26,13 @@ jobs: name: ${{ matrix.suite }} on ${{ matrix.architecture }} steps: - name: Checkout datadog-agent repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: path: go/src/github.com/DataDog/datadog-agent persist-credentials: false - name: Set up Node 20 - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3 + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 with: node-version: 20 @@ -40,7 +40,7 @@ jobs: run: sudo yarn global add serverless@^3.36.0 --prefix /usr/local - name: Checkout the datadog-lambda-extension repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: DataDog/datadog-lambda-extension path: go/src/github.com/DataDog/datadog-lambda-extension diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b2761698a9b16..82599a4b84956 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,6 @@ include: - .gitlab/check_deploy/check_deploy.yml - .gitlab/check_merge/do_not_merge.yml - .gitlab/choco_build/choco_build.yml - - .gitlab/choco_deploy/choco_deploy.yml - .gitlab/powershell_script_signing/powershell_script_signing.yml - .gitlab/powershell_script_deploy/powershell_script_deploy.yml - .gitlab/common/shared.yml @@ -86,7 +85,7 @@ stages: - deploy_dca - choco_and_install_script_build - trigger_release - - choco_and_install_script_deploy + - install_script_deploy - internal_image_deploy - e2e_deploy - install_script_testing @@ -276,6 +275,7 @@ variables: # Feature flags FF_SCRIPT_SECTIONS: 1 # Prevent multiline scripts log collapsing, see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3392 FF_KUBERNETES_HONOR_ENTRYPOINT: true # Honor the entrypoint in the Docker image when running Kubernetes jobs + FF_TIMESTAMPS: true # # Condition mixins for simplification of rules @@ -623,18 +623,6 @@ workflow: - <<: *if_main_branch - <<: *if_release_branch -.on_main_or_release_branch_or_all_builds_or_pkg_installer_changes: - - <<: *if_main_branch - - <<: *if_release_branch - - <<: *if_run_all_builds - - <<: *if_run_all_unit_tests - - changes: - paths: - - pkg/fleet/**/* - compare_to: main - variables: - FAST_TESTS: "true" - .only_main: - <<: *if_not_main_branch when: never @@ -841,7 +829,9 @@ workflow: - !reference [.on_e2e_main_release_or_rc] - changes: paths: - # TODO: Add paths that should trigger tests for ASC + - cmd/**/* + - pkg/**/* + - comp/**/* - test/new-e2e/tests/agent-shared-components/**/* compare_to: main # TODO: use a variable, when this is supported https://gitlab.com/gitlab-org/gitlab/-/issues/369916 diff --git a/.gitlab/.ci-linters.yml b/.gitlab/.ci-linters.yml index af5acfa72b74a..fd1af4a9868c5 100644 --- a/.gitlab/.ci-linters.yml +++ b/.gitlab/.ci-linters.yml @@ -37,6 +37,7 @@ needs-rules: - setup_agent_version - tests_ebpf_arm64 - tests_ebpf_x64 + - tests_macos_gitlab_amd64 - tests_windows_secagent_x64 - tests_windows_sysprobe_x64 - trigger_auto_staging_release @@ -56,8 +57,6 @@ job-owners: - build_otel_agent_binary_arm64 - build_otel_agent_binary_x64 - cancel-prev-pipelines - - check_pkg_size-amd64-a7 - - check_pkg_size-arm64-a7 - close_failing_tests_stale_issues - compute_gitlab_ci_config - deploy_cluster_agent_cloudfoundry @@ -83,7 +82,6 @@ job-owners: - new-e2e-unit-tests - ot_agent_deb-arm64-a7 - ot_agent_deb-x64-a7 - - publish_choco_7_x64 - publish_fakeintake - publish_fakeintake_latest - rc_kubernetes_deploy diff --git a/.gitlab/JOBOWNERS b/.gitlab/JOBOWNERS index e03f28a2d537d..2678bd7a5de6d 100644 --- a/.gitlab/JOBOWNERS +++ b/.gitlab/JOBOWNERS @@ -67,7 +67,7 @@ iot_agent_suse* @DataDog/agent-delivery dogstatsd_suse* @DataDog/agent-delivery agent_oci* @DataDog/agent-delivery installer_oci* @DataDog/agent-delivery -new_check_pkg_size @DataDog/agent-delivery +check_pkg_size @DataDog/agent-delivery # Testing package deploy deploy_deb_testing* @DataDog/agent-delivery diff --git a/.gitlab/choco_build/choco_build.yml b/.gitlab/choco_build/choco_build.yml index 8ec5bac814d18..06ff3c6bb5d0d 100644 --- a/.gitlab/choco_build/choco_build.yml +++ b/.gitlab/choco_build/choco_build.yml @@ -2,34 +2,8 @@ # choco_build stage # Contains jobs which build the chocolatey Agent package. -# Not currently used in the pipeline. -windows_choco_offline_7_x64: - rules: - !reference [.manual] - stage: choco_and_install_script_build - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["windows_msi_and_bosh_zip_x64-a7"] - variables: - ARCH: "x64" - script: - - $ErrorActionPreference = "Stop" - - Get-ChildItem omnibus\pkg - - copy omnibus\pkg\*.msi .\chocolatey\datadog-agent\offline\tools\ - - > - docker run --rm - -v "$(Get-Location):c:\mnt" - -e AWS_NETWORKING=true - registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -InstallMethod offline -Flavor $FLAVOR -InstallDeps 1" - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - copy build-out\*.nupkg omnibus\pkg - artifacts: - expire_in: 2 weeks - paths: - - omnibus/pkg - -# The online version of the choco job gets the msi package through the gitlab artifacts -.windows_choco_online_7_x64: +# The choco job gets the msi package through the gitlab artifacts +.windows_choco_7_x64: rules: !reference [.on_deploy_stable_or_beta_repo_branch] stage: choco_and_install_script_build @@ -39,11 +13,7 @@ windows_choco_offline_7_x64: script: - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - $ErrorActionPreference = "Stop" - - mkdir temp\ - - copy omnibus\pkg\*.msi temp\ - - if (Test-Path omnibus) { remove-item -recurse -force omnibus } - if (Test-Path build-out) { remove-item -recurse -force build-out } - - mkdir omnibus\pkg - > docker run --rm -v "$(Get-Location):c:\mnt" @@ -52,10 +22,8 @@ windows_choco_offline_7_x64: -e BUCKET_BRANCH="$BUCKET_BRANCH" -e AWS_NETWORKING=true registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -InstallMethod online -MSIDirectory c:\mnt\temp -Flavor $FLAVOR -InstallDeps 1" + powershell.exe -C "C:\mnt\tasks\winbuildscripts\Generate-Chocolatey-Package.ps1 -MSIDirectory c:\mnt\omnibus\pkg -Flavor $FLAVOR -InstallDeps 1" - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - Remove-Item -Path "temp\" -Recurse -Force - - copy build-out\*.nupkg omnibus\pkg - $CopyNupkgToS3 = "$S3_CP_CMD --recursive --exclude '*' --include '*.nupkg' build-out $S3_RELEASE_ARTIFACTS_URI/choco/nupkg" - Invoke-Expression $CopyNupkgToS3 artifacts: @@ -65,8 +33,8 @@ windows_choco_offline_7_x64: # Sometimes Chocolatey is flakey retry: 2 -windows_choco_online_7_x64: - extends: .windows_choco_online_7_x64 +windows_choco_7_x64: + extends: .windows_choco_7_x64 # On dev/PR branches: # - if the job is run manually it will create a package, but before the # package can be installed, the deploy_windows_testing-a7 job must @@ -75,8 +43,8 @@ windows_choco_online_7_x64: variables: FLAVOR: "datadog-agent" -windows_choco_online_7_x64-fips: - extends: .windows_choco_online_7_x64 +windows_choco_7_x64-fips: + extends: .windows_choco_7_x64 # On dev/PR branches: # - if the job is run manually it will create a package, but before the # package can be installed, the deploy_windows_testing-a7-fips job must diff --git a/.gitlab/choco_deploy/choco_deploy.yml b/.gitlab/choco_deploy/choco_deploy.yml deleted file mode 100644 index 05b0f0d83eba6..0000000000000 --- a/.gitlab/choco_deploy/choco_deploy.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# choco_build stage -# Contains a job which deploys the chocolatey Agent package. - -publish_choco_7_x64: - rules: !reference [.on_deploy_stable_or_beta_repo_branch_manual] - stage: choco_and_install_script_deploy - tags: ["runner:windows-docker", "windowsversion:1809"] - needs: ["windows_choco_online_7_x64"] - variables: - ARCH: "x64" - before_script: - - $tmpfile = [System.IO.Path]::GetTempFileName() - - (& "$CI_PROJECT_DIR\tools\ci\fetch_secret.ps1" -parameterName "$Env:CHOCOLATEY_API_KEY" -tempFile "$tmpfile") - - If ($lastExitCode -ne "0") { exit "$lastExitCode" } - - $chocolateyApiKey=$(cat "$tmpfile") - - Remove-Item "$tmpfile" - script: - - '$_instance_id = (iwr -UseBasicParsing http://169.254.169.254/latest/meta-data/instance-id).content ; Write-Host "Running on instance $($_instance_id)"' - - $ErrorActionPreference = "Stop" - - Get-ChildItem omnibus\pkg - - if (Test-Path nupkg) { remove-item -recurse -force nupkg } - - mkdir nupkg - - copy omnibus\pkg\*.nupkg nupkg\ - - Get-ChildItem nupkg - - > - docker run --rm - -v "$(Get-Location):c:\mnt" - -e CHOCOLATEY_API_KEY=${chocolateyApiKey} - registry.ddbuild.io/ci/datadog-agent-buildimages/windows_1809_${ARCH}${Env:DATADOG_AGENT_WINBUILDIMAGES_SUFFIX}:${Env:DATADOG_AGENT_WINBUILDIMAGES} - c:\mnt\tasks\winbuildscripts\chocopush.bat - - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } diff --git a/.gitlab/common/test_infra_version.yml b/.gitlab/common/test_infra_version.yml index 71934be35d15a..944343f20da2f 100644 --- a/.gitlab/common/test_infra_version.yml +++ b/.gitlab/common/test_infra_version.yml @@ -1,7 +1,4 @@ --- variables: - # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev - # and check the job creating the image to make sure you have the right SHA prefix - TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: "" - # Make sure to update test-infra-definitions version in go.mod as well - TEST_INFRA_DEFINITIONS_BUILDIMAGES: 4b4112f5f64d + TEST_INFRA_DEFINITIONS_BUILDIMAGES: 221bbc806266 + TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: '' diff --git a/.gitlab/deploy_packages/oci.yml b/.gitlab/deploy_packages/oci.yml index bcefacac8d134..4d8ecadebfdff 100644 --- a/.gitlab/deploy_packages/oci.yml +++ b/.gitlab/deploy_packages/oci.yml @@ -28,6 +28,10 @@ include: - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} ${VERSION} - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} ${CI_COMMIT_SHA} - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_PACKAGE} pipeline-${CI_PIPELINE_ID} + # necessary for ddstaging until we deprecate `-dev` packaging + - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_STAGING_PACKAGE} ${VERSION} + - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_STAGING_PACKAGE} ${CI_COMMIT_SHA} + - datadog-package replicate-s3 registry.ddbuild.io/ci/remote-updates/${OCI_PRODUCT}:pipeline-${CI_PIPELINE_ID} us-east-1 ${INSTALLER_TESTING_S3_BUCKET} ${S3_STAGING_PACKAGE} pipeline-${CI_PIPELINE_ID} variables: MAJOR_VERSION: 7 @@ -37,6 +41,7 @@ deploy_agent_oci: variables: OCI_PRODUCT: "datadog-agent" S3_PACKAGE: "agent-package" + S3_STAGING_PACKAGE: "agent-package-dev" deploy_installer_oci: extends: ".deploy_packages_oci" @@ -44,3 +49,4 @@ deploy_installer_oci: variables: OCI_PRODUCT: "datadog-installer" S3_PACKAGE: "installer-package" + S3_STAGING_PACKAGE: "installer-package-dev" diff --git a/.gitlab/dev_container_deploy/docker_linux.yml b/.gitlab/dev_container_deploy/docker_linux.yml index 5b61ffc403a2b..8994a6cbb9bce 100644 --- a/.gitlab/dev_container_deploy/docker_linux.yml +++ b/.gitlab/dev_container_deploy/docker_linux.yml @@ -80,6 +80,24 @@ dev_master-a7: - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-jmx-arm64 IMG_DESTINATIONS: agent-dev:master-py3-jmx +dev_master-fips: + extends: .docker_publish_job_definition + stage: dev_container_deploy + rules: !reference [.on_main] + needs: + - docker_build_fips_agent7 + - docker_build_fips_agent7_arm64 + - docker_build_fips_agent7_jmx + - docker_build_fips_agent7_arm64_jmx + variables: + IMG_REGISTRIES: dev + parallel: + matrix: + - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-arm64 + IMG_DESTINATIONS: agent-dev:master-fips + - IMG_SOURCES: ${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-amd64,${SRC_AGENT}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-7-fips-jmx-arm64 + IMG_DESTINATIONS: agent-dev:master-fips-jmx + dev_master-dogstatsd: extends: .docker_publish_job_definition stage: dev_container_deploy diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index b7f4b80cc6ed1..769ebe10138b8 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -216,26 +216,6 @@ kmt_run_secagent_tests_x64_ebpfless: - !reference [.collect_outcomes_kmt] - !reference [.upload_junit_kmt] -kmt_run_secagent_tests_x64_fentry: - extends: - - .kmt_run_secagent_tests - image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_x64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:amd64"] - needs: - - kmt_setup_env_secagent_x64 - - upload_dependencies_secagent_x64 - - upload_secagent_tests_x64 - variables: - ARCH: "x86_64" - parallel: - matrix: - - TAG: - - "amazon_2023" - TEST_SET: [cws_fentry] - after_script: - - !reference [.collect_outcomes_kmt] - - !reference [.upload_junit_kmt] - kmt_run_secagent_tests_x64_docker: extends: - .kmt_run_secagent_tests @@ -350,26 +330,6 @@ kmt_run_secagent_tests_arm64_ebpfless: - !reference [.collect_outcomes_kmt] - !reference [.upload_junit_kmt] -kmt_run_secagent_tests_arm64_fentry: - extends: - - .kmt_run_secagent_tests - image: registry.ddbuild.io/ci/datadog-agent-buildimages/system-probe_arm64$DATADOG_AGENT_SYSPROBE_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_SYSPROBE_BUILDIMAGES - tags: ["arch:arm64"] - needs: - - kmt_setup_env_secagent_arm64 - - upload_dependencies_secagent_arm64 - - upload_secagent_tests_arm64 - variables: - ARCH: "arm64" - parallel: - matrix: - - TAG: - - "ubuntu_24.04" - TEST_SET: [cws_fentry] - after_script: - - !reference [.collect_outcomes_kmt] - - !reference [.upload_junit_kmt] - kmt_run_secagent_tests_arm64_docker: extends: - .kmt_run_secagent_tests @@ -426,7 +386,6 @@ kmt_secagent_tests_join_arm64: - kmt_run_secagent_tests_arm64 - kmt_run_secagent_tests_arm64_ad - kmt_run_secagent_tests_arm64_ebpfless - - kmt_run_secagent_tests_arm64_fentry - kmt_run_secagent_tests_arm64_docker kmt_secagent_cleanup_arm64: @@ -449,7 +408,6 @@ kmt_secagent_tests_join_x64: - kmt_run_secagent_tests_x64_required - kmt_run_secagent_tests_x64_ad - kmt_run_secagent_tests_x64_ebpfless - - kmt_run_secagent_tests_x64_fentry - kmt_run_secagent_tests_x64_docker kmt_secagent_cleanup_x64: diff --git a/.gitlab/pkg_metrics/pkg_metrics.yml b/.gitlab/pkg_metrics/pkg_metrics.yml index ceed774b62f41..ed09fa8ed1ee2 100644 --- a/.gitlab/pkg_metrics/pkg_metrics.yml +++ b/.gitlab/pkg_metrics/pkg_metrics.yml @@ -76,118 +76,14 @@ send_pkg_size: - inv package.send-size --flavor "agent" --package-os "suse" --package-path $OMNIBUS_PACKAGE_DIR_SUSE/datadog-agent-7.*.aarch64.rpm --major-version "7" --git-ref "${CI_COMMIT_REF_SLUG}" --bucket-branch "${BUCKET_BRANCH}" --arch arm64 -.check_pkg_size: - stage: pkg_metrics - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - script: - - ls -l $OMNIBUS_PACKAGE_DIR - - if [[ "${ARCH}" == "amd64" ]]; then ls -l $OMNIBUS_PACKAGE_DIR_SUSE; fi - - - export failures=0 - - export last_stable=$(inv release.get-release-json-value "last_stable::${MAJOR_VERSION}" --no-worktree) - # Get stable packages from S3 buckets, send new package sizes & compare stable and new package sizes - # The loop assumes that all flavors start with "da", which is currently the case - # We want to run all package size comparisons before failing, so we set +e while doing the comparisons - # to get the error codes without exiting the shell. - - | - if [[ "${ARCH}" == "amd64" ]]; then ARCH_RPM_EXT="x86_64"; else ARCH_RPM_EXT="aarch64"; fi - for flavor in ${FLAVORS}; do - - if [[ "${ARCH}" == "amd64" && "$flavor" != "datadog-heroku-agent" ]]; then - mkdir -p "/tmp/stable/${flavor}/suse" - curl -sSL "https://s3.amazonaws.com/yum.datadoghq.com/suse/stable/${MAJOR_VERSION}/${ARCH_RPM_EXT}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" -o "/tmp/stable/${flavor}/suse/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - set +e - inv package.compare-size --package-type "${flavor} suse rpm" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR_SUSE/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm" --stable-package "/tmp/stable/${flavor}/suse/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - failures=$((${failures}+$?)) - set -e - fi - - mkdir -p "/tmp/stable/${flavor}" - - curl -sSL "https://s3.amazonaws.com/apt.datadoghq.com/pool/d/da/${flavor}_${last_stable}-1_${ARCH}.deb" -o "/tmp/stable/${flavor}/${flavor}_${last_stable}-1_${ARCH}.deb" - - set +e - inv package.compare-size --package-type "${flavor} deb" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR/${flavor}_${MAJOR_VERSION}*_${ARCH}.deb" --stable-package "/tmp/stable/${flavor}/${flavor}_${last_stable}-1_${ARCH}.deb" - failures=$((${failures}+$?)) - set -e - - if [[ "$flavor" != "datadog-heroku-agent" && ( "${ARCH}" == "amd64" || "$flavor" != "datadog-dogstatsd") ]]; then - # We don't build RPM packages for the heroku flavor - curl -sSL "https://s3.amazonaws.com/yum.datadoghq.com/stable/${MAJOR_VERSION}/${ARCH_RPM_EXT}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" -o "/tmp/stable/${flavor}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - set +e - inv package.compare-size --package-type "${flavor} rpm" --last-stable "${last_stable}" --threshold "${max_sizes[${flavor}]}" --new-package "$OMNIBUS_PACKAGE_DIR/${flavor}-${MAJOR_VERSION}.*.${ARCH_RPM_EXT}.rpm" --stable-package "/tmp/stable/${flavor}/${flavor}-${last_stable}-1.${ARCH_RPM_EXT}.rpm" - failures=$((${failures}+$?)) - set -e - fi - done - - # Make the job fail if at least one package is above threshold - - if [ "${failures}" -ne "0" ]; then false; fi - -check_pkg_size-amd64-a7: - extends: .check_pkg_size - rules: - - !reference [.except_mergequeue] - - when: on_success - needs: - - agent_deb-x64-a7 - - iot_agent_deb-x64 - - dogstatsd_deb-x64 - - agent_heroku_deb-x64-a7 - - agent_rpm-x64-a7 - - iot_agent_rpm-x64 - - dogstatsd_rpm-x64 - - agent_suse-x64-a7 - - dogstatsd_suse-x64 - - iot_agent_suse-x64 - variables: - MAJOR_VERSION: 7 - FLAVORS: "datadog-agent datadog-iot-agent datadog-dogstatsd datadog-heroku-agent" - ARCH: "amd64" - before_script: - # FIXME: ["datadog-agent"]="140000000" and ["datadog-heroku-agent"]="140000000" should - # be replaced by "50000000" - # "70000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="140000000" - ["datadog-iot-agent"]="10000000" - ["datadog-dogstatsd"]="10000000" - ["datadog-heroku-agent"]="70000000" - ) - -check_pkg_size-arm64-a7: - extends: .check_pkg_size - rules: !reference [.on_all_builds] - needs: - - agent_deb-arm64-a7 - - iot_agent_deb-arm64 - - dogstatsd_deb-arm64 - - agent_rpm-arm64-a7 - - iot_agent_rpm-arm64 - variables: - MAJOR_VERSION: 7 - FLAVORS: "datadog-agent datadog-iot-agent datadog-dogstatsd" - ARCH: "arm64" - before_script: - # FIXME: ["datadog-agent"]="140000000" should be replaced by "70000000" - # "140000000" is needed as of now because of multiple large additions in 7.45 - - | - declare -Ar max_sizes=( - ["datadog-agent"]="140000000" - ["datadog-iot-agent"]="10000000" - ["datadog-dogstatsd"]="10000000" - ) -new_check_pkg_size: +check_pkg_size: stage: pkg_metrics image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] rules: - !reference [.except_mergequeue] - when: on_success - allow_failure: true needs: - agent_deb-x64-a7 - iot_agent_deb-x64 diff --git a/.gitlab/powershell_script_deploy/powershell_script_deploy.yml b/.gitlab/powershell_script_deploy/powershell_script_deploy.yml index 4760dfdf18030..5269045adde05 100644 --- a/.gitlab/powershell_script_deploy/powershell_script_deploy.yml +++ b/.gitlab/powershell_script_deploy/powershell_script_deploy.yml @@ -5,7 +5,7 @@ powershell_script_deploy: image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] - stage: choco_and_install_script_deploy + stage: install_script_deploy rules: !reference [.manual] needs: ["powershell_script_signing"] @@ -19,7 +19,7 @@ powershell_script_deploy: windows_bootstrapper_deploy: image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES tags: ["arch:amd64"] - stage: choco_and_install_script_deploy + stage: install_script_deploy rules: !reference [.manual] needs: ["windows-installer-amd64"] diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml index da5ed4d68c8cf..1a92a354d47b8 100644 --- a/.gitlab/source_test/macos.yml +++ b/.gitlab/source_test/macos.yml @@ -2,44 +2,8 @@ include: - .gitlab/common/macos.yml -tests_macos: - stage: source_test - # HACK: Run macOS unit tests only on full pipelines, to limit the use of macOS GitHub runners. - rules: - - !reference [.on_main_or_release_branch_or_all_builds_or_pkg_installer_changes] - - when: manual - allow_failure: true - image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES - tags: ["arch:amd64"] - needs: ["setup_agent_version"] - script: - - !reference [.setup_macos_github_app] - - $S3_CP_CMD $S3_ARTIFACTS_URI/agent-version.cache . - - export VERSION_CACHE_CONTENT=$(cat agent-version.cache | base64 -) - # Python 3.12 changes default behavior how packages are installed. - # In particular, --break-system-packages command line option is - # required to use the old behavior or use a virtual env. https://github.com/actions/runner-images/issues/8615 - - python3 -m pip install -r tasks/libs/requirements-github.txt --break-system-packages - - FAST_TESTS_FLAG="" - - if [[ "$FAST_TESTS" = "true" ]]; then FAST_TESTS_FLAG="--fast-tests true"; fi - - inv -e github.trigger-macos --workflow-type "test" --datadog-agent-ref "$CI_COMMIT_SHA" --version-cache "$VERSION_CACHE_CONTENT" $FAST_TESTS_FLAG --test-washer - timeout: 6h - after_script: - - $CI_PROJECT_DIR/tools/ci/junit_upload.sh "junit-*-repacked.tgz" - artifacts: - expire_in: 2 weeks - when: always - paths: - - test_output.json - - junit-*-repacked.tgz - reports: - junit: "**/junit-out-*.xml" - .tests_macos_gitlab: stage: source_test - rules: - - !reference [.except_mergequeue] - - when: on_success extends: .macos_gitlab needs: ["go_deps", "go_tools_deps"] variables: diff --git a/CHANGELOG-DCA.rst b/CHANGELOG-DCA.rst index 569bda03c6a0d..7b5abd00a3bc2 100644 --- a/CHANGELOG-DCA.rst +++ b/CHANGELOG-DCA.rst @@ -2,6 +2,19 @@ Release Notes ============= +.. _Release Notes_7.60.1: + +7.60.1 +====== + +.. _Release Notes_7.60.1_Prelude: + +Prelude +------- + +Released on: 2024-12-19 +Pinned to datadog-agent v7.60.1: `CHANGELOG `_. + .. _Release Notes_7.60.0: 7.60.0 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ba9f97c930ee8..1def73a37b2e1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,27 @@ Release Notes ============= +.. _Release Notes_7.60.1: + +7.60.1 +====== + +.. _Release Notes_7.60.1_Prelude: + +Prelude +------- + +Release on: 2024-12-19 + + +.. _Release Notes_7.60.1_Security Notes: + +Security Notes +-------------- + +- Update ``golang.org/x/crypto`` to fix CVE-2024-45337. + + .. _Release Notes_7.60.0: 7.60.0 diff --git a/Dockerfiles/agent-ot/Dockerfile.agent-otel b/Dockerfiles/agent-ot/Dockerfile.agent-otel index 78534598a581c..49ea309af7019 100644 --- a/Dockerfiles/agent-ot/Dockerfile.agent-otel +++ b/Dockerfiles/agent-ot/Dockerfile.agent-otel @@ -1,5 +1,5 @@ -ARG AGENT_VERSION=7.57.0-v1.0-ot-beta-jmx -ARG AGENT_BRANCH=7.57.x-otel-beta-v1 +ARG AGENT_VERSION=7.59.0-v1.1.0-ot-beta-jmx +ARG AGENT_BRANCH=7.59.x # Use the Ubuntu Slim AMD64 base image FROM ubuntu:24.04 AS builder diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 92e5c072e3c8e..13955c7e8a940 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -2906,9 +2906,6 @@ core,google.golang.org/protobuf/types/known/timestamppb,BSD-3-Clause,Copyright ( core,google.golang.org/protobuf/types/known/wrapperspb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,google.golang.org/protobuf/types/pluginpb,BSD-3-Clause,Copyright (c) 2018 The Go Authors. All rights reserved core,gopkg.in/DataDog/dd-trace-go.v1/appsec/events,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." -core,gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/datastreams/options,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace,Apache-2.0,"Copyright 2016-Present Datadog, Inc." core,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext,Apache-2.0,"Copyright 2016-Present Datadog, Inc." diff --git a/chocolatey/datadog-agent/online/datadog-agent-online.nuspec b/chocolatey/datadog-agent/datadog-agent.nuspec similarity index 97% rename from chocolatey/datadog-agent/online/datadog-agent-online.nuspec rename to chocolatey/datadog-agent/datadog-agent.nuspec index 73f96bfb83871..f3d9ec86ca694 100644 --- a/chocolatey/datadog-agent/online/datadog-agent-online.nuspec +++ b/chocolatey/datadog-agent/datadog-agent.nuspec @@ -1,4 +1,4 @@ - + datadog-agent diff --git a/chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec b/chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec deleted file mode 100644 index aedf9555a1c51..0000000000000 --- a/chocolatey/datadog-agent/offline/datadog-agent-offline.nuspec +++ /dev/null @@ -1,31 +0,0 @@ - - - - datadog-agent-offline - $package_version$ - https://github.com/DataDog/datadog-agent/tree/main/chocolatey - Datadog - Datadog Agent Offline Install - Datadog - https://github.com/DataDog/datadog-agent - https://datadog-prod.imgix.net/img/dd_logo_70x75.png - $copyright$ - https://raw.githubusercontent.com/DataDog/datadog-agent/main/LICENSE - true - https://docs.datadoghq.com - datadog agent monitoring admin - The Datadog Agent for Microsoft Windows - The Datadog Agent faithfully collects events and metrics and brings them to Datadog on your behalf so that you can do something useful with your monitoring and performance data. - -## Package settings - -You may set [custom settings](https://docs.datadoghq.com/agent/basic_agent_usage/windows/?tab=commandline#installation) to the Agent when installing by using the [`--installer-arguments` option of `choco install`](https://chocolatey.org/docs/getting-started#overriding-default-install-directory-or-other-advanced-install-concepts). - -For example, to set the API key you may run: -`choco install -ia="APIKEY=""YOUR_DATADOG_API_KEY""" datadog-agent-offline` - $release_notes$ - - - - - diff --git a/chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 b/chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 deleted file mode 100644 index 98d39c92654e0..0000000000000 --- a/chocolatey/datadog-agent/offline/tools/chocolateyinstall.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -$ErrorActionPreference = 'Stop'; - -$toolsDir = "$(Split-Path -parent $MyInvocation.MyCommand.Definition)" -$nupkgs = Get-ChildItem $toolsDir\datadog-agent*.msi -if (($nupkgs | Measure-Object).Count -gt 1) { - Write-Host "More than 1 MSI installer exists - aborting" - exit -2 -} -$packageArgs = @{ - packageName = $env:ChocolateyPackageName - unzipLocation = $toolsDir - fileType = 'msi' - file = $nupkgs[0].FullName - softwareName = 'Datadog Agent' - silentArgs = "/qn /norestart /l*v `"$($env:TEMP)\$($packageName).$($env:chocolateyPackageVersion).MsiInstall.log`"" - validExitCodes= @(0, 3010, 1641) -} -Install-ChocolateyInstallPackage @packageArgs - -$installInfo = @" ---- -install_method: - tool: chocolatey - tool_version: chocolatey-$($env:CHOCOLATEY_VERSION) - installer_version: chocolatey_package-offline -"@ - -$appDataDir = (Get-ItemProperty -Path "HKLM:\SOFTWARE\Datadog\Datadog Agent").ConfigRoot -Out-File -FilePath $appDataDir\install_info -InputObject $installInfo diff --git a/chocolatey/datadog-agent/online/tools/VERIFICATION.txt b/chocolatey/datadog-agent/online/tools/VERIFICATION.txt deleted file mode 100644 index 5ef7f466e2cd6..0000000000000 --- a/chocolatey/datadog-agent/online/tools/VERIFICATION.txt +++ /dev/null @@ -1,5 +0,0 @@ -VERIFICATION -Verification is intended to assist the Chocolatey moderators and community in verifying that this package's contents are trustworthy. - -This package is published by Datadog itself. -The binaries are identical to other package types for the Datadog Agent. diff --git a/chocolatey/datadog-agent/offline/tools/VERIFICATION.txt b/chocolatey/datadog-agent/tools/VERIFICATION.txt similarity index 100% rename from chocolatey/datadog-agent/offline/tools/VERIFICATION.txt rename to chocolatey/datadog-agent/tools/VERIFICATION.txt diff --git a/chocolatey/datadog-agent/online/tools/chocolateyinstall.ps1 b/chocolatey/datadog-agent/tools/chocolateyinstall.ps1 similarity index 100% rename from chocolatey/datadog-agent/online/tools/chocolateyinstall.ps1 rename to chocolatey/datadog-agent/tools/chocolateyinstall.ps1 diff --git a/chocolatey/datadog-fips-agent/online/datadog-fips-agent-online.nuspec b/chocolatey/datadog-fips-agent/datadog-fips-agent.nuspec similarity index 100% rename from chocolatey/datadog-fips-agent/online/datadog-fips-agent-online.nuspec rename to chocolatey/datadog-fips-agent/datadog-fips-agent.nuspec diff --git a/chocolatey/datadog-fips-agent/online/tools/VERIFICATION.txt b/chocolatey/datadog-fips-agent/tools/VERIFICATION.txt similarity index 100% rename from chocolatey/datadog-fips-agent/online/tools/VERIFICATION.txt rename to chocolatey/datadog-fips-agent/tools/VERIFICATION.txt diff --git a/chocolatey/datadog-fips-agent/online/tools/chocolateyinstall.ps1 b/chocolatey/datadog-fips-agent/tools/chocolateyinstall.ps1 similarity index 100% rename from chocolatey/datadog-fips-agent/online/tools/chocolateyinstall.ps1 rename to chocolatey/datadog-fips-agent/tools/chocolateyinstall.ps1 diff --git a/cmd/agent/common/autodiscovery.go b/cmd/agent/common/autodiscovery.go index 8ff855aeca19f..67a220b706750 100644 --- a/cmd/agent/common/autodiscovery.go +++ b/cmd/agent/common/autodiscovery.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "time" "go.uber.org/atomic" @@ -41,6 +42,10 @@ var ( ) func setupAutoDiscovery(confSearchPaths []string, wmeta workloadmeta.Component, ac autodiscovery.Component) { + if pkgconfigsetup.Datadog().GetString("fleet_policies_dir") != "" { + confSearchPaths = append(confSearchPaths, filepath.Join(pkgconfigsetup.Datadog().GetString("fleet_policies_dir"), "conf.d")) + } + providers.InitConfigFilesReader(confSearchPaths) acTelemetryStore := ac.GetTelemetryStore() diff --git a/cmd/agent/dist/conf.d/cpu.d/conf.yaml.default b/cmd/agent/dist/conf.d/cpu.d/conf.yaml.default index 00d9a2dbba2c8..c338654dc4e71 100644 --- a/cmd/agent/dist/conf.d/cpu.d/conf.yaml.default +++ b/cmd/agent/dist/conf.d/cpu.d/conf.yaml.default @@ -1,2 +1,15 @@ +## This file is overwritten upon Agent upgrade. +## To make modifications to the check configuration, copy this file +## to `cpu.yaml` and make your changes on that file. + +init_config: + +## The Cpu check only supports one configured instance. + instances: - - {} + - + + ## @param report_total_percpu - boolean - required + ## Set to true to report cpu total metrics for each cpu. + # + # report_total_percpu: false diff --git a/cmd/agent/subcommands/flare/command.go b/cmd/agent/subcommands/flare/command.go index 1b4fb2d3bc1e3..a6da5391ae5d5 100644 --- a/cmd/agent/subcommands/flare/command.go +++ b/cmd/agent/subcommands/flare/command.go @@ -175,10 +175,18 @@ func readProfileData(seconds int) (flare.ProfileData, error) { type pprofGetter func(path string) ([]byte, error) - tcpGet := func(portConfig string) pprofGetter { - pprofURL := fmt.Sprintf("http://127.0.0.1:%d/debug/pprof", pkgconfigsetup.Datadog().GetInt(portConfig)) + tcpGet := func(portConfig string, onHTTPS bool) pprofGetter { + endpoint := url.URL{ + Scheme: "http", + Host: net.JoinHostPort("127.0.0.1", strconv.Itoa(pkgconfigsetup.Datadog().GetInt(portConfig))), + Path: "/debug/pprof", + } + if onHTTPS { + endpoint.Scheme = "https" + } + return func(path string) ([]byte, error) { - return util.DoGet(c, pprofURL+path, util.LeaveConnectionOpen) + return util.DoGet(c, endpoint.String()+path, util.LeaveConnectionOpen) } } @@ -228,15 +236,15 @@ func readProfileData(seconds int) (flare.ProfileData, error) { } agentCollectors := map[string]agentProfileCollector{ - "core": serviceProfileCollector(tcpGet("expvar_port"), seconds), - "security-agent": serviceProfileCollector(tcpGet("security_agent.expvar_port"), seconds), + "core": serviceProfileCollector(tcpGet("expvar_port", false), seconds), + "security-agent": serviceProfileCollector(tcpGet("security_agent.expvar_port", false), seconds), } if pkgconfigsetup.Datadog().GetBool("process_config.enabled") || pkgconfigsetup.Datadog().GetBool("process_config.container_collection.enabled") || pkgconfigsetup.Datadog().GetBool("process_config.process_collection.enabled") { - agentCollectors["process"] = serviceProfileCollector(tcpGet("process_config.expvar_port"), seconds) + agentCollectors["process"] = serviceProfileCollector(tcpGet("process_config.expvar_port", false), seconds) } if pkgconfigsetup.Datadog().GetBool("apm_config.enabled") { @@ -249,7 +257,7 @@ func readProfileData(seconds int) (flare.ProfileData, error) { traceCpusec = 4 } - agentCollectors["trace"] = serviceProfileCollector(tcpGet("apm_config.debug.port"), traceCpusec) + agentCollectors["trace"] = serviceProfileCollector(tcpGet("apm_config.debug.port", true), traceCpusec) } if pkgconfigsetup.SystemProbe().GetBool("system_probe_config.enabled") { diff --git a/cmd/agent/subcommands/flare/command_test.go b/cmd/agent/subcommands/flare/command_test.go index a27304e95b133..4c96cae079aa9 100644 --- a/cmd/agent/subcommands/flare/command_test.go +++ b/cmd/agent/subcommands/flare/command_test.go @@ -29,6 +29,7 @@ type commandTestSuite struct { suite.Suite sysprobeSocketPath string tcpServer *httptest.Server + tcpTLSServer *httptest.Server unixServer *httptest.Server systemProbeServer *httptest.Server } @@ -42,13 +43,17 @@ func (c *commandTestSuite) SetupSuite() { // This should be called by each test that requires them. func (c *commandTestSuite) startTestServers() { t := c.T() - c.tcpServer, c.unixServer, c.systemProbeServer = c.getPprofTestServer() + c.tcpServer, c.tcpTLSServer, c.unixServer, c.systemProbeServer = c.getPprofTestServer() t.Cleanup(func() { if c.tcpServer != nil { c.tcpServer.Close() c.tcpServer = nil } + if c.tcpTLSServer != nil { + c.tcpTLSServer.Close() + c.tcpTLSServer = nil + } if c.unixServer != nil { c.unixServer.Close() c.unixServer = nil @@ -82,12 +87,13 @@ func newMockHandler() http.HandlerFunc { }) } -func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, unixServer *httptest.Server, sysProbeServer *httptest.Server) { +func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, tcpTLSServer *httptest.Server, unixServer *httptest.Server, sysProbeServer *httptest.Server) { var err error t := c.T() handler := newMockHandler() tcpServer = httptest.NewServer(handler) + tcpTLSServer = httptest.NewTLSServer(handler) if runtime.GOOS == "linux" { unixServer = httptest.NewUnstartedServer(handler) unixServer.Listener, err = net.Listen("unix", c.sysprobeSocketPath) @@ -101,7 +107,7 @@ func (c *commandTestSuite) getPprofTestServer() (tcpServer *httptest.Server, uni sysProbeServer.Start() } - return tcpServer, unixServer, sysProbeServer + return tcpServer, tcpTLSServer, unixServer, sysProbeServer } func TestCommandTestSuite(t *testing.T) { @@ -116,10 +122,14 @@ func (c *commandTestSuite) TestReadProfileData() { require.NoError(t, err) port := u.Port() + u, err = url.Parse(c.tcpTLSServer.URL) + require.NoError(t, err) + httpsPort := u.Port() + mockConfig := configmock.New(t) mockConfig.SetWithoutSource("expvar_port", port) mockConfig.SetWithoutSource("apm_config.enabled", true) - mockConfig.SetWithoutSource("apm_config.debug.port", port) + mockConfig.SetWithoutSource("apm_config.debug.port", httpsPort) mockConfig.SetWithoutSource("apm_config.receiver_timeout", "10") mockConfig.SetWithoutSource("process_config.expvar_port", port) mockConfig.SetWithoutSource("security_agent.expvar_port", port) diff --git a/cmd/agent/subcommands/secret/command.go b/cmd/agent/subcommands/secret/command.go index e24f95ca6d693..d8b2a7048114f 100644 --- a/cmd/agent/subcommands/secret/command.go +++ b/cmd/agent/subcommands/secret/command.go @@ -101,7 +101,7 @@ func traceAgentSecretRefresh(conf config.Component) ([]byte, error) { c := apiutil.GetClient(false) c.Timeout = conf.GetDuration("server_timeout") * time.Second - url := fmt.Sprintf("http://127.0.0.1:%d/secret/refresh", port) + url := fmt.Sprintf("https://127.0.0.1:%d/secret/refresh", port) res, err := apiutil.DoGet(c, url, apiutil.CloseConnection) if err != nil { return nil, fmt.Errorf("could not contact trace-agent: %s", err) diff --git a/cmd/installer-downloader/main.go b/cmd/installer-downloader/main.go index f7b67910ae94e..60e2141af944e 100644 --- a/cmd/installer-downloader/main.go +++ b/cmd/installer-downloader/main.go @@ -44,12 +44,12 @@ func main() { ctx := context.Background() t := telemetry.NewTelemetry(env.HTTPClient(), env.APIKey, env.Site, fmt.Sprintf("datadog-installer-downloader-%s", Flavor)) - _ = t.Start(ctx) - defer func() { _ = t.Stop(ctx) }() var err error span, ctx := telemetry.StartSpanFromEnv(ctx, fmt.Sprintf("downloader-%s", Flavor)) - defer func() { span.Finish(err) }() err = runDownloader(ctx, env, Version, Flavor) + + span.Finish(err) + t.Stop() if err != nil { fmt.Fprintf(os.Stderr, "Installation failed: %v\n", err) os.Exit(1) diff --git a/cmd/installer/subcommands/installer/command.go b/cmd/installer/subcommands/installer/command.go index 66ec6b1c74c57..f394d9955d407 100644 --- a/cmd/installer/subcommands/installer/command.go +++ b/cmd/installer/subcommands/installer/command.go @@ -87,7 +87,7 @@ func UnprivilegedCommands(_ *command.GlobalParams) []*cobra.Command { type cmd struct { t *telemetry.Telemetry ctx context.Context - span telemetry.Span + span *telemetry.Span env *env.Env } @@ -107,10 +107,7 @@ func newCmd(operation string) *cmd { func (c *cmd) Stop(err error) { c.span.Finish(err) if c.t != nil { - err := c.t.Stop(context.Background()) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to stop telemetry: %v\n", err) - } + c.t.Stop() } } @@ -225,11 +222,6 @@ func newTelemetry(env *env.Env) *telemetry.Telemetry { site = config.Site } t := telemetry.NewTelemetry(env.HTTPClient(), apiKey, site, "datadog-installer") // No sampling rules for commands - err := t.Start(context.Background()) - if err != nil { - fmt.Printf("failed to start telemetry: %v\n", err) - return nil - } return t } diff --git a/cmd/installer/subcommands/installer/umask_nix.go b/cmd/installer/subcommands/installer/umask_nix.go index 1fd44c01ac405..dec64eccd6860 100644 --- a/cmd/installer/subcommands/installer/umask_nix.go +++ b/cmd/installer/subcommands/installer/umask_nix.go @@ -14,7 +14,7 @@ import ( ) // setInstallerUmask sets umask 0 to override any inherited umask -func setInstallerUmask(span telemetry.Span) { +func setInstallerUmask(span *telemetry.Span) { oldmask := syscall.Umask(0) span.SetTag("inherited_umask", oldmask) } diff --git a/cmd/installer/subcommands/installer/umask_windows.go b/cmd/installer/subcommands/installer/umask_windows.go index 1b076f92bc389..d8661700cd56e 100644 --- a/cmd/installer/subcommands/installer/umask_windows.go +++ b/cmd/installer/subcommands/installer/umask_windows.go @@ -10,4 +10,4 @@ package installer import "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" // setInstallerUmask no-op on Windows -func setInstallerUmask(_ telemetry.Span) {} +func setInstallerUmask(_ *telemetry.Span) {} diff --git a/cmd/otel-agent/subcommands/run/command.go b/cmd/otel-agent/subcommands/run/command.go index a414674620e44..d7958c6dabc59 100644 --- a/cmd/otel-agent/subcommands/run/command.go +++ b/cmd/otel-agent/subcommands/run/command.go @@ -25,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/hostname/hostnameinterface" "github.com/DataDog/datadog-agent/comp/core/hostname/remotehostnameimpl" log "github.com/DataDog/datadog-agent/comp/core/log/def" + logfx "github.com/DataDog/datadog-agent/comp/core/log/fx" logtracefx "github.com/DataDog/datadog-agent/comp/core/log/fx-trace" "github.com/DataDog/datadog-agent/comp/core/secrets" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" @@ -106,8 +107,20 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, fx.Provide(func() coreconfig.Component { return acfg }), + fx.Provide(func(_ coreconfig.Component) log.Params { + return log.ForDaemon(params.LoggerName, "log_file", pkgconfigsetup.DefaultOTelAgentLogFile) + }), + logfx.Module(), + fetchonlyimpl.Module(), + // TODO: don't rely on this pattern; remove this `ModuleWithParams` thing + // and instead adapt OptionalModule to allow parameter passing naturally. + // See: https://github.com/DataDog/datadog-agent/pull/28386 + configsyncimpl.ModuleWithParams(), + fx.Provide(func() configsyncimpl.Params { + return configsyncimpl.NewParams(params.SyncTimeout, params.SyncDelay, true) + }), converterfx.Module(), - fx.Provide(func(cp converter.Component) confmap.Converter { + fx.Provide(func(cp converter.Component, _ configsync.Component) confmap.Converter { return cp }), collectorcontribFx.Module(), @@ -178,10 +191,10 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, fx.Invoke(func(_ collectordef.Component, _ defaultforwarder.Forwarder, _ optional.Option[logsagentpipeline.Component]) { }), - // TODO: don't rely on this pattern; remove this `OptionalModuleWithParams` thing + // TODO: don't rely on this pattern; remove this `ModuleWithParams` thing // and instead adapt OptionalModule to allow parameter passing naturally. // See: https://github.com/DataDog/datadog-agent/pull/28386 - configsyncimpl.OptionalModuleWithParams(), + configsyncimpl.ModuleWithParams(), fx.Provide(func() configsyncimpl.Params { return configsyncimpl.NewParams(params.SyncTimeout, params.SyncDelay, true) }), @@ -205,7 +218,7 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, // TODO: consider adding configsync.Component as an explicit dependency for traceconfig // to avoid this sort of dependency tree hack. - fx.Provide(func(deps traceconfig.Dependencies, _ optional.Option[configsync.Component]) (traceconfig.Component, error) { + fx.Provide(func(deps traceconfig.Dependencies, _ configsync.Component) (traceconfig.Component, error) { // TODO: this would be much better if we could leverage traceconfig.Module // Must add a new parameter to traconfig.Module to handle this. return traceconfig.NewConfig(deps) @@ -223,13 +236,13 @@ func runOTelAgentCommand(ctx context.Context, params *subcommands.GlobalParams, // ForwarderBundle returns the fx.Option for the forwarder bundle. // TODO: cleanup the forwarder instantiation with fx. -// This is a bit of a hack because we need to enforce optional.Option[configsync.Component] +// This is a bit of a hack because we need to enforce configsync.Component // is passed to newForwarder to enforce the correct instantiation order. Currently, the // new forwarder.BundleWithProvider makes a few assumptions in its generic prototype, and // this is the current workaround to leverage it. func ForwarderBundle() fx.Option { return defaultforwarder.ModulWithOptionTMP( - fx.Provide(func(_ optional.Option[configsync.Component]) defaultforwarder.Params { + fx.Provide(func(_ configsync.Component) defaultforwarder.Params { return defaultforwarder.NewParams() })) } diff --git a/cmd/process-agent/command/main_common.go b/cmd/process-agent/command/main_common.go index dcb9287e74f04..4d6b33d5ad831 100644 --- a/cmd/process-agent/command/main_common.go +++ b/cmd/process-agent/command/main_common.go @@ -65,7 +65,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/fxutil" "github.com/DataDog/datadog-agent/pkg/util/fxutil/logging" "github.com/DataDog/datadog-agent/pkg/util/log" - "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/pkg/version" ) @@ -156,7 +155,7 @@ func runApp(ctx context.Context, globalParams *GlobalParams) error { fetchonlyimpl.Module(), // Provide configsync module - configsyncimpl.OptionalModule(), + configsyncimpl.Module(), // Provide autoexit module autoexitimpl.Module(), @@ -215,7 +214,7 @@ func runApp(ctx context.Context, globalParams *GlobalParams) error { _ expvars.Component, _ apiserver.Component, cfg config.Component, - _ optional.Option[configsync.Component], + _ configsync.Component, // TODO: This is needed by the container-provider which is not currently a component. // We should ensure the tagger is a dependency when converting to a component. _ tagger.Component, diff --git a/cmd/process-agent/subcommands/config/config.go b/cmd/process-agent/subcommands/config/config.go index 1acd0ec15e5e2..f7a9996ac10ab 100644 --- a/cmd/process-agent/subcommands/config/config.go +++ b/cmd/process-agent/subcommands/config/config.go @@ -181,6 +181,11 @@ func getConfigValue(deps dependencies, args []string) error { } func getClient(cfg model.Reader) (settings.Client, error) { + err := util.SetAuthToken(cfg) + if err != nil { + return nil, err + } + httpClient := apiutil.GetClient(false) ipcAddress, err := pkgconfigsetup.GetIPCAddress(pkgconfigsetup.Datadog()) diff --git a/cmd/process-agent/subcommands/status/status.go b/cmd/process-agent/subcommands/status/status.go index 8826c7547fb61..f98877e1979a2 100644 --- a/cmd/process-agent/subcommands/status/status.go +++ b/cmd/process-agent/subcommands/status/status.go @@ -21,6 +21,7 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" compStatus "github.com/DataDog/datadog-agent/comp/core/status" "github.com/DataDog/datadog-agent/comp/process" + "github.com/DataDog/datadog-agent/pkg/api/util" apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/collector/python" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -147,6 +148,11 @@ func runStatus(deps dependencies) error { return err } + err = util.SetAuthToken(deps.Config) + if err != nil { + return err + } + getAndWriteStatus(deps.Log, statusURL, os.Stdout) return nil } diff --git a/cmd/process-agent/subcommands/taggerlist/tagger_list.go b/cmd/process-agent/subcommands/taggerlist/tagger_list.go index e7ec238efdeea..235a5a2b958df 100644 --- a/cmd/process-agent/subcommands/taggerlist/tagger_list.go +++ b/cmd/process-agent/subcommands/taggerlist/tagger_list.go @@ -18,6 +18,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" "github.com/DataDog/datadog-agent/comp/core/tagger/api" + "github.com/DataDog/datadog-agent/pkg/api/util" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) @@ -58,6 +59,10 @@ func taggerList(deps dependencies) error { return err } + err = util.SetAuthToken(deps.Config) + if err != nil { + return err + } return api.GetTaggerList(color.Output, taggerURL) } diff --git a/cmd/security-agent/api/server.go b/cmd/security-agent/api/server.go index e776628e5ccd4..307a4e4c6cd43 100644 --- a/cmd/security-agent/api/server.go +++ b/cmd/security-agent/api/server.go @@ -12,9 +12,6 @@ package api import ( "crypto/tls" - "crypto/x509" - "encoding/pem" - "fmt" stdLog "log" "net" "net/http" @@ -23,10 +20,10 @@ import ( "github.com/gorilla/mux" "github.com/DataDog/datadog-agent/cmd/security-agent/api/agent" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/core/settings" "github.com/DataDog/datadog-agent/comp/core/status" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" - "github.com/DataDog/datadog-agent/pkg/api/security" "github.com/DataDog/datadog-agent/pkg/api/util" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -35,19 +32,21 @@ import ( // Server implements security agent API server type Server struct { - listener net.Listener - agent *agent.Agent + listener net.Listener + agent *agent.Agent + tlsConfig *tls.Config } // NewServer creates a new Server instance -func NewServer(statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component) (*Server, error) { +func NewServer(statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) (*Server, error) { listener, err := newListener() if err != nil { return nil, err } return &Server{ - listener: listener, - agent: agent.NewAgent(statusComponent, settings, wmeta), + listener: listener, + agent: agent.NewAgent(statusComponent, settings, wmeta), + tlsConfig: at.GetTLSServerConfig(), }, nil } @@ -62,43 +61,16 @@ func (s *Server) Start() error { // Validate token for every request r.Use(validateToken) - err := util.CreateAndSetAuthToken(pkgconfigsetup.Datadog()) - if err != nil { - return err - } - - hosts := []string{"127.0.0.1", "localhost"} - _, rootCertPEM, rootKey, err := security.GenerateRootCert(hosts, 2048) - if err != nil { - return fmt.Errorf("unable to start TLS server") - } - - // PEM encode the private key - rootKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey), - }) - - // Create a TLS cert using the private key and certificate - rootTLSCert, err := tls.X509KeyPair(rootCertPEM, rootKeyPEM) - if err != nil { - return fmt.Errorf("invalid key pair: %v", err) - } - - tlsConfig := tls.Config{ - Certificates: []tls.Certificate{rootTLSCert}, - MinVersion: tls.VersionTLS13, - } - // Use a stack depth of 4 on top of the default one to get a relevant filename in the stdlib logWriter, _ := pkglogsetup.NewLogWriter(4, log.ErrorLvl) srv := &http.Server{ Handler: r, ErrorLog: stdLog.New(logWriter, "Error from the agent http API server: ", 0), // log errors to seelog, - TLSConfig: &tlsConfig, + TLSConfig: s.tlsConfig, WriteTimeout: pkgconfigsetup.Datadog().GetDuration("server_timeout") * time.Second, } - tlsListener := tls.NewListener(s.listener, &tlsConfig) + tlsListener := tls.NewListener(s.listener, s.tlsConfig) go srv.Serve(tlsListener) //nolint:errcheck return nil diff --git a/cmd/security-agent/main_windows.go b/cmd/security-agent/main_windows.go index b81223e6aebc1..f6a2e3aedf28f 100644 --- a/cmd/security-agent/main_windows.go +++ b/cmd/security-agent/main_windows.go @@ -25,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands/start" "github.com/DataDog/datadog-agent/comp/agent/autoexit" "github.com/DataDog/datadog-agent/comp/agent/autoexit/autoexitimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" @@ -54,7 +55,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/defaultpaths" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/pkg/util/startstop" "github.com/DataDog/datadog-agent/pkg/util/winutil/servicemain" ) @@ -91,10 +91,11 @@ func (s *service) Run(svcctx context.Context) error { params := &cliParams{} err := fxutil.OneShot( func(log log.Component, config config.Component, _ secrets.Component, _ statsd.Component, _ sysprobeconfig.Component, - telemetry telemetry.Component, _ workloadmeta.Component, _ *cliParams, statusComponent status.Component, _ autoexit.Component, settings settings.Component, wmeta workloadmeta.Component) error { + telemetry telemetry.Component, _ workloadmeta.Component, _ *cliParams, statusComponent status.Component, _ autoexit.Component, + settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) error { defer start.StopAgent(log) - err := start.RunAgent(log, config, telemetry, statusComponent, settings, wmeta) + err := start.RunAgent(log, config, telemetry, statusComponent, settings, wmeta, at) if err != nil { if errors.Is(err, start.ErrAllComponentsDisabled) { // If all components are disabled, we should exit cleanly @@ -164,9 +165,9 @@ func (s *service) Run(svcctx context.Context) error { statusimpl.Module(), fetchonlyimpl.Module(), - configsyncimpl.OptionalModule(), + configsyncimpl.Module(), // Force the instantiation of the component - fx.Invoke(func(_ optional.Option[configsync.Component]) {}), + fx.Invoke(func(_ configsync.Component) {}), autoexitimpl.Module(), fx.Provide(func(c config.Component) settings.Params { return settings.Params{ diff --git a/cmd/security-agent/subcommands/start/command.go b/cmd/security-agent/subcommands/start/command.go index 12a93fc4560ff..933a44f4a0135 100644 --- a/cmd/security-agent/subcommands/start/command.go +++ b/cmd/security-agent/subcommands/start/command.go @@ -29,6 +29,7 @@ import ( "github.com/DataDog/datadog-agent/cmd/security-agent/subcommands/runtime" "github.com/DataDog/datadog-agent/comp/agent/autoexit" "github.com/DataDog/datadog-agent/comp/agent/autoexit/autoexitimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" "github.com/DataDog/datadog-agent/comp/core/config" @@ -65,7 +66,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/status/health" "github.com/DataDog/datadog-agent/pkg/util/coredump" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "github.com/DataDog/datadog-agent/pkg/util/optional" "github.com/DataDog/datadog-agent/pkg/util/profiling" "github.com/DataDog/datadog-agent/pkg/util/startstop" "github.com/DataDog/datadog-agent/pkg/version" @@ -173,9 +173,9 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { }), statusimpl.Module(), fetchonlyimpl.Module(), - configsyncimpl.OptionalModule(), + configsyncimpl.Module(), // Force the instantiation of the component - fx.Invoke(func(_ optional.Option[configsync.Component]) {}), + fx.Invoke(func(_ configsync.Component) {}), autoexitimpl.Module(), fx.Supply(pidimpl.NewParams(params.pidfilePath)), fx.Provide(func(c config.Component) settings.Params { @@ -201,10 +201,10 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command { // TODO(components): note how workloadmeta is passed anonymously, it is still required as it is used // as a global. This should eventually be fixed and all workloadmeta interactions should be via the // injected instance. -func start(log log.Component, config config.Component, _ secrets.Component, _ statsd.Component, _ sysprobeconfig.Component, telemetry telemetry.Component, statusComponent status.Component, _ pid.Component, _ autoexit.Component, settings settings.Component, wmeta workloadmeta.Component) error { +func start(log log.Component, config config.Component, _ secrets.Component, _ statsd.Component, _ sysprobeconfig.Component, telemetry telemetry.Component, statusComponent status.Component, _ pid.Component, _ autoexit.Component, settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) error { defer StopAgent(log) - err := RunAgent(log, config, telemetry, statusComponent, settings, wmeta) + err := RunAgent(log, config, telemetry, statusComponent, settings, wmeta, at) if errors.Is(err, ErrAllComponentsDisabled) || errors.Is(err, errNoAPIKeyConfigured) { return nil } @@ -256,7 +256,7 @@ var ErrAllComponentsDisabled = errors.New("all security-agent component are disa var errNoAPIKeyConfigured = errors.New("no API key configured") // RunAgent initialized resources and starts API server -func RunAgent(log log.Component, config config.Component, telemetry telemetry.Component, statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component) (err error) { +func RunAgent(log log.Component, config config.Component, telemetry telemetry.Component, statusComponent status.Component, settings settings.Component, wmeta workloadmeta.Component, at authtoken.Component) (err error) { if err := coredump.Setup(config); err != nil { log.Warnf("Can't setup core dumps: %v, core dumps might not be available after a crash", err) } @@ -299,7 +299,7 @@ func RunAgent(log log.Component, config config.Component, telemetry telemetry.Co } }() - srv, err = api.NewServer(statusComponent, settings, wmeta) + srv, err = api.NewServer(statusComponent, settings, wmeta, at) if err != nil { return log.Errorf("Error while creating api server, exiting: %v", err) } diff --git a/cmd/system-probe/api/debug/handlers_linux.go b/cmd/system-probe/api/debug/handlers_linux.go index d2bd7dfbd5f48..07ba06c49354f 100644 --- a/cmd/system-probe/api/debug/handlers_linux.go +++ b/cmd/system-probe/api/debug/handlers_linux.go @@ -17,19 +17,18 @@ import ( "time" ) -// HandleSelinuxSestatus reports the output of sestatus as an http result -func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - cmd := exec.CommandContext(ctx, "sestatus") +// handleCommand runs commandName with the provided arguments and writes it to the HTTP response. +// If the command exits with a failure or doesn't exist in the PATH, it will still 200 but report the failure. +// Any other kind of error will 500. +func handleCommand(ctx context.Context, w http.ResponseWriter, commandName string, args ...string) { + cmd := exec.CommandContext(ctx, commandName, args...) output, err := cmd.CombinedOutput() var execError *exec.Error var exitErr *exec.ExitError if err != nil { - // don't 500 for ExitErrors etc, to report "normal" failures to the selinux_sestatus.log file + // don't 500 for ExitErrors etc, to report "normal" failures to the flare log file if !errors.As(err, &execError) && !errors.As(err, &exitErr) { w.WriteHeader(500) } @@ -39,3 +38,19 @@ func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { w.Write(output) } + +// HandleSelinuxSestatus reports the output of sestatus as an http result +func HandleSelinuxSestatus(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + handleCommand(ctx, w, "sestatus") +} + +// HandleSelinuxSemoduleList reports the output of semodule -l as an http result +func HandleSelinuxSemoduleList(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + + handleCommand(ctx, w, "semodule", "-l") +} diff --git a/cmd/system-probe/api/debug/handlers_nolinux.go b/cmd/system-probe/api/debug/handlers_nolinux.go index 1475d821c1e6e..246f4a3a7c78a 100644 --- a/cmd/system-probe/api/debug/handlers_nolinux.go +++ b/cmd/system-probe/api/debug/handlers_nolinux.go @@ -18,3 +18,9 @@ func HandleSelinuxSestatus(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) io.WriteString(w, "HandleSelinuxSestatus is not supported on this platform") } + +// HandleSelinuxSemoduleList is not supported +func HandleSelinuxSemoduleList(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(500) + io.WriteString(w, "HandleSelinuxSemoduleList is not supported on this platform") +} diff --git a/cmd/system-probe/api/server.go b/cmd/system-probe/api/server.go index d81007a0c8f0d..f4d9e85522d91 100644 --- a/cmd/system-probe/api/server.go +++ b/cmd/system-probe/api/server.go @@ -60,6 +60,7 @@ func StartServer(cfg *sysconfigtypes.Config, telemetry telemetry.Component, wmet if runtime.GOOS == "linux" { mux.HandleFunc("/debug/ebpf_btf_loader_info", ebpf.HandleBTFLoaderInfo) mux.HandleFunc("/debug/selinux_sestatus", debug.HandleSelinuxSestatus) + mux.HandleFunc("/debug/selinux_semodule_list", debug.HandleSelinuxSemoduleList) } go func() { diff --git a/cmd/system-probe/modules/gpu.go b/cmd/system-probe/modules/gpu.go index 7ec273e7c2869..c7dbac874dccd 100644 --- a/cmd/system-probe/modules/gpu.go +++ b/cmd/system-probe/modules/gpu.go @@ -55,6 +55,7 @@ var GPUMonitoring = module.Factory{ //(https://github.com/NVIDIA/go-nvml/blob/main/pkg/nvml/lib.go#L30) NvmlLib: nvml.New(nvml.WithLibraryPath(c.NVMLLibraryPath)), ProcessMonitor: processEventConsumer, + WorkloadMeta: deps.WMeta, } ret := probeDeps.NvmlLib.Init() diff --git a/cmd/trace-agent/config/remote/config.go b/cmd/trace-agent/config/remote/config.go index d2c511a4586e7..6b650364743a7 100644 --- a/cmd/trace-agent/config/remote/config.go +++ b/cmd/trace-agent/config/remote/config.go @@ -43,7 +43,7 @@ func putBuffer(buffer *bytes.Buffer) { // ConfigHandler is the HTTP handler for configs func ConfigHandler(r *api.HTTPReceiver, cf rcclient.ConfigFetcher, cfg *config.AgentConfig, statsd statsd.ClientInterface, timing timing.Reporter) http.Handler { - cidProvider := api.NewIDProvider(cfg.ContainerProcRoot) + cidProvider := api.NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo) return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { defer timing.Since("datadog.trace_agent.receiver.config_process_ms", time.Now()) tags := r.TagStats(api.V07, req.Header, "").AsTags() diff --git a/cmd/trace-agent/subcommands/run/command.go b/cmd/trace-agent/subcommands/run/command.go index 1d0788501ffae..4526101362de1 100644 --- a/cmd/trace-agent/subcommands/run/command.go +++ b/cmd/trace-agent/subcommands/run/command.go @@ -113,9 +113,9 @@ func runTraceAgentProcess(ctx context.Context, cliParams *Params, defaultConfPat zstdfx.Module(), trace.Bundle(), fetchonlyimpl.Module(), - configsyncimpl.OptionalModule(), + configsyncimpl.Module(), // Force the instantiation of the components - fx.Invoke(func(_ traceagent.Component, _ optional.Option[configsync.Component], _ autoexit.Component) {}), + fx.Invoke(func(_ traceagent.Component, _ configsync.Component, _ autoexit.Component) {}), ) if err != nil && errors.Is(err, traceagentimpl.ErrAgentDisabled) { return nil diff --git a/comp/api/api/apiimpl/api_test.go b/comp/api/api/apiimpl/api_test.go index d92e8f1e4fa48..b248dc9531330 100644 --- a/comp/api/api/apiimpl/api_test.go +++ b/comp/api/api/apiimpl/api_test.go @@ -20,7 +20,7 @@ import ( "github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl" "github.com/DataDog/datadog-agent/comp/api/api/apiimpl/observability" api "github.com/DataDog/datadog-agent/comp/api/api/def" - "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl" "github.com/DataDog/datadog-agent/comp/collector/collector" "github.com/DataDog/datadog-agent/comp/core/autodiscovery" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/autodiscoveryimpl" @@ -76,7 +76,7 @@ func getTestAPIServer(t *testing.T, params config.MockParams) testdeps { demultiplexerimpl.MockModule(), fx.Supply(optional.NewNoneOption[rcservice.Component]()), fx.Supply(optional.NewNoneOption[rcservicemrf.Component]()), - fetchonlyimpl.MockModule(), + createandfetchimpl.Module(), fx.Supply(context.Background()), taggermock.Module(), fx.Provide(func(mock taggermock.Mock) tagger.Component { @@ -166,7 +166,6 @@ func TestStartBothServersWithObservability(t *testing.T) { req, err := http.NewRequest(http.MethodGet, url, nil) require.NoError(t, err) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", authTokenValue)) resp, err := util.GetClient(false).Do(req) require.NoError(t, err) defer resp.Body.Close() diff --git a/comp/api/api/apiimpl/security.go b/comp/api/api/apiimpl/security.go index f7176bbfbf99c..4b8b17d2ba33d 100644 --- a/comp/api/api/apiimpl/security.go +++ b/comp/api/api/apiimpl/security.go @@ -7,16 +7,9 @@ package apiimpl import ( "crypto/subtle" - "crypto/tls" - "crypto/x509" - "encoding/pem" "errors" - "fmt" "net/http" - "runtime" - "strings" - "github.com/DataDog/datadog-agent/pkg/api/security" "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/util/log" ) @@ -44,44 +37,3 @@ func parseToken(token string) (interface{}, error) { // type. return struct{}{}, nil } - -func buildSelfSignedKeyPair(additionalHostIdentities ...string) ([]byte, []byte) { - hosts := []string{"127.0.0.1", "localhost", "::1"} - hosts = append(hosts, additionalHostIdentities...) - _, rootCertPEM, rootKey, err := security.GenerateRootCert(hosts, 2048) - if err != nil { - return nil, nil - } - - // PEM encode the private key - rootKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey), - }) - - // Create and return TLS private cert and key - return rootCertPEM, rootKeyPEM -} - -func initializeTLS(additionalHostIdentities ...string) (*tls.Certificate, *x509.CertPool, error) { - // print the caller to identify what is calling this function - if _, file, line, ok := runtime.Caller(1); ok { - log.Infof("[%s:%d] Initializing TLS certificates for hosts %v", file, line, strings.Join(additionalHostIdentities, ", ")) - } - - cert, key := buildSelfSignedKeyPair(additionalHostIdentities...) - if cert == nil { - return nil, nil, errors.New("unable to generate certificate") - } - pair, err := tls.X509KeyPair(cert, key) - if err != nil { - return nil, nil, fmt.Errorf("unable to generate TLS key pair: %v", err) - } - - tlsCertPool := x509.NewCertPool() - ok := tlsCertPool.AppendCertsFromPEM(cert) - if !ok { - return nil, nil, fmt.Errorf("unable to add new certificate to pool") - } - - return &pair, tlsCertPool, nil -} diff --git a/comp/api/api/apiimpl/server.go b/comp/api/api/apiimpl/server.go index 762a67b8a4a63..6def35781eb1a 100644 --- a/comp/api/api/apiimpl/server.go +++ b/comp/api/api/apiimpl/server.go @@ -11,6 +11,7 @@ import ( stdLog "log" "net" "net/http" + "strconv" "github.com/DataDog/datadog-agent/comp/api/api/apiimpl/observability" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -47,34 +48,11 @@ func (server *apiServer) startServers() error { return fmt.Errorf("unable to get IPC address and port: %v", err) } - additionalHostIdentities := []string{apiAddr} - - ipcServerHost, ipcServerHostPort, ipcServerEnabled := getIPCServerAddressPort() - if ipcServerEnabled { - additionalHostIdentities = append(additionalHostIdentities, ipcServerHost) - } - - tlsKeyPair, tlsCertPool, err := initializeTLS(additionalHostIdentities...) - if err != nil { - return fmt.Errorf("unable to initialize TLS: %v", err) - } - - // tls.Config is written to when serving, so it has to be cloned for each server - tlsConfig := func() *tls.Config { - return &tls.Config{ - Certificates: []tls.Certificate{*tlsKeyPair}, - NextProtos: []string{"h2"}, - MinVersion: tls.VersionTLS12, - } - } - tmf := observability.NewTelemetryMiddlewareFactory(server.telemetry) // start the CMD server if err := server.startCMDServer( apiAddr, - tlsConfig(), - tlsCertPool, tmf, server.cfg, ); err != nil { @@ -82,8 +60,11 @@ func (server *apiServer) startServers() error { } // start the IPC server - if ipcServerEnabled { - if err := server.startIPCServer(ipcServerHostPort, tlsConfig(), tmf); err != nil { + if ipcServerPort := server.cfg.GetInt("agent_ipc.port"); ipcServerPort > 0 { + ipcServerHost := server.cfg.GetString("agent_ipc.host") + ipcServerHostPort := net.JoinHostPort(ipcServerHost, strconv.Itoa(ipcServerPort)) + + if err := server.startIPCServer(ipcServerHostPort, tmf); err != nil { // if we fail to start the IPC server, we should stop the CMD server server.stopServers() return fmt.Errorf("unable to start IPC API server: %v", err) diff --git a/comp/api/api/apiimpl/server_cmd.go b/comp/api/api/apiimpl/server_cmd.go index 2b806aa8726e3..2eca0fe7abd09 100644 --- a/comp/api/api/apiimpl/server_cmd.go +++ b/comp/api/api/apiimpl/server_cmd.go @@ -7,8 +7,6 @@ package apiimpl import ( "context" - "crypto/tls" - "crypto/x509" "fmt" "net/http" "time" @@ -35,8 +33,6 @@ const cmdServerShortName string = "CMD" func (server *apiServer) startCMDServer( cmdAddr string, - tlsConfig *tls.Config, - tlsCertPool *x509.CertPool, tmf observability.TelemetryMiddlewareFactory, cfg config.Component, ) (err error) { @@ -54,7 +50,7 @@ func (server *apiServer) startCMDServer( maxMessageSize := cfg.GetInt("cluster_agent.cluster_tagger.grpc_max_message_size") opts := []grpc.ServerOption{ - grpc.Creds(credentials.NewClientTLSFromCert(tlsCertPool, cmdAddr)), + grpc.Creds(credentials.NewTLS(server.authToken.GetTLSServerConfig())), grpc.StreamInterceptor(grpc_auth.StreamServerInterceptor(authInterceptor)), grpc.UnaryInterceptor(grpc_auth.UnaryServerInterceptor(authInterceptor)), grpc.MaxRecvMsgSize(maxMessageSize), @@ -79,11 +75,7 @@ func (server *apiServer) startCMDServer( autodiscovery: server.autoConfig, }) - dcreds := credentials.NewTLS(&tls.Config{ - ServerName: cmdAddr, - RootCAs: tlsCertPool, - }) - dopts := []grpc.DialOption{grpc.WithTransportCredentials(dcreds)} + dopts := []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(server.authToken.GetTLSClientConfig()))} // starting grpc gateway ctx := context.Background() @@ -133,7 +125,7 @@ func (server *apiServer) startCMDServer( srv := grpcutil.NewMuxedGRPCServer( cmdAddr, - tlsConfig, + server.authToken.GetTLSServerConfig(), s, grpcutil.TimeoutHandlerFunc(cmdMuxHandler, time.Duration(pkgconfigsetup.Datadog().GetInt64("server_timeout"))*time.Second), ) diff --git a/comp/api/api/apiimpl/server_ipc.go b/comp/api/api/apiimpl/server_ipc.go index 10e17509f75e3..634ec1810f35a 100644 --- a/comp/api/api/apiimpl/server_ipc.go +++ b/comp/api/api/apiimpl/server_ipc.go @@ -6,7 +6,6 @@ package apiimpl import ( - "crypto/tls" "net/http" "time" @@ -18,7 +17,7 @@ import ( const ipcServerName string = "IPC API Server" const ipcServerShortName string = "IPC" -func (server *apiServer) startIPCServer(ipcServerAddr string, tlsConfig *tls.Config, tmf observability.TelemetryMiddlewareFactory) (err error) { +func (server *apiServer) startIPCServer(ipcServerAddr string, tmf observability.TelemetryMiddlewareFactory) (err error) { server.ipcListener, err = getListener(ipcServerAddr) if err != nil { return err @@ -39,7 +38,7 @@ func (server *apiServer) startIPCServer(ipcServerAddr string, tlsConfig *tls.Con ipcServer := &http.Server{ Addr: ipcServerAddr, Handler: http.TimeoutHandler(ipcMuxHandler, time.Duration(pkgconfigsetup.Datadog().GetInt64("server_timeout"))*time.Second, "timeout"), - TLSConfig: tlsConfig, + TLSConfig: server.authToken.GetTLSServerConfig(), } startServer(server.ipcListener, ipcServer, ipcServerName) diff --git a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go index d8631536c3a44..6098dcf98f629 100644 --- a/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go +++ b/comp/checks/agentcrashdetect/agentcrashdetectimpl/agentcrashdetect.go @@ -168,11 +168,22 @@ func (wcd *AgentCrashDetect) Run() error { } log.Infof("Sending crash: %v", formatText(crash)) - lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), wcd.tconfig.TelemetryConfig.Endpoints, "ddnpm", true) + lts := internaltelemetry.NewClient(wcd.tconfig.NewHTTPClient(), toTelemEndpoints(wcd.tconfig.TelemetryConfig.Endpoints), "ddnpm", true) lts.SendLog("WARN", formatText(crash)) return nil } +func toTelemEndpoints(endpoints []*traceconfig.Endpoint) []*internaltelemetry.Endpoint { + telemEndpoints := make([]*internaltelemetry.Endpoint, 0, len(endpoints)) + for _, e := range endpoints { + telemEndpoints = append(telemEndpoints, &internaltelemetry.Endpoint{ + Host: e.Host, + APIKey: e.APIKey, + }) + } + return telemEndpoints +} + func newAgentCrashComponent(deps dependencies) agentcrashdetect.Component { instance := &agentCrashComponent{} instance.tconfig = deps.TConfig.Object() diff --git a/comp/core/configsync/configsyncimpl/module.go b/comp/core/configsync/configsyncimpl/module.go index 0b77c1370de20..a89bbacc91197 100644 --- a/comp/core/configsync/configsyncimpl/module.go +++ b/comp/core/configsync/configsyncimpl/module.go @@ -22,7 +22,6 @@ import ( log "github.com/DataDog/datadog-agent/comp/core/log/def" apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "github.com/DataDog/datadog-agent/pkg/util/optional" ) type dependencies struct { @@ -35,20 +34,20 @@ type dependencies struct { SyncParams Params } -// OptionalModule defines the fx options for this component. -func OptionalModule() fxutil.Module { +// Module defines the fx options for this component. +func Module() fxutil.Module { return fxutil.Component( - fx.Provide(newOptionalConfigSync), + fx.Provide(newComponent), fx.Supply(Params{}), ) } -// OptionalModuleWithParams defines the fx options for this component, but +// ModuleWithParams defines the fx options for this component, but // requires additionally specifying custom Params from the fx App, to be // passed to the constructor. -func OptionalModuleWithParams() fxutil.Module { +func ModuleWithParams() fxutil.Module { return fxutil.Component( - fx.Provide(newOptionalConfigSync), + fx.Provide(newComponent), ) } @@ -61,19 +60,21 @@ type configSync struct { client *http.Client connected bool ctx context.Context + enabled bool } -// newOptionalConfigSync checks if the component was enabled as per the config, and returns an optional.Option -func newOptionalConfigSync(deps dependencies) optional.Option[configsync.Component] { +// newComponent checks if the component was enabled as per the config and return a enable/disabled configsync +func newComponent(deps dependencies) configsync.Component { agentIPCPort := deps.Config.GetInt("agent_ipc.port") configRefreshIntervalSec := deps.Config.GetInt("agent_ipc.config_refresh_interval") if agentIPCPort <= 0 || configRefreshIntervalSec <= 0 { - return optional.NewNoneOption[configsync.Component]() + deps.Log.Infof("configsync disabled (agent_ipc.port: %d | agent_ipc.config_refresh_interval: %d)", agentIPCPort, configRefreshIntervalSec) + return configSync{} } - configSync := newConfigSync(deps, agentIPCPort, configRefreshIntervalSec) - return optional.NewOption(configSync) + deps.Log.Infof("configsync enabled (agent_ipc '%s:%d' | agent_ipc.config_refresh_interval: %d)", deps.Config.GetString("agent_ipc.host"), agentIPCPort, configRefreshIntervalSec) + return newConfigSync(deps, agentIPCPort, configRefreshIntervalSec) } // newConfigSync creates a new configSync component. @@ -98,6 +99,7 @@ func newConfigSync(deps dependencies, agentIPCPort int, configRefreshIntervalSec url: url, client: client, ctx: ctx, + enabled: true, } if deps.SyncParams.OnInit { diff --git a/comp/core/configsync/configsyncimpl/module_integration_test.go b/comp/core/configsync/configsyncimpl/module_integration_test.go index e4cfa8ec1ba63..5d0fde62a449a 100644 --- a/comp/core/configsync/configsyncimpl/module_integration_test.go +++ b/comp/core/configsync/configsyncimpl/module_integration_test.go @@ -22,7 +22,6 @@ import ( "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/configsync" "github.com/DataDog/datadog-agent/pkg/util/fxutil" - "github.com/DataDog/datadog-agent/pkg/util/optional" ) func TestOptionalModule(t *testing.T) { @@ -44,16 +43,14 @@ func TestOptionalModule(t *testing.T) { "agent_ipc.port": port, "agent_ipc.config_refresh_interval": 1, } - csopt := fxutil.Test[optional.Option[configsync.Component]](t, fx.Options( + comp := fxutil.Test[configsync.Component](t, fx.Options( core.MockBundle(), fetchonlyimpl.Module(), - OptionalModule(), + Module(), fx.Populate(&cfg), fx.Replace(config.MockParams{Overrides: overrides}), )) - - _, ok := csopt.Get() - require.True(t, ok) + require.True(t, comp.(configSync).enabled) require.EventuallyWithT(t, func(t *assert.CollectT) { assert.Equal(t, "value1", cfg.Get("key1")) diff --git a/comp/core/configsync/configsyncimpl/module_test.go b/comp/core/configsync/configsyncimpl/module_test.go index 076288737a65e..8051e4cce793c 100644 --- a/comp/core/configsync/configsyncimpl/module_test.go +++ b/comp/core/configsync/configsyncimpl/module_test.go @@ -8,34 +8,30 @@ package configsyncimpl import ( "testing" - "github.com/stretchr/testify/require" - pkgconfigmodel "github.com/DataDog/datadog-agent/pkg/config/model" + "github.com/stretchr/testify/assert" ) -func TestNewOptionalConfigSync(t *testing.T) { +func TestNewConfigSync(t *testing.T) { t.Run("enabled", func(t *testing.T) { deps := makeDeps(t) deps.Config.Set("agent_ipc.port", 1234, pkgconfigmodel.SourceFile) deps.Config.Set("agent_ipc.config_refresh_interval", 30, pkgconfigmodel.SourceFile) - optConfigSync := newOptionalConfigSync(deps) - _, ok := optConfigSync.Get() - require.True(t, ok) + comp := newComponent(deps) + assert.True(t, comp.(configSync).enabled) }) t.Run("disabled ipc port zero", func(t *testing.T) { deps := makeDeps(t) deps.Config.Set("agent_ipc.port", 0, pkgconfigmodel.SourceFile) - optConfigSync := newOptionalConfigSync(deps) - _, ok := optConfigSync.Get() - require.False(t, ok) + comp := newComponent(deps) + assert.False(t, comp.(configSync).enabled) }) t.Run("disabled config refresh interval zero", func(t *testing.T) { deps := makeDeps(t) deps.Config.Set("agent_ipc.config_refresh_interval", 0, pkgconfigmodel.SourceFile) - optConfigSync := newOptionalConfigSync(deps) - _, ok := optConfigSync.Get() - require.False(t, ok) + comp := newComponent(deps) + assert.False(t, comp.(configSync).enabled) }) } diff --git a/comp/core/flare/helpers/builder_test.go b/comp/core/flare/helpers/builder_test.go index 26508575eef92..ccc544eb6ecb9 100644 --- a/comp/core/flare/helpers/builder_test.go +++ b/comp/core/flare/helpers/builder_test.go @@ -213,8 +213,8 @@ func TestAddFileYamlDetection(t *testing.T) { - abcdef - abcdef`) redacted := `instances: -- host: 127.0.0.1 - token: "********"` + - host: 127.0.0.1 + token: "********"` fb.AddFile("test.yaml", clear) assertFileContent(t, fb, redacted, "test.yaml") @@ -297,8 +297,8 @@ func TestCopyFileYamlDetection(t *testing.T) { - abcdef - abcdef`) redacted := `instances: -- host: 127.0.0.1 - token: "********"` + - host: 127.0.0.1 + token: "********"` path1 := filepath.Join(t.TempDir(), "test.yaml") os.WriteFile(path1, []byte(input), os.ModePerm) diff --git a/comp/core/flare/providers.go b/comp/core/flare/providers.go index edb2c7ac86520..69ceae5ccf76f 100644 --- a/comp/core/flare/providers.go +++ b/comp/core/flare/providers.go @@ -53,6 +53,7 @@ func (f *flare) collectLogsFiles(fb types.FlareBuilder) error { func (f *flare) collectConfigFiles(fb types.FlareBuilder) error { confSearchPaths := map[string]string{ "": f.config.GetString("confd_path"), + "fleet": filepath.Join(f.config.GetString("fleet_policies_dir"), "conf.d"), "dist": filepath.Join(f.params.distPath, "conf.d"), "checksd": f.params.pythonChecksPath, } diff --git a/comp/core/gui/guiimpl/checks.go b/comp/core/gui/guiimpl/checks.go index 058a75b08f05c..f54c682f72b8b 100644 --- a/comp/core/gui/guiimpl/checks.go +++ b/comp/core/gui/guiimpl/checks.go @@ -42,9 +42,19 @@ var ( filepath.Join(defaultpaths.GetDistPath(), "checks.d"), // Custom checks pkgconfigsetup.Datadog().GetString("additional_checksd"), // Custom checks defaultpaths.PyChecksPath, // Integrations-core checks + getFleetPoliciesPath(), // Fleet Policies } ) +// getFleetPoliciesPath returns the path to the fleet policies directory if it is set in the configuration +// otherwise it returns an empty string +func getFleetPoliciesPath() string { + if len(pkgconfigsetup.Datadog().GetString("fleet_policies_dir")) > 0 { + return filepath.Join(pkgconfigsetup.Datadog().GetString("fleet_policies_dir"), "conf.d") + } + return "" +} + // Adds the specific handlers for /checks/ endpoints func checkHandler(r *mux.Router, collector collector.Component, ac autodiscovery.Component) { r.HandleFunc("/running", http.HandlerFunc(sendRunningChecks)).Methods("POST") @@ -208,6 +218,9 @@ func getCheckConfigFile(w http.ResponseWriter, r *http.Request) { var file []byte var e error for _, path := range configPaths { + if len(path) == 0 { + continue + } filePath, err := securejoin.SecureJoin(path, fileName) if err != nil { log.Errorf("Error: Unable to join config path with the file name: %s", fileName) @@ -443,7 +456,9 @@ func getConfigsInPath(path string) ([]string, error) { func listConfigs(w http.ResponseWriter, _ *http.Request) { filenames := []string{} for _, path := range configPaths { - + if len(path) == 0 { + continue + } configs, e := getConfigsInPath(path) if e != nil { log.Errorf("Unable to list configurations from %s: %v", path, e) diff --git a/comp/core/log/mock/go.mod b/comp/core/log/mock/go.mod index 8b744f1e85ca2..e4f0f87cabdcb 100644 --- a/comp/core/log/mock/go.mod +++ b/comp/core/log/mock/go.mod @@ -58,6 +58,7 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/config/structure => ../../../../pkg/config/structure diff --git a/comp/core/status/render_helpers.go b/comp/core/status/render_helpers.go index 7ee0830c50e1b..4c4769927d4c9 100644 --- a/comp/core/status/render_helpers.go +++ b/comp/core/status/render_helpers.go @@ -23,8 +23,6 @@ import ( "github.com/fatih/color" "github.com/spf13/cast" - "golang.org/x/text/cases" - "golang.org/x/text/language" "golang.org/x/text/unicode/norm" ) @@ -290,8 +288,12 @@ func formatTitle(title string) string { } title = strings.Join(words, " ") + if title == "" { + return "" + } + // Capitalize the first letter - return cases.Title(language.English, cases.NoLower).String(title) + return strings.ToUpper(string(title[0])) + title[1:] } func status(check map[string]interface{}) string { diff --git a/comp/core/tagger/collectors/pod_tag_extractor.go b/comp/core/tagger/collectors/pod_tag_extractor.go index 669a71584f609..aded502edaf4c 100644 --- a/comp/core/tagger/collectors/pod_tag_extractor.go +++ b/comp/core/tagger/collectors/pod_tag_extractor.go @@ -34,6 +34,8 @@ func (p *PodTagExtractor) Extract(podEntity *workloadmeta.KubernetesPod, cardina return append(tagInfos.LowCardTags, tagInfos.OrchestratorCardTags...) case types.LowCardinality: return tagInfos.LowCardTags + case types.NoneCardinality: + return []string{} default: log.Errorf("unsupported tag cardinality %v", cardinality) return []string{} diff --git a/comp/core/tagger/impl/tagger.go b/comp/core/tagger/impl/tagger.go index 7023106b1c118..de05b9f6eaa09 100644 --- a/comp/core/tagger/impl/tagger.go +++ b/comp/core/tagger/impl/tagger.go @@ -422,7 +422,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert // | none | empty || empty | // | empty | not empty || container prefix + originFromMsg | // | none | not empty || container prefix + originFromMsg | - if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == "none" { + if t.datadogConfig.dogstatsdOptOutEnabled && originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" @@ -460,8 +460,7 @@ func (t *TaggerWrapper) EnrichTags(tb tagset.TagsAccumulator, originInfo taggert } default: // Disable origin detection if cardinality is none - // TODO: The `none` cardinality should be directly supported by the Tagger. - if originInfo.Cardinality == "none" { + if originInfo.Cardinality == types.NoneCardinalityString { originInfo.ContainerIDFromSocket = packets.NoOrigin originInfo.PodUID = "" originInfo.ContainerID = "" diff --git a/comp/core/tagger/subscriber/subscription_manager_test.go b/comp/core/tagger/subscriber/subscription_manager_test.go index b9684af30e2d4..dbebd8ed23d98 100644 --- a/comp/core/tagger/subscriber/subscription_manager_test.go +++ b/comp/core/tagger/subscriber/subscription_manager_test.go @@ -109,6 +109,21 @@ func TestSubscriptionManager(t *testing.T) { highCardSubscription.Unsubscribe() + // None Cardinality Subscriber + noneCardSubID := "none-card-sub" + noneCardSubscription, err := sm.Subscribe(noneCardSubID, types.NewFilterBuilder().Include(types.EntityIDPrefix("foo")).Build(types.NoneCardinality), nil) + require.NoError(t, err) + + sm.Notify([]types.EntityEvent{ + events["added"], + events["modified"], + events["deleted"], + events["added-with-no-id"], + events["added-with-unmatched-prefix"], + }) + + noneCardSubscription.Unsubscribe() + // Verify low cardinality subscriber received events assertReceivedEvents(t, lowCardSubscription.EventsChan(), []types.EntityEvent{ { @@ -192,6 +207,28 @@ func TestSubscriptionManager(t *testing.T) { }, }, }) + + // Verify none cardinality subscriber received events + assertReceivedEvents(t, noneCardSubscription.EventsChan(), []types.EntityEvent{ + { + EventType: types.EventTypeAdded, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeModified, + Entity: types.Entity{ + ID: entityID, + }, + }, + { + EventType: types.EventTypeDeleted, + Entity: types.Entity{ + ID: entityID, + }, + }, + }) } func assertReceivedEvents(t *testing.T, ch chan []types.EntityEvent, expectedEvents []types.EntityEvent) { diff --git a/comp/core/tagger/tagstore/entity_tags.go b/comp/core/tagger/tagstore/entity_tags.go index 3488a8e345c16..ed62d45849109 100644 --- a/comp/core/tagger/tagstore/entity_tags.go +++ b/comp/core/tagger/tagstore/entity_tags.go @@ -109,12 +109,16 @@ func (e *EntityTagsWithMultipleSources) getStandard() []string { func (e *EntityTagsWithMultipleSources) getHashedTags(cardinality types.TagCardinality) tagset.HashedTags { e.computeCache() - if cardinality == types.HighCardinality { + switch cardinality { + case types.HighCardinality: return e.cachedAll - } else if cardinality == types.OrchestratorCardinality { + case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} + default: + return e.cachedLow } - return e.cachedLow } func (e *EntityTagsWithMultipleSources) computeCache() { @@ -302,6 +306,8 @@ func (e *EntityTagsWithSingleSource) getHashedTags(cardinality types.TagCardinal return e.cachedAll case types.OrchestratorCardinality: return e.cachedOrchestrator + case types.NoneCardinality: + return tagset.HashedTags{} default: return e.cachedLow } diff --git a/comp/core/tagger/tagstore/entity_tags_test.go b/comp/core/tagger/tagstore/entity_tags_test.go index c93d764377aaf..16e2d4b077f86 100644 --- a/comp/core/tagger/tagstore/entity_tags_test.go +++ b/comp/core/tagger/tagstore/entity_tags_test.go @@ -100,6 +100,12 @@ func TestGetHashedTags(t *testing.T) { []string{"l1:v1", "l2:v2", "service:s1", "o1:v1", "o2:v2", "h1:v1", "h2:v2"}, entityTags.getHashedTags(types.HighCardinality).Get(), ) + + assert.Equal( + t, + []string(nil), + entityTags.getHashedTags(types.NoneCardinality).Get(), + ) } func TestTagsForSource(t *testing.T) { diff --git a/comp/core/tagger/tagstore/tagstore_test.go b/comp/core/tagger/tagstore/tagstore_test.go index 4699aa6ad56cb..0d992b682ca68 100644 --- a/comp/core/tagger/tagstore/tagstore_test.go +++ b/comp/core/tagger/tagstore/tagstore_test.go @@ -88,10 +88,12 @@ func (s *StoreTestSuite) TestLookup() { tagsHigh := s.tagstore.Lookup(entityID, types.HighCardinality) tagsOrch := s.tagstore.Lookup(entityID, types.OrchestratorCardinality) tagsLow := s.tagstore.Lookup(entityID, types.LowCardinality) + tagsNone := s.tagstore.Lookup(entityID, types.NoneCardinality) assert.Len(s.T(), tagsHigh, 4) assert.Len(s.T(), tagsLow, 2) assert.Len(s.T(), tagsOrch, 3) + assert.Nil(s.T(), tagsNone) } func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { @@ -118,10 +120,12 @@ func (s *StoreTestSuite) TestLookupHashedWithEntityStr() { tagsLow := s.tagstore.LookupHashedWithEntityStr(entityID, types.LowCardinality) tagsOrch := s.tagstore.LookupHashedWithEntityStr(entityID, types.OrchestratorCardinality) tagsHigh := s.tagstore.LookupHashedWithEntityStr(entityID, types.HighCardinality) + tagsNone := s.tagstore.LookupHashedWithEntityStr(entityID, types.NoneCardinality) assert.ElementsMatch(s.T(), tagsLow.Get(), []string{"low1", "low2"}) assert.ElementsMatch(s.T(), tagsOrch.Get(), []string{"low1", "low2", "orchestrator1"}) assert.ElementsMatch(s.T(), tagsHigh.Get(), []string{"low1", "low2", "orchestrator1", "high1"}) + assert.ElementsMatch(s.T(), tagsNone.Get(), []string{}) } func (s *StoreTestSuite) TestLookupStandard() { diff --git a/comp/core/tagger/telemetry/telemetry.go b/comp/core/tagger/telemetry/telemetry.go index 97c7153d9c052..0682475eb6d8e 100644 --- a/comp/core/tagger/telemetry/telemetry.go +++ b/comp/core/tagger/telemetry/telemetry.go @@ -68,6 +68,7 @@ type Store struct { LowCardinalityQueries CardinalityTelemetry OrchestratorCardinalityQueries CardinalityTelemetry HighCardinalityQueries CardinalityTelemetry + NoneCardinalityQueries CardinalityTelemetry UnknownCardinalityQueries CardinalityTelemetry } @@ -144,6 +145,7 @@ func NewStore(telemetryComp telemetry.Component) *Store { LowCardinalityQueries: newCardinalityTelemetry(queries, types.LowCardinalityString), OrchestratorCardinalityQueries: newCardinalityTelemetry(queries, types.OrchestratorCardinalityString), HighCardinalityQueries: newCardinalityTelemetry(queries, types.HighCardinalityString), + NoneCardinalityQueries: newCardinalityTelemetry(queries, types.NoneCardinalityString), UnknownCardinalityQueries: newCardinalityTelemetry(queries, types.UnknownCardinalityString), } }) @@ -160,6 +162,8 @@ func (s *Store) QueriesByCardinality(card types.TagCardinality) *CardinalityTele return &s.OrchestratorCardinalityQueries case types.HighCardinality: return &s.HighCardinalityQueries + case types.NoneCardinality: + return &s.NoneCardinalityQueries default: return &s.UnknownCardinalityQueries } diff --git a/comp/core/tagger/types/types.go b/comp/core/tagger/types/types.go index e45f9e6daa084..dc53af7baf27b 100644 --- a/comp/core/tagger/types/types.go +++ b/comp/core/tagger/types/types.go @@ -78,6 +78,7 @@ const ( LowCardinality TagCardinality = iota OrchestratorCardinality HighCardinality + NoneCardinality ) // Entity is an entity ID + tags. @@ -92,6 +93,10 @@ type Entity struct { // GetTags flattens all tags from all cardinalities into a single slice of tag // strings. func (e Entity) GetTags(cardinality TagCardinality) []string { + if cardinality == NoneCardinality { + return []string{} + } + tagArrays := make([][]string, 0, 3) tagArrays = append(tagArrays, e.LowCardinalityTags) @@ -117,6 +122,11 @@ func (e Entity) Copy(cardinality TagCardinality) Entity { case LowCardinality: newEntity.HighCardinalityTags = nil newEntity.OrchestratorCardinalityTags = nil + case NoneCardinality: + newEntity.HighCardinalityTags = nil + newEntity.OrchestratorCardinalityTags = nil + newEntity.LowCardinalityTags = nil + newEntity.StandardTags = nil } return newEntity @@ -131,6 +141,8 @@ const ( ShortOrchestratorCardinalityString = "orch" // HighCardinalityString is the string representation of the high cardinality HighCardinalityString = "high" + // NoneCardinalityString is the string representation of the none cardinality + NoneCardinalityString = "none" // UnknownCardinalityString represents an unknown level of cardinality UnknownCardinalityString = "unknown" ) @@ -145,6 +157,8 @@ func StringToTagCardinality(c string) (TagCardinality, error) { return OrchestratorCardinality, nil case LowCardinalityString: return LowCardinality, nil + case NoneCardinalityString: + return NoneCardinality, nil default: return LowCardinality, fmt.Errorf("unsupported value %s received for tag cardinality", c) } @@ -160,6 +174,8 @@ func TagCardinalityToString(c TagCardinality) string { return OrchestratorCardinalityString case LowCardinality: return LowCardinalityString + case NoneCardinality: + return NoneCardinalityString default: return UnknownCardinalityString } diff --git a/comp/forwarder/defaultforwarder/go.mod b/comp/forwarder/defaultforwarder/go.mod index 97eb61b033263..26fee04e02f54 100644 --- a/comp/forwarder/defaultforwarder/go.mod +++ b/comp/forwarder/defaultforwarder/go.mod @@ -78,7 +78,6 @@ require ( github.com/stretchr/testify v1.10.0 go.uber.org/atomic v1.11.0 go.uber.org/fx v1.23.0 - golang.org/x/text v0.21.0 ) require ( @@ -146,8 +145,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/comp/forwarder/defaultforwarder/go.sum b/comp/forwarder/defaultforwarder/go.sum index 735da3a7badda..75d47f4e88e64 100644 --- a/comp/forwarder/defaultforwarder/go.sum +++ b/comp/forwarder/defaultforwarder/go.sum @@ -281,8 +281,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/forwarder/defaultforwarder/internal/retry/telemetry.go b/comp/forwarder/defaultforwarder/internal/retry/telemetry.go index ba3d2c7c35f53..f676c5bc75d8a 100644 --- a/comp/forwarder/defaultforwarder/internal/retry/telemetry.go +++ b/comp/forwarder/defaultforwarder/internal/retry/telemetry.go @@ -9,9 +9,6 @@ import ( "expvar" "strings" - "golang.org/x/text/cases" - "golang.org/x/text/language" - "github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder/transaction" "github.com/DataDog/datadog-agent/pkg/telemetry" ) @@ -308,11 +305,14 @@ func (t onDiskRetryQueueTelemetry) addDeserializeTransactionsCount(count int) { } func toCamelCase(s string) string { - caser := cases.Title(language.English) parts := strings.Split(s, "_") var camelCase string for _, p := range parts { - camelCase += caser.String(p) + if p == "" { + continue + } + camelCase += strings.ToUpper(string(p[0])) + camelCase += string(p[1:]) } return camelCase } diff --git a/comp/forwarder/orchestrator/orchestratorinterface/go.mod b/comp/forwarder/orchestrator/orchestratorinterface/go.mod index 7c55fcf139cf0..4fc6e333ce46b 100644 --- a/comp/forwarder/orchestrator/orchestratorinterface/go.mod +++ b/comp/forwarder/orchestrator/orchestratorinterface/go.mod @@ -153,7 +153,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/comp/forwarder/orchestrator/orchestratorinterface/go.sum b/comp/forwarder/orchestrator/orchestratorinterface/go.sum index c23eb9a608463..a055847bfd129 100644 --- a/comp/forwarder/orchestrator/orchestratorinterface/go.sum +++ b/comp/forwarder/orchestrator/orchestratorinterface/go.sum @@ -282,8 +282,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/otelcol/collector-contrib/def/go.mod b/comp/otelcol/collector-contrib/def/go.mod index b4b92cf6fe357..de371feafc0c9 100644 --- a/comp/otelcol/collector-contrib/def/go.mod +++ b/comp/otelcol/collector-contrib/def/go.mod @@ -107,9 +107,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect - golang.org/x/net v0.31.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect gonum.org/v1/gonum v0.15.1 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/otelcol/collector-contrib/def/go.sum b/comp/otelcol/collector-contrib/def/go.sum index b0340b1c9c43d..258b336c23443 100644 --- a/comp/otelcol/collector-contrib/def/go.sum +++ b/comp/otelcol/collector-contrib/def/go.sum @@ -265,8 +265,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -278,12 +278,12 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/comp/otelcol/collector-contrib/impl/go.mod b/comp/otelcol/collector-contrib/impl/go.mod index e816b6e25e500..9b14be986534c 100644 --- a/comp/otelcol/collector-contrib/impl/go.mod +++ b/comp/otelcol/collector-contrib/impl/go.mod @@ -312,15 +312,15 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.29.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.31.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.9.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/term v0.26.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.27.0 // indirect gonum.org/v1/gonum v0.15.1 // indirect diff --git a/comp/otelcol/collector-contrib/impl/go.sum b/comp/otelcol/collector-contrib/impl/go.sum index 5672b3b4c0c95..c97ea3c785b72 100644 --- a/comp/otelcol/collector-contrib/impl/go.sum +++ b/comp/otelcol/collector-contrib/impl/go.sum @@ -1120,8 +1120,8 @@ golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1218,8 +1218,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1241,8 +1241,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1319,8 +1319,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1334,8 +1334,8 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1352,8 +1352,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/comp/otelcol/ddflareextension/def/go.mod b/comp/otelcol/ddflareextension/def/go.mod index 7ac807821aee4..2160a62e2f369 100644 --- a/comp/otelcol/ddflareextension/def/go.mod +++ b/comp/otelcol/ddflareextension/def/go.mod @@ -16,7 +16,7 @@ require ( go.opentelemetry.io/otel/trace v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/otelcol/ddflareextension/def/go.sum b/comp/otelcol/ddflareextension/def/go.sum index cb77fcd0e24d3..e8348caae0f44 100644 --- a/comp/otelcol/ddflareextension/def/go.sum +++ b/comp/otelcol/ddflareextension/def/go.sum @@ -45,8 +45,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/otelcol/ddflareextension/impl/go.mod b/comp/otelcol/ddflareextension/impl/go.mod index 7af5d796d92fb..25448ad26a8c3 100644 --- a/comp/otelcol/ddflareextension/impl/go.mod +++ b/comp/otelcol/ddflareextension/impl/go.mod @@ -13,6 +13,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/tagger/tags => ../../../core/tagger/tags github.com/DataDog/datadog-agent/comp/core/tagger/types => ../../../core/tagger/types github.com/DataDog/datadog-agent/comp/core/tagger/utils => ../../../core/tagger/utils @@ -151,6 +152,7 @@ require ( require go.opentelemetry.io/collector/extension/extensiontest v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect @@ -505,7 +507,7 @@ require ( golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/comp/otelcol/ddflareextension/impl/go.sum b/comp/otelcol/ddflareextension/impl/go.sum index d4f8e67e79da5..1b8a5abd738cb 100644 --- a/comp/otelcol/ddflareextension/impl/go.sum +++ b/comp/otelcol/ddflareextension/impl/go.sum @@ -1139,8 +1139,8 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/comp/otelcol/ddflareextension/impl/server.go b/comp/otelcol/ddflareextension/impl/server.go index ce1d2ceadef52..233dcddee1977 100644 --- a/comp/otelcol/ddflareextension/impl/server.go +++ b/comp/otelcol/ddflareextension/impl/server.go @@ -116,7 +116,7 @@ func newServer(endpoint string, handler http.Handler, auth bool) (*server, error // no easy way currently to pass required bearer auth token to OSS collector; // skip the validation if running inside a separate collector // TODO: determine way to allow OSS collector to authenticate with agent, OTEL-2226 - if auth { + if auth && util.GetAuthToken() != "" { r.Use(validateToken) } diff --git a/comp/otelcol/logsagentpipeline/go.mod b/comp/otelcol/logsagentpipeline/go.mod index 2a575bf145e7e..302cf0cf4787c 100644 --- a/comp/otelcol/logsagentpipeline/go.mod +++ b/comp/otelcol/logsagentpipeline/go.mod @@ -153,7 +153,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/comp/otelcol/logsagentpipeline/go.sum b/comp/otelcol/logsagentpipeline/go.sum index b0f7d9ee40dfd..19075c11eaba4 100644 --- a/comp/otelcol/logsagentpipeline/go.sum +++ b/comp/otelcol/logsagentpipeline/go.sum @@ -282,8 +282,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod index 711aa095cc468..9e6bc558c4423 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.mod @@ -165,7 +165,7 @@ require ( go.uber.org/dig v1.18.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum index b0f7d9ee40dfd..19075c11eaba4 100644 --- a/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum +++ b/comp/otelcol/logsagentpipeline/logsagentpipelineimpl/go.sum @@ -282,8 +282,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod index c3e28a412d2e6..b7d7886ae3aac 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ../../../../../core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ../../../../../core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../../../../core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ../../../../../core/telemetry github.com/DataDog/datadog-agent/comp/def => ../../../../../def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../../../../forwarder/defaultforwarder @@ -138,6 +139,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/log/def v0.0.0-00010101000000-000000000000 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/core/status v0.56.0-rc.3 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder v0.56.0-rc.3 // indirect @@ -319,7 +321,7 @@ require ( go.uber.org/fx v1.23.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum index e9c507fbafa8a..a86b1d8b49cf7 100644 --- a/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/datadogexporter/go.sum @@ -534,8 +534,8 @@ golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod index 80cffa6a338c5..1bd29cf7cf9ea 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.mod @@ -157,7 +157,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum index c50808e9a40cd..d1856ceabaf7e 100644 --- a/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/logsagentexporter/go.sum @@ -377,8 +377,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= diff --git a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod index 1f8b9e2be8d1d..1c14c95bf9c38 100644 --- a/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod +++ b/comp/otelcol/otlp/components/exporter/serializerexporter/go.mod @@ -215,7 +215,7 @@ require ( go.uber.org/fx v1.23.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum b/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum index a33f1cc34269d..0b852b1fd8dd7 100644 --- a/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum +++ b/comp/otelcol/otlp/components/exporter/serializerexporter/go.sum @@ -460,8 +460,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/otelcol/otlp/components/metricsclient/go.sum b/comp/otelcol/otlp/components/metricsclient/go.sum index 08cca2d402cd6..a5181f913d8e8 100644 --- a/comp/otelcol/otlp/components/metricsclient/go.sum +++ b/comp/otelcol/otlp/components/metricsclient/go.sum @@ -64,8 +64,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.mod b/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.mod index 86f54c29c25ad..0faa1b34bcd0e 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.mod +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.mod @@ -79,9 +79,9 @@ require ( go.opentelemetry.io/otel/sdk v1.32.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.31.0 // indirect - golang.org/x/sys v0.27.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.sum b/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.sum index 30f9f975f43b2..05605cfdfae08 100644 --- a/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.sum +++ b/comp/otelcol/otlp/components/processor/infraattributesprocessor/go.sum @@ -107,20 +107,20 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/comp/otelcol/otlp/components/statsprocessor/go.mod b/comp/otelcol/otlp/components/statsprocessor/go.mod index 05eb43fdf4629..cff45d337dcd1 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.mod +++ b/comp/otelcol/otlp/components/statsprocessor/go.mod @@ -3,6 +3,7 @@ module github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsproces go 1.22.0 replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../metricsclient github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../../../../comp/trace/compression/impl-gzip @@ -33,6 +34,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.56.0-rc.3 // indirect @@ -89,7 +91,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.8.0 // indirect @@ -97,7 +99,6 @@ require ( google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/comp/otelcol/otlp/components/statsprocessor/go.sum b/comp/otelcol/otlp/components/statsprocessor/go.sum index 25a9e6fccac3c..abd898f9bed2a 100644 --- a/comp/otelcol/otlp/components/statsprocessor/go.sum +++ b/comp/otelcol/otlp/components/statsprocessor/go.sum @@ -196,8 +196,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -245,8 +245,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/comp/otelcol/otlp/testutil/go.mod b/comp/otelcol/otlp/testutil/go.mod index d51e4d63d2b36..609205dc9e161 100644 --- a/comp/otelcol/otlp/testutil/go.mod +++ b/comp/otelcol/otlp/testutil/go.mod @@ -98,7 +98,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/otelcol/otlp/testutil/go.sum b/comp/otelcol/otlp/testutil/go.sum index 0778f221b9bca..dca00de17ecd0 100644 --- a/comp/otelcol/otlp/testutil/go.sum +++ b/comp/otelcol/otlp/testutil/go.sum @@ -293,8 +293,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/process/apiserver/apiserver.go b/comp/process/apiserver/apiserver.go index 24bbe6353d06f..9c37154296f64 100644 --- a/comp/process/apiserver/apiserver.go +++ b/comp/process/apiserver/apiserver.go @@ -16,8 +16,10 @@ import ( "github.com/DataDog/datadog-agent/cmd/process-agent/api" "github.com/DataDog/datadog-agent/comp/api/authtoken" - log "github.com/DataDog/datadog-agent/comp/core/log/def" + logComp "github.com/DataDog/datadog-agent/comp/core/log/def" + "github.com/DataDog/datadog-agent/pkg/api/util" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/log" ) var _ Component = (*apiserver)(nil) @@ -31,7 +33,7 @@ type dependencies struct { Lc fx.Lifecycle - Log log.Component + Log logComp.Component At authtoken.Component @@ -41,6 +43,7 @@ type dependencies struct { //nolint:revive // TODO(PROC) Fix revive linter func newApiServer(deps dependencies) Component { r := mux.NewRouter() + r.Use(validateToken) api.SetupAPIServerHandlers(deps.APIServerDeps, r) // Set up routes addr, err := pkgconfigsetup.GetProcessAPIAddressPort(pkgconfigsetup.Datadog()) @@ -84,3 +87,13 @@ func newApiServer(deps dependencies) Component { return apiserver } + +func validateToken(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := util.Validate(w, r); err != nil { + log.Warnf("invalid auth token for %s request to %s: %s", r.Method, r.RequestURI, err) + return + } + next.ServeHTTP(w, r) + }) +} diff --git a/comp/process/apiserver/apiserver_test.go b/comp/process/apiserver/apiserver_test.go index a0d94db4db4c2..1131bad880776 100644 --- a/comp/process/apiserver/apiserver_test.go +++ b/comp/process/apiserver/apiserver_test.go @@ -6,15 +6,19 @@ package apiserver import ( + "fmt" + "net" "net/http" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/fx" - "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl" "github.com/DataDog/datadog-agent/comp/core" + "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/core/settings/settingsimpl" "github.com/DataDog/datadog-agent/comp/core/status" "github.com/DataDog/datadog-agent/comp/core/status/statusimpl" @@ -22,13 +26,21 @@ import ( taggerfx "github.com/DataDog/datadog-agent/comp/core/tagger/fx" workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" workloadmetafx "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx" + "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) func TestLifecycle(t *testing.T) { + listener, err := net.Listen("tcp", ":0") + require.NoError(t, err) + port := listener.Addr().(*net.TCPAddr).Port + _ = fxutil.Test[Component](t, fx.Options( Module(), core.MockBundle(), + fx.Replace(config.MockParams{Overrides: map[string]interface{}{ + "process_config.cmd_port": port, + }}), workloadmetafx.Module(workloadmeta.NewParams()), fx.Supply( status.Params{ @@ -40,16 +52,61 @@ func TestLifecycle(t *testing.T) { }), statusimpl.Module(), settingsimpl.MockModule(), - fetchonlyimpl.MockModule(), + createandfetchimpl.Module(), )) - assert.Eventually(t, func() bool { - res, err := http.Get("http://localhost:6162/config") - if err != nil { - return false - } + assert.EventuallyWithT(t, func(c *assert.CollectT) { + url := fmt.Sprintf("http://localhost:%d/agent/status", port) + req, err := http.NewRequest("GET", url, nil) + require.NoError(c, err) + req.Header.Set("Authorization", "Bearer "+util.GetAuthToken()) + res, err := util.GetClient(false).Do(req) + require.NoError(c, err) defer res.Body.Close() + assert.Equal(c, http.StatusOK, res.StatusCode) + }, 5*time.Second, time.Second) +} - return res.StatusCode == http.StatusOK +func TestPostAuthentication(t *testing.T) { + listener, err := net.Listen("tcp", ":0") + require.NoError(t, err) + port := listener.Addr().(*net.TCPAddr).Port + + _ = fxutil.Test[Component](t, fx.Options( + Module(), + core.MockBundle(), + fx.Replace(config.MockParams{Overrides: map[string]interface{}{ + "process_config.cmd_port": port, + }}), + workloadmetafx.Module(workloadmeta.NewParams()), + fx.Supply( + status.Params{ + PythonVersionGetFunc: func() string { return "n/a" }, + }, + ), + taggerfx.Module(tagger.Params{ + UseFakeTagger: true, + }), + statusimpl.Module(), + settingsimpl.MockModule(), + createandfetchimpl.Module(), + )) + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + // No authentication + url := fmt.Sprintf("http://localhost:%d/config/log_level?value=debug", port) + req, err := http.NewRequest("POST", url, nil) + require.NoError(c, err) + res, err := util.GetClient(false).Do(req) + require.NoError(c, err) + defer res.Body.Close() + assert.Equal(c, http.StatusUnauthorized, res.StatusCode) + + // With authentication + req.Header.Set("Authorization", "Bearer "+util.GetAuthToken()) + res, err = util.GetClient(false).Do(req) + require.NoError(c, err) + defer res.Body.Close() + assert.Equal(c, http.StatusOK, res.StatusCode) }, 5*time.Second, time.Second) } diff --git a/comp/remote-config/rcclient/rcclientimpl/rcclient.go b/comp/remote-config/rcclient/rcclientimpl/rcclient.go index 4e43c1d99b1aa..374bb41bfe459 100644 --- a/comp/remote-config/rcclient/rcclientimpl/rcclient.go +++ b/comp/remote-config/rcclient/rcclientimpl/rcclient.go @@ -168,6 +168,10 @@ func (rc rcClient) start() { } } +// mrfUpdateCallback is the callback function for the AGENT_FAILOVER configs. +// It fetches all the configs targeting the agent and applies the failover settings +// using an OR strategy. In case of nil the value is not updated, for a false it does not update if +// the setting is already set to true. func (rc rcClient) mrfUpdateCallback(updates map[string]state.RawConfig, applyStateCallback func(string, state.ApplyStatus)) { // If the updates map is empty, we should unset the failover settings if they were set via RC previously if len(updates) == 0 { @@ -188,8 +192,13 @@ func (rc rcClient) mrfUpdateCallback(updates map[string]state.RawConfig, applySt return } - applied := false + var enableLogs, enableMetrics *bool + var enableLogsCfgPth, enableMetricsCfgPth string for cfgPath, update := range updates { + if (enableLogs != nil && *enableLogs) && (enableMetrics != nil && *enableMetrics) { + break + } + mrfUpdate, err := parseMultiRegionFailoverConfig(update.Config) if err != nil { pkglog.Errorf("Multi-Region Failover update unmarshal failed: %s", err) @@ -200,42 +209,55 @@ func (rc rcClient) mrfUpdateCallback(updates map[string]state.RawConfig, applySt continue } - if mrfUpdate != nil && (mrfUpdate.FailoverMetrics != nil || mrfUpdate.FailoverLogs != nil) { - // If we've received multiple config files updating the failover settings, we should disregard all but the first update and log it, as this is unexpected - if applied { - pkglog.Warnf("Multiple Multi-Region Failover updates received, disregarding update of `multi_region_failover.failover_metrics` to %v and `multi_region_failover.failover_logs` to %v", mrfUpdate.FailoverMetrics, mrfUpdate.FailoverLogs) - applyStateCallback(cfgPath, state.ApplyStatus{ - State: state.ApplyStateError, - Error: "Multiple Multi-Region Failover updates received. Only the first was applied.", - }) - continue - } + if mrfUpdate == nil || (mrfUpdate.FailoverMetrics == nil && mrfUpdate.FailoverLogs == nil) { + continue + } - if mrfUpdate.FailoverMetrics != nil { - err = rc.applyMRFRuntimeSetting("multi_region_failover.failover_metrics", *mrfUpdate.FailoverMetrics, cfgPath, applyStateCallback) - if err != nil { - continue - } - change := "disabled" - if *mrfUpdate.FailoverMetrics { - change = "enabled" - } - pkglog.Infof("Received remote update for Multi-Region Failover configuration: %s failover for metrics", change) - } - if mrfUpdate.FailoverLogs != nil { - err = rc.applyMRFRuntimeSetting("multi_region_failover.failover_logs", *mrfUpdate.FailoverLogs, cfgPath, applyStateCallback) - if err != nil { - continue - } - change := "disabled" - if *mrfUpdate.FailoverLogs { - change = "enabled" - } - pkglog.Infof("Received remote update for Multi-Region Failover configuration: %s failover for logs", change) - } - applyStateCallback(cfgPath, state.ApplyStatus{State: state.ApplyStateAcknowledged}) - applied = true + if !(enableMetrics != nil && *enableMetrics) && mrfUpdate.FailoverMetrics != nil { + enableMetrics = mrfUpdate.FailoverMetrics + enableMetricsCfgPth = cfgPath + } + + if !(enableLogs != nil && *enableLogs) && mrfUpdate.FailoverLogs != nil { + enableLogs = mrfUpdate.FailoverLogs + enableLogsCfgPth = cfgPath + } + } + + if enableMetrics != nil { + err := rc.applyMRFRuntimeSetting("multi_region_failover.failover_metrics", *enableMetrics, enableMetricsCfgPth, applyStateCallback) + if err != nil { + pkglog.Errorf("Multi-Region Failover failed to apply new metrics settings : %s", err) + applyStateCallback(enableMetricsCfgPth, state.ApplyStatus{ + State: state.ApplyStateError, + Error: err.Error(), + }) + return + } + change := "disabled" + if *enableMetrics { + change = "enabled" + } + pkglog.Infof("Received remote update for Multi-Region Failover configuration: %s failover for metrics", change) + applyStateCallback(enableMetricsCfgPth, state.ApplyStatus{State: state.ApplyStateAcknowledged}) + } + + if enableLogs != nil { + err := rc.applyMRFRuntimeSetting("multi_region_failover.failover_logs", *enableLogs, enableLogsCfgPth, applyStateCallback) + if err != nil { + pkglog.Errorf("Multi-Region Failover failed to apply new logs settings : %s", err) + applyStateCallback(enableMetricsCfgPth, state.ApplyStatus{ + State: state.ApplyStateError, + Error: err.Error(), + }) + return + } + change := "disabled" + if *enableLogs { + change = "enabled" } + pkglog.Infof("Received remote update for Multi-Region Failover configuration: %s failover for logs", change) + applyStateCallback(enableLogsCfgPth, state.ApplyStatus{State: state.ApplyStateAcknowledged}) } } diff --git a/comp/remote-config/rcclient/rcclientimpl/rcclient_test.go b/comp/remote-config/rcclient/rcclientimpl/rcclient_test.go index f6974273d3325..71b8bca9de2f2 100644 --- a/comp/remote-config/rcclient/rcclientimpl/rcclient_test.go +++ b/comp/remote-config/rcclient/rcclientimpl/rcclient_test.go @@ -6,6 +6,7 @@ package rcclientimpl import ( + "fmt" "testing" "time" @@ -61,6 +62,39 @@ func (m *mockLogLevelRuntimeSettings) Hidden() bool { func applyEmpty(_ string, _ state.ApplyStatus) {} +type MockComponent interface { + settings.Component + + SetRuntimeSetting(setting string, value interface{}, source model.Source) error +} + +type MockComponentImplMrf struct { + settings.Component + + logs *bool + metrics *bool + traces *bool +} + +func (m *MockComponentImplMrf) SetRuntimeSetting(setting string, value interface{}, _ model.Source) error { + v, ok := value.(bool) + if !ok { + return fmt.Errorf("unexpected value type %T", value) + } + + switch setting { + case "multi_region_failover.failover_metrics": + m.metrics = &v + case "multi_region_failover.failover_logs": + m.logs = &v + case "multi_region_failover.failover_traces": + m.traces = &v + default: + return &settings.SettingNotFoundError{Name: setting} + } + return nil +} + func TestRCClientCreate(t *testing.T) { _, err := newRemoteConfigClient( fxutil.Test[dependencies]( @@ -189,3 +223,61 @@ func TestAgentConfigCallback(t *testing.T) { assert.Equal(t, "debug", cfg.Get("log_level")) assert.Equal(t, model.SourceCLI, cfg.GetSource("log_level")) } + +func TestAgentMRFConfigCallback(t *testing.T) { + pkglog.SetupLogger(pkglog.Default(), "info") + cfg := configmock.New(t) + + rc := fxutil.Test[rcclient.Component](t, + fx.Options( + Module(), + fx.Provide(func() log.Component { return logmock.New(t) }), + fx.Provide(func() config.Component { return cfg }), + sysprobeconfig.NoneModule(), + fx.Supply( + rcclient.Params{ + AgentName: "test-agent", + AgentVersion: "7.0.0", + }, + ), + fx.Supply( + settings.Params{ + Settings: map[string]settings.RuntimeSetting{ + "log_level": &mockLogLevelRuntimeSettings{logLevel: "info"}, + }, + Config: cfg, + }, + ), + settingsimpl.Module(), + ), + ) + + allInactive := state.RawConfig{Config: []byte(`{"name": "none"}`)} + noLogs := state.RawConfig{Config: []byte(`{"name": "nologs", "failover_logs": false}`)} + activeMetrics := state.RawConfig{Config: []byte(`{"name": "yesmetrics", "failover_metrics": true}`)} + + structRC := rc.(rcClient) + + ipcAddress, err := pkgconfigsetup.GetIPCAddress(cfg) + assert.NoError(t, err) + + structRC.client, _ = client.NewUnverifiedGRPCClient( + ipcAddress, pkgconfigsetup.GetIPCPort(), func() (string, error) { return security.FetchAuthToken(cfg) }, + client.WithAgent("test-agent", "9.99.9"), + client.WithProducts(state.ProductAgentConfig), + client.WithPollInterval(time.Hour), + ) + structRC.settingsComponent = &MockComponentImplMrf{} + + // Should enable metrics failover and disable logs failover + structRC.mrfUpdateCallback(map[string]state.RawConfig{ + "datadog/2/AGENT_FAILOVER/none/configname": allInactive, + "datadog/2/AGENT_FAILOVER/nologs/configname": noLogs, + "datadog/2/AGENT_FAILOVER/yesmetrics/configname": activeMetrics, + }, applyEmpty) + + cmpntSettings := structRC.settingsComponent.(*MockComponentImplMrf) + assert.True(t, *cmpntSettings.metrics) + assert.False(t, *cmpntSettings.logs) + assert.Nil(t, cmpntSettings.traces) +} diff --git a/comp/trace/agent/def/go.mod b/comp/trace/agent/def/go.mod index 56d13f69c88cf..1bfc5a3ba75af 100644 --- a/comp/trace/agent/def/go.mod +++ b/comp/trace/agent/def/go.mod @@ -26,7 +26,7 @@ require ( go.opentelemetry.io/otel/trace v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect diff --git a/comp/trace/agent/def/go.sum b/comp/trace/agent/def/go.sum index 7b3dbbc7fe0b3..92539a14a3bd1 100644 --- a/comp/trace/agent/def/go.sum +++ b/comp/trace/agent/def/go.sum @@ -82,8 +82,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/comp/trace/agent/impl/agent.go b/comp/trace/agent/impl/agent.go index b3d783288ceb6..de9237dedeea5 100644 --- a/comp/trace/agent/impl/agent.go +++ b/comp/trace/agent/impl/agent.go @@ -24,6 +24,7 @@ import ( "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/api/authtoken" "github.com/DataDog/datadog-agent/comp/core/secrets" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" "github.com/DataDog/datadog-agent/comp/dogstatsd/statsd" @@ -68,6 +69,7 @@ type dependencies struct { Statsd statsd.Component Tagger tagger.Component Compressor compression.Component + At authtoken.Component } var _ traceagent.Component = (*component)(nil) @@ -93,6 +95,7 @@ type component struct { params *Params tagger tagger.Component telemetryCollector telemetry.TelemetryCollector + at authtoken.Component wg *sync.WaitGroup } @@ -115,6 +118,7 @@ func NewAgent(deps dependencies) (traceagent.Component, error) { params: deps.Params, telemetryCollector: deps.TelemetryCollector, tagger: deps.Tagger, + at: deps.At, wg: &sync.WaitGroup{}, } statsdCl, err := setupMetrics(deps.Statsd, c.config, c.telemetryCollector) diff --git a/comp/trace/agent/impl/run.go b/comp/trace/agent/impl/run.go index 20ecbc6496a1b..f40b36e29ae7f 100644 --- a/comp/trace/agent/impl/run.go +++ b/comp/trace/agent/impl/run.go @@ -98,6 +98,9 @@ func runAgentSidekicks(ag component) error { })) } + // Configure the Trace Agent Debug server to use the IPC certificate + ag.Agent.DebugServer.SetTLSConfig(ag.at.GetTLSServerConfig()) + log.Infof("Trace agent running on host %s", tracecfg.Hostname) if pcfg := profilingConfig(tracecfg); pcfg != nil { if err := profiling.Start(*pcfg); err != nil { diff --git a/comp/trace/bundle_test.go b/comp/trace/bundle_test.go index e9874a4b40077..692e7c255f473 100644 --- a/comp/trace/bundle_test.go +++ b/comp/trace/bundle_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/fx" + "github.com/DataDog/datadog-agent/comp/api/authtoken/createandfetchimpl" + "github.com/DataDog/datadog-agent/comp/api/authtoken/fetchonlyimpl" "github.com/DataDog/datadog-agent/comp/core" coreconfig "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" @@ -45,6 +47,7 @@ func TestBundleDependencies(t *testing.T) { zstdfx.Module(), taggerfx.Module(tagger.Params{}), fx.Supply(&traceagentimpl.Params{}), + createandfetchimpl.Module(), ) } @@ -75,6 +78,7 @@ func TestMockBundleDependencies(t *testing.T) { fx.Invoke(func(_ traceagent.Component) {}), MockBundle(), taggerfx.Module(tagger.Params{}), + fetchonlyimpl.MockModule(), )) require.NotNil(t, cfg.Object()) diff --git a/comp/trace/config/config_test.go b/comp/trace/config/config_test.go index 9c9d45eb284ac..9706e2c784f4c 100644 --- a/comp/trace/config/config_test.go +++ b/comp/trace/config/config_test.go @@ -9,6 +9,7 @@ import ( "bufio" "bytes" "context" + _ "embed" "encoding/json" "errors" "net/http" @@ -245,6 +246,9 @@ func TestTelemetryEndpointsConfig(t *testing.T) { }) } +//go:embed testdata/stringcode.go.tmpl +var stringCodeBody string + func TestConfigHostname(t *testing.T) { t.Run("fail", func(t *testing.T) { overrides := map[string]interface{}{ @@ -360,11 +364,6 @@ func TestConfigHostname(t *testing.T) { }) t.Run("external", func(t *testing.T) { - body, err := os.ReadFile("testdata/stringcode.go.tmpl") - if err != nil { - t.Fatal(err) - } - // makeProgram creates a new binary file which returns the given response and exits to the OS // given the specified code, returning the path of the program. makeProgram := func(t *testing.T, response string, code int) string { @@ -372,7 +371,7 @@ func TestConfigHostname(t *testing.T) { if err != nil { t.Fatal(err) } - tmpl, err := template.New("program").Parse(string(body)) + tmpl, err := template.New("program").Parse(stringCodeBody) if err != nil { t.Fatal(err) } @@ -399,6 +398,7 @@ func TestConfigHostname(t *testing.T) { fallbackHostnameFunc = func() (string, error) { return "fallback.host", nil } t.Run("good", func(t *testing.T) { + t.Skip("Skip flaky test while we explore fixes.") bin := makeProgram(t, "host.name", 0) defer os.Remove(bin) diff --git a/comp/trace/config/setup.go b/comp/trace/config/setup.go index f19c5eeaf8a4f..06a157dc6418b 100644 --- a/comp/trace/config/setup.go +++ b/comp/trace/config/setup.go @@ -19,13 +19,14 @@ import ( "go.opentelemetry.io/collector/component/componenttest" - apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" corecompcfg "github.com/DataDog/datadog-agent/comp/core/config" tagger "github.com/DataDog/datadog-agent/comp/core/tagger/def" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/comp/otelcol/otlp/configcheck" + apiutil "github.com/DataDog/datadog-agent/pkg/api/util" "github.com/DataDog/datadog-agent/pkg/config/env" "github.com/DataDog/datadog-agent/pkg/config/model" pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" @@ -121,6 +122,9 @@ func prepareConfig(c corecompcfg.Component, tagger tagger.Component) (*config.Ag cfg.ContainerTags = func(cid string) ([]string, error) { return tagger.Tag(types.NewEntityID(types.ContainerID, cid), types.HighCardinality) } + cfg.ContainerIDFromOriginInfo = func(originInfo origindetection.OriginInfo) (string, error) { + return tagger.GenerateContainerIDFromOriginInfo(originInfo) + } cfg.ContainerProcRoot = coreConfigObject.GetString("container_proc_root") cfg.GetAgentAuthToken = apiutil.GetAuthToken return cfg, nil diff --git a/comp/trace/status/statusimpl/status.go b/comp/trace/status/statusimpl/status.go index e476ee0281d7a..00a8730b87da8 100644 --- a/comp/trace/status/statusimpl/status.go +++ b/comp/trace/status/statusimpl/status.go @@ -95,7 +95,7 @@ func (s statusProvider) populateStatus() map[string]interface{} { port := s.Config.GetInt("apm_config.debug.port") c := client() - url := fmt.Sprintf("http://localhost:%d/debug/vars", port) + url := fmt.Sprintf("https://localhost:%d/debug/vars", port) resp, err := apiutil.DoGet(c, url, apiutil.CloseConnection) if err != nil { return map[string]interface{}{ diff --git a/comp/updater/telemetry/telemetryimpl/telemetry.go b/comp/updater/telemetry/telemetryimpl/telemetry.go index d8f961e8398e6..ee1bab4efdd56 100644 --- a/comp/updater/telemetry/telemetryimpl/telemetry.go +++ b/comp/updater/telemetry/telemetryimpl/telemetry.go @@ -7,10 +7,10 @@ package telemetryimpl import ( + "context" "net/http" "go.uber.org/fx" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "github.com/DataDog/datadog-agent/comp/core/config" "github.com/DataDog/datadog-agent/comp/updater/telemetry" @@ -38,13 +38,7 @@ func newTelemetry(deps dependencies) (telemetry.Component, error) { client := &http.Client{ Transport: httputils.CreateHTTPTransport(deps.Config), } - telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon", - fleettelemetry.WithSamplingRules( - tracer.NameServiceRule("cdn.*", "datadog-installer-daemon", 0.1), - tracer.NameServiceRule("*garbage_collect*", "datadog-installer-daemon", 0.05), - tracer.NameServiceRule("HTTPClient.*", "datadog-installer-daemon", 0.05), - ), - ) - deps.Lc.Append(fx.Hook{OnStart: telemetry.Start, OnStop: telemetry.Stop}) + telemetry := fleettelemetry.NewTelemetry(client, utils.SanitizeAPIKey(deps.Config.GetString("api_key")), deps.Config.GetString("site"), "datadog-installer-daemon") + deps.Lc.Append(fx.Hook{OnStop: func(context.Context) error { telemetry.Stop(); return nil }}) return telemetry, nil } diff --git a/docs/cloud-workload-security/backend_linux.md b/docs/cloud-workload-security/backend_linux.md index 7b0794f82fc9d..e4e1af8c22c37 100644 --- a/docs/cloud-workload-security/backend_linux.md +++ b/docs/cloud-workload-security/backend_linux.md @@ -1718,9 +1718,6 @@ CSM Threats event for Linux systems have the following JSON schema: "container": { "$ref": "#/$defs/ContainerContext" }, - "cgroup": { - "$ref": "#/$defs/CGroupContext" - }, "network": { "$ref": "#/$defs/NetworkContext" }, @@ -1730,6 +1727,9 @@ CSM Threats event for Linux systems have the following JSON schema: "security_profile": { "$ref": "#/$defs/SecurityProfileContext" }, + "cgroup": { + "$ref": "#/$defs/CGroupContext" + }, "selinux": { "$ref": "#/$defs/SELinuxEvent" }, @@ -1802,10 +1802,10 @@ CSM Threats event for Linux systems have the following JSON schema: | `exit` | $ref | Please see [ExitEvent](#exitevent) | | `process` | $ref | Please see [ProcessContext](#processcontext) | | `container` | $ref | Please see [ContainerContext](#containercontext) | -| `cgroup` | $ref | Please see [CGroupContext](#cgroupcontext) | | `network` | $ref | Please see [NetworkContext](#networkcontext) | | `dd` | $ref | Please see [DDContext](#ddcontext) | | `security_profile` | $ref | Please see [SecurityProfileContext](#securityprofilecontext) | +| `cgroup` | $ref | Please see [CGroupContext](#cgroupcontext) | | `selinux` | $ref | Please see [SELinuxEvent](#selinuxevent) | | `bpf` | $ref | Please see [BPFEvent](#bpfevent) | | `mmap` | $ref | Please see [MMapEvent](#mmapevent) | diff --git a/docs/cloud-workload-security/backend_linux.schema.json b/docs/cloud-workload-security/backend_linux.schema.json index b6f9cddc6cbb7..b4aa6b761fbfb 100644 --- a/docs/cloud-workload-security/backend_linux.schema.json +++ b/docs/cloud-workload-security/backend_linux.schema.json @@ -1707,9 +1707,6 @@ "container": { "$ref": "#/$defs/ContainerContext" }, - "cgroup": { - "$ref": "#/$defs/CGroupContext" - }, "network": { "$ref": "#/$defs/NetworkContext" }, @@ -1719,6 +1716,9 @@ "security_profile": { "$ref": "#/$defs/SecurityProfileContext" }, + "cgroup": { + "$ref": "#/$defs/CGroupContext" + }, "selinux": { "$ref": "#/$defs/SELinuxEvent" }, diff --git a/flakes.yaml b/flakes.yaml index ab2af6b55f3ea..8c95e78ab01e0 100644 --- a/flakes.yaml +++ b/flakes.yaml @@ -11,3 +11,4 @@ test/new-e2e/tests/containers: - TestECSSuite/TestCPU/metric___container.cpu.usage{^ecs_container_name:stress-ng$} - TestEKSSuite/TestCPU/metric___container.cpu.usage{^kube_deployment:stress-ng$,^kube_namespace:workload-cpustress$} - TestKindSuite/TestCPU/metric___container.cpu.usage{^kube_deployment:stress-ng$,^kube_namespace:workload-cpustress$} + - TestKindSuite/TestAdmissionControllerWithAutoDetectedLanguage diff --git a/go.mod b/go.mod index 30981dbec9ce2..744c460efe7c6 100644 --- a/go.mod +++ b/go.mod @@ -159,7 +159,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/pointer v0.59.0 github.com/DataDog/datadog-agent/pkg/util/scrubber v0.59.1 github.com/DataDog/datadog-go/v5 v5.6.0 - github.com/DataDog/datadog-operator v1.11.0-rc.2 + github.com/DataDog/datadog-operator v0.7.1-0.20241219210556-f517775059d1 github.com/DataDog/ebpf-manager v0.7.4 github.com/DataDog/gopsutil v1.2.2 github.com/DataDog/nikos v1.12.8 @@ -308,7 +308,7 @@ require ( go4.org/netipx v0.0.0-20220812043211-3cc044ffd68d golang.org/x/arch v0.12.0 golang.org/x/exp v0.0.0-20241210194714-1829a127f884 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/sync v0.10.0 golang.org/x/sys v0.28.0 golang.org/x/text v0.21.0 @@ -654,7 +654,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 github.com/DataDog/datadog-agent/comp/core/status v0.59.0-rc.6 github.com/DataDog/datadog-agent/comp/core/status/statusimpl v0.56.0-rc.3 - github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-00010101000000-000000000000 + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/DataDog/datadog-agent/comp/core/tagger/tags v0.0.0-00010101000000-000000000000 github.com/DataDog/datadog-agent/comp/core/tagger/types v0.59.0 github.com/DataDog/datadog-agent/comp/core/telemetry v0.59.0 diff --git a/go.sum b/go.sum index 26375333ef0cf..60055b97b4ad9 100644 --- a/go.sum +++ b/go.sum @@ -130,8 +130,8 @@ github.com/DataDog/datadog-api-client-go/v2 v2.33.0/go.mod h1:d3tOEgUd2kfsr9uuHQ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= -github.com/DataDog/datadog-operator v1.11.0-rc.2 h1:4tMZlxbYE0WEpRYAhoqEe8nLP67C/PDq7utJJcD8RM8= -github.com/DataDog/datadog-operator v1.11.0-rc.2/go.mod h1:mD+3PWR0wOSVJGaXjkpzsYEK/7PhqjOipx2usgfsxM0= +github.com/DataDog/datadog-operator v0.7.1-0.20241219210556-f517775059d1 h1:EFGXjo7iSZO1f0ZpLE2H2UIcPH2x4yhxQnKd8UKrs3k= +github.com/DataDog/datadog-operator v0.7.1-0.20241219210556-f517775059d1/go.mod h1:mD+3PWR0wOSVJGaXjkpzsYEK/7PhqjOipx2usgfsxM0= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42 h1:RoH7VLzTnxHEugRPIgnGlxwDFszFGI7b3WZZUtWuPRM= github.com/DataDog/dd-sensitive-data-scanner/sds-go/go v0.0.0-20240816154533-f7f9beb53a42/go.mod h1:TX7CTOQ3LbQjfAi4SwqUoR5gY1zfUk7VRBDTuArjaDc= github.com/DataDog/dd-trace-go/v2 v2.0.0-beta.11 h1:6vwU//TjBIghQKMgIP9UyIRhN/LWS1y8tYzvRnu8JZw= @@ -2257,8 +2257,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/internal/tools/go.mod b/internal/tools/go.mod index 79be9ca61a043..3b03d5fa6b2af 100644 --- a/internal/tools/go.mod +++ b/internal/tools/go.mod @@ -227,7 +227,7 @@ require ( golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/internal/tools/go.sum b/internal/tools/go.sum index 2bf7f8b759bd6..e3f2dd4d19710 100644 --- a/internal/tools/go.sum +++ b/internal/tools/go.sum @@ -643,8 +643,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20170207211851-4464e7848382/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/perf v0.0.0-20210220033136-40a54f11e909 h1:rWw0Gj4DMl/2otJ8CnfTcwOWkpROAc6qhXXoMrYOCgo= golang.org/x/perf v0.0.0-20210220033136-40a54f11e909/go.mod h1:KRSrLY7jerMEa0Ih7gBheQ3FYDiSx6liMnniX1o3j2g= diff --git a/internal/tools/proto/go.mod b/internal/tools/proto/go.mod index 12d0a2fb58004..4c81cd0d2261a 100644 --- a/internal/tools/proto/go.mod +++ b/internal/tools/proto/go.mod @@ -19,7 +19,7 @@ require ( github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect diff --git a/internal/tools/proto/go.sum b/internal/tools/proto/go.sum index 3d105f38523d1..43d99cf5a50b1 100644 --- a/internal/tools/proto/go.sum +++ b/internal/tools/proto/go.sum @@ -71,8 +71,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/modules.yml b/modules.yml index 291db46bc53cc..5d5e5cb2057ff 100644 --- a/modules.yml +++ b/modules.yml @@ -35,7 +35,8 @@ modules: comp/core/status: used_by_otel: true comp/core/status/statusimpl: default - comp/core/tagger/origindetection: default + comp/core/tagger/origindetection: + used_by_otel: true comp/core/tagger/tags: used_by_otel: true comp/core/tagger/types: diff --git a/omnibus/config/software/openssl-fips-provider.rb b/omnibus/config/software/openssl-fips-provider.rb index 776d7e81531b3..7d9584b2e3767 100644 --- a/omnibus/config/software/openssl-fips-provider.rb +++ b/omnibus/config/software/openssl-fips-provider.rb @@ -22,14 +22,16 @@ relative_path "openssl-#{OPENSSL_FIPS_MODULE_VERSION}" build do + env = with_standard_compiler_flags() + env["MAKEFLAGS"] = "-j#{workers}" # Exact build steps from security policy: # https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp4282.pdf # # ---------------- DO NOT MODIFY LINES BELOW HERE ---------------- - command "./Configure enable-fips" + command "./Configure enable-fips", env: env - command "make" - command "make install" + command "make", env: env + command "make install", env: env # ---------------- DO NOT MODIFY LINES ABOVE HERE ---------------- mkdir "#{install_dir}/embedded/ssl" diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 2fe37a032fc24..1cd3c096363f2 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -54,11 +54,11 @@ # Don't forward CC and CXX to python extensions Makefile, it's quite unlikely that any non default # compiler we use would end up being available in the system/docker image used by customers - if linux_target? && ENV["CC"] + if linux_target? && env["CC"] command "sed -i \"s/^CC=[[:space:]]*${CC}/CC=gcc/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CC}/gcc/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env end - if linux_target? && ENV["CXX"] + if linux_target? && env["CXX"] command "sed -i \"s/^CXX=[[:space:]]*${CXX}/CC=g++/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-3.12-*-linux-gnu/Makefile", :env => env command "sed -i \"s/${CXX}/g++/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env end diff --git a/omnibus/config/templates/openssl-fips-provider/openssl.cnf.erb b/omnibus/config/templates/openssl-fips-provider/openssl.cnf.erb index 1304d1021b08d..23b1637b9c017 100644 --- a/omnibus/config/templates/openssl-fips-provider/openssl.cnf.erb +++ b/omnibus/config/templates/openssl-fips-provider/openssl.cnf.erb @@ -5,6 +5,7 @@ openssl_conf = openssl_init [openssl_init] providers = provider_sect +alg_section = algorithm_sect [provider_sect] fips = fips_sect @@ -12,3 +13,6 @@ base = base_sect [base_sect] activate = 1 + +[algorithm_sect] +default_properties = fips=yes diff --git a/pkg/api/security/security.go b/pkg/api/security/security.go index 40b1310c2f79c..93867572572d0 100644 --- a/pkg/api/security/security.go +++ b/pkg/api/security/security.go @@ -239,7 +239,7 @@ func validateAuthToken(authToken string) error { // writes auth token(s) to a file with the same permissions as datadog.yaml func saveAuthToken(token, tokenPath string) error { log.Infof("Saving a new authentication token in %s", tokenPath) - if err := os.WriteFile(tokenPath, []byte(token), 0o600); err != nil { + if err := writeFileAndSync(tokenPath, []byte(token), 0o600); err != nil { return err } @@ -256,3 +256,21 @@ func saveAuthToken(token, tokenPath string) error { log.Infof("Wrote auth token in %s", tokenPath) return nil } + +// call the file.Sync method to flush the file to disk and catch potential I/O errors +func writeFileAndSync(name string, data []byte, perm os.FileMode) error { + f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) + if err != nil { + return err + } + _, err = f.Write(data) + // only sync data if the write was successful + if err == nil { + err = f.Sync() + } + // but always close the file and report the first error that occurred + if err1 := f.Close(); err1 != nil && err == nil { + err = err1 + } + return err +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go index 06c936d6d0684..302fa2f36b0e0 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar.go @@ -34,6 +34,7 @@ import ( apiCommon "github.com/DataDog/datadog-agent/pkg/util/kubernetes/apiserver/common" "github.com/DataDog/datadog-agent/pkg/util/kubernetes/clustername" "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) const webhookName = "agent_sidecar" @@ -59,7 +60,7 @@ type Webhook struct { // These fields store datadog agent config parameters // to avoid calling the config resolution each time the webhook // receives requests because the resolution is CPU expensive. - profilesJSON string + profileOverrides []ProfileOverride provider string imageName string imageTag string @@ -72,7 +73,12 @@ type Webhook struct { // NewWebhook returns a new Webhook func NewWebhook(datadogConfig config.Component) *Webhook { - nsSelector, objSelector := labelSelectors(datadogConfig) + profileOverrides, err := loadSidecarProfiles(datadogConfig.GetString("admission_controller.agent_sidecar.profiles")) + if err != nil { + log.Errorf("encountered issue when loading sidecar profiles: %s", err) + } + + nsSelector, objSelector := labelSelectors(datadogConfig, profileOverrides) containerRegistry := mutatecommon.ContainerRegistry(datadogConfig, "admission_controller.agent_sidecar.container_registry") @@ -86,8 +92,8 @@ func NewWebhook(datadogConfig config.Component) *Webhook { namespaceSelector: nsSelector, objectSelector: objSelector, containerRegistry: containerRegistry, + profileOverrides: profileOverrides, - profilesJSON: datadogConfig.GetString("admission_controller.agent_sidecar.profiles"), provider: datadogConfig.GetString("admission_controller.agent_sidecar.provider"), imageName: datadogConfig.GetString("admission_controller.agent_sidecar.image_name"), imageTag: datadogConfig.GetString("admission_controller.agent_sidecar.image_tag"), @@ -150,6 +156,18 @@ func (w *Webhook) WebhookFunc() admission.WebhookFunc { } } +// isReadOnlyRootFilesystem returns whether the agent sidecar should have the readOnlyRootFilesystem security setup +func (w *Webhook) isReadOnlyRootFilesystem() bool { + if len(w.profileOverrides) == 0 { + return false + } + securityContext := w.profileOverrides[0].SecurityContext + if securityContext != nil && securityContext.ReadOnlyRootFilesystem != nil { + return *securityContext.ReadOnlyRootFilesystem + } + return false // default to false (temp) +} + func (w *Webhook) injectAgentSidecar(pod *corev1.Pod, _ string, _ dynamic.Interface) (bool, error) { if pod == nil { return false, errors.New(metrics.InvalidInput) @@ -163,6 +181,16 @@ func (w *Webhook) injectAgentSidecar(pod *corev1.Pod, _ string, _ dynamic.Interf if !agentSidecarExists { agentSidecarContainer := w.getDefaultSidecarTemplate() + if w.isReadOnlyRootFilesystem() { + volumes := w.getSecurityVolumeTemplates() + pod.Spec.Volumes = append(pod.Spec.Volumes, *volumes...) + w.addSecurityConfigToAgent(agentSidecarContainer) + // Don't want to apply any overrides to the agent sidecar init container + defer func() { + initContainer := w.getSecurityInitTemplate() + pod.Spec.InitContainers = append(pod.Spec.InitContainers, *initContainer) + }() + } pod.Spec.Containers = append(pod.Spec.Containers, *agentSidecarContainer) podUpdated = true } @@ -190,7 +218,7 @@ func (w *Webhook) injectAgentSidecar(pod *corev1.Pod, _ string, _ dynamic.Interf } podUpdated = podUpdated || updated - updated, err = applyProfileOverrides(&pod.Spec.Containers[i], w.profilesJSON) + updated, err = applyProfileOverrides(&pod.Spec.Containers[i], w.profileOverrides) if err != nil { log.Errorf("Failed to apply profile overrides: %v", err) return podUpdated, errors.New(metrics.InvalidInput) @@ -204,6 +232,77 @@ func (w *Webhook) injectAgentSidecar(pod *corev1.Pod, _ string, _ dynamic.Interf return podUpdated, nil } +func (w *Webhook) getSecurityInitTemplate() *corev1.Container { + return &corev1.Container{ + Image: fmt.Sprintf("%s/%s:%s", w.containerRegistry, w.imageName, w.imageTag), + ImagePullPolicy: corev1.PullIfNotPresent, + Name: "init-copy-agent-config", + Command: []string{"sh", "-c", "cp -R /etc/datadog-agent/* /agent-config/"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: agentConfigVolumeName, + MountPath: "/agent-config", + }, + }, + } +} + +func (w *Webhook) getSecurityVolumeTemplates() *[]corev1.Volume { + return &[]corev1.Volume{ + { + Name: agentConfigVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: agentOptionsVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: agentTmpVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: agentLogsVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } +} + +func (w *Webhook) addSecurityConfigToAgent(agentContainer *corev1.Container) { + volumeMounts := []corev1.VolumeMount{ + { + Name: agentConfigVolumeName, + MountPath: "/etc/datadog-agent", + }, + { + Name: agentOptionsVolumeName, + MountPath: "/opt/datadog-agent/run", + }, + { + Name: agentTmpVolumeName, + MountPath: "/tmp", + }, + { + Name: agentLogsVolumeName, + MountPath: "/var/log/datadog", + }, + } + agentContainer.VolumeMounts = append(agentContainer.VolumeMounts, volumeMounts...) + + if agentContainer.SecurityContext == nil { + agentContainer.SecurityContext = &corev1.SecurityContext{} + } + agentContainer.SecurityContext.ReadOnlyRootFilesystem = pointer.Ptr(true) +} + func (w *Webhook) getDefaultSidecarTemplate() *corev1.Container { ddSite := os.Getenv("DD_SITE") if ddSite == "" { @@ -288,21 +387,19 @@ func (w *Webhook) getDefaultSidecarTemplate() *corev1.Container { } // labelSelectors returns the mutating webhooks object selectors based on the configuration -func labelSelectors(datadogConfig config.Component) (namespaceSelector, objectSelector *metav1.LabelSelector) { +func labelSelectors(datadogConfig config.Component, profileOverrides []ProfileOverride) (namespaceSelector, objectSelector *metav1.LabelSelector) { // Read and parse selectors selectorsJSON := datadogConfig.GetString("admission_controller.agent_sidecar.selectors") - profilesJSON := datadogConfig.GetString("admission_controller.agent_sidecar.profiles") // Get sidecar profiles - _, err := loadSidecarProfiles(profilesJSON) - if err != nil { - log.Errorf("encountered issue when loading sidecar profiles: %s", err) + if profileOverrides == nil { + log.Error("sidecar profiles are not loaded") return nil, nil } var selectors []Selector - err = json.Unmarshal([]byte(selectorsJSON), &selectors) + err := json.Unmarshal([]byte(selectorsJSON), &selectors) if err != nil { log.Errorf("failed to parse selectors for admission controller agent sidecar injection webhook: %s", err) return nil, nil diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go index aac87fce23cba..9d6dd44be2d90 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/agent_sidecar_test.go @@ -59,19 +59,67 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - provider: "", - profilesJSON: "[]", + provider: "", + profilesJSON: `[{ + "securityContext": { + "readOnlyRootFilesystem": true + } + }]`, ExpectError: false, ExpectInjection: true, ExpectedPodAfterInjection: func() *corev1.Pod { + webhook := NewWebhook(mockConfig) + sidecar := webhook.getDefaultSidecarTemplate() + webhook.addSecurityConfigToAgent(sidecar) return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name", }, Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{*webhook.getSecurityInitTemplate()}, Containers: []corev1.Container{ {Name: "container-name"}, - *NewWebhook(mockConfig).getDefaultSidecarTemplate(), + *sidecar, + }, + Volumes: *webhook.getSecurityVolumeTemplates(), + }, + } + }, + }, + { + Name: "should inject sidecar, no security features if default overridden to false", + Pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + }, + }, + }, + provider: "", + profilesJSON: `[{ + "securityContext": { + "readOnlyRootFilesystem": false + } + }]`, + ExpectError: false, + ExpectInjection: true, + ExpectedPodAfterInjection: func() *corev1.Pod { + sidecar := *NewWebhook(mockConfig).getDefaultSidecarTemplate() + // Records the false readOnlyRootFilesystem but doesn't add the initContainers, volumes and mounts + sidecar.SecurityContext = &corev1.SecurityContext{ + ReadOnlyRootFilesystem: pointer.Ptr(false), + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-name", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "container-name"}, + sidecar, }, }, } @@ -157,8 +205,12 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - provider: "", - profilesJSON: "[]", + provider: "", + profilesJSON: `[{ + "securityContext": { + "readOnlyRootFilesystem": false + } + }]`, ExpectError: false, ExpectInjection: true, ExpectedPodAfterInjection: func() *corev1.Pod { @@ -168,6 +220,9 @@ func TestInjectAgentSidecar(t *testing.T) { Name: "DD_AUTO_EXIT_NOPROCESS_ENABLED", Value: "true", }) + defaultContainer.SecurityContext = &corev1.SecurityContext{ + ReadOnlyRootFilesystem: pointer.Ptr(false), + } return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "pod-name", @@ -199,14 +254,20 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - provider: "fargate", - profilesJSON: "[]", + provider: "fargate", + profilesJSON: `[{ + "securityContext": { + "readOnlyRootFilesystem": true + } + }]`, ExpectError: false, ExpectInjection: true, ExpectedPodAfterInjection: func() *corev1.Pod { - sidecar := *NewWebhook(mockConfig).getDefaultSidecarTemplate() + webhook := NewWebhook(mockConfig) + sidecar := webhook.getDefaultSidecarTemplate() + webhook.addSecurityConfigToAgent(sidecar) _, _ = withEnvOverrides( - &sidecar, + sidecar, corev1.EnvVar{ Name: "DD_EKS_FARGATE", Value: "true", @@ -221,13 +282,13 @@ func TestInjectAgentSidecar(t *testing.T) { }, ) - sidecar.VolumeMounts = []corev1.VolumeMount{ + sidecar.VolumeMounts = append(sidecar.VolumeMounts, []corev1.VolumeMount{ { Name: "ddsockets", MountPath: "/var/run/datadog", ReadOnly: false, }, - } + }...) return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -238,6 +299,7 @@ func TestInjectAgentSidecar(t *testing.T) { }, Spec: corev1.PodSpec{ ShareProcessNamespace: pointer.Ptr(true), + InitContainers: []corev1.Container{*webhook.getSecurityInitTemplate()}, Containers: []corev1.Container{ { Name: "container-name", @@ -259,16 +321,16 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - sidecar, + *sidecar, }, - Volumes: []corev1.Volume{ - { + Volumes: append(*webhook.getSecurityVolumeTemplates(), + corev1.Volume{ Name: "ddsockets", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, - }, + ), }, } }, @@ -287,28 +349,33 @@ func TestInjectAgentSidecar(t *testing.T) { }, provider: "fargate", profilesJSON: `[{ - "env": [ - {"name": "ENV_VAR_1", "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "secret-key"}}}, - {"name": "ENV_VAR_2", "value": "value2"} - ], - "resources": { - "limits": { - "cpu": "1", - "memory": "512Mi" - }, - "requests": { - "cpu": "0.5", - "memory": "256Mi" - } - } - }]`, + "env": [ + {"name": "ENV_VAR_1", "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "secret-key"}}}, + {"name": "ENV_VAR_2", "value": "value2"} + ], + "resources": { + "limits": { + "cpu": "1", + "memory": "512Mi" + }, + "requests": { + "cpu": "0.5", + "memory": "256Mi" + } + }, + "securityContext": { + "readOnlyRootFilesystem": true + } + }]`, ExpectError: false, ExpectInjection: true, ExpectedPodAfterInjection: func() *corev1.Pod { - sidecar := *NewWebhook(mockConfig).getDefaultSidecarTemplate() + webhook := NewWebhook(mockConfig) + sidecar := webhook.getDefaultSidecarTemplate() + webhook.addSecurityConfigToAgent(sidecar) _, _ = withEnvOverrides( - &sidecar, + sidecar, corev1.EnvVar{ Name: "DD_EKS_FARGATE", Value: "true", @@ -336,18 +403,18 @@ func TestInjectAgentSidecar(t *testing.T) { }, ) - _ = withResourceLimits(&sidecar, corev1.ResourceRequirements{ + _ = withResourceLimits(sidecar, corev1.ResourceRequirements{ Limits: corev1.ResourceList{"cpu": resource.MustParse("1"), "memory": resource.MustParse("512Mi")}, Requests: corev1.ResourceList{"cpu": resource.MustParse("0.5"), "memory": resource.MustParse("256Mi")}, }) - sidecar.VolumeMounts = []corev1.VolumeMount{ + sidecar.VolumeMounts = append(sidecar.VolumeMounts, []corev1.VolumeMount{ { Name: "ddsockets", MountPath: "/var/run/datadog", ReadOnly: false, }, - } + }...) return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -358,6 +425,7 @@ func TestInjectAgentSidecar(t *testing.T) { }, Spec: corev1.PodSpec{ ShareProcessNamespace: pointer.Ptr(true), + InitContainers: []corev1.Container{*webhook.getSecurityInitTemplate()}, Containers: []corev1.Container{ { Name: "container-name", @@ -379,16 +447,16 @@ func TestInjectAgentSidecar(t *testing.T) { }, }, }, - sidecar, + *sidecar, }, - Volumes: []corev1.Volume{ - { + Volumes: append(*webhook.getSecurityVolumeTemplates(), + corev1.Volume{ Name: "ddsockets", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, - }, + ), }, } }, @@ -644,3 +712,80 @@ func TestDefaultSidecarTemplateClusterAgentEnvVars(t *testing.T) { }) } } + +func TestIsReadOnlyRootFilesystem(t *testing.T) { + tests := []struct { + name string + profile string + expected bool + }{ + { + name: "no profile", + profile: "", + expected: false, + }, + { + name: "empty or default profile", + profile: "[]", + expected: false, + }, + { + name: "profile without security context", + profile: `[{ + "env": [ + {"name": "ENV_VAR_2", "value": "value2"} + ], + }]`, + expected: false, + }, + { + name: "profile with security context, readOnlyRootFilesystem empty", + profile: `[{ + "securityContext": {} + }]`, + expected: false, + }, + { + name: "profile with security context, readOnlyRootFilesystem true", + profile: `[{ + "securityContext": { + "readOnlyRootFilesystem": true + } + }]`, + expected: true, + }, + { + name: "profile with security context, readOnlyRootFilesystem false", + profile: `[{ + "securityContext": { + "readOnlyRootFilesystem": false + } + }]`, + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + mockConfig := configmock.New(t) + mockConfig.SetWithoutSource("admission_controller.agent_sidecar.profiles", test.profile) + webhook := NewWebhook(mockConfig) + sidecar := webhook.getDefaultSidecarTemplate() + + // Webhook properly parses profile config + assert.Equal(tt, test.expected, webhook.isReadOnlyRootFilesystem()) + + if test.expected { + // Webhook properly applies the security context to the sidecar + webhook.addSecurityConfigToAgent(sidecar) + assert.NotNil(t, sidecar.SecurityContext) + assert.NotNil(t, sidecar.SecurityContext.ReadOnlyRootFilesystem) + assert.Equal(t, test.expected, *sidecar.SecurityContext.ReadOnlyRootFilesystem) + } else { + assert.Nil(t, sidecar.SecurityContext) + profile, _ := loadSidecarProfiles(test.profile) + applyProfileOverrides(sidecar, profile) + } + }) + } +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/constants.go b/pkg/clusteragent/admission/mutate/agent_sidecar/constants.go index fd73e16272e54..dfd44618d3206 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/constants.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/constants.go @@ -11,3 +11,10 @@ const ( agentSidecarContainerName = "datadog-agent-injected" providerFargate = "fargate" ) + +const ( + agentConfigVolumeName = "agent-config" + agentOptionsVolumeName = "agent-option" + agentTmpVolumeName = "agent-tmp" + agentLogsVolumeName = "agent-log" +) diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/overrides.go b/pkg/clusteragent/admission/mutate/agent_sidecar/overrides.go index 2a42c1d8947f1..9389d381a6c01 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/overrides.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/overrides.go @@ -54,3 +54,19 @@ func withResourceLimits(container *corev1.Container, resourceLimits corev1.Resou container.Resources = resourceLimits return nil } + +// withSecurityContextOverrides applies the security context overrides to the container +func withSecurityContextOverrides(container *corev1.Container, securityContext *corev1.SecurityContext) (bool, error) { + if container == nil { + return false, fmt.Errorf("can't apply security context overrides to nil container") + } + + mutated := false + + if securityContext != nil { + container.SecurityContext = securityContext + mutated = true + } + + return mutated, nil +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/overrides_test.go b/pkg/clusteragent/admission/mutate/agent_sidecar/overrides_test.go index bde80aea94ab2..13ab63c9fd77e 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/overrides_test.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/overrides_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) func TestWithEnvOverrides(t *testing.T) { @@ -144,3 +146,92 @@ func TestWithResourceLimits(t *testing.T) { }) } } + +func TestWithSecurityContextOverrides(t *testing.T) { + tests := []struct { + name string + baseContainer *corev1.Container + securityOverride *corev1.SecurityContext + expectError bool + expectMutated bool + containerAfterOverride *corev1.Container + }{ + { + name: "nil container", + baseContainer: nil, + securityOverride: &corev1.SecurityContext{}, + expectError: true, + expectMutated: false, + containerAfterOverride: nil, + }, + { + name: "no overrides", + baseContainer: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(true), + }, + }, + securityOverride: nil, + expectError: false, + expectMutated: false, + containerAfterOverride: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(true), + }, + }, + }, + { + name: "apply overrides", + baseContainer: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + ReadOnlyRootFilesystem: pointer.Ptr(true), + }, + }, + securityOverride: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(false), + }, + expectError: false, + expectMutated: true, + containerAfterOverride: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(false), + }, + }, + }, + { + name: "apply blank overrides", + baseContainer: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{ + RunAsUser: pointer.Ptr(int64(1000)), + ReadOnlyRootFilesystem: pointer.Ptr(true), + }, + }, + securityOverride: &corev1.SecurityContext{}, + expectError: false, + expectMutated: true, + containerAfterOverride: &corev1.Container{ + SecurityContext: &corev1.SecurityContext{}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(tt *testing.T) { + mutated, err := withSecurityContextOverrides(test.baseContainer, test.securityOverride) + + assert.Equal(tt, test.expectMutated, mutated) + assert.Equal(tt, test.containerAfterOverride, test.baseContainer) + + if test.expectError { + assert.Error(tt, err) + } else { + assert.NoError(tt, err) + } + + }) + } +} diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/profiles.go b/pkg/clusteragent/admission/mutate/agent_sidecar/profiles.go index 101f54bd059eb..79430f8d8cbca 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/profiles.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/profiles.go @@ -25,6 +25,7 @@ import ( type ProfileOverride struct { EnvVars []corev1.EnvVar `json:"env,omitempty"` ResourceRequirements corev1.ResourceRequirements `json:"resources,omitempty"` + SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` } // loadSidecarProfiles returns the profile overrides provided by the user @@ -48,16 +49,15 @@ func loadSidecarProfiles(profilesJSON string) ([]ProfileOverride, error) { // applyProfileOverrides applies the profile overrides to the container. It // returns a boolean that indicates if the container was mutated -func applyProfileOverrides(container *corev1.Container, profilesJSON string) (bool, error) { +func applyProfileOverrides(container *corev1.Container, profiles []ProfileOverride) (bool, error) { if container == nil { return false, fmt.Errorf("can't apply profile overrides to nil containers") } - profiles, err := loadSidecarProfiles(profilesJSON) - - if err != nil { - return false, err + if profiles == nil { + return false, fmt.Errorf("can't apply nil profiles") } + if len(profiles) == 0 { return false, nil } @@ -82,5 +82,12 @@ func applyProfileOverrides(container *corev1.Container, profilesJSON string) (bo mutated = true } + // Apply security context overrides + overridden, err = withSecurityContextOverrides(container, overrides.SecurityContext) + if err != nil { + return false, err + } + mutated = mutated || overridden + return mutated, nil } diff --git a/pkg/clusteragent/admission/mutate/agent_sidecar/profiles_test.go b/pkg/clusteragent/admission/mutate/agent_sidecar/profiles_test.go index 772e8b784a78d..4b8c007d47fe5 100644 --- a/pkg/clusteragent/admission/mutate/agent_sidecar/profiles_test.go +++ b/pkg/clusteragent/admission/mutate/agent_sidecar/profiles_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + + "github.com/DataDog/datadog-agent/pkg/util/pointer" ) func TestLoadSidecarProfiles(t *testing.T) { @@ -29,6 +31,12 @@ func TestLoadSidecarProfiles(t *testing.T) { expectedProfiles: nil, expectError: true, }, + { + name: "default profile", + profilesJSON: "[]", + expectedProfiles: []ProfileOverride{}, + expectError: false, + }, { name: "single valid profile", profilesJSON: `[ @@ -46,6 +54,9 @@ func TestLoadSidecarProfiles(t *testing.T) { "cpu": "0.5", "memory": "256Mi" } + }, + "securityContext": { + "readOnlyRootFilesystem": true } } ]`, @@ -65,6 +76,7 @@ func TestLoadSidecarProfiles(t *testing.T) { "memory": resource.MustParse("256Mi"), }, }, + SecurityContext: &corev1.SecurityContext{ReadOnlyRootFilesystem: pointer.Ptr(true)}, }, }, expectError: false, @@ -166,6 +178,11 @@ func TestApplyProfileOverrides(t *testing.T) { "cpu": "0.5", "memory": "256Mi" } + }, + "securityContext": { + "readOnlyRootFilesystem": true, + "runAsNonRoot": false, + "runAsUser": 1000 } } ]`, @@ -185,6 +202,7 @@ func TestApplyProfileOverrides(t *testing.T) { "memory": resource.MustParse("256Mi"), }, }, + SecurityContext: &corev1.SecurityContext{ReadOnlyRootFilesystem: pointer.Ptr(true), RunAsNonRoot: pointer.Ptr(false), RunAsUser: pointer.Ptr(int64(1000))}, }, expectError: false, expectMutated: true, @@ -247,7 +265,10 @@ func TestApplyProfileOverrides(t *testing.T) { "cpu": "0.5", "memory": "256Mi" } - } + }, + "securityContext": { + "readOnlyRootFilesystem": true + } }]`, baseContainer: &corev1.Container{}, expectedContainer: &corev1.Container{ @@ -261,6 +282,7 @@ func TestApplyProfileOverrides(t *testing.T) { Limits: corev1.ResourceList{"cpu": resource.MustParse("1"), "memory": resource.MustParse("512Mi")}, Requests: corev1.ResourceList{"cpu": resource.MustParse("0.5"), "memory": resource.MustParse("256Mi")}, }, + SecurityContext: &corev1.SecurityContext{ReadOnlyRootFilesystem: pointer.Ptr(true)}, }, expectError: false, expectMutated: true, @@ -326,7 +348,10 @@ func TestApplyProfileOverrides(t *testing.T) { "cpu": "0.5", "memory": "256Mi" } - } + }, + "securityContext": { + "readOnlyRootFilesystem": true + } }, { "env": [ @@ -341,7 +366,10 @@ func TestApplyProfileOverrides(t *testing.T) { "cpu": "1", "memory": "512Mi" } - } + }, + "securityContext": { + "readOnlyRootFilesystem": false + } } ]`, baseContainer: &corev1.Container{}, @@ -353,7 +381,8 @@ func TestApplyProfileOverrides(t *testing.T) { for _, test := range tests { t.Run(test.name, func(tt *testing.T) { - mutated, err := applyProfileOverrides(test.baseContainer, test.profilesJSON) + profiles, _ := loadSidecarProfiles(test.profilesJSON) + mutated, err := applyProfileOverrides(test.baseContainer, profiles) assert.Equal(tt, test.expectMutated, mutated) diff --git a/pkg/clusteragent/autoscaling/workload/controller_horizontal.go b/pkg/clusteragent/autoscaling/workload/controller_horizontal.go index 555cf50b181d3..5bd044a18885c 100644 --- a/pkg/clusteragent/autoscaling/workload/controller_horizontal.go +++ b/pkg/clusteragent/autoscaling/workload/controller_horizontal.go @@ -246,6 +246,27 @@ func (hr *horizontalController) computeScaleAction( evalAfter = rulesNextEvalAfter.Truncate(time.Second) + time.Second } + // Stabilize recommendation + var stabilizationLimitReason string + var stabilizationLimitedReplicas int32 + upscaleStabilizationSeconds := int32(0) + downscaleStabilizationSeconds := int32(0) + + if policy := autoscalerInternal.Spec().Policy; policy != nil { + if upscalePolicy := policy.Upscale; upscalePolicy != nil { + upscaleStabilizationSeconds = int32(upscalePolicy.StabilizationWindowSeconds) + } + if downscalePolicy := policy.Downscale; downscalePolicy != nil { + downscaleStabilizationSeconds = int32(downscalePolicy.StabilizationWindowSeconds) + } + } + + stabilizationLimitedReplicas, stabilizationLimitReason = stabilizeRecommendations(scalingTimestamp, autoscalerInternal.HorizontalLastActions(), currentDesiredReplicas, targetDesiredReplicas, upscaleStabilizationSeconds, downscaleStabilizationSeconds, scaleDirection) + if stabilizationLimitReason != "" { + limitReason = stabilizationLimitReason + targetDesiredReplicas = stabilizationLimitedReplicas + } + horizontalAction := &datadoghq.DatadogPodAutoscalerHorizontalAction{ FromReplicas: currentDesiredReplicas, ToReplicas: targetDesiredReplicas, @@ -457,3 +478,44 @@ func accumulateReplicasChange(currentTime time.Time, events []datadoghq.DatadogP } return } + +func stabilizeRecommendations(currentTime time.Time, pastActions []datadoghq.DatadogPodAutoscalerHorizontalAction, currentReplicas int32, originalTargetDesiredReplicas int32, stabilizationWindowUpscaleSeconds int32, stabilizationWindowDownscaleSeconds int32, scaleDirection scaleDirection) (int32, string) { + limitReason := "" + + if len(pastActions) == 0 { + return originalTargetDesiredReplicas, limitReason + } + + upRecommendation := originalTargetDesiredReplicas + upCutoff := currentTime.Add(-time.Duration(stabilizationWindowUpscaleSeconds) * time.Second) + + downRecommendation := originalTargetDesiredReplicas + downCutoff := currentTime.Add(-time.Duration(stabilizationWindowDownscaleSeconds) * time.Second) + + for _, a := range pastActions { + if scaleDirection == scaleUp && a.Time.Time.After(upCutoff) { + upRecommendation = min(upRecommendation, *a.RecommendedReplicas) + } + + if scaleDirection == scaleDown && a.Time.Time.After(downCutoff) { + downRecommendation = max(downRecommendation, *a.RecommendedReplicas) + } + + if (scaleDirection == scaleUp && a.Time.Time.Before(upCutoff)) || (scaleDirection == scaleDown && a.Time.Time.Before(downCutoff)) { + break + } + } + + recommendation := currentReplicas + if recommendation < upRecommendation { + recommendation = upRecommendation + } + if recommendation > downRecommendation { + recommendation = downRecommendation + } + if recommendation != originalTargetDesiredReplicas { + limitReason = fmt.Sprintf("desired replica count limited to %d (originally %d) due to stabilization window", recommendation, originalTargetDesiredReplicas) + } + + return recommendation, limitReason +} diff --git a/pkg/clusteragent/autoscaling/workload/controller_horizontal_test.go b/pkg/clusteragent/autoscaling/workload/controller_horizontal_test.go index fc2eb1b1fe88c..89014b196a066 100644 --- a/pkg/clusteragent/autoscaling/workload/controller_horizontal_test.go +++ b/pkg/clusteragent/autoscaling/workload/controller_horizontal_test.go @@ -77,6 +77,15 @@ func (f *horizontalControllerFixture) runSync(fakePai *model.FakePodAutoscalerIn return autoscalerInternal, res, err } +func newHorizontalAction(time time.Time, fromReplicas, toReplicas, recommendedReplicas int32) datadoghq.DatadogPodAutoscalerHorizontalAction { + return datadoghq.DatadogPodAutoscalerHorizontalAction{ + Time: metav1.NewTime(time), + FromReplicas: fromReplicas, + ToReplicas: toReplicas, + RecommendedReplicas: pointer.Ptr[int32](recommendedReplicas), + } +} + type horizontalScalingTestArgs struct { fakePai *model.FakePodAutoscalerInternal dataSource datadoghq.DatadogPodAutoscalerValueSource @@ -181,6 +190,14 @@ func TestHorizontalControllerSyncPrerequisites(t *testing.T) { Kind: expectedGVK.Kind, APIVersion: expectedGVK.Group + "/" + expectedGVK.Version, }, + Policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 0, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 0, + }, + }, } fakePai.Error = nil f.scaler.On("get", mock.Anything, autoscalerNamespace, autoscalerName, expectedGVK).Return( @@ -1119,3 +1136,352 @@ func TestHorizontalControllerSyncScaleDecisionsWithRules(t *testing.T) { assert.Equal(t, autoscaling.NoRequeue, result) assert.NoError(t, err) } + +func TestStabilizeRecommendations(t *testing.T) { + currentTime := time.Now() + + tests := []struct { + name string + actions []datadoghq.DatadogPodAutoscalerHorizontalAction + currentReplicas int32 + recReplicas int32 + expected int32 + expectedReason string + upscaleWindow int32 + downscaleWindow int32 + scaleDirection scaleDirection + }{ + { + name: "no downscale stabilization - constant upscale", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-60*time.Second), 4, 5, 6), + newHorizontalAction(currentTime.Add(-30*time.Second), 6, 4, 4), + }, + currentReplicas: 4, + recReplicas: 8, + expected: 8, + expectedReason: "", + upscaleWindow: 0, + downscaleWindow: 300, + scaleDirection: scaleUp, + }, + { + name: "downscale stabilization", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-60*time.Second), 8, 6, 6), + newHorizontalAction(currentTime.Add(-30*time.Second), 6, 5, 5), + }, + currentReplicas: 5, + recReplicas: 4, + expected: 5, + expectedReason: "desired replica count limited to 5 (originally 4) due to stabilization window", + upscaleWindow: 0, + downscaleWindow: 300, + scaleDirection: scaleDown, + }, + { + name: "downscale stabilization, recommendation flapping", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-90*time.Second), 8, 6, 6), + newHorizontalAction(currentTime.Add(-60*time.Second), 6, 9, 9), + newHorizontalAction(currentTime.Add(-30*time.Second), 9, 7, 7), + }, + currentReplicas: 7, + recReplicas: 5, + expected: 7, + expectedReason: "desired replica count limited to 7 (originally 5) due to stabilization window", + upscaleWindow: 0, + downscaleWindow: 300, + scaleDirection: scaleDown, + }, + { + name: "upscale stabilization", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-60*time.Second), 5, 6, 6), + newHorizontalAction(currentTime.Add(-30*time.Second), 6, 8, 8), + }, + currentReplicas: 8, + recReplicas: 12, + expected: 8, + expectedReason: "desired replica count limited to 8 (originally 12) due to stabilization window", + upscaleWindow: 300, + downscaleWindow: 0, + scaleDirection: scaleUp, + }, + { + name: "upscale stabilization, recommendation flapping", + actions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(currentTime.Add(-90*time.Second), 6, 8, 8), + newHorizontalAction(currentTime.Add(-60*time.Second), 8, 7, 7), + newHorizontalAction(currentTime.Add(-30*time.Second), 7, 9, 9), + }, + currentReplicas: 9, + recReplicas: 12, + expected: 9, + expectedReason: "desired replica count limited to 9 (originally 12) due to stabilization window", + upscaleWindow: 300, + downscaleWindow: 0, + scaleDirection: scaleUp, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + recommendedReplicas, limitReason := stabilizeRecommendations(currentTime, tt.actions, tt.currentReplicas, tt.recReplicas, tt.upscaleWindow, tt.downscaleWindow, tt.scaleDirection) + assert.Equal(t, tt.expected, recommendedReplicas) + assert.Equal(t, tt.expectedReason, limitReason) + }) + } +} + +func TestHorizontalControllerSyncDownscaleWithStabilization(t *testing.T) { + testTime := time.Now() + startTime := testTime.Add(-time.Hour) + defaultStepDuration := 30 * time.Second + + f := newHorizontalControllerFixture(t, startTime) + autoscalerNamespace := "default" + autoscalerName := "test" + + expectedGVK := schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + } + + fakePai := &model.FakePodAutoscalerInternal{ + Namespace: autoscalerNamespace, + Name: autoscalerName, + Spec: &datadoghq.DatadogPodAutoscalerSpec{ + TargetRef: v2.CrossVersionObjectReference{ + Name: autoscalerName, + Kind: expectedGVK.Kind, + APIVersion: expectedGVK.Group + "/" + expectedGVK.Version, + }, + Constraints: &datadoghq.DatadogPodAutoscalerConstraints{ + MinReplicas: pointer.Ptr[int32](90), + MaxReplicas: 120, + }, + Policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 0, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 300, + }, + }, + }, + HorizontalLastActions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(f.clock.Now().Add(-60*time.Second), 90, 94, 94), + newHorizontalAction(f.clock.Now().Add(-30*time.Second), 94, 97, 97), + }, + ScalingValues: model.ScalingValues{ + Horizontal: &model.HorizontalScalingValues{ + Source: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + Timestamp: f.clock.Now().Add(-defaultStepDuration), + Replicas: 100, + }, + }, + TargetGVK: expectedGVK, + HorizontalEventsRetention: 5 * time.Minute, + } + + // Test upscale to 100 replicas (not limited) + f.clock.Step(defaultStepDuration) + result, err := f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 97, + statusReplicas: 97, + recReplicas: 100, + scaleReplicas: 100, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test downscale to 97 replicas, limited to 100 + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 97, + scaleReplicas: 100, + scaleLimitReason: "desired replica count limited to 100 (originally 97) due to stabilization window", + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test downscale to 95 replicas, still limited to 100 + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 95, + scaleReplicas: 100, + scaleLimitReason: "desired replica count limited to 100 (originally 95) due to stabilization window", + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test downscale to 92 replicas (not limited) + // Moving clock 4 minutes forward to get past stabilization window + f.clock.Step(4 * time.Minute) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 92, + scaleReplicas: 92, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test upscale to 100 replicas (not limited) + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 92, + statusReplicas: 92, + recReplicas: 100, + scaleReplicas: 100, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) +} + +func TestHorizontalControllerSyncUpscaleWithStabilization(t *testing.T) { + testTime := time.Now() + startTime := testTime.Add(-time.Hour) + defaultStepDuration := 30 * time.Second + + f := newHorizontalControllerFixture(t, startTime) + autoscalerNamespace := "default" + autoscalerName := "test" + + expectedGVK := schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + } + + fakePai := &model.FakePodAutoscalerInternal{ + Namespace: autoscalerNamespace, + Name: autoscalerName, + Spec: &datadoghq.DatadogPodAutoscalerSpec{ + TargetRef: v2.CrossVersionObjectReference{ + Name: autoscalerName, + Kind: expectedGVK.Kind, + APIVersion: expectedGVK.Group + "/" + expectedGVK.Version, + }, + Constraints: &datadoghq.DatadogPodAutoscalerConstraints{ + MinReplicas: pointer.Ptr[int32](90), + MaxReplicas: 120, + }, + Policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 300, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + StabilizationWindowSeconds: 0, + }, + }, + }, + HorizontalLastActions: []datadoghq.DatadogPodAutoscalerHorizontalAction{ + newHorizontalAction(f.clock.Now().Add(-60*time.Second), 120, 110, 110), + newHorizontalAction(f.clock.Now().Add(-30*time.Second), 110, 104, 104), + }, + ScalingValues: model.ScalingValues{ + Horizontal: &model.HorizontalScalingValues{ + Source: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + Timestamp: f.clock.Now().Add(-defaultStepDuration), + Replicas: 100, + }, + }, + TargetGVK: expectedGVK, + HorizontalEventsRetention: 5 * time.Minute, + } + + // Test downscale to 100 replicas (not limited) + f.clock.Step(defaultStepDuration) + result, err := f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 104, + statusReplicas: 104, + recReplicas: 100, + scaleReplicas: 100, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test upscale to 102 replicas, limited to 100 + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 102, + scaleReplicas: 100, + scaleLimitReason: "desired replica count limited to 100 (originally 102) due to stabilization window", + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test upscale to 105 replicas, still limited to 100 + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 105, + scaleReplicas: 100, + scaleLimitReason: "desired replica count limited to 100 (originally 105) due to stabilization window", + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test upscale to 102 replicas (not limited) + // Moving clock 4 minutes forward to get past stabilization window + f.clock.Step(4 * time.Minute) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 100, + statusReplicas: 100, + recReplicas: 102, + scaleReplicas: 102, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) + + // Test downscale to 100 replicas (not limited) + f.clock.Step(defaultStepDuration) + result, err = f.testScalingDecision(horizontalScalingTestArgs{ + fakePai: fakePai, + dataSource: datadoghq.DatadogPodAutoscalerAutoscalingValueSource, + dataOffset: defaultStepDuration, + currentReplicas: 102, + statusReplicas: 102, + recReplicas: 100, + scaleReplicas: 100, + }) + assert.Equal(t, autoscaling.NoRequeue, result) + assert.NoError(t, err) +} diff --git a/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler.go b/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler.go index a1dc2f722beb1..0b36ab17ae2ef 100644 --- a/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler.go +++ b/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler.go @@ -589,16 +589,14 @@ func getHorizontalEventsRetention(policy *datadoghq.DatadogPodAutoscalerPolicy, if policy.Upscale != nil { upscaleRetention := getLongestScalingRulesPeriod(policy.Upscale.Rules) - if upscaleRetention > longestRetention { - longestRetention = upscaleRetention - } + upscaleStabilizationWindow := time.Second * time.Duration(policy.Upscale.StabilizationWindowSeconds) + longestRetention = max(longestRetention, upscaleRetention, upscaleStabilizationWindow) } if policy.Downscale != nil { downscaleRetention := getLongestScalingRulesPeriod(policy.Downscale.Rules) - if downscaleRetention > longestRetention { - longestRetention = downscaleRetention - } + downscaleStabilizationWindow := time.Second * time.Duration(policy.Downscale.StabilizationWindowSeconds) + longestRetention = max(longestRetention, downscaleRetention, downscaleStabilizationWindow) } if longestRetention > longestLookbackAllowed { diff --git a/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler_test.go b/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler_test.go index 15dc2a99bea8d..966f23b46c641 100644 --- a/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler_test.go +++ b/pkg/clusteragent/autoscaling/workload/model/pod_autoscaler_test.go @@ -76,3 +76,193 @@ func TestAddHorizontalAction(t *testing.T) { *addedAction2, }, horizontalLastActions) } + +func TestGetHorizontalEventsRetention(t *testing.T) { + tests := []struct { + name string + policy *datadoghq.DatadogPodAutoscalerPolicy + longestLookbackAllowed time.Duration + expectedRetention time.Duration + }{ + { + name: "No policy, no retention", + policy: nil, + longestLookbackAllowed: 0, + expectedRetention: 0, + }, + { + name: "No policy, 15 minutes retention", + policy: nil, + longestLookbackAllowed: 15 * time.Minute, + expectedRetention: 0, + }, + { + name: "Upscale policy with rules, 30 minutes retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 900, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 15 * time.Minute, + }, + { + name: "Upscale policy with rules, 10 minutes max retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 900, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 10 * time.Minute, + expectedRetention: 10 * time.Minute, + }, + { + name: "Upscale and downscale policy with rules, 30 minutes retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 900, + Value: 2, + }, + }, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 960, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 16 * time.Minute, + }, + { + name: "Upscale and downscale policy with rules, 10 minutes max retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 900, + Value: 2, + }, + }, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 960, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 10 * time.Minute, + expectedRetention: 10 * time.Minute, + }, + { + name: "Upscale stabilization window 5 minutes", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 180, + Value: 2, + }, + }, + StabilizationWindowSeconds: 300, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 180, + Value: 2, + }, + }, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 5 * time.Minute, + }, + { + name: "Downscale stabilization window 5 minutes", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 180, + Value: 2, + }, + }, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 180, + Value: 2, + }, + }, + StabilizationWindowSeconds: 300, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 5 * time.Minute, + }, + { + name: "Stabilization, rules, max retention", + policy: &datadoghq.DatadogPodAutoscalerPolicy{ + Upscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 360, + Value: 2, + }, + }, + StabilizationWindowSeconds: 300, + }, + Downscale: &datadoghq.DatadogPodAutoscalerScalingPolicy{ + Rules: []datadoghq.DatadogPodAutoscalerScalingRule{ + { + Type: "Pods", + PeriodSeconds: 420, + Value: 2, + }, + }, + StabilizationWindowSeconds: 180, + }, + }, + longestLookbackAllowed: 30 * time.Minute, + expectedRetention: 7 * time.Minute, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + horizontalEventsRetention := getHorizontalEventsRetention(tt.policy, tt.longestLookbackAllowed) + assert.Equal(t, tt.expectedRetention, horizontalEventsRetention) + }) + } +} diff --git a/pkg/collector/corechecks/embed/apm/apm.go b/pkg/collector/corechecks/embed/apm/apm.go index 28162165eee6d..b724e77720b63 100644 --- a/pkg/collector/corechecks/embed/apm/apm.go +++ b/pkg/collector/corechecks/embed/apm/apm.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build apm && !windows && !linux +//go:build darwin // Package apm contains the APM check package apm diff --git a/pkg/collector/corechecks/embed/apm/apm_nix.go b/pkg/collector/corechecks/embed/apm/apm_nix.go index a024e45a11844..3dd1535ce8432 100644 --- a/pkg/collector/corechecks/embed/apm/apm_nix.go +++ b/pkg/collector/corechecks/embed/apm/apm_nix.go @@ -3,9 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build apm && !windows && !linux - -// linux handled by systemd/upstart +//go:build darwin package apm diff --git a/pkg/collector/corechecks/embed/apm/stub.go b/pkg/collector/corechecks/embed/apm/stub.go index b4c5ea584872b..5ed387c9a2d8f 100644 --- a/pkg/collector/corechecks/embed/apm/stub.go +++ b/pkg/collector/corechecks/embed/apm/stub.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !apm || windows || linux +//go:build !darwin // Package apm provides a stub for the APM check package apm diff --git a/pkg/collector/corechecks/embed/process/process_agent.go b/pkg/collector/corechecks/embed/process/process_agent.go index c27d5a4b8f975..0c4f971467a9e 100644 --- a/pkg/collector/corechecks/embed/process/process_agent.go +++ b/pkg/collector/corechecks/embed/process/process_agent.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build process && (darwin || freebsd) +//go:build darwin //nolint:revive // TODO(PLINT) Fix revive linter package process diff --git a/pkg/collector/corechecks/embed/process/stub.go b/pkg/collector/corechecks/embed/process/stub.go index 1faeaac1b12a6..2279f0ea3c370 100644 --- a/pkg/collector/corechecks/embed/process/stub.go +++ b/pkg/collector/corechecks/embed/process/stub.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !process || (!darwin && !freebsd) +//go:build !darwin //nolint:revive // TODO(Process) Fix revive linter package process diff --git a/pkg/collector/corechecks/snmp/integration_topology_test.go b/pkg/collector/corechecks/snmp/integration_topology_test.go index b4f0f42ceb6d0..c9298738ccf56 100644 --- a/pkg/collector/corechecks/snmp/integration_topology_test.go +++ b/pkg/collector/corechecks/snmp/integration_topology_test.go @@ -661,6 +661,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:101.1", "source_type": "lldp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -689,6 +690,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:102.2", "source_type": "lldp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -1363,6 +1365,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:1.5", "source_type": "cdp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -1386,6 +1389,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:2.3", "source_type": "cdp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -2056,6 +2060,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:101.1", "source_type": "lldp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" @@ -2084,6 +2089,7 @@ profiles: { "id": "profile-metadata:1.2.3.4:102.2", "source_type": "lldp", + "integration": "snmp", "local": { "device": { "dd_id": "profile-metadata:1.2.3.4" diff --git a/pkg/collector/corechecks/snmp/internal/report/report_device_metadata.go b/pkg/collector/corechecks/snmp/internal/report/report_device_metadata.go index 3e9ad39677d4c..6bdcec852a1d8 100644 --- a/pkg/collector/corechecks/snmp/internal/report/report_device_metadata.go +++ b/pkg/collector/corechecks/snmp/internal/report/report_device_metadata.go @@ -385,8 +385,9 @@ func buildNetworkTopologyMetadataWithLLDP(deviceID string, store *metadata.Store remEntryUniqueID := localPortNum + "." + lldpRemIndex newLink := devicemetadata.TopologyLinkMetadata{ - ID: deviceID + ":" + remEntryUniqueID, - SourceType: topologyLinkSourceTypeLLDP, + ID: deviceID + ":" + remEntryUniqueID, + SourceType: topologyLinkSourceTypeLLDP, + Integration: common.SnmpIntegrationName, Remote: &devicemetadata.TopologyLinkSide{ Device: &devicemetadata.TopologyLinkDevice{ Name: store.GetColumnAsString("lldp_remote.device_name", strIndex), @@ -446,8 +447,9 @@ func buildNetworkTopologyMetadataWithCDP(deviceID string, store *metadata.Store, remEntryUniqueID := cdpCacheIfIndex + "." + cdpCacheDeviceIndex newLink := devicemetadata.TopologyLinkMetadata{ - ID: deviceID + ":" + remEntryUniqueID, - SourceType: topologyLinkSourceTypeCDP, + ID: deviceID + ":" + remEntryUniqueID, + SourceType: topologyLinkSourceTypeCDP, + Integration: common.SnmpIntegrationName, Remote: &devicemetadata.TopologyLinkSide{ Device: &devicemetadata.TopologyLinkDevice{ Name: store.GetColumnAsString("cdp_remote.device_name", strIndex), diff --git a/pkg/collector/corechecks/system/cpu/cpu/context_switches.go b/pkg/collector/corechecks/system/cpu/cpu/context_switches.go new file mode 100644 index 0000000000000..ca83b826dd331 --- /dev/null +++ b/pkg/collector/corechecks/system/cpu/cpu/context_switches.go @@ -0,0 +1,15 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. +//go:build !linux && !windows + +package cpu + +import "fmt" + +// GetContextSwitches retrieves the number of context switches for the current process. +// It returns an integer representing the count and an error if the retrieval fails. +func GetContextSwitches() (int64, error) { + return 0, fmt.Errorf("context switches not supported on macOS") +} diff --git a/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches_linux.go b/pkg/collector/corechecks/system/cpu/cpu/context_switches_linux.go similarity index 64% rename from pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches_linux.go rename to pkg/collector/corechecks/system/cpu/cpu/context_switches_linux.go index 97dd5eab205e5..b1442821092c4 100644 --- a/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches_linux.go +++ b/pkg/collector/corechecks/system/cpu/cpu/context_switches_linux.go @@ -9,17 +9,23 @@ package cpu import ( "bufio" "fmt" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/util/log" "os" - "path/filepath" "strconv" "strings" - - "github.com/DataDog/datadog-agent/pkg/aggregator/sender" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" ) -func readCtxSwitches(procStatPath string) (ctxSwitches int64, err error) { - file, err := os.Open(procStatPath) +// GetContextSwitches retrieves the number of context switches for the current process. +// It returns an integer representing the count and an error if the retrieval fails. +func GetContextSwitches() (ctxSwitches int64, err error) { + log.Debug("collecting ctx switches") + procfsPath := "/proc" + if pkgconfigsetup.Datadog().IsSet("procfs_path") { + procfsPath = pkgconfigsetup.Datadog().GetString("procfs_path") + } + filePath := procfsPath + "/stat" + file, err := os.Open(filePath) if err != nil { return 0, err } @@ -32,24 +38,10 @@ func readCtxSwitches(procStatPath string) (ctxSwitches int64, err error) { elemts := strings.Split(txt, " ") ctxSwitches, err = strconv.ParseInt(elemts[1], 10, 64) if err != nil { - return 0, fmt.Errorf("%s in '%s' at line %d", err, procStatPath, i) + return 0, fmt.Errorf("%s in '%s' at line %d", err, filePath, i) } return ctxSwitches, nil } } - return 0, fmt.Errorf("could not find the context switches in stat file") } - -func collectCtxSwitches(sender sender.Sender) error { - procfsPath := "/proc" - if pkgconfigsetup.Datadog().IsSet("procfs_path") { - procfsPath = pkgconfigsetup.Datadog().GetString("procfs_path") - } - ctxSwitches, err := readCtxSwitches(filepath.Join(procfsPath, "/stat")) - if err != nil { - return err - } - sender.MonotonicCount("system.cpu.context_switches", float64(ctxSwitches), "", nil) - return nil -} diff --git a/pkg/collector/corechecks/system/cpu/cpu/cpu.go b/pkg/collector/corechecks/system/cpu/cpu/cpu.go index a098b35a39647..827f032bba9e5 100644 --- a/pkg/collector/corechecks/system/cpu/cpu/cpu.go +++ b/pkg/collector/corechecks/system/cpu/cpu/cpu.go @@ -11,6 +11,7 @@ import ( "fmt" "github.com/shirou/gopsutil/v4/cpu" + "gopkg.in/yaml.v2" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator/sender" @@ -23,15 +24,20 @@ import ( const CheckName = "cpu" // For testing purpose -var cpuTimesFunc = cpu.Times -var cpuInfoFunc = cpu.Info +var getCpuTimes = cpu.Times +var getCpuInfo = cpu.Info +var getContextSwitches = GetContextSwitches // Check doesn't need additional fields type Check struct { core.CheckBase - nbCPU float64 - lastNbCycle float64 - lastTimes cpu.TimesStat + instanceConfig cpuInstanceConfig + lastNbCycle float64 + lastTimes cpu.TimesStat +} + +type cpuInstanceConfig struct { + ReportTotalPerCPU bool `yaml:"report_total_percpu"` } // Run executes the check @@ -40,40 +46,81 @@ func (c *Check) Run() error { if err != nil { return err } + c.reportContextSwitches(sender) + numCores, err := c.reportCpuInfo(sender) + if err != nil { + return err + } + err = c.reportCpuMetricsPercent(sender, numCores) + if err != nil { + return err + } + err = c.reportCpuMetricsTotal(sender) + if err != nil { + return err + } + sender.Commit() + return nil +} - err = collectCtxSwitches(sender) +func (c *Check) reportContextSwitches(sender sender.Sender) { + ctxSwitches, err := getContextSwitches() if err != nil { - log.Debugf("cpu.Check could not read context switches: %s", err.Error()) - // Don't return here, we still want to collect the CPU metrics even if we could not + log.Debugf("could not read context switches: %s", err.Error()) + // Don't return error here, we still want to collect the CPU metrics even if we could not // read the context switches + } else { + log.Debugf("getContextSwitches: %d", ctxSwitches) + sender.MonotonicCount("system.cpu.context_switches", float64(ctxSwitches), "", nil) + } +} + +func (c *Check) reportCpuInfo(sender sender.Sender) (numCores int32, err error) { + cpuInfo, err := getCpuInfo() + if err != nil { + log.Errorf("could not retrieve cpu info: %s", err.Error()) + return 0, err + } + log.Debugf("getCpuInfo: %s", cpuInfo) + numCores = 0 + for _, i := range cpuInfo { + numCores += i.Cores } + sender.Gauge("system.cpu.num_cores", float64(numCores), "", nil) + return numCores, nil +} - cpuTimes, err := cpuTimesFunc(false) +func (c *Check) reportCpuMetricsPercent(sender sender.Sender, numCores int32) (err error) { + cpuTimes, err := getCpuTimes(false) if err != nil { - log.Errorf("cpu.Check: could not retrieve cpu stats: %s", err) + log.Errorf("could not retrieve cpu times: %s", err.Error()) + return err + } + log.Debugf("getCpuTimes(false): %s", cpuTimes) + if len(cpuTimes) == 0 { + err = fmt.Errorf("no cpu stats retrieve (empty results)") + log.Errorf("%s", err.Error()) return err - } else if len(cpuTimes) < 1 { - errEmpty := fmt.Errorf("no cpu stats retrieve (empty results)") - log.Errorf("cpu.Check: %s", errEmpty) - return errEmpty } t := cpuTimes[0] - total := t.User + t.System + t.Idle + t.Nice + t.Iowait + t.Irq + t.Softirq + t.Steal - nbCycle := total / c.nbCPU - + log.Debugf("total: %f", total) + nbCycle := total / float64(numCores) + log.Debugf("nbCycle: %f", nbCycle) if c.lastNbCycle != 0 { // gopsutil return the sum of every CPU + log.Debugf("c.lastNbCycle: %f", c.lastNbCycle) toPercent := 100 / (nbCycle - c.lastNbCycle) + log.Debugf("toPercent: %f", toPercent) - user := ((t.User + t.Nice) - (c.lastTimes.User + c.lastTimes.Nice)) / c.nbCPU - system := ((t.System + t.Irq + t.Softirq) - (c.lastTimes.System + c.lastTimes.Irq + c.lastTimes.Softirq)) / c.nbCPU - interrupt := ((t.Irq + t.Softirq) - (c.lastTimes.Irq + c.lastTimes.Softirq)) / c.nbCPU - iowait := (t.Iowait - c.lastTimes.Iowait) / c.nbCPU - idle := (t.Idle - c.lastTimes.Idle) / c.nbCPU - stolen := (t.Steal - c.lastTimes.Steal) / c.nbCPU - guest := (t.Guest - c.lastTimes.Guest) / c.nbCPU + user := ((t.User + t.Nice) - (c.lastTimes.User + c.lastTimes.Nice)) / float64(numCores) + system := ((t.System + t.Irq + t.Softirq) - (c.lastTimes.System + c.lastTimes.Irq + c.lastTimes.Softirq)) / float64(numCores) + interrupt := ((t.Irq + t.Softirq) - (c.lastTimes.Irq + c.lastTimes.Softirq)) / float64(numCores) + iowait := (t.Iowait - c.lastTimes.Iowait) / float64(numCores) + idle := (t.Idle - c.lastTimes.Idle) / float64(numCores) + stolen := (t.Steal - c.lastTimes.Steal) / float64(numCores) + guest := (t.Guest - c.lastTimes.Guest) / float64(numCores) sender.Gauge("system.cpu.user", user*toPercent, "", nil) sender.Gauge("system.cpu.system", system*toPercent, "", nil) @@ -83,32 +130,43 @@ func (c *Check) Run() error { sender.Gauge("system.cpu.stolen", stolen*toPercent, "", nil) sender.Gauge("system.cpu.guest", guest*toPercent, "", nil) } - - sender.Gauge("system.cpu.num_cores", c.nbCPU, "", nil) - sender.Commit() - c.lastNbCycle = nbCycle c.lastTimes = t return nil } -// Configure the CPU check -func (c *Check) Configure(senderManager sender.SenderManager, _ uint64, data integration.Data, initConfig integration.Data, source string) error { - err := c.CommonConfigure(senderManager, initConfig, data, source) +func (c *Check) reportCpuMetricsTotal(sender sender.Sender) (err error) { + cpuTimes, err := getCpuTimes(c.instanceConfig.ReportTotalPerCPU) if err != nil { + log.Errorf("could not retrieve cpu times: %s", err.Error()) return err } - // NOTE: This runs before the python checks, so we should be good, but cpuInfo() - // on windows initializes COM to the multithreaded model. Therefore, - // if a python check has run on this native windows thread prior and - // CoInitialized() the thread to a different model (ie. single-threaded) - // This will cause cpuInfo() to fail. - info, err := cpuInfoFunc() + log.Debugf("getCpuTimes(%t): %s", c.instanceConfig.ReportTotalPerCPU, cpuTimes) + for _, t := range cpuTimes { + tags := []string{fmt.Sprintf("core:%s", t.CPU)} + sender.Gauge("system.cpu.user.total", t.User, "", tags) + sender.Gauge("system.cpu.nice.total", t.Nice, "", tags) + sender.Gauge("system.cpu.system.total", t.System, "", tags) + sender.Gauge("system.cpu.idle.total", t.Idle, "", tags) + sender.Gauge("system.cpu.iowait.total", t.Iowait, "", tags) + sender.Gauge("system.cpu.irq.total", t.Irq, "", tags) + sender.Gauge("system.cpu.softirq.total", t.Softirq, "", tags) + sender.Gauge("system.cpu.steal.total", t.Steal, "", tags) + sender.Gauge("system.cpu.guest.total", t.Guest, "", tags) + sender.Gauge("system.cpu.guestnice.total", t.GuestNice, "", tags) + } + return nil +} + +// Configure configures the network checks +func (c *Check) Configure(senderManager sender.SenderManager, _ uint64, rawInstance integration.Data, rawInitConfig integration.Data, source string) error { + err := c.CommonConfigure(senderManager, rawInitConfig, rawInstance, source) if err != nil { - return fmt.Errorf("cpu.Check: could not query CPU info") + return err } - for _, i := range info { - c.nbCPU += float64(i.Cores) + err = yaml.Unmarshal(rawInstance, &c.instanceConfig) + if err != nil { + return err } return nil } diff --git a/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches.go b/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches.go deleted file mode 100644 index 21a57e5ac4cde..0000000000000 --- a/pkg/collector/corechecks/system/cpu/cpu/cpu_ctx_switches.go +++ /dev/null @@ -1,14 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. -//go:build !linux && !windows - -package cpu - -import "github.com/DataDog/datadog-agent/pkg/aggregator/sender" - -func collectCtxSwitches(_ sender.Sender) error { - // On non-linux systems, do nothing - return nil -} diff --git a/pkg/collector/corechecks/system/cpu/cpu/cpu_test.go b/pkg/collector/corechecks/system/cpu/cpu/cpu_test.go index 43a243f36d10d..c656765354022 100644 --- a/pkg/collector/corechecks/system/cpu/cpu/cpu_test.go +++ b/pkg/collector/corechecks/system/cpu/cpu/cpu_test.go @@ -8,52 +8,70 @@ package cpu import ( "errors" - "os" - "path/filepath" - "runtime" "testing" "github.com/DataDog/datadog-agent/comp/core/autodiscovery/integration" "github.com/DataDog/datadog-agent/pkg/aggregator" "github.com/DataDog/datadog-agent/pkg/aggregator/mocksender" "github.com/DataDog/datadog-agent/pkg/collector/check" - pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" - "github.com/DataDog/datadog-agent/pkg/metrics" - "github.com/shirou/gopsutil/v4/cpu" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) var ( - firstSample = []cpu.TimesStat{ + firstTotalSample = cpu.TimesStat{ + CPU: "cpu-total", + User: 29386, + Nice: 623, + System: 63584, + Idle: 96761, + Iowait: 12113, + Irq: 10, + Softirq: 1151, + Steal: 0, + Guest: 0, + GuestNice: 0, + } + secondTotalSample = cpu.TimesStat{ + CPU: "cpu-total", + User: 29586, + Nice: 625, + System: 68584, + Idle: 96761, + Iowait: 12153, + Irq: 15, + Softirq: 1451, + Steal: 2, + Guest: 0, + GuestNice: 0, + } + perCPUSamples = []cpu.TimesStat{ { - CPU: "cpu-total", - User: 1229386, - Nice: 623, - System: 263584, - Idle: 25496761, - Iowait: 12113, - Irq: 10, - Softirq: 1151, - Steal: 0, - Guest: 0, - GuestNice: 0, + CPU: "cpu0", + User: 83970.9, + Nice: 0.0, + System: 64060.9, + Idle: 208877.4, + Iowait: 12.1, + Irq: 43.5, + Softirq: 8.6, + Steal: 65.9, + Guest: 2.4, + GuestNice: 5.1, }, - } - secondSample = []cpu.TimesStat{ { - CPU: "cpu-total", - User: 1229586, - Nice: 625, - System: 268584, - Idle: 25596761, - Iowait: 12153, - Irq: 15, - Softirq: 1451, - Steal: 2, - Guest: 0, - GuestNice: 0, + CPU: "cpu1", + User: 82638.9, + Nice: 50.0, + System: 61564.1, + Idle: 212758.8, + Iowait: 1.2, + Irq: 2.3, + Softirq: 3.4, + Steal: 4.5, + Guest: 5.6, + GuestNice: 6.7, }, } cpuInfo = []cpu.InfoStat{ @@ -81,207 +99,328 @@ func createCheck() check.Check { return cpuCheck } -func TestCPUCheckLinuxFirstRunOk(t *testing.T) { - cpuInfoFunc = func() ([]cpu.InfoStat, error) { +func setupDefaultMocks() { + getContextSwitches = func() (int64, error) { + return 4, nil + } + getCpuInfo = func() ([]cpu.InfoStat, error) { return cpuInfo, nil } + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + return []cpu.TimesStat{firstTotalSample}, nil + } +} + +func TestCPUCheckLinuxErrorInInstanceConfig(t *testing.T) { + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + + err := cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`min_collection_interval: "string_value"`), nil, "test") + + assert.NotNil(t, err) +} + +func TestCPUCheckLinuxErrorReportTotalPerCPUConfigNotBoolean(t *testing.T) { cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - if runtime.GOOS == "linux" { - m.On(metrics.MonotonicCountType.String(), "system.cpu.context_switches", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) + + err := cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`report_total_percpu: "string_value"`), nil, "test") + + assert.NotNil(t, err) +} + +func TestCPUCheckLinuxErrorStoppedSender(t *testing.T) { + stoppedSenderError := errors.New("demultiplexer is stopped") + getCpuInfo = func() ([]cpu.InfoStat, error) { + return cpuInfo, nil } - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", float64(1), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) - m.On("Commit").Return().Times(1) + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil - } + m.GetSenderManager().(*aggregator.AgentDemultiplexer).Stop(false) err := cpuCheck.Run() - assert.Nil(t, err) - m.AssertExpectations(t) + assert.Equal(t, stoppedSenderError, err) } -func TestCPUCheckLinuxTwoRunsOk(t *testing.T) { - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil +func TestContextSwitchesError(t *testing.T) { + setupDefaultMocks() + getContextSwitches = func() (int64, error) { + return 0, errors.New("GetContextSwitches error") } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - if runtime.GOOS == "linux" { - m.On(metrics.MonotonicCountType.String(), "system.cpu.context_switches", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(2) - } - m.On(metrics.GaugeType.String(), "system.cpu.user", 0.1913803067769472, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.system", 5.026101621048045, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.interrupt", 0.2889653146879648, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.iowait", 0.03789709045088063, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.idle", 94.74272612720159, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.stolen", 0.0018948545225440318, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.guest", 0.0, "", []string(nil)).Return().Times(1) - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", 1.0, "", []string(nil)).Return().Times(2) - m.On("Commit").Return().Times(2) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil - } - cpuCheck.Run() - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return secondSample, nil - } err := cpuCheck.Run() assert.Nil(t, err) - m.AssertExpectations(t) + m.AssertNotCalled(t, "MonotonicCount", "system.cpu.context_switches", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorInInstanceConfig(t *testing.T) { +func TestContextSwitchesOk(t *testing.T) { + setupDefaultMocks() cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() - err := cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`min_collection_interval: "string_value"`), nil, "test") + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") + err := cpuCheck.Run() - assert.NotNil(t, err) + assert.Nil(t, err) + m.AssertMetric(t, "MonotonicCount", "system.cpu.context_switches", 4, "", []string(nil)) } -func TestCPUCheckLinuxErrorInCpuInfo(t *testing.T) { +func TestNumCoresError(t *testing.T) { + setupDefaultMocks() cpuInfoError := errors.New("cpu.Check: could not query CPU info") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { + getCpuInfo = func() ([]cpu.InfoStat, error) { return nil, cpuInfoError } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() - err := cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") + err := cpuCheck.Run() assert.Equal(t, cpuInfoError, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.num_cores", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorStoppedSender(t *testing.T) { - stoppedSenderError := errors.New("demultiplexer is stopped") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } +func TestNumCoresOk(t *testing.T) { + setupDefaultMocks() cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - m.GetSenderManager().(*aggregator.AgentDemultiplexer).Stop(false) err := cpuCheck.Run() - assert.Equal(t, stoppedSenderError, err) + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.num_cores", 1, "", nil) } -func TestCPUCheckLinuxErrorProcFsPathNoExists(t *testing.T) { - pkgconfigsetup.Datadog().SetDefault("procfs_path", "/tmp") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil +func TestSystemCpuMetricsError(t *testing.T) { + setupDefaultMocks() + cpuTimesError := errors.New("cpu.Check: could not query CPU times") + getCpuTimes = func(bool) ([]cpu.TimesStat, error) { + return nil, cpuTimesError } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", float64(1), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) - m.On("Commit").Return().Times(1) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") err := cpuCheck.Run() - assert.Nil(t, err) - m.AssertExpectations(t) - m.AssertNumberOfCalls(t, metrics.MonotonicCountType.String(), 0) + assert.Equal(t, cpuTimesError, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.user", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.system", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.interrupt", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.iowait", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.idle", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.stolen", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guest", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorProcFsPathEmptyFile(t *testing.T) { - tempFilePath := filepath.Join(os.TempDir(), "stat") - tempFile, err := os.Create(tempFilePath) - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(tempFile.Name()) - pkgconfigsetup.Datadog().SetDefault("procfs_path", os.TempDir()) - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil +func TestSystemCpuMetricsEmpty(t *testing.T) { + setupDefaultMocks() + expectedError := errors.New("no cpu stats retrieve (empty results)") + getCpuTimes = func(bool) ([]cpu.TimesStat, error) { + return []cpu.TimesStat{}, nil } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", float64(1), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) - m.On("Commit").Return().Times(1) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - err = cpuCheck.Run() + err := cpuCheck.Run() - assert.Nil(t, err) - m.AssertExpectations(t) - m.AssertNumberOfCalls(t, metrics.MonotonicCountType.String(), 0) + assert.Equal(t, expectedError, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.user", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.system", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.interrupt", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.iowait", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.idle", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.stolen", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guest", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorProcFsPathWrongFormat(t *testing.T) { - tempFilePath := filepath.Join(os.TempDir(), "stat") - tempFile, err := os.Create(tempFilePath) - if err != nil { - t.Fatal("Error creating temporary file:", err) - } - defer os.Remove(tempFile.Name()) - _, err = tempFile.WriteString("ctxt string_value\n") - if err != nil { - t.Fatal("Error writing to temporary file:", err) - } - pkgconfigsetup.Datadog().SetDefault("procfs_path", os.TempDir()) - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return firstSample, nil - } +func TestSystemCpuMetricsNotReportedOnFirstCheck(t *testing.T) { + setupDefaultMocks() cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) - m.On(metrics.GaugeType.String(), "system.cpu.num_cores", float64(1), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")).Return().Times(1) - m.On("Commit").Return().Times(1) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") - err = cpuCheck.Run() + err := cpuCheck.Run() - assert.Nil(t, err) - m.AssertExpectations(t) - m.AssertNumberOfCalls(t, metrics.MonotonicCountType.String(), 0) + assert.Equal(t, nil, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.user", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.system", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.interrupt", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.iowait", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.idle", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.stolen", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guest", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) } -func TestCPUCheckLinuxErrorInCpuTimes(t *testing.T) { - cpuTimesError := errors.New("cpu Times error") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil - } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return nil, cpuTimesError +func TestSystemCpuMetricsReportedOnSecondCheck(t *testing.T) { + setupDefaultMocks() + firstCall := true + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + if firstCall { + firstCall = false + return []cpu.TimesStat{firstTotalSample}, nil + } + return []cpu.TimesStat{secondTotalSample}, nil } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") + cpuCheck.Run() err := cpuCheck.Run() - assert.Equal(t, cpuTimesError, err) + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.user", 3.640295548747522, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.system", 95.60281131735448, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.interrupt", 5.496485853306902, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.iowait", 0.7208506037123806, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.idle", 0.0, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.stolen", 0.03604253018561903, "", []string(nil)) + m.AssertMetric(t, "Gauge", "system.cpu.guest", 0.0, "", []string(nil)) } -func TestCPUCheckLinuxEmptyCpuTimes(t *testing.T) { - expectedError := errors.New("no cpu stats retrieve (empty results)") - cpuInfoFunc = func() ([]cpu.InfoStat, error) { - return cpuInfo, nil +func TestSystemCpuMetricsPerCpuError(t *testing.T) { + setupDefaultMocks() + cpuTimesError := errors.New("cpu.Check: could not query CPU times") + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return nil, cpuTimesError + } + return []cpu.TimesStat{firstTotalSample}, nil } - cpuTimesFunc = func(bool) ([]cpu.TimesStat, error) { - return []cpu.TimesStat{}, nil + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() + + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`report_total_percpu: true`), nil, "test") + err := cpuCheck.Run() + + assert.Equal(t, cpuTimesError, err) + m.AssertNotCalled(t, "Gauge", "system.cpu.user.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.nice.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.system.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.idle.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.iowait.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.irq.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.softirq.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.steal.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guest.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + m.AssertNotCalled(t, "Gauge", "system.cpu.guestnice.total", mock.AnythingOfType("float64"), mock.AnythingOfType("string"), mock.AnythingOfType("[]string")) + +} + +func TestSystemCpuMetricsPerCpuDefault(t *testing.T) { + setupDefaultMocks() + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + return []cpu.TimesStat{firstTotalSample}, nil } cpuCheck := createCheck() m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, nil, nil, "test") err := cpuCheck.Run() - assert.Equal(t, expectedError, err) + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.user.total", 29386, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.nice.total", 623, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.system.total", 63584, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.idle.total", 96761, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.iowait.total", 12113, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.irq.total", 10, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.softirq.total", 1151, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.steal.total", 0.0, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.guest.total", 0.0, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.guestnice.total", 0.0, "", []string{"core:cpu-total"}) +} +func TestSystemCpuMetricsPerCpuFalse(t *testing.T) { + setupDefaultMocks() + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + return []cpu.TimesStat{firstTotalSample}, nil + } + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() + + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`report_total_percpu: false`), nil, "test") + err := cpuCheck.Run() + + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.user.total", 29386, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.nice.total", 623, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.system.total", 63584, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.idle.total", 96761, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.iowait.total", 12113, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.irq.total", 10, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.softirq.total", 1151, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.steal.total", 0.0, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.guest.total", 0.0, "", []string{"core:cpu-total"}) + m.AssertMetric(t, "Gauge", "system.cpu.guestnice.total", 0.0, "", []string{"core:cpu-total"}) +} +func TestSystemCpuMetricsPerCpuTrue(t *testing.T) { + setupDefaultMocks() + getCpuTimes = func(perCpu bool) ([]cpu.TimesStat, error) { + if perCpu { + return perCPUSamples, nil + } + return []cpu.TimesStat{firstTotalSample}, nil + } + cpuCheck := createCheck() + m := mocksender.NewMockSender(cpuCheck.ID()) + m.SetupAcceptAll() + + cpuCheck.Configure(m.GetSenderManager(), integration.FakeConfigHash, []byte(`report_total_percpu: true`), nil, "test") + err := cpuCheck.Run() + + assert.Equal(t, nil, err) + m.AssertMetric(t, "Gauge", "system.cpu.user.total", 83970.9, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.user.total", 82638.9, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.nice.total", 0.0, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.nice.total", 50.0, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.system.total", 64060.9, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.system.total", 61564.1, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.idle.total", 208877.4, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.idle.total", 212758.8, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.iowait.total", 12.1, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.iowait.total", 1.2, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.irq.total", 43.5, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.irq.total", 2.3, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.softirq.total", 8.6, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.softirq.total", 3.4, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.steal.total", 65.9, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.steal.total", 4.5, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.guest.total", 2.4, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.guest.total", 5.6, "", []string{"core:cpu1"}) + m.AssertMetric(t, "Gauge", "system.cpu.guestnice.total", 5.1, "", []string{"core:cpu0"}) + m.AssertMetric(t, "Gauge", "system.cpu.guestnice.total", 6.7, "", []string{"core:cpu1"}) } diff --git a/pkg/config/config_template.yaml b/pkg/config/config_template.yaml index a858e50ce8b7f..95d7f8c806e39 100644 --- a/pkg/config/config_template.yaml +++ b/pkg/config/config_template.yaml @@ -1024,13 +1024,13 @@ api_key: ## @env DD_LOGS_CONFIG_INTEGRATIONS_LOGS_FILES_MAX_SIZE - integer - optional - default: 10 ## The max size in MB that an integration logs file is allowed to use # - # integrations_logs_files_max_size + # integrations_logs_files_max_size: 10 ## @param integrations_logs_total_usage - integer - optional - default: 100 ## @env DD_LOGS_CONFIG_INTEGRATIONS_LOGS_TOTAL_USAGE - integer - optional - default: 100 ## The total combined usage all integrations logs files can use # - # integrations_logs_total_usage + # integrations_logs_total_usage: 100 {{ end -}} {{- if .TraceAgent }} @@ -1197,7 +1197,7 @@ api_key: # remove_query_string: false ## @param DD_APM_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS - boolean - optional ## If enabled, path segments in URLs containing digits are replaced by "?" - # remove_path_with_digits: false + # remove_paths_with_digits: false # # memcached: ## @param DD_APM_OBFUSCATION_MEMCACHED_ENABLED - boolean - optional @@ -1260,8 +1260,8 @@ api_key: ## The set of keys for which their values will be passed through SQL obfuscation # obfuscate_sql_values: # - val1 - # cache: - ## @param DD_APM_CACHE_ENABLED - boolean - optional + # cache: + ## @param DD_APM_OBFUSCATION_CACHE_ENABLED - boolean - optional ## Enables caching obfuscated statements. Currently supported for SQL and MongoDB queries. ## Enabled by default. # enabled: true @@ -1404,29 +1404,34 @@ api_key: # # port: 5012 - ## @param instrumentation_enabled - boolean - default: false - ## @env DD_APM_INSTRUMENTATION_ENABLED - boolean - default: false - ## Enables Single Step Instrumentation in the cluster (in beta) + ## @param instrumentation - custom object - optional + ## Specifies settings for Single Step Instrumentation. # - # instrumentation_enabled: false + # instrumentation: - ## @param instrumentation_enabled_namespaces - list of strings - optional - ## @env DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES - space separated list of strings - optional - ## Enables Single Step Instrumentation in specific namespaces, while Single Step Instrumentation is off in the whole cluster (in beta) - ## Can only be set if DD_APM_INSTRUMENTATION_ENABLED is false. Cannot be set together with DD_APM_INSTRUMENTATION_DISABLED_NAMESPACES. - # - # instrumentation_enabled_namespaces: - # - ns1 - # - apps + ## @param enabled - boolean - default: false + ## @env DD_APM_INSTRUMENTATION_ENABLED - boolean - default: false + ## Enables Single Step Instrumentation in the cluster (in beta) + # + # enabled: false - ## @param instrumentation_disabled_namespaces - list of strings - optional - ## @env DD_APM_INSTRUMENTATION_DISABLED_NAMESPACES - space separated list of strings - optional - ## Disables Single Step Instrumentation in specific namespaces, while Single Step Instrumentation is enabled in the whole cluster (in beta) - ## Can only be set if DD_APM_INSTRUMENTATION_ENABLED is true. Cannot be set together with DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES. - # - # instrumentation_disabled_namespaces: - # - ns2 - # - system-ns + ## @param enabled_namespaces - list of strings - optional + ## @env DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES - space separated list of strings - optional + ## Enables Single Step Instrumentation in specific namespaces, while Single Step Instrumentation is off in the whole cluster (in beta) + ## Can only be set if DD_APM_INSTRUMENTATION_ENABLED is false. Cannot be set together with DD_APM_INSTRUMENTATION_DISABLED_NAMESPACES. + # + # enabled_namespaces: + # - ns1 + # - apps + + ## @param disabled_namespaces - list of strings - optional + ## @env DD_APM_INSTRUMENTATION_DISABLED_NAMESPACES - space separated list of strings - optional + ## Disables Single Step Instrumentation in specific namespaces, while Single Step Instrumentation is enabled in the whole cluster (in beta) + ## Can only be set if DD_APM_INSTRUMENTATION_ENABLED is true. Cannot be set together with DD_APM_INSTRUMENTATION_ENABLED_NAMESPACES. + # + # disabled_namespaces: + # - ns2 + # - system-ns ## @param trace_buffer - integer - optional - default: 0 ## @env DD_APM_TRACE_BUFFER - integer - optional - default: 0 @@ -1448,25 +1453,26 @@ api_key: ## Enables and configures the Probabilistic Sampler, compatible with the ## OTel Probabilistic Sampler Processor ( https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/probabilisticsamplerprocessor#probabilistic-sampling-processor ) ## - #probabilistic_sampler: - ## @env DD_APM_PROBABILISTIC_SAMPLER_ENABLED - boolean - optional - default: false - ## Enables or disables the probabilistic sampler - # enabled: false - # - ## @env DD_APM_PROBABILISTIC_SAMPLER_SAMPLING_PERCENTAGE - float - optional - default: 0 - ## Samples this percentage (0-100) of traffic - # sampling_percentage: 0 - # - ## @env DD_APM_PROBABILISTIC_SAMPLER_HASH_SEED - integer - optional - default: 0 - ## hash_seed: A seed used for the hash algorithm. This must match other agents and OTel - ## collectors using the probabilistic sampler to ensure consistent sampling. - # hash_seed: 0 + # probabilistic_sampler: + + ## @env DD_APM_PROBABILISTIC_SAMPLER_ENABLED - boolean - optional - default: false + ## Enables or disables the probabilistic sampler + # enabled: false + # + ## @env DD_APM_PROBABILISTIC_SAMPLER_SAMPLING_PERCENTAGE - float - optional - default: 0 + ## Samples this percentage (0-100) of traffic + # sampling_percentage: 0 + # + ## @env DD_APM_PROBABILISTIC_SAMPLER_HASH_SEED - integer - optional - default: 0 + ## hash_seed: A seed used for the hash algorithm. This must match other agents and OTel + ## collectors using the probabilistic sampler to ensure consistent sampling. + # hash_seed: 0 ## @param error_tracking_standalone - object - optional ## Enables Error Tracking Standalone ## - #error_tracking_standalone: - # + # error_tracking_standalone: + ## @param enabled - boolean - optional - default: false ## @env DD_APM_ERROR_TRACKING_STANDALONE_ENABLED - boolean - optional - default: false ## Enables or disables Error Tracking Standalone @@ -3613,7 +3619,7 @@ api_key: ## @param prometheus_scrape - custom object - optional ## This section configures the Autodiscovery based on the Prometheus annotations # -# prometheus: +# prometheus_scrape: ## @param enabled - boolean - optional - default: false ## Enables the prometheus config provider @@ -3864,50 +3870,50 @@ api_key: ## @param ignored_ip_addresses - list of strings - optional ## A list of IP addresses to ignore when scanning the network. # - # ignored_ip_addresses: - # - - # - + # ignored_ip_addresses: + # - + # - ## @param port - integer - optional - default: 161 ## The UDP port to use when connecting to SNMP devices. # - # port: 161 + # port: 161 ## @param snmp_version - integer - optional - default: ## Set the version of the SNMP protocol. Available options are: `1`, `2` or `3`. ## If unset, the Agent tries to guess the correct version based on other configuration ## parameters, for example: if `user` is set, the Agent uses SNMP v3. # - # snmp_version: + # snmp_version: ## @param timeout - integer - optional - default: 5 ## The number of seconds before timing out. # - # timeout: 5 + # timeout: 5 ## @param retries - integer - optional - default: 3 ## The number of retries before failure. # - # retries: 3 + # retries: 3 ## @param community_string - string - optional ## Required for SNMP v1 & v2. ## Enclose the community string with single quote like below (to avoid special characters being interpreted). ## Ex: 'public' # - # community_string: '' + # community_string: '' ## @param user - string - optional ## The username to connect to your SNMP devices. ## SNMPv3 only. # - # user: + # user: ## @param authKey - string - optional ## The passphrase to use with your Authentication type. ## SNMPv3 only. # - # authKey: + # authKey: ## @param authProtocol - string - optional ## The authentication protocol to use when connecting to your SNMP devices. @@ -3915,13 +3921,13 @@ api_key: ## Defaults to MD5 when `authentication_key` is specified. ## SNMPv3 only. # - # authProtocol: + # authProtocol: ## @param privKey - string - optional ## The passphrase to use with your privacy protocol. ## SNMPv3 only. # - # privKey: + # privKey: ## @param privProtocol - string - optional ## The privacy protocol to use when connecting to your SNMP devices. @@ -3929,35 +3935,35 @@ api_key: ## Defaults to DES when `privacy_key` is specified. ## SNMPv3 only. # - # privProtocol: + # privProtocol: ## @param context_name - string - optional ## The name of your context (optional SNMP v3-only parameter). # - # context_name: + # context_name: ## @param tags - list of strings - optional ## A list of tags to attach to every metric and service check of all devices discovered in the subnet. ## ## Learn more about tagging at https://docs.datadoghq.com/tagging # - # tags: - # - : - # - : + # tags: + # - : + # - : ## @param ad_identifier - string - optional - default: snmp ## A unique identifier to attach to devices from that subnetwork. ## When configuring the SNMP integration in snmp.d/auto_conf.yaml, ## specify the corresponding ad_identifier at the top of the file. # - # ad_identifier: snmp + # ad_identifier: snmp ## @param loader - string - optional - default: python ## Check loader to use. Available loaders: ## - core: will use corecheck SNMP integration ## - python: will use python SNMP integration # - # loader: core + # loader: core ## @param min_collection_interval - number - optional - default: 15 ## This changes the collection interval for the check instances created from @@ -3966,7 +3972,7 @@ api_key: ## For more information, see: ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval # - # min_collection_interval: 15 + # min_collection_interval: 15 ## @param use_device_id_as_hostname - boolean - optional - default: false ## Use `device:` (device_id is composed of `:`) as `hostname` @@ -3974,12 +3980,12 @@ api_key: ## `host:device:` as tag). ## This option is needed for custom tags. # - # use_device_id_as_hostname: true + # use_device_id_as_hostname: true ## @param oid_batch_size - integer - optional - default: 5 ## The number of OIDs handled by each batch. # - # oid_batch_size: 5 + # oid_batch_size: 5 ## @param interface_configs - map - optional ## This option is used to override interface inbound/outbound speed and add interface tags @@ -3994,27 +4000,27 @@ api_key: ## - "testTagKey:testTagValue" ## - "tagKey2:tagValue2" # - # interface_config: - # "10.0.0.1": - # - match_field: name - # match_value: eth0 - # in_speed: 50 - # out_speed: 25 - # - match_field: index - # match_value: '10' - # in_speed: 50 - # out_speed: 25 - # "10.0.0.2": - # - match_field: name - # match_value: eth3 - # in_speed: 50 - # out_speed: 25 - # "10.0.0.3": - # - match_field: name - # match_value: eth4 - # tags: - # - "monitored:true" - # - "customKey:customValue" + # interface_config: + # "10.0.0.1": + # - match_field: name + # match_value: eth0 + # in_speed: 50 + # out_speed: 25 + # - match_field: index + # match_value: '10' + # in_speed: 50 + # out_speed: 25 + # "10.0.0.2": + # - match_field: name + # match_value: eth3 + # in_speed: 50 + # out_speed: 25 + # "10.0.0.3": + # - match_field: name + # match_value: eth4 + # tags: + # - "monitored:true" + # - "customKey:customValue" ## @param ping - custom object - optional ## Configure ICMP pings for all hosts in SNMP autodiscovery @@ -4028,17 +4034,17 @@ api_key: ## of system-probe for elevated privileges. See ## system-probe.yaml.example for details. # - # ping: - # enabled: true # Disabled by default - # timeout: 3000 # Timeout in milliseconds - # count: 2 # Number of ping packets to send per check run - # interval: 10 # Time between sending pings (up to `count` packets) in milliseconds - # linux: # Linux-specific configuration - # use_raw_socket: true # Send pings in a privileged fashion using a raw socket. - # # This may be required if your system doesn't support - # # sending pings in an unprivileged fashion (using a UDP socket). - # # If `use_raw_socket` is set to true, you MUST also enable - # # system-probe which has elevated privileges. To enable it, see system-probe.yaml.example. + # ping: + # enabled: true # Disabled by default + # timeout: 3000 # Timeout in milliseconds + # count: 2 # Number of ping packets to send per check run + # interval: 10 # Time between sending pings (up to `count` packets) in milliseconds + # linux: # Linux-specific configuration + # use_raw_socket: true # Send pings in a privileged fashion using a raw socket. + # # This may be required if your system doesn't support + # # sending pings in an unprivileged fashion (using a UDP socket). + # # If `use_raw_socket` is set to true, you MUST also enable + # # system-probe which has elevated privileges. To enable it, see system-probe.yaml.example. ## @param snmp_traps - custom object - optional diff --git a/pkg/config/fetcher/from_processes.go b/pkg/config/fetcher/from_processes.go index 5b8088f41d970..95c24e283fb46 100644 --- a/pkg/config/fetcher/from_processes.go +++ b/pkg/config/fetcher/from_processes.go @@ -71,7 +71,7 @@ func TraceAgentConfig(config config.Reader) (string, error) { c := util.GetClient(false) c.Timeout = config.GetDuration("server_timeout") * time.Second - ipcAddressWithPort := fmt.Sprintf("http://127.0.0.1:%d/config", port) + ipcAddressWithPort := fmt.Sprintf("https://127.0.0.1:%d/config", port) client := settingshttp.NewClient(c, ipcAddressWithPort, "trace-agent", settingshttp.NewHTTPClientOptions(util.CloseConnection)) return client.FullConfig() diff --git a/pkg/config/remote/go.mod b/pkg/config/remote/go.mod index a1c4a4ebbc561..3b809d121d819 100644 --- a/pkg/config/remote/go.mod +++ b/pkg/config/remote/go.mod @@ -141,7 +141,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/tinylib/msgp v1.2.4 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect diff --git a/pkg/config/remote/go.sum b/pkg/config/remote/go.sum index ca537b0d1660e..176ccf5fcd4b4 100644 --- a/pkg/config/remote/go.sum +++ b/pkg/config/remote/go.sum @@ -378,8 +378,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= diff --git a/pkg/config/remote/service/service.go b/pkg/config/remote/service/service.go index a379ac5b387ec..84377477da44c 100644 --- a/pkg/config/remote/service/service.go +++ b/pkg/config/remote/service/service.go @@ -111,50 +111,21 @@ func (s *Service) getNewDirectorRoots(uptane uptaneClient, currentVersion uint64 return roots, nil } -func (s *Service) getTargetFiles(uptane uptaneClient, products []rdata.Product, cachedTargetFiles []*pbgo.TargetFileMeta) ([]*pbgo.File, error) { - productSet := make(map[rdata.Product]struct{}) - for _, product := range products { - productSet[product] = struct{}{} - } - targets, err := uptane.Targets() +func (s *Service) getTargetFiles(uptaneClient uptaneClient, targetFilePaths []string) ([]*pbgo.File, error) { + files, err := uptaneClient.TargetFiles(targetFilePaths) if err != nil { return nil, err } - cachedTargets := make(map[string]data.FileMeta) - for _, cachedTarget := range cachedTargetFiles { - hashes := make(data.Hashes) - for _, hash := range cachedTarget.Hashes { - h, err := hex.DecodeString(hash.Hash) - if err != nil { - return nil, err - } - hashes[hash.Algorithm] = h - } - cachedTargets[cachedTarget.Path] = data.FileMeta{ - Hashes: hashes, - Length: cachedTarget.Length, - } - } + var configFiles []*pbgo.File - for targetPath, targetMeta := range targets { - configPathMeta, err := rdata.ParseConfigPath(targetPath) - if err != nil { - return nil, err - } - if _, inClientProducts := productSet[rdata.Product(configPathMeta.Product)]; inClientProducts { - if notEqualErr := tufutil.FileMetaEqual(cachedTargets[targetPath], targetMeta.FileMeta); notEqualErr == nil { - continue - } - fileContents, err := uptane.TargetFile(targetPath) - if err != nil { - return nil, err - } - configFiles = append(configFiles, &pbgo.File{ - Path: targetPath, - Raw: fileContents, - }) - } + for path, contents := range files { + // Note: This unconditionally succeeds as long as we don't change bufferDestination earlier + configFiles = append(configFiles, &pbgo.File{ + Path: path, + Raw: contents, + }) } + return configFiles, nil } @@ -211,6 +182,7 @@ type uptaneClient interface { StoredOrgUUID() (string, error) Targets() (data.TargetFiles, error) TargetFile(path string) ([]byte, error) + TargetFiles(files []string) (map[string][]byte, error) TargetsMeta() ([]byte, error) TargetsCustom() ([]byte, error) TUFVersionState() (uptane.TUFVersions, error) @@ -828,36 +800,30 @@ func (s *CoreAgentService) ClientGetConfigs(_ context.Context, request *pbgo.Cli if err != nil { return nil, err } - targetsRaw, err := s.uptane.TargetsMeta() + + directorTargets, err := s.uptane.Targets() if err != nil { return nil, err } - targetFiles, err := s.getTargetFiles(s.uptane, rdata.StringListToProduct(request.Client.Products), request.CachedTargetFiles) + matchedClientConfigs, err := executeTracerPredicates(request.Client, directorTargets) if err != nil { return nil, err } - directorTargets, err := s.uptane.Targets() + neededFiles, err := filterNeededTargetFiles(matchedClientConfigs, request.CachedTargetFiles, directorTargets) if err != nil { return nil, err } - matchedClientConfigs, err := executeTracerPredicates(request.Client, directorTargets) + + targetFiles, err := s.getTargetFiles(s.uptane, neededFiles) if err != nil { return nil, err } - // filter files to only return the ones that predicates marked for this client - matchedConfigsMap := make(map[string]interface{}) - for _, configPointer := range matchedClientConfigs { - matchedConfigsMap[configPointer] = struct{}{} - } - filteredFiles := make([]*pbgo.File, 0, len(matchedClientConfigs)) - for _, targetFile := range targetFiles { - if _, ok := matchedConfigsMap[targetFile.Path]; ok { - filteredFiles = append(filteredFiles, targetFile) - } + targetsRaw, err := s.uptane.TargetsMeta() + if err != nil { + return nil, err } - canonicalTargets, err := enforceCanonicalJSON(targetsRaw) if err != nil { return nil, err @@ -866,11 +832,42 @@ func (s *CoreAgentService) ClientGetConfigs(_ context.Context, request *pbgo.Cli return &pbgo.ClientGetConfigsResponse{ Roots: roots, Targets: canonicalTargets, - TargetFiles: filteredFiles, + TargetFiles: targetFiles, ClientConfigs: matchedClientConfigs, }, nil } +func filterNeededTargetFiles(neededConfigs []string, cachedTargetFiles []*pbgo.TargetFileMeta, tufTargets data.TargetFiles) ([]string, error) { + // Build an O(1) lookup of cached target files + cachedTargetsMap := make(map[string]data.FileMeta) + for _, cachedTarget := range cachedTargetFiles { + hashes := make(data.Hashes) + for _, hash := range cachedTarget.Hashes { + h, err := hex.DecodeString(hash.Hash) + if err != nil { + return nil, err + } + hashes[hash.Algorithm] = h + } + cachedTargetsMap[cachedTarget.Path] = data.FileMeta{ + Hashes: hashes, + Length: cachedTarget.Length, + } + } + + // We don't need to pull the raw contents if the client already has the exact version of the file cached + filteredList := make([]string, 0, len(neededConfigs)) + for _, path := range neededConfigs { + if notEqualErr := tufutil.FileMetaEqual(cachedTargetsMap[path], tufTargets[path].FileMeta); notEqualErr == nil { + continue + } + + filteredList = append(filteredList, path) + } + + return filteredList, nil +} + // ConfigGetState returns the state of the configuration and the director repos in the local store func (s *CoreAgentService) ConfigGetState() (*pbgo.GetStateConfigResponse, error) { state, err := s.uptane.State() @@ -1117,29 +1114,14 @@ func (c *HTTPClient) getUpdate( if tufVersions.DirectorTargets == currentTargetsVersion { return nil, nil } - roots, err := c.getNewDirectorRoots(c.uptane, currentRootVersion, tufVersions.DirectorRoot) - if err != nil { - return nil, err - } - targetsRaw, err := c.uptane.TargetsMeta() - if err != nil { - return nil, err - } - targetFiles, err := c.getTargetFiles(c.uptane, rdata.StringListToProduct(products), cachedTargetFiles) - if err != nil { - return nil, err - } - - canonicalTargets, err := enforceCanonicalJSON(targetsRaw) - if err != nil { - return nil, err - } + // Filter out files that either: + // - don't correspond to the product list the client is requesting + // - have expired directorTargets, err := c.uptane.Targets() if err != nil { return nil, err } - productsMap := make(map[string]struct{}) for _, product := range products { productsMap[product] = struct{}{} @@ -1165,14 +1147,33 @@ func (c *HTTPClient) getUpdate( configs = append(configs, path) } + span.SetTag("configs.returned", configs) + span.SetTag("configs.expired", expiredConfigs) + // Gather the files and map-ify them for the state data structure + targetFiles, err := c.getTargetFiles(c.uptane, configs) + if err != nil { + return nil, err + } fileMap := make(map[string][]byte, len(targetFiles)) for _, f := range targetFiles { fileMap[f.Path] = f.Raw } - span.SetTag("configs.returned", configs) - span.SetTag("configs.expired", expiredConfigs) + // Gather some TUF metadata files we need to send down + roots, err := c.getNewDirectorRoots(c.uptane, currentRootVersion, tufVersions.DirectorRoot) + if err != nil { + return nil, err + } + targetsRaw, err := c.uptane.TargetsMeta() + if err != nil { + return nil, err + } + canonicalTargets, err := enforceCanonicalJSON(targetsRaw) + if err != nil { + return nil, err + } + return &state.Update{ TUFRoots: roots, TUFTargets: canonicalTargets, diff --git a/pkg/config/remote/service/service_test.go b/pkg/config/remote/service/service_test.go index 40d8302db3192..34175eea2feba 100644 --- a/pkg/config/remote/service/service_test.go +++ b/pkg/config/remote/service/service_test.go @@ -116,6 +116,11 @@ func (m *mockUptane) TargetFile(path string) ([]byte, error) { return args.Get(0).([]byte), args.Error(1) } +func (m *mockUptane) TargetFiles(files []string) (map[string][]byte, error) { + args := m.Called(files) + return args.Get(0).(map[string][]byte), args.Error(1) +} + func (m *mockUptane) TargetsMeta() ([]byte, error) { args := m.Called() return args.Get(0).([]byte), args.Error(1) @@ -486,8 +491,8 @@ func TestService(t *testing.T) { }, nil) uptaneClient.On("DirectorRoot", uint64(3)).Return(root3, nil) uptaneClient.On("DirectorRoot", uint64(4)).Return(root4, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/1").Return(fileAPM1, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/2").Return(fileAPM2, nil) + + uptaneClient.On("TargetFiles", mock.MatchedBy(listsEqual([]string{"datadog/2/APM_SAMPLING/id/1", "datadog/2/APM_SAMPLING/id/2"}))).Return(map[string][]byte{"datadog/2/APM_SAMPLING/id/1": fileAPM1, "datadog/2/APM_SAMPLING/id/2": fileAPM2}, nil) uptaneClient.On("Update", lastConfigResponse).Return(nil) api.On("Fetch", mock.Anything, &pbgo.LatestConfigsRequest{ Hostname: service.hostname, @@ -513,7 +518,7 @@ func TestService(t *testing.T) { configResponse, err := service.ClientGetConfigs(context.Background(), &pbgo.ClientGetConfigsRequest{Client: client}) assert.NoError(t, err) assert.ElementsMatch(t, [][]byte{canonicalRoot3, canonicalRoot4}, configResponse.Roots) - assert.ElementsMatch(t, []*pbgo.File{{Path: "datadog/2/APM_SAMPLING/id/1", Raw: fileAPM1}, {Path: "datadog/2/APM_SAMPLING/id/2", Raw: fileAPM2}}, configResponse.TargetFiles) + assert.ElementsMatch(t, []*pbgo.File{{Path: "datadog/2/APM_SAMPLING/id/1", Raw: fileAPM1}, {Path: "datadog/2/APM_SAMPLING/id/2", Raw: fileAPM2}}, configResponse.TargetFiles, nil) assert.Equal(t, canonicalTargets, configResponse.Targets) assert.ElementsMatch(t, configResponse.ClientConfigs, @@ -597,8 +602,7 @@ func TestServiceClientPredicates(t *testing.T) { DirectorRoot: 1, DirectorTargets: 5, }, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/1").Return([]byte(``), nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/2").Return([]byte(``), nil) + uptaneClient.On("TargetFiles", mock.MatchedBy(listsEqual([]string{"datadog/2/APM_SAMPLING/id/1", "datadog/2/APM_SAMPLING/id/2"}))).Return(map[string][]byte{"datadog/2/APM_SAMPLING/id/1": []byte(``), "datadog/2/APM_SAMPLING/id/2": []byte(``)}, nil) uptaneClient.On("Update", lastConfigResponse).Return(nil) api.On("Fetch", mock.Anything, &pbgo.LatestConfigsRequest{ Hostname: service.hostname, @@ -887,8 +891,7 @@ func TestConfigExpiration(t *testing.T) { DirectorRoot: 1, DirectorTargets: 5, }, nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/1").Return([]byte(``), nil) - uptaneClient.On("TargetFile", "datadog/2/APM_SAMPLING/id/2").Return([]byte(``), nil) + uptaneClient.On("TargetFiles", []string{"datadog/2/APM_SAMPLING/id/1"}).Return(map[string][]byte{"datadog/2/APM_SAMPLING/id/1": []byte(``), "datadog/2/APM_SAMPLING/id/2": []byte(``)}, nil) uptaneClient.On("Update", lastConfigResponse).Return(nil) api.On("Fetch", mock.Anything, &pbgo.LatestConfigsRequest{ Hostname: service.hostname, @@ -1190,7 +1193,7 @@ func TestHTTPClientRecentUpdate(t *testing.T) { }, nil, ) - uptaneClient.On("TargetFile", "datadog/2/TESTING1/id/1").Return([]byte(`testing_1`), nil) + uptaneClient.On("TargetFiles", []string{"datadog/2/TESTING1/id/1"}).Return(map[string][]byte{"datadog/2/TESTING1/id/1": []byte(`testing_1`)}, nil) client := setupCDNClient(t, uptaneClient) defer client.Close() @@ -1236,7 +1239,7 @@ func TestHTTPClientUpdateSuccess(t *testing.T) { }, nil, ) - uptaneClient.On("TargetFile", "datadog/2/TESTING1/id/1").Return([]byte(`testing_1`), nil) + uptaneClient.On("TargetFiles", []string{"datadog/2/TESTING1/id/1"}).Return(map[string][]byte{"datadog/2/TESTING1/id/1": []byte(`testing_1`)}, nil) updateErr := fmt.Errorf("uh oh") if tt.updateSucceeds { @@ -1261,3 +1264,24 @@ func TestHTTPClientUpdateSuccess(t *testing.T) { }) } } + +func listsEqual(mustMatch []string) func(candidate []string) bool { + return func(candidate []string) bool { + if len(candidate) != len(mustMatch) { + return false + } + + candidateSet := make(map[string]struct{}) + for _, item := range candidate { + candidateSet[item] = struct{}{} + } + + for _, item := range mustMatch { + if _, ok := candidateSet[item]; !ok { + return false + } + } + + return true + } +} diff --git a/pkg/config/remote/uptane/client.go b/pkg/config/remote/uptane/client.go index b4da1c675b296..fcd1d3834b478 100644 --- a/pkg/config/remote/uptane/client.go +++ b/pkg/config/remote/uptane/client.go @@ -342,6 +342,37 @@ func (c *Client) TargetFile(path string) ([]byte, error) { return c.unsafeTargetFile(path) } +// TargetFiles returns the content of various multiple target files if the repository is in a +// verified state. +func (c *Client) TargetFiles(targetFiles []string) (map[string][]byte, error) { + c.Lock() + defer c.Unlock() + + err := c.verify() + if err != nil { + return nil, err + } + + // Build the storage space + destinations := make(map[string]client.Destination) + for _, path := range targetFiles { + destinations[path] = &bufferDestination{} + } + + err = c.directorTUFClient.DownloadBatch(destinations) + if err != nil { + return nil, err + } + + // Build the return type + files := make(map[string][]byte) + for path, contents := range destinations { + files[path] = contents.(*bufferDestination).Bytes() + } + + return files, nil +} + // TargetsMeta returns the current raw targets.json meta of this uptane client func (c *Client) TargetsMeta() ([]byte, error) { c.Lock() diff --git a/pkg/config/setup/config.go b/pkg/config/setup/config.go index 24f8d81503459..fdb1d427e9d15 100644 --- a/pkg/config/setup/config.go +++ b/pkg/config/setup/config.go @@ -79,7 +79,7 @@ const ( DefaultRuntimePoliciesDir = "/etc/datadog-agent/runtime-security.d" // DefaultCompressorKind is the default compressor. Options available are 'zlib' and 'zstd' - DefaultCompressorKind = "zlib" + DefaultCompressorKind = "zstd" // DefaultZstdCompressionLevel is the default compression level for `zstd`. // Compression level 1 provides the lowest compression ratio, but uses much less RSS especially diff --git a/pkg/config/setup/system_probe.go b/pkg/config/setup/system_probe.go index a68e01f5bb606..12a94b923c7b7 100644 --- a/pkg/config/setup/system_probe.go +++ b/pkg/config/setup/system_probe.go @@ -372,13 +372,12 @@ func InitSystemProbeConfig(cfg pkgconfigmodel.Config) { eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.classifier_handle"), 0) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.raw_classifier_handle"), 0) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_ring_buffer"), true) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry"), false) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry_amd64"), false) - eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry_arm64"), false) + eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "event_stream.use_fentry"), true) + eventMonitorBindEnv(cfg, join(evNS, "event_stream.use_fentry_amd64")) + eventMonitorBindEnv(cfg, join(evNS, "event_stream.use_fentry_arm64")) eventMonitorBindEnv(cfg, join(evNS, "event_stream.buffer_size")) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "envs_with_value"), []string{"LD_PRELOAD", "LD_LIBRARY_PATH", "PATH", "HISTSIZE", "HISTFILESIZE", "GLIBC_TUNABLES"}) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "runtime_compilation.enabled"), false) - eventMonitorBindEnv(cfg, join(evNS, "runtime_compilation.compiled_constants_enabled")) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.enabled"), true) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.ingress.enabled"), false) eventMonitorBindEnvAndSetDefault(cfg, join(evNS, "network.raw_packet.enabled"), false) diff --git a/pkg/config/setup/system_probe_cws.go b/pkg/config/setup/system_probe_cws.go index d2cc6276907d3..c689e1fd68e28 100644 --- a/pkg/config/setup/system_probe_cws.go +++ b/pkg/config/setup/system_probe_cws.go @@ -57,6 +57,7 @@ func initCWSSystemProbeConfig(cfg pkgconfigmodel.Config) { cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.min_timeout", "10m") cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.max_dump_size", 1750) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_cgroups_count", 5) + cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.cgroup_managers", []string{"docker", "podman", "containerd", "cri-o"}) cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.traced_event_types", []string{"exec", "open", "dns", "imds"}) cfg.BindEnv("runtime_security_config.activity_dump.cgroup_dump_timeout") // deprecated in favor of dump_duration cfg.BindEnvAndSetDefault("runtime_security_config.activity_dump.dump_duration", "900s") diff --git a/pkg/config/teeconfig/go.mod b/pkg/config/teeconfig/go.mod index 2484fb9cd55e3..daec9f976da68 100644 --- a/pkg/config/teeconfig/go.mod +++ b/pkg/config/teeconfig/go.mod @@ -33,6 +33,7 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/ebpf/maps/generic_map.go b/pkg/ebpf/maps/generic_map.go index 6355837e0fe3a..e2fb14f9bd9d3 100644 --- a/pkg/ebpf/maps/generic_map.go +++ b/pkg/ebpf/maps/generic_map.go @@ -149,8 +149,17 @@ func Map[K any, V any](m *ebpf.Map) (*GenericMap[K, V], error) { return nil, err } + // See if we can perform binary.Read on the key type. If we can't we can't use the batch API + // for this map + var kval K + keySupportsBatchAPI := canBinaryReadKey[K]() + if !keySupportsBatchAPI { + log.Warnf("Key type %T does not support binary.Read, batch API will not be used for this map", kval) + } + return &GenericMap[K, V]{ - m: m, + m: m, + keySupportsBatchAPI: keySupportsBatchAPI, }, nil } diff --git a/pkg/ebpf/maps/generic_map_test.go b/pkg/ebpf/maps/generic_map_test.go index 96be7e988e677..52b568c367d66 100644 --- a/pkg/ebpf/maps/generic_map_test.go +++ b/pkg/ebpf/maps/generic_map_test.go @@ -592,3 +592,19 @@ func TestValidateMapKeyValueSize(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, gm2) } + +func TestGenericHashMapCanUseBatchAPI(t *testing.T) { + hash, err := ebpf.NewMap(&ebpf.MapSpec{ + Type: ebpf.Hash, + KeySize: 4, + ValueSize: 4, + MaxEntries: 10, + }) + require.NoError(t, err) + t.Cleanup(func() { hash.Close() }) + + gm, err := Map[int32, int32](hash) + require.NoError(t, err) + + require.Equal(t, BatchAPISupported(), gm.CanUseBatchAPI()) +} diff --git a/pkg/ebpf/uprobes/testutil.go b/pkg/ebpf/uprobes/testutil.go index 93b109b499aa3..2d4e8b85d4b25 100644 --- a/pkg/ebpf/uprobes/testutil.go +++ b/pkg/ebpf/uprobes/testutil.go @@ -13,6 +13,7 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "sync" "testing" "time" @@ -132,6 +133,21 @@ type FakeProcFSEntry struct { Command string Exe string Maps string + Env map[string]string +} + +// getEnvironContents returns the formatted contents of the /proc//environ file for the entry. +func (f *FakeProcFSEntry) getEnvironContents() string { + if len(f.Env) == 0 { + return "" + } + + formattedEnvVars := make([]string, 0, len(f.Env)) + for k, v := range f.Env { + formattedEnvVars = append(formattedEnvVars, fmt.Sprintf("%s=%s", k, v)) + } + + return strings.Join(formattedEnvVars, "\x00") + "\x00" } // CreateFakeProcFS creates a fake /proc filesystem with the given entries, useful for testing attachment to processes. @@ -145,16 +161,13 @@ func CreateFakeProcFS(t *testing.T, entries []FakeProcFSEntry) string { createFile(t, filepath.Join(baseDir, "comm"), entry.Command) createFile(t, filepath.Join(baseDir, "maps"), entry.Maps) createSymlink(t, entry.Exe, filepath.Join(baseDir, "exe")) + createFile(t, filepath.Join(baseDir, "environ"), entry.getEnvironContents()) } return procRoot } func createFile(t *testing.T, path, data string) { - if data == "" { - return - } - dir := filepath.Dir(path) require.NoError(t, os.MkdirAll(dir, 0775)) require.NoError(t, os.WriteFile(path, []byte(data), 0775)) diff --git a/pkg/flare/archive.go b/pkg/flare/archive.go index c3e5dffe00ca8..8c2ad86688e97 100644 --- a/pkg/flare/archive.go +++ b/pkg/flare/archive.go @@ -214,7 +214,7 @@ func getExpVar(fb flaretypes.FlareBuilder) error { apmDebugPort := pkgconfigsetup.Datadog().GetInt("apm_config.debug.port") f := filepath.Join("expvar", "trace-agent") - resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/debug/vars", apmDebugPort)) + resp, err := http.Get(fmt.Sprintf("https://127.0.0.1:%d/debug/vars", apmDebugPort)) if err != nil { return fb.AddFile(f, []byte(fmt.Sprintf("Error retrieving vars: %v", err))) } @@ -394,6 +394,11 @@ func getProcessAgentTaggerList() ([]byte, error) { return nil, fmt.Errorf("wrong configuration to connect to process-agent") } + err = apiutil.SetAuthToken(pkgconfigsetup.Datadog()) + if err != nil { + return nil, err + } + taggerListURL := fmt.Sprintf("http://%s/agent/tagger-list", addressPort) return getTaggerList(taggerListURL) } diff --git a/pkg/flare/archive_linux.go b/pkg/flare/archive_linux.go index dafe8bd41d1bc..9a3aea87a0ac0 100644 --- a/pkg/flare/archive_linux.go +++ b/pkg/flare/archive_linux.go @@ -39,6 +39,7 @@ func addSystemProbePlatformSpecificEntries(fb flaretypes.FlareBuilder) { _ = fb.AddFileFromFunc(filepath.Join("system-probe", "conntrack_host.log"), getSystemProbeConntrackHost) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "ebpf_btf_loader.log"), getSystemProbeBTFLoaderInfo) _ = fb.AddFileFromFunc(filepath.Join("system-probe", "selinux_sestatus.log"), getSystemProbeSelinuxSestatus) + _ = fb.AddFileFromFunc(filepath.Join("system-probe", "selinux_semodule_list.log"), getSystemProbeSelinuxSemoduleList) } } @@ -155,3 +156,9 @@ func getSystemProbeSelinuxSestatus() ([]byte, error) { url := sysprobeclient.DebugURL("/selinux_sestatus") return getHTTPData(sysProbeClient, url) } + +func getSystemProbeSelinuxSemoduleList() ([]byte, error) { + sysProbeClient := sysprobeclient.Get(getSystemProbeSocketPath()) + url := sysprobeclient.DebugURL("/selinux_semodule_list") + return getHTTPData(sysProbeClient, url) +} diff --git a/pkg/fleet/daemon/daemon.go b/pkg/fleet/daemon/daemon.go index ba79d5456dd3e..5023393d99e5b 100644 --- a/pkg/fleet/daemon/daemon.go +++ b/pkg/fleet/daemon/daemon.go @@ -570,7 +570,7 @@ type requestState struct { ErrorCode installerErrors.InstallerErrorCode } -func newRequestContext(request remoteAPIRequest) (telemetry.Span, context.Context) { +func newRequestContext(request remoteAPIRequest) (*telemetry.Span, context.Context) { ctx := context.WithValue(context.Background(), requestStateKey, &requestState{ Package: request.Package, ID: request.ID, diff --git a/pkg/fleet/installer/installer.go b/pkg/fleet/installer/installer.go index d9a2a910f45ef..179cad03a2834 100644 --- a/pkg/fleet/installer/installer.go +++ b/pkg/fleet/installer/installer.go @@ -631,6 +631,8 @@ func (i *installerImpl) promoteExperiment(ctx context.Context, pkg string) error func (i *installerImpl) preparePackage(ctx context.Context, pkg string, _ []string) error { switch pkg { + case packageDatadogInstaller: + return packages.PrepareInstaller(ctx) case packageDatadogAgent: return packages.PrepareAgent(ctx) default: diff --git a/pkg/fleet/installer/oci/download.go b/pkg/fleet/installer/oci/download.go index 7fcce0234512c..e557cf8b53d7f 100644 --- a/pkg/fleet/installer/oci/download.go +++ b/pkg/fleet/installer/oci/download.go @@ -62,8 +62,8 @@ const ( ) const ( - layerMaxSize = 3 << 30 // 3GiB - extractLayerRetries = 3 + layerMaxSize = 3 << 30 // 3GiB + networkRetries = 3 ) var ( @@ -318,30 +318,32 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) return fmt.Errorf("could not get layer media type: %w", err) } if layerMediaType == mediaType { - // Retry stream reset errors - for i := 0; i < extractLayerRetries; i++ { - if i > 0 { - time.Sleep(time.Second) - } - uncompressedLayer, err := layer.Uncompressed() - if err != nil { - return fmt.Errorf("could not uncompress layer: %w", err) - } - err = tar.Extract(uncompressedLayer, dir, layerMaxSize) - uncompressedLayer.Close() - if err != nil { - if !isStreamResetError(err) && !isConnectionResetByPeerError(err) { - return fmt.Errorf("could not extract layer: %w", err) + err = withNetworkRetries( + func() error { + var err error + defer func() { + if err != nil { + deferErr := tar.Clean(dir) + if deferErr != nil { + err = deferErr + } + } + }() + uncompressedLayer, err := layer.Uncompressed() + if err != nil { + return err } - log.Warnf("network error while extracting layer, retrying") - // Clean up the directory before retrying to avoid partial extraction - err = tar.Clean(dir) + err = tar.Extract(uncompressedLayer, dir, layerMaxSize) + uncompressedLayer.Close() if err != nil { - return fmt.Errorf("could not clean directory: %w", err) + return err } - } else { - break - } + + return nil + }, + ) + if err != nil { + return fmt.Errorf("could not extract layer: %w", err) } } } @@ -349,16 +351,22 @@ func (d *DownloadedPackage) ExtractLayers(mediaType types.MediaType, dir string) } // WriteOCILayout writes the image as an OCI layout to the given directory. -func (d *DownloadedPackage) WriteOCILayout(dir string) error { - layoutPath, err := layout.Write(dir, empty.Index) - if err != nil { - return fmt.Errorf("could not write layout: %w", err) - } - err = layoutPath.AppendImage(d.Image) - if err != nil { - return fmt.Errorf("could not append image to layout: %w", err) - } - return nil +func (d *DownloadedPackage) WriteOCILayout(dir string) (err error) { + var layoutPath layout.Path + return withNetworkRetries( + func() error { + layoutPath, err = layout.Write(dir, empty.Index) + if err != nil { + return fmt.Errorf("could not write layout: %w", err) + } + + err = layoutPath.AppendImage(d.Image) + if err != nil { + return fmt.Errorf("could not append image to layout: %w", err) + } + return nil + }, + ) } // PackageURL returns the package URL for the given site, package and version. @@ -371,15 +379,50 @@ func PackageURL(env *env.Env, pkg string, version string) string { } } +func withNetworkRetries(f func() error) error { + var err error + for i := 0; i < networkRetries; i++ { + err = f() + if err == nil { + return nil + } + if !isRetryableNetworkError(err) { + return err + } + log.Warnf("retrying after network error: %s", err) + time.Sleep(time.Second) + } + return err +} + +// isRetryableNetworkError returns true if the error is a network error we should retry on +func isRetryableNetworkError(err error) bool { + if err == nil { + return false + } + + if netErr, ok := err.(*net.OpError); ok { + if netErr.Temporary() { + // Temporary errors, such as "connection timed out" + return true + } + if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { + if errno, ok := syscallErr.Err.(syscall.Errno); ok { + // Connection reset errors, such as "connection reset by peer" + return errno == syscall.ECONNRESET + } + } + } + + return isStreamResetError(err) +} + // isStreamResetError returns true if the given error is a stream reset error. // Sometimes, in GCR, the tar extract fails with "stream error: stream ID x; INTERNAL_ERROR; received from peer". // This happens because the uncompressed layer reader is a http/2 response body under the hood. That body is // streamed and receives a "reset stream frame", with the code 0x2 (INTERNAL_ERROR). This is an error from the server // that we need to retry. func isStreamResetError(err error) bool { - if err == nil { - return false - } serr := http2.StreamError{} if errors.As(err, &serr) { return serr.Code == http2.ErrCodeInternal @@ -391,18 +434,6 @@ func isStreamResetError(err error) bool { return false } -// isConnectionResetByPeer returns true if the error is a connection reset by peer error -func isConnectionResetByPeerError(err error) bool { - if netErr, ok := err.(*net.OpError); ok { - if syscallErr, ok := netErr.Err.(*os.SyscallError); ok { - if errno, ok := syscallErr.Err.(syscall.Errno); ok { - return errno == syscall.ECONNRESET - } - } - } - return false -} - type usernamePasswordKeychain struct { username string password string diff --git a/pkg/fleet/installer/packages/datadog_agent.go b/pkg/fleet/installer/packages/datadog_agent.go index c3b56f0e9c87b..7a07b99389ee0 100644 --- a/pkg/fleet/installer/packages/datadog_agent.go +++ b/pkg/fleet/installer/packages/datadog_agent.go @@ -23,7 +23,6 @@ import ( const ( agentPackage = "datadog-agent" - pathOldAgent = "/opt/datadog-agent" agentSymlink = "/usr/bin/datadog-agent" agentUnit = "datadog-agent.service" traceAgentUnit = "datadog-agent-trace.service" @@ -76,13 +75,18 @@ func PrepareAgent(ctx context.Context) (err error) { span, ctx := telemetry.StartSpanFromContext(ctx, "prepare_agent") defer func() { span.Finish(err) }() - // Check if the agent has been installed by a package manager, if yes remove it - if !oldAgentInstalled() { - return nil // Nothing to do - } - err = stopOldAgentUnits(ctx) + err = removeDebRPMPackage(ctx, "datadog-agent") if err != nil { - return fmt.Errorf("failed to stop old agent units: %w", err) + return fmt.Errorf("failed to remove deb/rpm datadog-agent package: %w", err) + } + + for _, unit := range stableUnits { + if err := stopUnit(ctx, unit); err != nil { + log.Warnf("Failed to stop %s: %s", unit, err) + } + if err := disableUnit(ctx, unit); err != nil { + log.Warnf("Failed to disable %s: %s", unit, err) + } } return removeDebRPMPackage(ctx, agentPackage) } @@ -205,28 +209,6 @@ func RemoveAgent(ctx context.Context) error { return nil } -func oldAgentInstalled() bool { - _, err := os.Stat(pathOldAgent) - return err == nil -} - -func stopOldAgentUnits(ctx context.Context) (err error) { - if !oldAgentInstalled() { - return nil - } - span, ctx := telemetry.StartSpanFromContext(ctx, "remove_old_agent_units") - defer span.Finish(err) - for _, unit := range stableUnits { - if err := stopUnit(ctx, unit); err != nil { - return fmt.Errorf("failed to stop %s: %v", unit, err) - } - if err := disableUnit(ctx, unit); err != nil { - return fmt.Errorf("failed to disable %s: %v", unit, err) - } - } - return nil -} - func chownRecursive(path string, uid int, gid int, ignorePaths []string) error { return filepath.Walk(path, func(p string, _ os.FileInfo, err error) error { if err != nil { diff --git a/pkg/fleet/installer/packages/datadog_installer.go b/pkg/fleet/installer/packages/datadog_installer.go index 14d0f042cd0e6..1e36bb462c1fd 100644 --- a/pkg/fleet/installer/packages/datadog_installer.go +++ b/pkg/fleet/installer/packages/datadog_installer.go @@ -41,6 +41,17 @@ func addDDAgentGroup(ctx context.Context) error { return exec.CommandContext(ctx, "groupadd", "--system", "dd-agent").Run() } +// PrepareInstaller prepares the installer +func PrepareInstaller(ctx context.Context) error { + if err := stopUnit(ctx, installerUnit); err != nil { + log.Warnf("Failed to stop unit %s: %s", installerUnit, err) + } + if err := disableUnit(ctx, installerUnit); err != nil { + log.Warnf("Failed to disable %s: %s", installerUnit, err) + } + return nil +} + // SetupInstaller installs and starts the installer systemd units func SetupInstaller(ctx context.Context) (err error) { defer func() { diff --git a/pkg/fleet/installer/packages/datadog_installer_windows.go b/pkg/fleet/installer/packages/datadog_installer_windows.go index 6a4b313120840..6df65c397dd69 100644 --- a/pkg/fleet/installer/packages/datadog_installer_windows.go +++ b/pkg/fleet/installer/packages/datadog_installer_windows.go @@ -11,16 +11,22 @@ package packages import ( "context" "fmt" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/msi" - "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" "os" "path" + + "github.com/DataDog/datadog-agent/pkg/fleet/internal/msi" + "github.com/DataDog/datadog-agent/pkg/fleet/internal/paths" ) const ( datadogInstaller = "datadog-installer" ) +// PrepareInstaller prepares the installer +func PrepareInstaller(_ context.Context) error { + return nil +} + // SetupInstaller installs and starts the installer func SetupInstaller(_ context.Context) error { rootPath := "" diff --git a/pkg/fleet/installer/packages/pkg_manager.go b/pkg/fleet/installer/packages/pkg_manager.go index 98b9d76b29d59..90eb34a5b2f52 100644 --- a/pkg/fleet/installer/packages/pkg_manager.go +++ b/pkg/fleet/installer/packages/pkg_manager.go @@ -8,44 +8,62 @@ package packages import ( - "bytes" "context" + "errors" "fmt" "os/exec" "github.com/DataDog/datadog-agent/pkg/fleet/telemetry" ) +func dpkgInstalled() (bool, error) { + _, err := exec.LookPath("dpkg") + if err != nil && !errors.Is(err, exec.ErrNotFound) { + return false, err + } + return err == nil, nil +} + +func rpmInstalled() (bool, error) { + _, err := exec.LookPath("rpm") + if err != nil && !errors.Is(err, exec.ErrNotFound) { + return false, err + } + return err == nil, nil +} + // removeDebRPMPackage removes a package installed via deb/rpm package manager // It doesn't remove dependencies or purge as we want to keep existing configuration files // and reinstall the package using the installer. // Note: we don't run the pre/post remove scripts as we want to avoid surprises for older agent versions (like removing config) func removeDebRPMPackage(ctx context.Context, pkg string) (err error) { - span, _ := telemetry.StartSpanFromContext(ctx, "remove_deb_rpm_package") + span, _ := telemetry.StartSpanFromContext(ctx, "removeDebRPMPackage") defer func() { span.Finish(err) }() - // Compute the right command depending on the package manager - var cmd *exec.Cmd - if _, pathErr := exec.LookPath("dpkg"); pathErr == nil { - // Doesn't fail if the package isn't installed - cmd = exec.Command("dpkg", "-r", "--no-triggers", agentPackage) - } else if _, pathErr := exec.LookPath("rpm"); pathErr == nil { - // Check if package exist, else the command will fail - pkgErr := exec.Command("rpm", "-q", agentPackage).Run() - if pkgErr == nil { - cmd = exec.Command("rpm", "-e", "--nodeps", "--noscripts", agentPackage) - } - } - - if cmd == nil { - // If we can't find a package manager or the package is not installed, ignore this step + + dpkgInstalled, err := dpkgInstalled() + if err != nil { + return err + } + rpmInstalled, err := rpmInstalled() + if err != nil { + return err + } + var packageInstalled bool + var removeCmd *exec.Cmd + if dpkgInstalled { + removeCmd = exec.Command("dpkg", "-r", pkg) + packageInstalled = exec.Command("dpkg", "-s", pkg).Run() == nil + } + if rpmInstalled { + removeCmd = exec.Command("rpm", "-e", pkg) + packageInstalled = exec.Command("rpm", "-q", pkg).Run() == nil + } + if !packageInstalled { return nil } - - // Run the command - var buf bytes.Buffer - cmd.Stderr = &buf - if err := cmd.Run(); err != nil { - return fmt.Errorf("failed to uninstall deb/rpm package %s (%w): %s", pkg, err, buf.String()) + out, err := removeCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to uninstall deb/rpm package %s (%w): %s", pkg, err, out) } return nil } diff --git a/pkg/fleet/installer/setup/common/setup.go b/pkg/fleet/installer/setup/common/setup.go index 0a7b93447c22f..b6e512d7dc5d6 100644 --- a/pkg/fleet/installer/setup/common/setup.go +++ b/pkg/fleet/installer/setup/common/setup.go @@ -41,7 +41,7 @@ type Setup struct { Out *Output Env *env.Env Ctx context.Context - Span telemetry.Span + Span *telemetry.Span Packages Packages Config Config } @@ -130,7 +130,7 @@ func (s *Setup) installPackage(name string, url string) (err error) { span, ctx := telemetry.StartSpanFromContext(s.Ctx, "install") defer func() { span.Finish(err) }() span.SetTag("url", url) - span.SetTag("_top_level", 1) + span.SetTopLevel() s.Out.WriteString(fmt.Sprintf("Installing %s...\n", name)) err = s.installer.Install(ctx, url, nil) diff --git a/pkg/fleet/internal/cdn/cdn.go b/pkg/fleet/internal/cdn/cdn.go index f35418cc0b7bc..dcb57bf0ee17d 100644 --- a/pkg/fleet/internal/cdn/cdn.go +++ b/pkg/fleet/internal/cdn/cdn.go @@ -165,28 +165,17 @@ func (c *CDN) Close() error { } // writePolicyMetadata writes the policy metadata to the given directory -// and makes it readable to dd-agent +// and makes it world-readable func writePolicyMetadata(config Config, dir string) error { - ddAgentUID, ddAgentGID, err := getAgentIDs() - if err != nil { - return fmt.Errorf("error getting dd-agent user and group IDs: %w", err) - } - state := config.State() stateBytes, err := json.Marshal(state) if err != nil { return fmt.Errorf("could not marshal state: %w", err) } - err = os.WriteFile(filepath.Join(dir, policyMetadataFilename), stateBytes, 0440) + err = os.WriteFile(filepath.Join(dir, policyMetadataFilename), stateBytes, 0444) if err != nil { return fmt.Errorf("could not write %s: %w", policyMetadataFilename, err) } - if runtime.GOOS != "windows" { - err = os.Chown(filepath.Join(dir, policyMetadataFilename), ddAgentUID, ddAgentGID) - if err != nil { - return fmt.Errorf("could not chown %s: %w", policyMetadataFilename, err) - } - } return nil } diff --git a/pkg/fleet/internal/exec/installer_exec.go b/pkg/fleet/internal/exec/installer_exec.go index 834f54b3d8a53..92f5880723fa9 100644 --- a/pkg/fleet/internal/exec/installer_exec.go +++ b/pkg/fleet/internal/exec/installer_exec.go @@ -40,7 +40,7 @@ func NewInstallerExec(env *env.Env, installerBinPath string) *InstallerExec { type installerCmd struct { *exec.Cmd - span telemetry.Span + span *telemetry.Span ctx context.Context } diff --git a/pkg/fleet/telemetry/http_wrapper.go b/pkg/fleet/telemetry/http_wrapper.go new file mode 100644 index 0000000000000..2f94bda79fe1f --- /dev/null +++ b/pkg/fleet/telemetry/http_wrapper.go @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "net/http" + "strconv" + "strings" +) + +// WrapRoundTripper wraps the round tripper with the telemetry round tripper. +func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { + if rt == nil { + rt = http.DefaultTransport + } + if wrapped, ok := rt.(*roundTripper); ok { + rt = wrapped.base + } + return &roundTripper{ + base: rt, + } +} + +type roundTripper struct { + base http.RoundTripper +} + +func (rt *roundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + span, _ := StartSpanFromContext(req.Context(), "http.request") + defer func() { span.Finish(err) }() + + url := *req.URL + url.User = nil + + span.span.Type = "http" + span.SetResourceName(req.Method + " " + urlFromRequest(req)) + span.span.Meta["http.method"] = req.Method + span.span.Meta["http.url"] = req.URL.String() + span.span.Meta["span.kind"] = "client" + span.span.Meta["network.destination.name"] = url.Hostname() + res, err = rt.base.RoundTrip(req) + if err != nil { + span.SetTag("http.errors", err.Error()) + return res, err + } + span.SetTag("http.status_code", strconv.Itoa(res.StatusCode)) + if res.StatusCode >= 400 { + span.SetTag("http.errors", res.Status) + } + return res, err +} + +// urlFromRequest returns the URL from the HTTP request. The URL query string is included in the return object iff queryString is true +// See https://docs.datadoghq.com/tracing/configure_data_security#redacting-the-query-in-the-url for more information. +func urlFromRequest(r *http.Request) string { + // Quoting net/http comments about net.Request.URL on server requests: + // "For most requests, fields other than Path and RawQuery will be + // empty. (See RFC 7230, Section 5.3)" + // This is why we don't rely on url.URL.String(), url.URL.Host, url.URL.Scheme, etc... + var url string + path := r.URL.EscapedPath() + scheme := r.URL.Scheme + if r.TLS != nil { + scheme = "https" + } + if r.Host != "" { + url = strings.Join([]string{scheme, "://", r.Host, path}, "") + } else { + url = path + } + // Collect the query string if we are allowed to report it and obfuscate it if possible/allowed + if r.URL.RawQuery != "" { + query := r.URL.RawQuery + url = strings.Join([]string{url, query}, "?") + } + if frag := r.URL.EscapedFragment(); frag != "" { + url = strings.Join([]string{url, frag}, "#") + } + return url +} diff --git a/pkg/fleet/telemetry/span.go b/pkg/fleet/telemetry/span.go index ac8f79516c736..ac5f00b9ade0a 100644 --- a/pkg/fleet/telemetry/span.go +++ b/pkg/fleet/telemetry/span.go @@ -7,20 +7,144 @@ package telemetry import ( - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "context" + "fmt" + "math/rand/v2" + "runtime/debug" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" ) -// Span is an alias for ddtrace.Span until we phase ddtrace out. -type Span struct{ ddtrace.Span } +const spanKey = spanContextKey("span_context") + +type spanContextKey string + +// Span represents a span. +type Span struct { + mu sync.Mutex + span internaltelemetry.Span + finished atomic.Bool +} + +func newSpan(name string, parentID, traceID uint64) *Span { + if traceID == 0 { + traceID = rand.Uint64() + if !headSamplingKeep(name, traceID) { + traceID = dropTraceID + } + } + s := &Span{ + span: internaltelemetry.Span{ + TraceID: traceID, + ParentID: parentID, + SpanID: rand.Uint64(), + Name: name, + Resource: name, + Start: time.Now().UnixNano(), + Meta: make(map[string]string), + Metrics: make(map[string]float64), + }, + } + if parentID == 0 { + s.SetTopLevel() + } + + globalTracer.registerSpan(s) + return s +} // Finish finishes the span with an error. func (s *Span) Finish(err error) { - s.Span.Finish(tracer.WithError(err)) + s.finished.Store(true) + s.mu.Lock() + defer s.mu.Unlock() + s.span.Duration = time.Now().UnixNano() - s.span.Start + if err != nil { + s.span.Error = 1 + s.span.Meta = map[string]string{ + "error.message": err.Error(), + "error.stack": string(debug.Stack()), + } + } + globalTracer.finishSpan(s) } // SetResourceName sets the resource name of the span. func (s *Span) SetResourceName(name string) { - s.Span.SetTag(ext.ResourceName, name) + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.span.Resource = name +} + +// SetTopLevel sets the span as a top level span. +func (s *Span) SetTopLevel() { + s.SetTag("_top_level", 1) +} + +// SetTag sets a tag on the span. +func (s *Span) SetTag(key string, value interface{}) { + if s.finished.Load() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if value == nil { + s.span.Meta[key] = "nil" + } + switch v := value.(type) { + case string: + s.span.Meta[key] = v + case bool: + s.span.Meta[key] = strconv.FormatBool(v) + case int: + s.span.Metrics[key] = float64(v) + case int8: + s.span.Metrics[key] = float64(v) + case int16: + s.span.Metrics[key] = float64(v) + case int32: + s.span.Metrics[key] = float64(v) + case int64: + s.span.Metrics[key] = float64(v) + case uint: + s.span.Metrics[key] = float64(v) + case uint8: + s.span.Metrics[key] = float64(v) + case uint16: + s.span.Metrics[key] = float64(v) + case uint32: + s.span.Metrics[key] = float64(v) + case uint64: + s.span.Metrics[key] = float64(v) + case float32: + s.span.Metrics[key] = float64(v) + case float64: + s.span.Metrics[key] = v + default: + s.span.Meta[key] = fmt.Sprintf("not_supported_type %T", v) + } +} + +type spanIDs struct { + traceID uint64 + spanID uint64 +} + +func getSpanIDsFromContext(ctx context.Context) (spanIDs, bool) { + sIDs, ok := ctx.Value(spanKey).(spanIDs) + if !ok { + return spanIDs{}, false + } + return sIDs, true +} + +func setSpanIDsInContext(ctx context.Context, span *Span) context.Context { + return context.WithValue(ctx, spanKey, spanIDs{traceID: span.span.TraceID, spanID: span.span.SpanID}) } diff --git a/pkg/fleet/telemetry/telemetry.go b/pkg/fleet/telemetry/telemetry.go index 5bbfd8ca773c4..66174ad18fc32 100644 --- a/pkg/fleet/telemetry/telemetry.go +++ b/pkg/fleet/telemetry/telemetry.go @@ -8,280 +8,178 @@ package telemetry import ( "context" - "errors" "fmt" - "io" - "math/rand/v2" - "net" "net/http" "os" "strconv" "strings" - "sync" - - httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" - "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" - - "github.com/gorilla/mux" + "time" "github.com/DataDog/datadog-agent/pkg/internaltelemetry" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - traceconfig "github.com/DataDog/datadog-agent/pkg/trace/config" - "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" ) const ( - // EnvTraceID is the environment variable key for the trace ID - EnvTraceID = "DATADOG_TRACE_ID" - // EnvParentID is the environment variable key for the parent ID - EnvParentID = "DATADOG_PARENT_ID" -) - -const ( + envTraceID = "DATADOG_TRACE_ID" + envParentID = "DATADOG_PARENT_ID" telemetrySubdomain = "instrumentation-telemetry-intake" - telemetryEndpoint = "/v0.4/traces" ) // Telemetry handles the telemetry for fleet components. type Telemetry struct { telemetryClient internaltelemetry.Client + done chan struct{} + flushed chan struct{} - site string + env string service string - - listener *telemetryListener - server *http.Server - client *http.Client - - samplingRules []tracer.SamplingRule } -// Option is a functional option for telemetry. -type Option func(*Telemetry) - // NewTelemetry creates a new telemetry instance -func NewTelemetry(client *http.Client, apiKey string, site string, service string, opts ...Option) *Telemetry { - endpoint := &traceconfig.Endpoint{ +func NewTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + t := newTelemetry(client, apiKey, site, service) + t.Start() + return t +} + +func newTelemetry(client *http.Client, apiKey string, site string, service string) *Telemetry { + endpoint := &internaltelemetry.Endpoint{ Host: fmt.Sprintf("https://%s.%s", telemetrySubdomain, strings.TrimSpace(site)), APIKey: apiKey, } - listener := newTelemetryListener() - t := &Telemetry{ - telemetryClient: internaltelemetry.NewClient(client, []*traceconfig.Endpoint{endpoint}, service, site == "datad0g.com"), - site: site, - service: service, - listener: listener, - server: &http.Server{}, - client: &http.Client{ - Transport: &http.Transport{ - Dial: listener.Dial, - }, - }, + env := "prod" + if site == "datad0g.com" { + env = "staging" } - for _, opt := range opts { - opt(t) + + return &Telemetry{ + telemetryClient: internaltelemetry.NewClient(client, []*internaltelemetry.Endpoint{endpoint}, service, site == "datad0g.com"), + done: make(chan struct{}), + flushed: make(chan struct{}), + env: env, + service: service, } - t.server.Handler = t.handler() - return t } // Start starts the telemetry -func (t *Telemetry) Start(_ context.Context) error { +func (t *Telemetry) Start() { + ticker := time.Tick(1 * time.Minute) go func() { - err := t.server.Serve(t.listener) - if err != nil { - log.Infof("telemetry server stopped: %v", err) + for { + select { + case <-ticker: + t.sendCompletedSpans() + case <-t.done: + t.sendCompletedSpans() + close(t.flushed) + return + } } }() - env := "prod" - if t.site == "datad0g.com" { - env = "staging" - } - - tracer.Start( - tracer.WithService(t.service), - tracer.WithServiceVersion(version.AgentVersion), - tracer.WithEnv(env), - tracer.WithGlobalTag("site", t.site), - tracer.WithHTTPClient(t.client), - tracer.WithLogStartup(false), - - // We don't need the value, we just need to enforce that it's not - // the default. If it is, then the tracer will try to use the socket - // if it exists -- and it always exists for newer agents. - // If the agent address is the socket, the tracer overrides WithHTTPClient to use it. - tracer.WithAgentAddr("192.0.2.42:12345"), // 192.0.2.0/24 is reserved - tracer.WithSamplingRules(t.samplingRules), - ) - return nil } // Stop stops the telemetry -func (t *Telemetry) Stop(ctx context.Context) error { - tracer.Flush() - tracer.Stop() - t.listener.Close() - err := t.server.Shutdown(ctx) - if err != nil { - log.Errorf("error shutting down telemetry server: %v", err) - } - return nil -} - -func (t *Telemetry) handler() http.Handler { - r := mux.NewRouter().Headers("Content-Type", "application/msgpack").Subrouter() - r.HandleFunc(telemetryEndpoint, func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - log.Errorf("error reading request body: %v", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - var traces pb.Traces - _, err = traces.UnmarshalMsg(body) - if err != nil { - log.Errorf("error unmarshalling traces: %v", err) - w.WriteHeader(http.StatusBadRequest) - return - } - t.telemetryClient.SendTraces(traces) - w.WriteHeader(http.StatusOK) - }) - return r +func (t *Telemetry) Stop() { + close(t.done) + <-t.flushed } -type telemetryListener struct { - conns chan net.Conn - - close chan struct{} - closeOnce sync.Once -} - -func newTelemetryListener() *telemetryListener { - return &telemetryListener{ - conns: make(chan net.Conn), - close: make(chan struct{}), +func (t *Telemetry) extractCompletedSpans() internaltelemetry.Traces { + spans := globalTracer.flushCompletedSpans() + if len(spans) == 0 { + return internaltelemetry.Traces{} } -} - -func (l *telemetryListener) Close() error { - l.closeOnce.Do(func() { - close(l.close) - }) - return nil -} - -func (l *telemetryListener) Accept() (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - case conn := <-l.conns: - return conn, nil + traces := make(map[uint64][]*internaltelemetry.Span) + for _, span := range spans { + span.span.Service = t.service + span.span.Meta["env"] = t.env + span.span.Meta["version"] = version.AgentVersion + span.span.Metrics["_sampling_priority_v1"] = 2 + traces[span.span.TraceID] = append(traces[span.span.TraceID], &span.span) + } + tracesArray := make([]internaltelemetry.Trace, 0, len(traces)) + for _, trace := range traces { + tracesArray = append(tracesArray, internaltelemetry.Trace(trace)) } + return internaltelemetry.Traces(tracesArray) } -func (l *telemetryListener) Addr() net.Addr { - return addr(0) +func (t *Telemetry) sendCompletedSpans() { + tracesArray := t.extractCompletedSpans() + if len(tracesArray) == 0 { + return + } + t.telemetryClient.SendTraces(tracesArray) } -func (l *telemetryListener) Dial(_, _ string) (net.Conn, error) { - select { - case <-l.close: - return nil, errors.New("listener closed") - default: +// SpanFromContext returns the span from the context if available. +func SpanFromContext(ctx context.Context) (*Span, bool) { + spanIDs, ok := getSpanIDsFromContext(ctx) + if !ok { + return nil, false } - server, client := net.Pipe() - l.conns <- server - return client, nil + return globalTracer.getSpan(spanIDs.spanID) } -type addr int - -func (addr) Network() string { - return "memory" +// StartSpanFromEnv starts a span using the environment variables to find the parent span. +func StartSpanFromEnv(ctx context.Context, operationName string) (*Span, context.Context) { + traceID, parentID := extractIDsFromEnv() + return StartSpanFromIDs(ctx, operationName, traceID, parentID) } -func (addr) String() string { - return "local" +func extractIDsFromEnv() (string, string) { + parentID, ok := os.LookupEnv(envParentID) + if !ok { + return "0", "0" + } + traceID, ok := os.LookupEnv(envTraceID) + if !ok { + return "0", "0" + } + return traceID, parentID } -// StartSpanFromIDs starts a span using the trace and parent -// IDs provided. -func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - ctxCarrier := tracer.TextMapCarrier{ - tracer.DefaultTraceIDHeader: traceID, - tracer.DefaultParentIDHeader: parentID, - tracer.DefaultPriorityHeader: "2", +func converIDsToUint64(traceID, parentID string) (uint64, uint64) { + traceIDInt, err := strconv.ParseUint(traceID, 10, 64) + if err != nil { + return 0, 0 } - spanCtx, err := tracer.Extract(ctxCarrier) + parentIDInt, err := strconv.ParseUint(parentID, 10, 64) if err != nil { - log.Debugf("failed to extract span context from install script params: %v", err) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return 0, 0 } - spanOptions = append([]ddtrace.StartSpanOption{tracer.ChildOf(spanCtx)}, spanOptions...) - return StartSpanFromContext(ctx, operationName, spanOptions...) + return traceIDInt, parentIDInt } -// SpanFromContext returns the span from the context if available. -func SpanFromContext(ctx context.Context) (Span, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return Span{}, false - } - return Span{span}, true +// StartSpanFromIDs starts a span using the trace and parent +// IDs provided. +func StartSpanFromIDs(ctx context.Context, operationName, traceID, parentID string) (*Span, context.Context) { + traceIDInt, parentIDInt := converIDsToUint64(traceID, parentID) + span, ctx := startSpanFromIDs(ctx, operationName, traceIDInt, parentIDInt) + span.SetTopLevel() + return span, ctx } -// StartSpanFromContext starts a span using the context to find the parent span. -func StartSpanFromContext(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - span, ctx := tracer.StartSpanFromContext(ctx, operationName, spanOptions...) - return Span{span}, ctx +func startSpanFromIDs(ctx context.Context, operationName string, traceID, parentID uint64) (*Span, context.Context) { + s := newSpan(operationName, parentID, traceID) + ctx = setSpanIDsInContext(ctx, s) + return s, ctx } -// StartSpanFromEnv starts a span using the environment variables to find the parent span. -func StartSpanFromEnv(ctx context.Context, operationName string, spanOptions ...ddtrace.StartSpanOption) (Span, context.Context) { - traceID, ok := os.LookupEnv(EnvTraceID) - if !ok { - traceID = strconv.FormatUint(rand.Uint64(), 10) - } - parentID, ok := os.LookupEnv(EnvParentID) - if !ok { - parentID = "0" - } - return StartSpanFromIDs(ctx, operationName, traceID, parentID, spanOptions...) +// StartSpanFromContext starts a span using the context to find the parent span. +func StartSpanFromContext(ctx context.Context, operationName string) (*Span, context.Context) { + spanIDs, _ := getSpanIDsFromContext(ctx) + return startSpanFromIDs(ctx, operationName, spanIDs.traceID, spanIDs.spanID) } // EnvFromContext returns the environment variables for the context. func EnvFromContext(ctx context.Context) []string { - spanCtx, ok := SpanContextFromContext(ctx) + sIDs, ok := getSpanIDsFromContext(ctx) if !ok { return []string{} } return []string{ - fmt.Sprintf("%s=%d", EnvTraceID, spanCtx.TraceID()), - fmt.Sprintf("%s=%d", EnvParentID, spanCtx.SpanID()), + fmt.Sprintf("%s=%s", envTraceID, strconv.FormatUint(sIDs.traceID, 10)), + fmt.Sprintf("%s=%s", envParentID, strconv.FormatUint(sIDs.spanID, 10)), } } - -// SpanContextFromContext extracts the span context from the context if available. -func SpanContextFromContext(ctx context.Context) (ddtrace.SpanContext, bool) { - span, ok := tracer.SpanFromContext(ctx) - if !ok { - return nil, false - } - return span.Context(), true -} - -// WithSamplingRules sets the sampling rules for the telemetry. -func WithSamplingRules(rules ...tracer.SamplingRule) Option { - return func(t *Telemetry) { - t.samplingRules = rules - } -} - -// WrapRoundTripper wraps the round tripper with the telemetry round tripper. -func WrapRoundTripper(rt http.RoundTripper) http.RoundTripper { - return httptrace.WrapRoundTripper(rt) -} diff --git a/pkg/fleet/telemetry/telemetry_test.go b/pkg/fleet/telemetry/telemetry_test.go new file mode 100644 index 0000000000000..d0866ee4d4a13 --- /dev/null +++ b/pkg/fleet/telemetry/telemetry_test.go @@ -0,0 +1,222 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "context" + "errors" + "net/http" + "os" + "testing" + + "github.com/DataDog/datadog-agent/pkg/internaltelemetry" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFreshSpan(t *testing.T) { + ctx := context.Background() + _, ok := SpanFromContext(ctx) + require.False(t, ok) + + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + s.SetResourceName("new") + + span, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, s, span) + + assert.Equal(t, "test", s.span.Name) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", s.span.Resource) + assert.Equal(t, "new", span.span.Resource) +} + +func TestInheritence(t *testing.T) { + ctx := context.Background() + s, ctx := StartSpanFromContext(ctx, "test") + require.NotNil(t, s) + + child, _ := StartSpanFromContext(ctx, "child") + require.NotNil(t, child) + + assert.Equal(t, s.span.SpanID, child.span.ParentID) + assert.Equal(t, s.span.TraceID, child.span.TraceID) +} + +func TestStartSpanFromIDs(t *testing.T) { + ctx := context.Background() + traceID := "100" + parentID := "200" + + span, ctx := StartSpanFromIDs(ctx, "ids-operation", traceID, parentID) + require.NotNil(t, span, "Expected a span") + require.Equal(t, uint64(100), span.span.TraceID) + require.Equal(t, uint64(200), span.span.ParentID) + + val, ok := span.span.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + + spanFromCtx, ok := SpanFromContext(ctx) + require.True(t, ok) + require.Equal(t, span, spanFromCtx) +} + +func strPtr(s string) *string { + return &s +} + +func TestSpanFromEnv(t *testing.T) { + randTraceID := uint64(9) + tt := []struct { + name string + envTraceID *string + envParentID *string + expectedTraceID uint64 + expectedParentID uint64 + }{ + { + name: "no parent env", + envTraceID: strPtr("100"), + envParentID: nil, + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "no trace env", + envTraceID: nil, + envParentID: strPtr("100"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "traceID malformed", + envTraceID: strPtr("not-a-number"), + envParentID: strPtr("200"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "parentID malformed", + envTraceID: strPtr("100"), + envParentID: strPtr("not-a-number"), + expectedTraceID: randTraceID, + expectedParentID: 0, + }, + { + name: "inheritance", + envTraceID: strPtr("100"), + envParentID: strPtr("200"), + expectedTraceID: 100, + expectedParentID: 200, + }, + } + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + if tc.envTraceID != nil { + os.Setenv(envTraceID, *tc.envTraceID) + defer os.Unsetenv(envTraceID) + } + if tc.envParentID != nil { + os.Setenv(envParentID, *tc.envParentID) + defer os.Unsetenv(envParentID) + } + + span, ctx := StartSpanFromEnv(context.Background(), "env-operation") + require.NotNil(t, span, "Expected a span") + s, ok := SpanFromContext(ctx) + assert.True(t, ok) + assert.Equal(t, span, s) + + assert.Equal(t, tc.expectedParentID, span.span.ParentID) + if tc.expectedTraceID != randTraceID { + assert.Equal(t, tc.expectedTraceID, span.span.TraceID) + } else { + assert.NotEqual(t, 0, span.span.TraceID) + } + + }) + } +} + +func TestLimit(t *testing.T) { + totalSpans := maxSpansInFlight + 2 + ctx := context.Background() + for i := 0; i < totalSpans; i++ { + _, ctx = StartSpanFromContext(ctx, "test") + } + assert.Len(t, globalTracer.spans, maxSpansInFlight) +} + +func TestEnvFromContext(t *testing.T) { + s, ctx := StartSpanFromContext(context.Background(), "test") + s.span.TraceID = 456 + s.span.SpanID = 123 + ctx = setSpanIDsInContext(ctx, s) + env := EnvFromContext(ctx) + assert.ElementsMatch(t, []string{"DATADOG_TRACE_ID=456", "DATADOG_PARENT_ID=123"}, env) + + env = EnvFromContext(context.Background()) + assert.ElementsMatch(t, []string{}, env) +} + +func TestSpanFinished(t *testing.T) { + s, _ := StartSpanFromContext(context.Background(), "test") + s.Finish(nil) + s.SetResourceName("new") + s.SetTag("key", "value") + + assert.Equal(t, "test", s.span.Resource) + _, ok := s.span.Meta["key"] + assert.False(t, ok) +} + +func TestRemapOnFlush(t *testing.T) { + const testService = "test-service" + const numTraces = 10 + telem := newTelemetry(&http.Client{}, "api", "datad0g.com", testService) + globalTracer = &tracer{spans: make(map[uint64]*Span)} + + // traces with 2 spans + for i := 0; i < numTraces; i++ { + parentSpan, ctx := StartSpanFromContext(context.Background(), "parent") + childSpan, _ := StartSpanFromContext(ctx, "child") + childSpan.Finish(errors.New("test_error")) + parentSpan.Finish(nil) + } + resTraces := telem.extractCompletedSpans() + require.Len(t, resTraces, numTraces) + + for _, trace := range resTraces { + assert.Len(t, trace, 2) + for _, span := range trace { + assert.Equal(t, testService, span.Service) + assert.Equal(t, "staging", span.Meta["env"]) + assert.Equal(t, 2.0, span.Metrics["_sampling_priority_v1"]) + } + var parent, child *internaltelemetry.Span + if trace[0].Name == "parent" { + parent = trace[0] + child = trace[1] + } else { + parent = trace[1] + child = trace[0] + } + assert.Equal(t, parent.SpanID, child.ParentID) + val, ok := parent.Metrics["_top_level"] + require.True(t, ok) + require.Equal(t, 1.0, val) + _, ok = child.Metrics["_top_level"] + require.False(t, ok) + + require.Equal(t, int32(1), child.Error) + require.Equal(t, "test_error", child.Meta["error.message"]) + require.Contains(t, child.Meta["error.stack"], "telemetry_test.go") + } +} diff --git a/pkg/fleet/telemetry/tracer.go b/pkg/fleet/telemetry/tracer.go new file mode 100644 index 0000000000000..0923b3563e457 --- /dev/null +++ b/pkg/fleet/telemetry/tracer.go @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package telemetry provides the telemetry for fleet components. +package telemetry + +import ( + "math" + "strings" + "sync" +) + +const ( + dropTraceID = 1 + maxSpansInFlight = 1000 +) + +var ( + globalTracer *tracer + samplingRates = map[string]float64{ + "cdn": 0.1, + "garbage_collect": 0.05, + "HTTPClient": 0.05, + } +) + +func init() { + globalTracer = &tracer{ + spans: make(map[uint64]*Span), + } +} + +type tracer struct { + mu sync.Mutex + spans map[uint64]*Span + completedSpans []*Span +} + +func (t *tracer) registerSpan(span *Span) { + if span.span.TraceID == dropTraceID { + return + } + t.mu.Lock() + defer t.mu.Unlock() + // naive maxSpansInFlight check as this is just telemetry + // next iteration if needed would be to flush long running spans to troubleshoot + if len(t.spans) >= maxSpansInFlight { + return + } + t.spans[span.span.SpanID] = span +} + +func (t *tracer) getSpan(spanID uint64) (*Span, bool) { + t.mu.Lock() + defer t.mu.Unlock() + span, ok := t.spans[spanID] + return span, ok +} + +func (t *tracer) finishSpan(span *Span) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.spans, span.span.SpanID) + t.completedSpans = append(t.completedSpans, span) +} + +func (t *tracer) flushCompletedSpans() []*Span { + t.mu.Lock() + defer t.mu.Unlock() + newSpanArray := make([]*Span, 0) + completedSpans := t.completedSpans + t.completedSpans = newSpanArray + return completedSpans +} + +func headSamplingKeep(spanName string, traceID uint64) bool { + for k, r := range samplingRates { + if strings.Contains(spanName, k) { + return sampledByRate(traceID, r) + } + } + return true +} + +func sampledByRate(n uint64, rate float64) bool { + if rate < 1 { + return n*uint64(1111111111111111111) < uint64(rate*math.MaxUint64) + } + return true +} diff --git a/pkg/gohai/go.mod b/pkg/gohai/go.mod index 45edc82c08f38..e7dee142ec26f 100644 --- a/pkg/gohai/go.mod +++ b/pkg/gohai/go.mod @@ -26,7 +26,6 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/gohai/go.sum b/pkg/gohai/go.sum index 99c1b368ae068..828633a9495c7 100644 --- a/pkg/gohai/go.sum +++ b/pkg/gohai/go.sum @@ -44,7 +44,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/gpu/consumer.go b/pkg/gpu/consumer.go index f4ebb7f04b0f2..55148350b12e3 100644 --- a/pkg/gpu/consumer.go +++ b/pkg/gpu/consumer.go @@ -220,7 +220,7 @@ func (c *cudaEventConsumer) getStreamKey(header *gpuebpf.CudaEventHeader) stream // Try to get the GPU device if we can, but do not fail if we can't as we want to report // the data even if we can't get the GPU UUID - gpuDevice, err := c.sysCtx.getCurrentActiveGpuDevice(int(pid), int(tid)) + gpuDevice, err := c.sysCtx.getCurrentActiveGpuDevice(int(pid), int(tid), containerID) if err != nil { log.Warnf("Error getting GPU device for process %d: %v", pid, err) } else { diff --git a/pkg/gpu/consumer_test.go b/pkg/gpu/consumer_test.go index 6530f2bb49738..c7c7aa04837d7 100644 --- a/pkg/gpu/consumer_test.go +++ b/pkg/gpu/consumer_test.go @@ -22,7 +22,7 @@ import ( func TestConsumerCanStartAndStop(t *testing.T) { handler := ddebpf.NewRingBufferHandler(consumerChannelSize) cfg := config.New() - ctx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) + ctx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) consumer := newCudaEventConsumer(ctx, handler, cfg) diff --git a/pkg/gpu/context.go b/pkg/gpu/context.go index 77bac5185339f..59d778bfea5c1 100644 --- a/pkg/gpu/context.go +++ b/pkg/gpu/context.go @@ -16,10 +16,15 @@ import ( "github.com/NVIDIA/go-nvml/pkg/nvml" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/errors" "github.com/DataDog/datadog-agent/pkg/gpu/cuda" "github.com/DataDog/datadog-agent/pkg/util/ktime" + "github.com/DataDog/datadog-agent/pkg/util/log" ) +const nvidiaResourceName = "nvidia.com/gpu" + // systemContext holds certain attributes about the system that are used by the GPU probe. type systemContext struct { // maxGpuThreadsPerDevice maps each device index to the maximum number of threads it can run in parallel @@ -60,6 +65,9 @@ type systemContext struct { // visibleDevicesCache is a cache of visible devices for each process, to avoid // looking into the environment variables every time visibleDevicesCache map[int][]nvml.Device + + // workloadmeta is the workloadmeta component that we use to get necessary container metadata + workloadmeta workloadmeta.Component } // symbolsEntry embeds cuda.Symbols adding a field for keeping track of the last @@ -73,7 +81,7 @@ func (e *symbolsEntry) updateLastUsedTime() { e.lastUsedTime = time.Now() } -func getSystemContext(nvmlLib nvml.Interface, procRoot string) (*systemContext, error) { +func getSystemContext(nvmlLib nvml.Interface, procRoot string, wmeta workloadmeta.Component) (*systemContext, error) { ctx := &systemContext{ maxGpuThreadsPerDevice: make(map[int]int), deviceSmVersions: make(map[int]int), @@ -83,6 +91,7 @@ func getSystemContext(nvmlLib nvml.Interface, procRoot string) (*systemContext, procRoot: procRoot, selectedDeviceByPIDAndTID: make(map[int]map[int]int32), visibleDevicesCache: make(map[int][]nvml.Device), + workloadmeta: wmeta, } if err := ctx.fillDeviceInfo(); err != nil { @@ -209,13 +218,75 @@ func (ctx *systemContext) cleanupOldEntries() { } } +// filterDevicesForContainer filters the available GPU devices for the given +// container. If the ID is not empty, we check the assignment of GPU resources +// to the container and return only the devices that are available to the +// container. +func (ctx *systemContext) filterDevicesForContainer(devices []nvml.Device, containerID string) ([]nvml.Device, error) { + if containerID == "" { + // If the process is not running in a container, we assume all devices are available. + return devices, nil + } + + container, err := ctx.workloadmeta.GetContainer(containerID) + if err != nil { + // If we don't find the container, we assume all devices are available. + // This can happen sometimes, e.g. if we don't have the container in the + // store yet. Do not block metrics on that. + if errors.IsNotFound(err) { + return devices, nil + } + + // Some error occurred while retrieving the container, this could be a + // general error with the store, report it. + return nil, fmt.Errorf("cannot retrieve data for container %s: %s", containerID, err) + } + + var filteredDevices []nvml.Device + for _, resource := range container.AllocatedResources { + // Only consider NVIDIA GPUs + if resource.Name != nvidiaResourceName { + continue + } + + for _, device := range devices { + uuid, ret := device.GetUUID() + if ret != nvml.SUCCESS { + log.Warnf("Error getting GPU UUID for device %s: %s", device, nvml.ErrorString(ret)) + continue + } + + if resource.ID == uuid { + filteredDevices = append(filteredDevices, device) + break + } + } + } + + // We didn't find any devices assigned to the container, report it as an error. + if len(filteredDevices) == 0 { + return nil, fmt.Errorf("no GPU devices found for container %s that matched its allocated resources %+v", containerID, container.AllocatedResources) + } + + return filteredDevices, nil +} + // getCurrentActiveGpuDevice returns the active GPU device for a given process and thread, based on the // last selection (via cudaSetDevice) this thread made and the visible devices for the process. -func (ctx *systemContext) getCurrentActiveGpuDevice(pid int, tid int) (nvml.Device, error) { +func (ctx *systemContext) getCurrentActiveGpuDevice(pid int, tid int, containerID string) (nvml.Device, error) { visibleDevices, ok := ctx.visibleDevicesCache[pid] if !ok { - var err error - visibleDevices, err = cuda.GetVisibleDevicesForProcess(ctx.gpuDevices, pid, ctx.procRoot) + // Order is important! We need to filter the devices for the container + // first. In a container setting, the environment variable acts as a + // filter on the devices that are available to the process, not on the + // devices available on the host system. + var err error // avoid shadowing visibleDevices, declare error before so we can use = instead of := + visibleDevices, err = ctx.filterDevicesForContainer(ctx.gpuDevices, containerID) + if err != nil { + return nil, fmt.Errorf("error filtering devices for container %s: %w", containerID, err) + } + + visibleDevices, err = cuda.GetVisibleDevicesForProcess(visibleDevices, pid, ctx.procRoot) if err != nil { return nil, fmt.Errorf("error getting visible devices for process %d: %w", pid, err) } diff --git a/pkg/gpu/context_test.go b/pkg/gpu/context_test.go new file mode 100644 index 0000000000000..dbb6e1735df24 --- /dev/null +++ b/pkg/gpu/context_test.go @@ -0,0 +1,176 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build linux_bpf + +package gpu + +import ( + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + "github.com/DataDog/datadog-agent/pkg/ebpf/uprobes" + "github.com/DataDog/datadog-agent/pkg/gpu/testutil" + "github.com/DataDog/datadog-agent/pkg/util/kernel" +) + +func TestFilterDevicesForContainer(t *testing.T) { + wmetaMock := testutil.GetWorkloadMetaMock(t) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), wmetaMock) + require.NotNil(t, sysCtx) + require.NoError(t, err) + + // Create a container with a single GPU and add it to the store + containerID := "abcdef" + deviceIndex := 2 + gpuUUID := testutil.GPUUUIDs[deviceIndex] + container := &workloadmeta.Container{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindContainer, + ID: containerID, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: containerID, + }, + AllocatedResources: []workloadmeta.ContainerAllocatedResource{ + { + Name: nvidiaResourceName, + ID: gpuUUID, + }, + }, + } + + wmetaMock.Set(container) + storeContainer, err := wmetaMock.GetContainer(containerID) + require.NoError(t, err, "container should be found in the store") + require.NotNil(t, storeContainer, "container should be found in the store") + + t.Run("NoContainer", func(t *testing.T) { + filtered, err := sysCtx.filterDevicesForContainer(sysCtx.gpuDevices, "") + require.NoError(t, err) + testutil.RequireDeviceListsEqual(t, filtered, sysCtx.gpuDevices) // With no container, all devices should be returned + }) + + t.Run("NonExistentContainer", func(t *testing.T) { + filtered, err := sysCtx.filterDevicesForContainer(sysCtx.gpuDevices, "non-existent-at-all") + require.NoError(t, err) + testutil.RequireDeviceListsEqual(t, filtered, sysCtx.gpuDevices) // If we can't find the container, all devices should be returned + }) + + t.Run("ContainerWithGPU", func(t *testing.T) { + filtered, err := sysCtx.filterDevicesForContainer(sysCtx.gpuDevices, containerID) + require.NoError(t, err) + require.Len(t, filtered, 1) + testutil.RequireDeviceListsEqual(t, filtered, sysCtx.gpuDevices[deviceIndex:deviceIndex+1]) + }) +} + +func TestGetCurrentActiveGpuDevice(t *testing.T) { + pidNoContainer := 1234 + pidNoContainerButEnv := 2235 + pidContainer := 3238 + pidContainerAndEnv := 3239 + + envVisibleDevices := []int32{1, 2, 3} + envVisibleDevicesStr := make([]string, len(envVisibleDevices)) + for i, idx := range envVisibleDevices { + envVisibleDevicesStr[i] = strconv.Itoa(int(idx)) + } + envVisibleDevicesValue := strings.Join(envVisibleDevicesStr, ",") + + procFs := uprobes.CreateFakeProcFS(t, []uprobes.FakeProcFSEntry{ + {Pid: uint32(pidNoContainer)}, + {Pid: uint32(pidContainer)}, + {Pid: uint32(pidContainerAndEnv), Env: map[string]string{"CUDA_VISIBLE_DEVICES": envVisibleDevicesValue}}, + {Pid: uint32(pidNoContainerButEnv), Env: map[string]string{"CUDA_VISIBLE_DEVICES": envVisibleDevicesValue}}, + }) + + wmetaMock := testutil.GetWorkloadMetaMock(t) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), procFs, wmetaMock) + require.NotNil(t, sysCtx) + require.NoError(t, err) + + // Create a container with a single GPU and add it to the store + containerID := "abcdef" + container := &workloadmeta.Container{ + EntityID: workloadmeta.EntityID{ + Kind: workloadmeta.KindContainer, + ID: containerID, + }, + EntityMeta: workloadmeta.EntityMeta{ + Name: containerID, + }, + } + + containerDeviceIndexes := []int32{1, 2, 3, 4} + for _, idx := range containerDeviceIndexes { + gpuUUID := testutil.GPUUUIDs[idx] + resource := workloadmeta.ContainerAllocatedResource{ + Name: nvidiaResourceName, + ID: gpuUUID, + } + container.AllocatedResources = append(container.AllocatedResources, resource) + } + + wmetaMock.Set(container) + storeContainer, err := wmetaMock.GetContainer(containerID) + require.NoError(t, err, "container should be found in the store") + require.NotNil(t, storeContainer, "container should be found in the store") + + cases := []struct { + name string + pid int + containerID string + configuredDeviceIdx []int32 + expectedDeviceIdx []int32 + }{ + { + name: "NoContainer", + containerID: "", + pid: pidNoContainer, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{1, 2}, + }, + { + name: "NoContainerButEnv", + containerID: "", + pid: pidNoContainerButEnv, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{envVisibleDevices[1], envVisibleDevices[2]}, + }, + { + name: "WithContainer", + containerID: containerID, + pid: pidContainer, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{containerDeviceIndexes[1], containerDeviceIndexes[2]}, + }, + { + name: "WithContainerAndEnv", + pid: pidContainerAndEnv, + containerID: containerID, + configuredDeviceIdx: []int32{1, 2}, + expectedDeviceIdx: []int32{containerDeviceIndexes[envVisibleDevices[1]], containerDeviceIndexes[envVisibleDevices[2]]}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + for i, idx := range c.configuredDeviceIdx { + sysCtx.setDeviceSelection(c.pid, c.pid+i, idx) + } + + for i, idx := range c.expectedDeviceIdx { + activeDevice, err := sysCtx.getCurrentActiveGpuDevice(c.pid, c.pid+i, c.containerID) + require.NoError(t, err) + testutil.RequireDevicesEqual(t, sysCtx.gpuDevices[idx], activeDevice, "invalid device at index %d (real index is %d, selected index is %d)", i, idx, c.configuredDeviceIdx[i]) + } + }) + } +} diff --git a/pkg/gpu/probe.go b/pkg/gpu/probe.go index 3ab58ede54f35..fafc90df19c28 100644 --- a/pkg/gpu/probe.go +++ b/pkg/gpu/probe.go @@ -22,6 +22,7 @@ import ( "github.com/cilium/ebpf" "github.com/DataDog/datadog-agent/comp/core/telemetry" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu/model" ddebpf "github.com/DataDog/datadog-agent/pkg/ebpf" ebpftelemetry "github.com/DataDog/datadog-agent/pkg/ebpf/telemetry" @@ -81,6 +82,10 @@ type ProbeDependencies struct { // ProcessMonitor is the process monitor interface ProcessMonitor uprobes.ProcessMonitor + + // WorkloadMeta used to retrieve data about workloads (containers, processes) running + // on the host + WorkloadMeta workloadmeta.Component } // Probe represents the GPU monitoring probe @@ -109,7 +114,7 @@ func NewProbe(cfg *config.Config, deps ProbeDependencies) (*Probe, error) { } attachCfg := getAttacherConfig(cfg) - sysCtx, err := getSystemContext(deps.NvmlLib, cfg.ProcRoot) + sysCtx, err := getSystemContext(deps.NvmlLib, cfg.ProcRoot, deps.WorkloadMeta) if err != nil { return nil, fmt.Errorf("error getting system context: %w", err) } diff --git a/pkg/gpu/probe_stub.go b/pkg/gpu/probe_stub.go index 734a11919abb9..2f4ebf544c984 100644 --- a/pkg/gpu/probe_stub.go +++ b/pkg/gpu/probe_stub.go @@ -11,6 +11,7 @@ import ( "github.com/NVIDIA/go-nvml/pkg/nvml" "github.com/DataDog/datadog-agent/comp/core/telemetry" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/collector/corechecks/gpu/model" "github.com/DataDog/datadog-agent/pkg/ebpf" "github.com/DataDog/datadog-agent/pkg/gpu/config" @@ -21,6 +22,7 @@ type ProbeDependencies struct { Telemetry telemetry.Component NvmlLib nvml.Interface ProcessMonitor any // uprobes.ProcessMonitor is only compiled with the linux_bpf build tag, so we need to use type any here + WorkloadMeta workloadmeta.Component } // Probe is not implemented on non-linux systems diff --git a/pkg/gpu/probe_test.go b/pkg/gpu/probe_test.go index f3eda56e2d2d9..ff2834b0327bc 100644 --- a/pkg/gpu/probe_test.go +++ b/pkg/gpu/probe_test.go @@ -47,6 +47,7 @@ func (s *probeTestSuite) getProbe() *Probe { deps := ProbeDependencies{ NvmlLib: testutil.GetBasicNvmlMock(), ProcessMonitor: consumerstestutil.NewTestProcessConsumer(t), + WorkloadMeta: testutil.GetWorkloadMetaMock(t), } probe, err := NewProbe(cfg, deps) require.NoError(t, err) diff --git a/pkg/gpu/stats_test.go b/pkg/gpu/stats_test.go index c0445e6c4e869..5aa4b2304cf3a 100644 --- a/pkg/gpu/stats_test.go +++ b/pkg/gpu/stats_test.go @@ -31,7 +31,7 @@ func getMetricsEntry(key model.StatsKey, stats *model.GPUStats) *model.Utilizati } func getStatsGeneratorForTest(t *testing.T) (*statsGenerator, map[streamKey]*StreamHandler, int64) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) require.NotNil(t, sysCtx) diff --git a/pkg/gpu/stream_test.go b/pkg/gpu/stream_test.go index f33e4f342e98b..0e4f0a5b734c8 100644 --- a/pkg/gpu/stream_test.go +++ b/pkg/gpu/stream_test.go @@ -21,10 +21,16 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/kernel" ) -func TestKernelLaunchesHandled(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) +func getSystemContextForTest(t *testing.T) *systemContext { + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot(), testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + require.NotNil(t, sysCtx) + + return sysCtx +} + +func TestKernelLaunchesHandled(t *testing.T) { + stream := newStreamHandler(0, "", getSystemContextForTest(t)) kernStartTime := uint64(1) launch := &gpuebpf.CudaKernelLaunch{ @@ -81,9 +87,7 @@ func TestKernelLaunchesHandled(t *testing.T) { } func TestMemoryAllocationsHandled(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime := uint64(1) memFreeTime := uint64(2) @@ -152,9 +156,7 @@ func TestMemoryAllocationsHandled(t *testing.T) { } func TestMemoryAllocationsDetectLeaks(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime := uint64(1) memAddr := uint64(42) @@ -187,9 +189,7 @@ func TestMemoryAllocationsDetectLeaks(t *testing.T) { } func TestMemoryAllocationsNoCrashOnInvalidFree(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime := uint64(1) memFreeTime := uint64(2) @@ -231,9 +231,7 @@ func TestMemoryAllocationsNoCrashOnInvalidFree(t *testing.T) { } func TestMemoryAllocationsMultipleAllocsHandled(t *testing.T) { - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), kernel.ProcFSRoot()) - require.NoError(t, err) - stream := newStreamHandler(0, "", sysCtx) + stream := newStreamHandler(0, "", getSystemContextForTest(t)) memAllocTime1, memAllocTime2 := uint64(1), uint64(10) memFreeTime1, memFreeTime2 := uint64(15), uint64(20) @@ -324,7 +322,7 @@ func TestMemoryAllocationsMultipleAllocsHandled(t *testing.T) { func TestKernelLaunchesIncludeEnrichedKernelData(t *testing.T) { proc := kernel.ProcFSRoot() - sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), proc) + sysCtx, err := getSystemContext(testutil.GetBasicNvmlMock(), proc, testutil.GetWorkloadMetaMock(t)) require.NoError(t, err) // Set up the caches in system context so no actual queries are done diff --git a/pkg/gpu/testutil/mocks.go b/pkg/gpu/testutil/mocks.go index a7dadd39215ad..009385125600a 100644 --- a/pkg/gpu/testutil/mocks.go +++ b/pkg/gpu/testutil/mocks.go @@ -9,8 +9,19 @@ package testutil import ( + "fmt" + "testing" + "github.com/NVIDIA/go-nvml/pkg/nvml" nvmlmock "github.com/NVIDIA/go-nvml/pkg/nvml/mock" + "github.com/stretchr/testify/require" + "go.uber.org/fx" + + "github.com/DataDog/datadog-agent/comp/core" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" + workloadmetafxmock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/fx-mock" + workloadmetamock "github.com/DataDog/datadog-agent/comp/core/workloadmeta/mock" + "github.com/DataDog/datadog-agent/pkg/util/fxutil" ) // DefaultGpuCores is the default number of cores for a GPU device in the mock. @@ -18,13 +29,18 @@ const DefaultGpuCores = 10 // GPUUUIDs is a list of UUIDs for the devices returned by the mock var GPUUUIDs = []string{ - "GPU-12345678-1234-1234-1234-123456789012", - "GPU-99999999-1234-1234-1234-123456789013", - "GPU-00000000-1234-1234-1234-123456789014", + "GPU-00000000-1234-1234-1234-123456789012", + "GPU-11111111-1234-1234-1234-123456789013", + "GPU-22222222-1234-1234-1234-123456789014", + "GPU-33333333-1234-1234-1234-123456789015", + "GPU-44444444-1234-1234-1234-123456789016", + "GPU-55555555-1234-1234-1234-123456789017", + "GPU-66666666-1234-1234-1234-123456789018", } -// GPUCores is a list of number of cores for the devices returned by the mock, should be the same length as GPUUUIDs -var GPUCores = []int{DefaultGpuCores, 20, 30} +// GPUCores is a list of number of cores for the devices returned by the mock, +// should be the same length as GPUUUIDs. If not, GetBasicNvmlMock will panic. +var GPUCores = []int{DefaultGpuCores, 20, 30, 40, 50, 60, 70} // DefaultGpuUUID is the UUID for the default device returned by the mock var DefaultGpuUUID = GPUUUIDs[0] @@ -47,6 +63,11 @@ func GetDeviceMock(deviceIdx int) *nvmlmock.Device { // GetBasicNvmlMock returns a mock of the nvml.Interface with a single device with 10 cores, // useful for basic tests that need only the basic interaction with NVML to be working. func GetBasicNvmlMock() *nvmlmock.Interface { + if len(GPUUUIDs) != len(GPUCores) { + // Make it really easy to spot errors if we change any of the arrays. + panic("GPUUUIDs and GPUCores must have the same length, please fix it") + } + return &nvmlmock.Interface{ DeviceGetCountFunc: func() (int, nvml.Return) { return len(GPUUUIDs), nvml.SUCCESS @@ -59,3 +80,49 @@ func GetBasicNvmlMock() *nvmlmock.Interface { }, } } + +// GetWorkloadMetaMock returns a mock of the workloadmeta.Component. +func GetWorkloadMetaMock(t *testing.T) workloadmetamock.Mock { + return fxutil.Test[workloadmetamock.Mock](t, fx.Options( + core.MockBundle(), + workloadmetafxmock.MockModule(workloadmeta.NewParams()), + )) +} + +// RequireDevicesEqual checks that the two devices are equal by comparing their UUIDs, which gives a better +// output than using require.Equal on the devices themselves +func RequireDevicesEqual(t *testing.T, expected, actual nvml.Device, msgAndArgs ...interface{}) { + extraFmt := "" + if len(msgAndArgs) > 0 { + extraFmt = fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + ": " + } + + expectedUUID, ret := expected.GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%s%scannot retrieve UUID for expected device %v%s", extraFmt, expected) + + actualUUID, ret := actual.GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%scannot retrieve UUID for actual device %v%s", extraFmt, actual) + + require.Equal(t, expectedUUID, actualUUID, "%sUUIDs do not match", extraFmt) +} + +// RequireDeviceListsEqual checks that the two device lists are equal by comparing their UUIDs, which gives a better +// output than using require.ElementsMatch on the lists themselves +func RequireDeviceListsEqual(t *testing.T, expected, actual []nvml.Device, msgAndArgs ...interface{}) { + extraFmt := "" + if len(msgAndArgs) > 0 { + extraFmt = fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) + ": " + } + + require.Len(t, actual, len(expected), "%sdevice lists have different lengths", extraFmt) + + for i := range expected { + expectedUUID, ret := expected[i].GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%scannot retrieve UUID for expected device index %d", extraFmt, i) + + actualUUID, ret := actual[i].GetUUID() + require.Equal(t, ret, nvml.SUCCESS, "%scannot retrieve UUID for actual device index %d", extraFmt, i) + + require.Equal(t, expectedUUID, actualUUID, "%sUUIDs do not match for element %d", extraFmt, i) + } +} diff --git a/pkg/internaltelemetry/client.go b/pkg/internaltelemetry/client.go index 52b1b2010edf0..ee7754fad190b 100644 --- a/pkg/internaltelemetry/client.go +++ b/pkg/internaltelemetry/client.go @@ -21,8 +21,6 @@ import ( "go.uber.org/atomic" - pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" - "github.com/DataDog/datadog-agent/pkg/trace/config" "github.com/DataDog/datadog-agent/pkg/util/log" "github.com/DataDog/datadog-agent/pkg/version" "github.com/shirou/gopsutil/v4/host" @@ -36,13 +34,19 @@ const ( // Client defines the interface for a telemetry client type Client interface { SendLog(level string, message string) - SendTraces(traces pb.Traces) + SendTraces(traces Traces) +} + +// Endpoint defines the endpoint object +type Endpoint struct { + APIKey string `json:"-"` + Host string } type client struct { m sync.Mutex client httpClient - endpoints []*config.Endpoint + endpoints []*Endpoint sendPayloadTimeout time.Duration // we can pre-calculate the host payload structure at init time @@ -94,7 +98,7 @@ type Application struct { // TracePayload defines the trace payload object type TracePayload struct { - Traces []pb.Trace `json:"traces"` + Traces []Trace `json:"traces"` } // LogPayload defines the log payload object @@ -115,7 +119,7 @@ type httpClient interface { } // NewClient creates a new telemetry client -func NewClient(httpClient httpClient, endpoints []*config.Endpoint, service string, debug bool) Client { +func NewClient(httpClient httpClient, endpoints []*Endpoint, service string, debug bool) Client { info, err := host.Info() if err != nil { log.Errorf("failed to retrieve host info: %v", err) @@ -158,7 +162,7 @@ func (c *client) SendLog(level, message string) { c.sendPayload(RequestTypeLogs, payload) } -func (c *client) SendTraces(traces pb.Traces) { +func (c *client) SendTraces(traces Traces) { c.m.Lock() defer c.m.Unlock() payload := TracePayload{ @@ -170,8 +174,8 @@ func (c *client) SendTraces(traces pb.Traces) { // sampleTraces is a simple uniform sampling function that samples traces based // on the sampling rate, given that there is no trace agent to sample the traces // We try to keep the tracer behaviour: the first rule that matches apply its rate to the whole trace -func (c *client) sampleTraces(traces pb.Traces) pb.Traces { - tracesWithSampling := pb.Traces{} +func (c *client) sampleTraces(traces Traces) Traces { + tracesWithSampling := Traces{} for _, trace := range traces { samplingRate := 1.0 for _, span := range trace { @@ -206,7 +210,7 @@ func (c *client) sendPayload(requestType RequestType, payload interface{}) { group := sync.WaitGroup{} for _, endpoint := range c.endpoints { group.Add(1) - go func(endpoint *config.Endpoint) { + go func(endpoint *Endpoint) { defer group.Done() url := fmt.Sprintf("%s%s", endpoint.Host, telemetryEndpoint) req, err := http.NewRequest("POST", url, bytes.NewReader(serializedPayload)) diff --git a/pkg/internaltelemetry/traces.go b/pkg/internaltelemetry/traces.go new file mode 100644 index 0000000000000..54c8545a604b7 --- /dev/null +++ b/pkg/internaltelemetry/traces.go @@ -0,0 +1,41 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2023-present Datadog, Inc. + +// Package internaltelemetry full description in README.md +package internaltelemetry + +// Traces is a collection of traces +type Traces []Trace + +// Trace is a collection of spans with the same trace ID +type Trace []*Span + +// Span used for installation telemetry +type Span struct { + // Service is the name of the service that handled this span. + Service string `json:"service"` + // Name is the name of the operation this span represents. + Name string `json:"name"` + // Resource is the name of the resource this span represents. + Resource string `json:"resource"` + // TraceID is the ID of the trace to which this span belongs. + TraceID uint64 `json:"trace_id"` + // SpanID is the ID of this span. + SpanID uint64 `json:"span_id"` + // ParentID is the ID of the parent span. + ParentID uint64 `json:"parent_id"` + // Start is the start time of this span in nanoseconds since the Unix epoch. + Start int64 `json:"start"` + // Duration is the duration of this span in nanoseconds. + Duration int64 `json:"duration"` + // Error is the error status of this span. + Error int32 `json:"error"` + // Meta is a mapping from tag name to tag value for string-valued tags. + Meta map[string]string `json:"meta,omitempty"` + // Metrics is a mapping from metric name to metric value for numeric metrics. + Metrics map[string]float64 `json:"metrics,omitempty"` + // Type is the type of the span. + Type string `json:"type"` +} diff --git a/pkg/kubestatemetrics/builder/builder.go b/pkg/kubestatemetrics/builder/builder.go index d0c5ebd1ea998..ea26602567f56 100644 --- a/pkg/kubestatemetrics/builder/builder.go +++ b/pkg/kubestatemetrics/builder/builder.go @@ -9,6 +9,7 @@ package builder import ( "context" + "fmt" "reflect" "time" @@ -18,7 +19,11 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + apiwatch "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" ksmbuild "k8s.io/kube-state-metrics/v2/pkg/builder" ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types" @@ -196,6 +201,14 @@ func GenerateStores[T any]( isPod = true } else if u, ok := expectedType.(*unstructured.Unstructured); ok { isPod = u.GetAPIVersion() == "v1" && u.GetKind() == "Pod" + } else if _, ok := expectedType.(*corev1.ConfigMap); ok { + configMapStore, err := generateConfigMapStores(b, metricFamilies, useAPIServerCache) + if err != nil { + log.Debugf("Defaulting to kube-state-metrics for configmap collection: %v", err) + } else { + log.Debug("Using meta.k8s.io API for configmap collection") + return configMapStore + } } if b.namespaces.IsAllNamespaces() { @@ -340,3 +353,101 @@ func handlePodCollection[T any](b *Builder, store cache.Store, client T, listWat listWatcher := listWatchFunc(client, namespace, fieldSelector) b.startReflector(&corev1.Pod{}, store, listWatcher, useAPIServerCache) } + +func generateConfigMapStores( + b *Builder, + metricFamilies []generator.FamilyGenerator, + useAPIServerCache bool, +) ([]cache.Store, error) { + restConfig, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to create in-cluster config for metadata client: %w", err) + } + + metadataClient, err := metadata.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("failed to create metadata client: %w", err) + } + + gvr := schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "configmaps", + } + + filteredMetricFamilies := generator.FilterFamilyGenerators(b.allowDenyList, metricFamilies) + composedMetricGenFuncs := generator.ComposeMetricGenFuncs(filteredMetricFamilies) + + stores := make([]cache.Store, 0) + + if b.namespaces.IsAllNamespaces() { + log.Infof("Using NamespaceAll for ConfigMap collection.") + store := store.NewMetricsStore(composedMetricGenFuncs, "configmap") + listWatcher := createConfigMapListWatch(metadataClient, gvr, v1.NamespaceAll) + b.startReflector(&corev1.ConfigMap{}, store, listWatcher, useAPIServerCache) + return []cache.Store{store}, nil + } + + for _, ns := range b.namespaces { + store := store.NewMetricsStore(composedMetricGenFuncs, "configmap") + listWatcher := createConfigMapListWatch(metadataClient, gvr, ns) + b.startReflector(&corev1.ConfigMap{}, store, listWatcher, useAPIServerCache) + stores = append(stores, store) + } + + return stores, nil +} + +func createConfigMapListWatch(metadataClient metadata.Interface, gvr schema.GroupVersionResource, namespace string) *cache.ListWatch { + return &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + result, err := metadataClient.Resource(gvr).Namespace(namespace).List(context.TODO(), options) + if err != nil { + return nil, err + } + + configMapList := &corev1.ConfigMapList{} + for _, item := range result.Items { + configMapList.Items = append(configMapList.Items, corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + UID: item.GetUID(), + ResourceVersion: item.GetResourceVersion(), + }, + }) + } + + return configMapList, nil + }, + WatchFunc: func(options v1.ListOptions) (apiwatch.Interface, error) { + watcher, err := metadataClient.Resource(gvr).Namespace(namespace).Watch(context.TODO(), options) + if err != nil { + return nil, err + } + + return apiwatch.Filter(watcher, func(event apiwatch.Event) (apiwatch.Event, bool) { + if event.Object == nil { + return event, false + } + + partialObject, ok := event.Object.(*v1.PartialObjectMetadata) + if !ok { + return event, false + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: partialObject.GetName(), + Namespace: partialObject.GetNamespace(), + UID: partialObject.GetUID(), + ResourceVersion: partialObject.GetResourceVersion(), + }, + } + + event.Object = configMap + return event, true + }), nil + }, + } +} diff --git a/pkg/logs/client/go.mod b/pkg/logs/client/go.mod index 49dad213dc24f..07b8c9ec2e3be 100644 --- a/pkg/logs/client/go.mod +++ b/pkg/logs/client/go.mod @@ -63,7 +63,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/DataDog/datadog-agent/pkg/version v0.59.1 github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 ) require ( diff --git a/pkg/logs/client/go.sum b/pkg/logs/client/go.sum index 5f7384266b821..b14b43571558e 100644 --- a/pkg/logs/client/go.sum +++ b/pkg/logs/client/go.sum @@ -263,8 +263,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/logs/client/http/destination.go b/pkg/logs/client/http/destination.go index 2a554e5476504..93bf35ec67167 100644 --- a/pkg/logs/client/http/destination.go +++ b/pkg/logs/client/http/destination.go @@ -396,6 +396,13 @@ func httpClientFactory(timeout time.Duration, cfg pkgconfigmodel.Reader) func() var transport *http.Transport transportConfig := cfg.Get("logs_config.http_protocol") + + // If any proxy is set, use http1 + // This will be removed in a future version + if cfg.GetProxies() != nil { + transportConfig = "http1" + } + // Configure transport based on user setting switch transportConfig { case "http1": diff --git a/pkg/logs/client/http/destination_test.go b/pkg/logs/client/http/destination_test.go index 92bf984a79e5a..3f65bba9b9e2c 100644 --- a/pkg/logs/client/http/destination_test.go +++ b/pkg/logs/client/http/destination_test.go @@ -494,3 +494,34 @@ func TestTransportProtocol_HTTP1FallBack(t *testing.T) { // Assert that the server automatically falls back to HTTP/1.1 assert.Equal(t, "HTTP/1.1", resp.Proto) } + +func TestTransportProtocol_HTTP1WhenUsingProxy(t *testing.T) { + c := configmock.New(t) + + // Force client to use ALNP + c.SetWithoutSource("logs_config.http_protocol", "auto") + c.SetWithoutSource("skip_ssl_validation", true) + + // The test server uses TLS, so if we set the http proxy (not https), it still makes + // a request to the test server, but disable HTTP/2 since a proxy is configured. + c.SetWithoutSource("proxy.http", "http://foo.bar") + + server := NewTestHTTPSServer(false) + defer server.Close() + + timeout := 5 * time.Second + client := httpClientFactory(timeout, c)() + + req, err := http.NewRequest("POST", server.URL, nil) + if err != nil { + t.Fatalf("Failed to create request: %v", err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatalf("Failed to send request: %v", err) + } + defer resp.Body.Close() + + // Assert that the server chose HTTP/1.1 because a proxy was configured + assert.Equal(t, "HTTP/1.1", resp.Proto) +} diff --git a/pkg/logs/pipeline/go.mod b/pkg/logs/pipeline/go.mod index 10ad9ab1983b5..dbb50c6aec2e7 100644 --- a/pkg/logs/pipeline/go.mod +++ b/pkg/logs/pipeline/go.mod @@ -150,7 +150,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/pkg/logs/pipeline/go.sum b/pkg/logs/pipeline/go.sum index b0f7d9ee40dfd..19075c11eaba4 100644 --- a/pkg/logs/pipeline/go.sum +++ b/pkg/logs/pipeline/go.sum @@ -282,8 +282,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/logs/sender/go.mod b/pkg/logs/sender/go.mod index 437b2592b6e94..28b1f5738002a 100644 --- a/pkg/logs/sender/go.mod +++ b/pkg/logs/sender/go.mod @@ -131,7 +131,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/pkg/logs/sender/go.sum b/pkg/logs/sender/go.sum index 5f7384266b821..b14b43571558e 100644 --- a/pkg/logs/sender/go.sum +++ b/pkg/logs/sender/go.sum @@ -263,8 +263,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/logs/util/testutils/go.mod b/pkg/logs/util/testutils/go.mod index 46e01043de4bf..1c02f6f2f8775 100644 --- a/pkg/logs/util/testutils/go.mod +++ b/pkg/logs/util/testutils/go.mod @@ -95,4 +95,5 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/metrics/metricsource.go b/pkg/metrics/metricsource.go index f3afa888d60b4..5fcc0ff956952 100644 --- a/pkg/metrics/metricsource.go +++ b/pkg/metrics/metricsource.go @@ -308,6 +308,7 @@ const ( MetricSourceAnyscale MetricSourceMilvus MetricSourceNvidiaNim + MetricSourceQuarkus ) // String returns a string representation of MetricSource @@ -879,6 +880,8 @@ func (ms MetricSource) String() string { return "aws_neuron" case MetricSourceMilvus: return "milvus" + case MetricSourceQuarkus: + return "quarkus" default: return "" } @@ -1421,6 +1424,8 @@ func CheckNameToMetricSource(name string) MetricSource { return MetricSourceTibcoEMS case "milvus": return MetricSourceMilvus + case "quarkus": + return MetricSourceQuarkus default: return MetricSourceUnknown } diff --git a/pkg/network/tracer/tracer_linux_test.go b/pkg/network/tracer/tracer_linux_test.go index 29a8b6e34965a..46f33ac7fa405 100644 --- a/pkg/network/tracer/tracer_linux_test.go +++ b/pkg/network/tracer/tracer_linux_test.go @@ -109,7 +109,7 @@ func (s *TracerSuite) TestTCPRemoveEntries() { defer c2.Close() assert.EventuallyWithT(t, func(ct *assert.CollectT) { - conn, ok := findConnection(c2.LocalAddr(), c2.RemoteAddr(), getConnections(t, tr)) + conn, ok := findConnection(c2.LocalAddr(), c2.RemoteAddr(), getConnections(ct, tr)) if !assert.True(ct, ok) { return } @@ -123,9 +123,9 @@ func (s *TracerSuite) TestTCPRemoveEntries() { }, 3*time.Second, 100*time.Millisecond) // Make sure the first connection got cleaned up - assert.Eventually(t, func() bool { - _, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), getConnections(t, tr)) - return !ok + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + _, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), getConnections(ct, tr)) + require.False(ct, ok) }, 5*time.Second, 100*time.Millisecond) } @@ -735,10 +735,10 @@ func (s *TracerSuite) TestGatewayLookupEnabled() { dnsServerAddr := &net.UDPAddr{IP: dnsAddr, Port: 53} var conn *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(ct *assert.CollectT) { var ok bool - conn, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok + conn, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(ct, tr)) + require.True(ct, ok, "connection not found") }, 3*time.Second, 100*time.Millisecond) require.NotNil(t, conn.Via, "connection is missing via: %s", conn) @@ -791,10 +791,10 @@ func (s *TracerSuite) TestGatewayLookupSubnetLookupError() { dnsClientAddr := &net.UDPAddr{IP: net.ParseIP(clientIP), Port: clientPort} dnsServerAddr := &net.UDPAddr{IP: destAddr, Port: 53} var c *network.ConnectionStats - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(ct *assert.CollectT) { var ok bool - c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok + c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(ct, tr)) + require.True(ct, ok, "connection not found") }, 3*time.Second, 100*time.Millisecond, "connection not found") require.Nil(t, c.Via) @@ -804,10 +804,10 @@ func (s *TracerSuite) TestGatewayLookupSubnetLookupError() { }, 6*time.Second, 100*time.Millisecond, "failed to send dns query") dnsClientAddr = &net.UDPAddr{IP: net.ParseIP(clientIP), Port: clientPort} - require.Eventually(t, func() bool { + require.EventuallyWithT(t, func(ct *assert.CollectT) { var ok bool - c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(t, tr)) - return ok + c, ok = findConnection(dnsClientAddr, dnsServerAddr, getConnections(ct, tr)) + require.True(ct, ok, "connection not found") }, 3*time.Second, 100*time.Millisecond, "connection not found") require.Nil(t, c.Via) @@ -1529,25 +1529,25 @@ func testUDPReusePort(t *testing.T, udpnet string, ip string) { assert.EventuallyWithT(t, func(ct *assert.CollectT) { //nolint:revive // TODO // use t instead of ct because getConnections uses require (not assert), and we get a better error message that way - connections := getConnections(t, tr) + connections := getConnections(ct, tr) incoming, ok := findConnection(c.RemoteAddr(), c.LocalAddr(), connections) - if assert.True(t, ok, "unable to find incoming connection") { - assert.Equal(t, network.INCOMING, incoming.Direction) + if assert.True(ct, ok, "unable to find incoming connection") { + assert.Equal(ct, network.INCOMING, incoming.Direction) // make sure the inverse values are seen for the other message - assert.Equal(t, serverMessageSize, int(incoming.Monotonic.SentBytes), "incoming sent") - assert.Equal(t, clientMessageSize, int(incoming.Monotonic.RecvBytes), "incoming recv") - assert.True(t, incoming.IntraHost, "incoming intrahost") + assert.Equal(ct, serverMessageSize, int(incoming.Monotonic.SentBytes), "incoming sent") + assert.Equal(ct, clientMessageSize, int(incoming.Monotonic.RecvBytes), "incoming recv") + assert.True(ct, incoming.IntraHost, "incoming intrahost") } outgoing, ok := findConnection(c.LocalAddr(), c.RemoteAddr(), connections) - if assert.True(t, ok, "unable to find outgoing connection") { - assert.Equal(t, network.OUTGOING, outgoing.Direction) + if assert.True(ct, ok, "unable to find outgoing connection") { + assert.Equal(ct, network.OUTGOING, outgoing.Direction) - assert.Equal(t, clientMessageSize, int(outgoing.Monotonic.SentBytes), "outgoing sent") - assert.Equal(t, serverMessageSize, int(outgoing.Monotonic.RecvBytes), "outgoing recv") - assert.True(t, outgoing.IntraHost, "outgoing intrahost") + assert.Equal(ct, clientMessageSize, int(outgoing.Monotonic.SentBytes), "outgoing sent") + assert.Equal(ct, serverMessageSize, int(outgoing.Monotonic.RecvBytes), "outgoing recv") + assert.True(ct, outgoing.IntraHost, "outgoing intrahost") } }, 3*time.Second, 100*time.Millisecond) @@ -1660,8 +1660,8 @@ func (s *TracerSuite) TestSendfileRegression() { t.Logf("looking for connections %+v <-> %+v", c.LocalAddr(), c.RemoteAddr()) var outConn, inConn *network.ConnectionStats - assert.Eventually(t, func() bool { - conns := getConnections(t, tr) + assert.EventuallyWithT(t, func(ct *assert.CollectT) { + conns := getConnections(ct, tr) t.Log(conns) if outConn == nil { outConn = network.FirstConnection(conns, network.ByType(connType), network.ByFamily(family), network.ByTuple(c.LocalAddr(), c.RemoteAddr())) @@ -1669,7 +1669,8 @@ func (s *TracerSuite) TestSendfileRegression() { if inConn == nil { inConn = network.FirstConnection(conns, network.ByType(connType), network.ByFamily(family), network.ByTuple(c.RemoteAddr(), c.LocalAddr())) } - return outConn != nil && inConn != nil + require.NotNil(ct, outConn) + require.NotNil(ct, inConn) }, 3*time.Second, 100*time.Millisecond, "couldn't find connections used by sendfile(2)") if assert.NotNil(t, outConn, "couldn't find outgoing connection used by sendfile(2)") { @@ -2433,7 +2434,7 @@ LOOP: require.NoError(t, c.Close(), "error closing client connection") require.EventuallyWithT(t, func(collect *assert.CollectT) { - conn, found := findConnection(c.LocalAddr(), srv.Addr(), getConnections(t, tr)) + conn, found := findConnection(c.LocalAddr(), srv.Addr(), getConnections(collect, tr)) require.True(collect, found, "could not find connection") require.True(collect, conn.IsClosed, "connection should be closed") // after closing the client connection, the duration should be diff --git a/pkg/network/usm/debugger/README.md b/pkg/network/usm/debugger/README.md new file mode 100644 index 0000000000000..27d71055a8e31 --- /dev/null +++ b/pkg/network/usm/debugger/README.md @@ -0,0 +1,31 @@ +# USM debugger + +A minimal, self-contained build of USM for faster iterations of debugging eBPF +code on remote machines. + +Prepare the build for the target architectures you need (the `foo` KMT stack +does not need to exist). This is needed in order to be able to build the eBPF +object files for a different architecture than your host machine. + +``` +inv -e kmt.prepare system-probe --compile-only --stack=foo --arch=x86_64 +inv -e kmt.prepare system-probe --compile-only --stack=foo --arch=arm64 +``` + +Build the binary with one of the following commands for the architecture of +your target machine: + +``` +inv -e system-probe.build-usm-debugger --arch=x86_64 +inv -e system-probe.build-usm-debugger --arch=arm64 +``` + +Copy the `bin/usm-debugger` to the `system-probe` container in the +`datadog-agent` pod on your target machine. Open a shell in that container and +execute the binary. + +The eBPF programs are always built with debug logs enabled so you can view them +with `cat /sys/kernel/tracing/trace_pipe`. + +If you need to change the system-probe config, edit `cmd/usm_debugger.go` and +rebuild. diff --git a/pkg/network/usm/debugger/cmd/.gitignore b/pkg/network/usm/debugger/cmd/.gitignore new file mode 100644 index 0000000000000..5761abcfdf0c2 --- /dev/null +++ b/pkg/network/usm/debugger/cmd/.gitignore @@ -0,0 +1 @@ +*.o diff --git a/pkg/network/usm/debugger/cmd/ebpf_bytecode.go b/pkg/network/usm/debugger/cmd/ebpf_bytecode.go new file mode 100644 index 0000000000000..e924386d9278d --- /dev/null +++ b/pkg/network/usm/debugger/cmd/ebpf_bytecode.go @@ -0,0 +1,62 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build usm_debugger + +package main + +import ( + _ "embed" + "os" + "path" + + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// The code below is essentially responsible for embedding the CO-RE artifacts +// during compilation time and writing them to a temporary folder during +// runtime, so they can be loaded by the `usm.Monitor` as regular compilation +// assets. + +//go:embed usm-debug.o +var usmProgram []byte + +//go:embed shared-libraries-debug.o +var sharedLibrariesProgram []byte + +func setupBytecode() func() { + type program struct { + filePath string + bytecode []byte + } + + var ( + bytecodeDir = os.TempDir() + coreDir = path.Join(bytecodeDir, "co-re") + ) + + os.Setenv("DD_SYSTEM_PROBE_BPF_DIR", bytecodeDir) + err := os.MkdirAll(coreDir, os.ModePerm) + checkError(err) + + programs := []program{ + {path.Join(coreDir, "usm-debug.o"), usmProgram}, + {path.Join(coreDir, "shared-libraries-debug.o"), sharedLibrariesProgram}, + } + + for _, p := range programs { + f, err := os.Create(p.filePath) + checkError(err) + _, err = f.Write(p.bytecode) + checkError(err) + log.Debugf("writing ebpf bytecode to %s", p.filePath) + } + + return func() { + for _, p := range programs { + os.Remove(p.filePath) + } + } +} diff --git a/pkg/network/usm/debugger/cmd/usm_debugger.go b/pkg/network/usm/debugger/cmd/usm_debugger.go new file mode 100644 index 0000000000000..70fb985c169be --- /dev/null +++ b/pkg/network/usm/debugger/cmd/usm_debugger.go @@ -0,0 +1,89 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build usm_debugger + +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/DataDog/datadog-agent/cmd/system-probe/config" + pkgconfigsetup "github.com/DataDog/datadog-agent/pkg/config/setup" + networkconfig "github.com/DataDog/datadog-agent/pkg/network/config" + "github.com/DataDog/datadog-agent/pkg/network/usm" + pkglogsetup "github.com/DataDog/datadog-agent/pkg/util/log/setup" +) + +func main() { + err := pkglogsetup.SetupLogger( + "usm-debugger", + "debug", + "", + pkglogsetup.GetSyslogURI(pkgconfigsetup.Datadog()), + false, + true, + false, + pkgconfigsetup.Datadog(), + ) + checkError(err) + + cleanupFn := setupBytecode() + defer cleanupFn() + + monitor, err := usm.NewMonitor(getConfiguration(), nil) + checkError(err) + + err = monitor.Start() + checkError(err) + + go func() { + t := time.NewTicker(10 * time.Second) + for range t.C { + _ = monitor.GetProtocolStats() + } + }() + + defer monitor.Stop() + done := make(chan os.Signal, 1) + signal.Notify(done, syscall.SIGINT, syscall.SIGTERM) + <-done +} + +func checkError(err error) { + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(-1) + } +} + +func getConfiguration() *networkconfig.Config { + // done for the purposes of initializing the configuration values + _, err := config.New("", "") + checkError(err) + + c := networkconfig.New() + + // run debug version of the eBPF program + c.BPFDebug = true + c.EnableUSMEventStream = false + + // don't buffer data in userspace + // this is to ensure that we won't inadvertently trigger an OOM kill + // by enabling the debugger inside a system-probe container. + c.MaxHTTPStatsBuffered = 0 + c.MaxKafkaStatsBuffered = 0 + + // make sure we use the CO-RE compilation artifact embedded + // in this build (see `ebpf_bytecode.go`) + c.EnableCORE = true + c.EnableRuntimeCompiler = false + + return c +} diff --git a/pkg/network/usm/utils/debugger.go b/pkg/network/usm/utils/debugger.go index 4da685bf2fbc9..8094b2e02c6c3 100644 --- a/pkg/network/usm/utils/debugger.go +++ b/pkg/network/usm/utils/debugger.go @@ -31,6 +31,7 @@ type Attacher interface { type TracedProgram struct { ProgramType string FilePath string + PathID PathIdentifier PIDs []uint32 } @@ -131,6 +132,7 @@ func (d *tlsDebugger) GetTracedPrograms(moduleName string) []TracedProgram { program.ProgramType = programType program.FilePath = registration.sampleFilePath + program.PathID = pathID } registry.m.Unlock() diff --git a/pkg/networkdevice/metadata/payload.go b/pkg/networkdevice/metadata/payload.go index 54b2f45eb0996..5f67824728cbe 100644 --- a/pkg/networkdevice/metadata/payload.go +++ b/pkg/networkdevice/metadata/payload.go @@ -131,10 +131,11 @@ type TopologyLinkSide struct { // TopologyLinkMetadata contains topology interface to interface links metadata type TopologyLinkMetadata struct { - ID string `json:"id"` - SourceType string `json:"source_type"` - Local *TopologyLinkSide `json:"local"` - Remote *TopologyLinkSide `json:"remote"` + ID string `json:"id"` + SourceType string `json:"source_type"` + Integration string `json:"integration,omitempty"` + Local *TopologyLinkSide `json:"local"` + Remote *TopologyLinkSide `json:"remote"` } // NetflowExporter contains netflow exporters info diff --git a/pkg/obfuscate/go.mod b/pkg/obfuscate/go.mod index b34df591521b5..df547c8beb777 100644 --- a/pkg/obfuscate/go.mod +++ b/pkg/obfuscate/go.mod @@ -20,7 +20,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/pkg/obfuscate/go.sum b/pkg/obfuscate/go.sum index 063bd88a07005..87558fb1a5f79 100644 --- a/pkg/obfuscate/go.sum +++ b/pkg/obfuscate/go.sum @@ -62,8 +62,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pkg/orchestrator/model/go.mod b/pkg/orchestrator/model/go.mod index 96f90197c3fe2..f941c7960e5ae 100644 --- a/pkg/orchestrator/model/go.mod +++ b/pkg/orchestrator/model/go.mod @@ -17,7 +17,7 @@ require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/orchestrator/model/go.sum b/pkg/orchestrator/model/go.sum index 3c93bebd8c6dd..01c4e89a188d8 100644 --- a/pkg/orchestrator/model/go.sum +++ b/pkg/orchestrator/model/go.sum @@ -19,7 +19,5 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/proto/go.mod b/pkg/proto/go.mod index 283fed1972634..890f1f5360dcf 100644 --- a/pkg/proto/go.mod +++ b/pkg/proto/go.mod @@ -25,7 +25,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/pkg/proto/go.sum b/pkg/proto/go.sum index 899e2f639d24b..bbbf81762e92d 100644 --- a/pkg/proto/go.sum +++ b/pkg/proto/go.sum @@ -91,8 +91,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/proto/pbgo/core/model.pb.go b/pkg/proto/pbgo/core/model.pb.go index 016a2cf53e260..9262b4dc43a5a 100644 --- a/pkg/proto/pbgo/core/model.pb.go +++ b/pkg/proto/pbgo/core/model.pb.go @@ -1155,8 +1155,8 @@ type GenerateContainerIDFromOriginInfoRequest_ExternalData struct { unknownFields protoimpl.UnknownFields Init *bool `protobuf:"varint,1,opt,name=init,proto3,oneof" json:"init,omitempty"` // Init is true if the container is an init container. - ContainerName *string `protobuf:"bytes,2,opt,name=containerName,proto3,oneof" json:"containerName,omitempty"` // Container name as seen by the Admission Controller. - PodUID *string `protobuf:"bytes,3,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID as seen by the Admission Controller. + ContainerName *string `protobuf:"bytes,2,opt,name=containerName,proto3,oneof" json:"containerName,omitempty"` // Container name in the Kubernetes Pod spec. + PodUID *string `protobuf:"bytes,3,opt,name=podUID,proto3,oneof" json:"podUID,omitempty"` // Pod UID in the Kubernetes Pod spec. } func (x *GenerateContainerIDFromOriginInfoRequest_ExternalData) Reset() { diff --git a/pkg/security/agent/agent_nix.go b/pkg/security/agent/agent_nix.go index bc444d6f049fe..8083e3259b87e 100644 --- a/pkg/security/agent/agent_nix.go +++ b/pkg/security/agent/agent_nix.go @@ -12,9 +12,10 @@ import ( "go.uber.org/atomic" + "github.com/DataDog/datadog-go/v5/statsd" + workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def" "github.com/DataDog/datadog-agent/pkg/security/security_profile/dump" - "github.com/DataDog/datadog-go/v5/statsd" ) // NewRuntimeSecurityAgent instantiates a new RuntimeSecurityAgent @@ -30,7 +31,7 @@ func NewRuntimeSecurityAgent(statsdClient statsd.ClientInterface, hostname strin } // on windows do no storage manager - storage, err := dump.NewAgentStorageManager() + storage, err := dump.NewAgentStorageManager(statsdClient) if err != nil { return nil, err } diff --git a/pkg/security/config/config.go b/pkg/security/config/config.go index ab6b0b6b591e1..9d165a4ae1900 100644 --- a/pkg/security/config/config.go +++ b/pkg/security/config/config.go @@ -101,6 +101,9 @@ type RuntimeSecurityConfig struct { // ActivityDumpTracedCgroupsCount defines the maximum count of cgroups that should be monitored concurrently. Leave this parameter to 0 to prevent the generation // of activity dumps based on cgroups. ActivityDumpTracedCgroupsCount int + // ActivityDumpCgroupsManagers defines the cgroup managers we generate dumps for. + ActivityDumpCgroupsManagers []string + // ActivityDumpTracedEventTypes defines the list of events that should be captured in an activity dump. Leave this // parameter empty to monitor all event types. If not already present, the `exec` event will automatically be added // to this list. @@ -368,6 +371,7 @@ func NewRuntimeSecurityConfig() (*RuntimeSecurityConfig, error) { ActivityDumpLoadControlPeriod: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.load_controller_period"), ActivityDumpLoadControlMinDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.min_timeout"), ActivityDumpTracedCgroupsCount: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.traced_cgroups_count"), + ActivityDumpCgroupsManagers: pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.cgroup_managers"), ActivityDumpTracedEventTypes: parseEventTypeStringSlice(pkgconfigsetup.SystemProbe().GetStringSlice("runtime_security_config.activity_dump.traced_event_types")), ActivityDumpCgroupDumpTimeout: pkgconfigsetup.SystemProbe().GetDuration("runtime_security_config.activity_dump.dump_duration"), ActivityDumpRateLimiter: pkgconfigsetup.SystemProbe().GetInt("runtime_security_config.activity_dump.rate_limiter"), diff --git a/pkg/security/ebpf/c/include/constants/custom.h b/pkg/security/ebpf/c/include/constants/custom.h index 88be17fa3c80b..1e2311bf8fc7e 100644 --- a/pkg/security/ebpf/c/include/constants/custom.h +++ b/pkg/security/ebpf/c/include/constants/custom.h @@ -196,4 +196,8 @@ static __attribute__((always_inline)) u64 get_imds_ip() { #define CGROUP_MANAGER_CRI 4 #define CGROUP_MANAGER_SYSTEMD 5 +#define CGROUP_MANAGER_MASK 0b111 +#define CGROUP_SYSTEMD_SERVICE (0 << 8) +#define CGROUP_SYSTEMD_SCOPE (1 << 8) + #endif diff --git a/pkg/security/ebpf/c/include/constants/offsets/filesystem.h b/pkg/security/ebpf/c/include/constants/offsets/filesystem.h index 24b2d5f769af1..797070a8534e7 100644 --- a/pkg/security/ebpf/c/include/constants/offsets/filesystem.h +++ b/pkg/security/ebpf/c/include/constants/offsets/filesystem.h @@ -9,8 +9,11 @@ struct mount; unsigned long __attribute__((always_inline)) get_inode_ino(struct inode *inode) { + u64 inode_ino_offset; + LOAD_CONSTANT("inode_ino_offset", inode_ino_offset); + unsigned long ino; - bpf_probe_read(&ino, sizeof(inode), &inode->i_ino); + bpf_probe_read(&ino, sizeof(inode), (void *)inode + inode_ino_offset); return ino; } diff --git a/pkg/security/ebpf/c/include/helpers/activity_dump.h b/pkg/security/ebpf/c/include/helpers/activity_dump.h index c0f8d246006bf..70e776cca8161 100644 --- a/pkg/security/ebpf/c/include/helpers/activity_dump.h +++ b/pkg/security/ebpf/c/include/helpers/activity_dump.h @@ -53,10 +53,16 @@ __attribute__((always_inline)) struct cgroup_tracing_event_t *get_cgroup_tracing return evt; } +__attribute__((always_inline)) u32 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { + u32 cgroup_manager = cgroup->cgroup_flags & CGROUP_MANAGER_MASK; + u32 supported = (cgroup->cgroup_flags != 0) && (bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_manager) != NULL); + return supported; +} + __attribute__((always_inline)) bool reserve_traced_cgroup_spot(struct cgroup_context_t *cgroup, u64 now, u64 cookie, struct activity_dump_config *config) { // insert dump config defaults - u32 defaults_key = 0; - struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &defaults_key); + u32 cgroup_flags = cgroup->cgroup_flags; + struct activity_dump_config *defaults = bpf_map_lookup_elem(&activity_dump_config_defaults, &cgroup_flags); if (defaults == NULL) { // should never happen, ignore return false; @@ -102,11 +108,15 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return 0; } - if ((container->cgroup_context.cgroup_flags & 0b111) == CGROUP_MANAGER_SYSTEMD) { + if (!is_cgroup_activity_dumps_supported(&container->cgroup_context)) { return 0; } - copy_container_id(container->container_id, evt->container.container_id); + if ((container->cgroup_context.cgroup_flags&CGROUP_MANAGER_MASK) != CGROUP_MANAGER_SYSTEMD) { + copy_container_id(container->container_id, evt->container.container_id); + } else { + evt->container.container_id[0] = '\0'; + } evt->container.cgroup_context = container->cgroup_context; evt->cookie = cookie; evt->config = config; @@ -115,10 +125,6 @@ __attribute__((always_inline)) u64 trace_new_cgroup(void *ctx, u64 now, struct c return cookie; } -__attribute__((always_inline)) u64 is_cgroup_activity_dumps_supported(struct cgroup_context_t *cgroup) { - return (cgroup->cgroup_flags != 0) && ((cgroup->cgroup_flags&0b111) != CGROUP_MANAGER_SYSTEMD); -} - __attribute__((always_inline)) u64 should_trace_new_process_cgroup(void *ctx, u64 now, u32 pid, struct container_context_t *container) { // should we start tracing this cgroup ? struct cgroup_context_t cgroup_context; diff --git a/pkg/security/ebpf/c/include/helpers/filesystem.h b/pkg/security/ebpf/c/include/helpers/filesystem.h index 07ca0431d7382..94210013c123e 100644 --- a/pkg/security/ebpf/c/include/helpers/filesystem.h +++ b/pkg/security/ebpf/c/include/helpers/filesystem.h @@ -102,10 +102,15 @@ void __attribute__((always_inline)) fill_file(struct dentry *dentry, struct file file->dev = get_dentry_dev(dentry); - bpf_probe_read(&file->metadata.nlink, sizeof(file->metadata.nlink), (void *)&d_inode->i_nlink); + u64 inode_nlink_offset; + LOAD_CONSTANT("inode_nlink_offset", inode_nlink_offset); + u64 inode_gid_offset; + LOAD_CONSTANT("inode_gid_offset", inode_gid_offset); + + bpf_probe_read(&file->metadata.nlink, sizeof(file->metadata.nlink), (void *)d_inode + inode_nlink_offset); bpf_probe_read(&file->metadata.mode, sizeof(file->metadata.mode), &d_inode->i_mode); bpf_probe_read(&file->metadata.uid, sizeof(file->metadata.uid), &d_inode->i_uid); - bpf_probe_read(&file->metadata.gid, sizeof(file->metadata.gid), &d_inode->i_gid); + bpf_probe_read(&file->metadata.gid, sizeof(file->metadata.gid), (void *)d_inode + inode_gid_offset); u64 inode_ctime_sec_offset; LOAD_CONSTANT("inode_ctime_sec_offset", inode_ctime_sec_offset); @@ -118,10 +123,10 @@ void __attribute__((always_inline)) fill_file(struct dentry *dentry, struct file bpf_probe_read(&nsec, sizeof(nsec), (void *)d_inode + inode_ctime_nsec_offset); file->metadata.ctime.tv_nsec = nsec; } else { -#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 6, 0) - bpf_probe_read(&file->metadata.ctime, sizeof(file->metadata.ctime), &d_inode->i_ctime); -#elif LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) - bpf_probe_read(&file->metadata.ctime, sizeof(file->metadata.ctime), &d_inode->__i_ctime); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) + u64 inode_ctime_offset; + LOAD_CONSTANT("inode_ctime_offset", inode_ctime_offset); + bpf_probe_read(&file->metadata.ctime, sizeof(file->metadata.ctime), (void *)d_inode + inode_ctime_offset); #else bpf_probe_read(&file->metadata.ctime.tv_sec, sizeof(file->metadata.ctime.tv_sec), &d_inode->i_ctime_sec); bpf_probe_read(&file->metadata.ctime.tv_nsec, sizeof(file->metadata.ctime.tv_nsec), &d_inode->i_ctime_nsec); @@ -139,10 +144,10 @@ void __attribute__((always_inline)) fill_file(struct dentry *dentry, struct file bpf_probe_read(&nsec, sizeof(nsec), (void *)d_inode + inode_mtime_nsec_offset); file->metadata.mtime.tv_nsec = nsec; } else { -#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0) - bpf_probe_read(&file->metadata.mtime, sizeof(file->metadata.mtime), &d_inode->i_mtime); -#elif LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) - bpf_probe_read(&file->metadata.mtime, sizeof(file->metadata.mtime), &d_inode->__i_mtime); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 11, 0) + u64 inode_mtime_offset; + LOAD_CONSTANT("inode_mtime_offset", inode_mtime_offset); + bpf_probe_read(&file->metadata.mtime, sizeof(file->metadata.mtime), (void *)d_inode + inode_mtime_offset); #else bpf_probe_read(&file->metadata.mtime.tv_sec, sizeof(file->metadata.mtime.tv_sec), &d_inode->i_mtime_sec); bpf_probe_read(&file->metadata.mtime.tv_nsec, sizeof(file->metadata.mtime.tv_nsec), &d_inode->i_mtime_nsec); diff --git a/pkg/security/ebpf/c/include/hooks/cgroup.h b/pkg/security/ebpf/c/include/hooks/cgroup.h index a231142b8f90a..b7ce66a870b92 100644 --- a/pkg/security/ebpf/c/include/hooks/cgroup.h +++ b/pkg/security/ebpf/c/include/hooks/cgroup.h @@ -189,13 +189,13 @@ static __attribute__((always_inline)) int trace__cgroup_write(ctx_t *ctx) { #endif int length = bpf_probe_read_str(prefix, sizeof(cgroup_prefix_t), container_id) & 0xff; - if (cgroup_flags == 0 && ( - (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') - || - (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') - )) { - cgroup_flags = CGROUP_MANAGER_SYSTEMD; - } else if (cgroup_flags != 0) { + if (cgroup_flags == 0) { + if (length >= 9 && (*prefix)[length-9] == '.' && (*prefix)[length-8] == 's' && (*prefix)[length-7] == 'e' && (*prefix)[length-6] == 'r' && (*prefix)[length-5] == 'v' && (*prefix)[length-4] == 'i' && (*prefix)[length-3] == 'c' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SERVICE; + } else if (length >= 7 && (*prefix)[length-7] == '.' && (*prefix)[length-6] == 's' && (*prefix)[length-5] == 'c' && (*prefix)[length-4] == 'o' && (*prefix)[length-3] == 'p' && (*prefix)[length-2] == 'e') { + cgroup_flags = CGROUP_MANAGER_SYSTEMD | CGROUP_SYSTEMD_SCOPE; + } + } else { bpf_probe_read(&new_entry.container.container_id, sizeof(new_entry.container.container_id), container_id); } diff --git a/pkg/security/ebpf/c/include/maps.h b/pkg/security/ebpf/c/include/maps.h index d6ab3ceb74dfb..3b821b19fe5cf 100644 --- a/pkg/security/ebpf/c/include/maps.h +++ b/pkg/security/ebpf/c/include/maps.h @@ -28,7 +28,7 @@ BPF_ARRAY_MAP(syscall_ctx_gen_id, u32, 1) BPF_ARRAY_MAP(syscall_ctx, char[MAX_SYSCALL_CTX_SIZE], MAX_SYSCALL_CTX_ENTRIES) BPF_HASH_MAP(activity_dumps_config, u64, struct activity_dump_config, 1) // max entries will be overridden at runtime -BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 1) +BPF_HASH_MAP(activity_dump_config_defaults, u32, struct activity_dump_config, 5) BPF_HASH_MAP(traced_cgroups, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(cgroup_wait_list, struct path_key_t, u64, 1) // max entries will be overridden at runtime BPF_HASH_MAP(traced_pids, u32, u64, 8192) // max entries will be overridden at runtime diff --git a/pkg/security/ebpf/kernel/kernel.go b/pkg/security/ebpf/kernel/kernel.go index 7ff8ef846ac9e..646163b104936 100644 --- a/pkg/security/ebpf/kernel/kernel.go +++ b/pkg/security/ebpf/kernel/kernel.go @@ -103,6 +103,8 @@ var ( Kernel6_5 = kernel.VersionCode(6, 5, 0) // Kernel6_6 is the KernelVersion representation of kernel version 6.6 Kernel6_6 = kernel.VersionCode(6, 6, 0) + // Kernel6_7 is the KernelVersion representation of kernel version 6.7 + Kernel6_7 = kernel.VersionCode(6, 7, 0) // Kernel6_10 is the KernelVersion representation of kernel version 6.10 Kernel6_10 = kernel.VersionCode(6, 10, 0) // Kernel6_11 is the KernelVersion representation of kernel version 6.11 diff --git a/pkg/security/probe/config/config.go b/pkg/security/probe/config/config.go index c312bbdea3181..c597eebe84e0e 100644 --- a/pkg/security/probe/config/config.go +++ b/pkg/security/probe/config/config.go @@ -101,12 +101,6 @@ type Config struct { // RuntimeCompilationEnabled defines if the runtime-compilation is enabled RuntimeCompilationEnabled bool - // EnableRuntimeCompiledConstants defines if the runtime compilation based constant fetcher is enabled - RuntimeCompiledConstantsEnabled bool - - // RuntimeCompiledConstantsIsSet is set if the runtime compiled constants option is user-set - RuntimeCompiledConstantsIsSet bool - // NetworkLazyInterfacePrefixes is the list of interfaces prefix that aren't explicitly deleted by the container // runtime, and that are lazily deleted by the kernel when a network namespace is cleaned up. This list helps the // agent detect when a network namespace should be purged from all caches. @@ -190,9 +184,7 @@ func NewConfig() (*Config, error) { EventServerBurst: pkgconfigsetup.SystemProbe().GetInt(join(evNS, "event_server.burst")), // runtime compilation - RuntimeCompilationEnabled: getBool("runtime_compilation.enabled"), - RuntimeCompiledConstantsEnabled: getBool("runtime_compilation.compiled_constants_enabled"), - RuntimeCompiledConstantsIsSet: isSet("runtime_compilation.compiled_constants_enabled"), + RuntimeCompilationEnabled: getBool("runtime_compilation.enabled"), } if err := c.sanitize(); err != nil { @@ -225,10 +217,6 @@ func (c *Config) sanitize() error { c.RuntimeCompilationEnabled = false } - if !c.RuntimeCompilationEnabled { - c.RuntimeCompiledConstantsEnabled = false - } - if c.EventStreamBufferSize%os.Getpagesize() != 0 || c.EventStreamBufferSize&(c.EventStreamBufferSize-1) != 0 { return fmt.Errorf("runtime_security_config.event_stream.buffer_size must be a power of 2 and a multiple of %d", os.Getpagesize()) } diff --git a/pkg/security/probe/constantfetch/available.go b/pkg/security/probe/constantfetch/available.go index bdbb9edcf4620..1035fb497e600 100644 --- a/pkg/security/probe/constantfetch/available.go +++ b/pkg/security/probe/constantfetch/available.go @@ -12,7 +12,6 @@ import ( "errors" "fmt" - "github.com/DataDog/datadog-go/v5/statsd" "github.com/cilium/ebpf/btf" pkgebpf "github.com/DataDog/datadog-agent/pkg/ebpf" @@ -22,18 +21,13 @@ import ( ) // GetAvailableConstantFetchers returns available constant fetchers -func GetAvailableConstantFetchers(config *config.Config, kv *kernel.Version, statsdClient statsd.ClientInterface) []ConstantFetcher { +func GetAvailableConstantFetchers(config *config.Config, kv *kernel.Version) []ConstantFetcher { fetchers := make([]ConstantFetcher, 0) if coreFetcher, err := NewBTFConstantFetcherFromCurrentKernel(); err == nil { fetchers = append(fetchers, coreFetcher) } - if config.RuntimeCompiledConstantsEnabled { - rcConstantFetcher := NewRuntimeCompilationConstantFetcher(&config.Config, statsdClient) - fetchers = append(fetchers, rcConstantFetcher) - } - btfhubFetcher, err := NewBTFHubConstantFetcher(kv) if err != nil { seclog.Debugf("failed to create btfhub constant fetcher: %v", err) diff --git a/pkg/security/probe/constantfetch/available_unsupported.go b/pkg/security/probe/constantfetch/available_unsupported.go index 3e003cbc28601..6195452074bbc 100644 --- a/pkg/security/probe/constantfetch/available_unsupported.go +++ b/pkg/security/probe/constantfetch/available_unsupported.go @@ -11,15 +11,13 @@ package constantfetch import ( "errors" - "github.com/DataDog/datadog-go/v5/statsd" - "github.com/DataDog/datadog-agent/pkg/security/ebpf/kernel" "github.com/DataDog/datadog-agent/pkg/security/probe/config" "github.com/DataDog/datadog-agent/pkg/security/seclog" ) // GetAvailableConstantFetchers returns available constant fetchers -func GetAvailableConstantFetchers(_ *config.Config, kv *kernel.Version, _ statsd.ClientInterface) []ConstantFetcher { +func GetAvailableConstantFetchers(_ *config.Config, kv *kernel.Version) []ConstantFetcher { fetchers := make([]ConstantFetcher, 0) btfhubFetcher, err := NewBTFHubConstantFetcher(kv) diff --git a/pkg/security/probe/constantfetch/btfhub.go b/pkg/security/probe/constantfetch/btfhub.go index c81a2ee968153..1d7aaaa191454 100644 --- a/pkg/security/probe/constantfetch/btfhub.go +++ b/pkg/security/probe/constantfetch/btfhub.go @@ -79,12 +79,12 @@ func (f *BTFHubConstantFetcher) appendRequest(id string) { } // AppendSizeofRequest appends a sizeof request -func (f *BTFHubConstantFetcher) AppendSizeofRequest(id, _, _ string) { +func (f *BTFHubConstantFetcher) AppendSizeofRequest(id, _ string) { f.appendRequest(id) } // AppendOffsetofRequest appends an offset request -func (f *BTFHubConstantFetcher) AppendOffsetofRequest(id, _, _, _ string) { +func (f *BTFHubConstantFetcher) AppendOffsetofRequest(id, _, _ string) { f.appendRequest(id) } diff --git a/pkg/security/probe/constantfetch/btfhub/constants.json b/pkg/security/probe/constantfetch/btfhub/constants.json index abf445a40119a..90238609bab82 100644 --- a/pkg/security/probe/constantfetch/btfhub/constants.json +++ b/pkg/security/probe/constantfetch/btfhub/constants.json @@ -20,6 +20,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -66,6 +71,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -112,6 +122,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -158,6 +173,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -204,6 +224,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -250,6 +275,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -296,6 +326,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -342,6 +377,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -388,6 +428,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -434,6 +479,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -477,6 +527,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -520,6 +575,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -566,6 +626,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -612,6 +677,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -658,6 +728,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -704,6 +779,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -750,6 +830,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -792,6 +877,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -833,6 +923,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -873,6 +968,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -915,6 +1015,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -959,6 +1064,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1003,6 +1113,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1050,6 +1165,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1095,6 +1215,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1140,6 +1265,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1185,6 +1315,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1230,6 +1365,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1277,6 +1417,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1323,6 +1468,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1363,6 +1513,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1404,6 +1559,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1451,6 +1611,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -1498,6 +1663,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -1545,6 +1715,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1592,6 +1767,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1640,6 +1820,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1686,6 +1871,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1732,6 +1922,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -1780,6 +1975,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1827,6 +2027,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1876,6 +2081,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -1925,6 +2135,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -1973,6 +2188,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2021,6 +2241,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2069,6 +2294,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2117,6 +2347,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2165,6 +2400,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2213,6 +2453,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2261,6 +2506,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2309,6 +2559,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2357,6 +2612,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -2406,6 +2666,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -2455,6 +2720,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -2504,6 +2774,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -2553,6 +2828,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2601,6 +2881,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2649,6 +2934,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2697,6 +2987,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2745,6 +3040,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2793,6 +3093,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2841,6 +3146,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2889,6 +3199,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2937,6 +3252,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -2985,6 +3305,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -3034,6 +3359,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -3083,6 +3413,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -3132,6 +3467,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -3175,6 +3515,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3218,6 +3563,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3261,6 +3611,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3305,6 +3660,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3348,6 +3708,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3394,6 +3759,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3437,6 +3807,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3481,6 +3856,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3529,6 +3909,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3578,6 +3963,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3626,6 +4016,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3675,6 +4070,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -3722,6 +4122,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3771,6 +4176,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -3819,6 +4229,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -3868,6 +4283,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -3918,6 +4338,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -3968,6 +4393,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4017,6 +4447,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4066,6 +4501,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4116,6 +4556,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4163,6 +4608,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4212,6 +4662,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4262,6 +4717,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4312,6 +4772,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4359,6 +4824,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4401,6 +4871,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4447,6 +4922,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4493,6 +4973,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -4542,6 +5027,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4592,6 +5082,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4642,6 +5137,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4692,6 +5192,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4742,6 +5247,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -4789,6 +5299,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -4838,6 +5353,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -4888,6 +5408,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -4935,6 +5460,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -4984,6 +5514,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5031,6 +5566,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5077,6 +5617,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5123,6 +5668,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5170,6 +5720,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5217,6 +5772,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5264,6 +5824,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5313,6 +5878,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5363,6 +5933,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5413,6 +5988,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5463,6 +6043,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5513,6 +6098,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5563,6 +6153,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -5608,6 +6203,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5652,6 +6252,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5698,6 +6303,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5745,6 +6355,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5793,6 +6408,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -5840,6 +6460,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -5889,6 +6514,11 @@ "flowi4_uli_offset": 64, "flowi6_saddr_offset": 72, "flowi6_uli_offset": 92, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 320, "linux_binprm_envc_offset": 324, "linux_binprm_p_offset": 280, @@ -5932,6 +6562,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -5975,6 +6610,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6021,6 +6661,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6067,6 +6712,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6114,6 +6764,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6160,6 +6815,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6207,6 +6867,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6256,6 +6921,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -6306,6 +6976,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -6356,6 +7031,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -6406,6 +7086,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -6451,6 +7136,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6495,6 +7185,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6539,6 +7234,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6583,6 +7283,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6629,6 +7334,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6677,6 +7387,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6725,6 +7440,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6773,6 +7493,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6821,6 +7546,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6869,6 +7599,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6917,6 +7652,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -6965,6 +7705,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7014,6 +7759,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7057,6 +7807,11 @@ "flowi4_uli_offset": 28, "flowi6_saddr_offset": 36, "flowi6_uli_offset": 56, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7100,6 +7855,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7143,6 +7903,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7186,6 +7951,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7229,6 +7999,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7272,6 +8047,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7315,6 +8095,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7358,6 +8143,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7401,6 +8191,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7444,6 +8239,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7487,6 +8287,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7530,6 +8335,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7573,6 +8383,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7616,6 +8431,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7659,6 +8479,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7702,6 +8527,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7745,6 +8575,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7791,6 +8626,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7839,6 +8679,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7887,6 +8732,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7936,6 +8786,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -7985,6 +8840,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8033,6 +8893,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8081,6 +8946,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8129,6 +8999,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8179,6 +9054,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8229,6 +9109,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8279,6 +9164,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8329,6 +9219,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -8376,6 +9271,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8424,6 +9324,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8472,6 +9377,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8520,6 +9430,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8568,6 +9483,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8616,6 +9536,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8664,6 +9589,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8713,6 +9643,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8762,6 +9697,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8811,6 +9751,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 192, "linux_binprm_envc_offset": 196, "linux_binprm_p_offset": 152, @@ -8860,6 +9805,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8908,6 +9858,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -8956,6 +9911,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -9004,6 +9964,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -9052,6 +10017,11 @@ "flowi4_uli_offset": 40, "flowi6_saddr_offset": 48, "flowi6_uli_offset": 68, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "linux_binprm_argc_offset": 200, "linux_binprm_envc_offset": 204, "linux_binprm_p_offset": 152, @@ -9100,6 +10070,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9150,6 +10125,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9200,6 +10180,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9250,6 +10235,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9300,6 +10290,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9350,6 +10345,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9400,6 +10400,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9450,6 +10455,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9500,6 +10510,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9550,6 +10565,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9600,6 +10620,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9650,6 +10675,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9700,6 +10730,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9750,6 +10785,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -9799,6 +10839,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -9848,6 +10893,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9898,6 +10948,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9948,6 +11003,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -9998,6 +11058,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10048,6 +11113,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10098,6 +11168,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10147,6 +11222,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10196,6 +11276,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10245,6 +11330,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10294,6 +11384,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -10343,6 +11438,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10393,6 +11493,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10443,6 +11548,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10493,6 +11603,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10543,6 +11658,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10593,6 +11713,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 88, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10643,6 +11768,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10693,6 +11823,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10743,6 +11878,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10793,6 +11933,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10843,6 +11988,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10893,6 +12043,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10943,6 +12098,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -10993,6 +12153,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -11043,6 +12208,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 96, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 72, @@ -11093,6 +12263,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11142,6 +12317,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11191,6 +12371,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11240,6 +12425,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11289,6 +12479,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, @@ -11338,6 +12533,11 @@ "flowi4_uli_offset": 48, "flowi6_saddr_offset": 56, "flowi6_uli_offset": 76, + "inode_ctime_offset": 120, + "inode_gid_offset": 8, + "inode_ino_offset": 64, + "inode_mtime_offset": 104, + "inode_nlink_offset": 72, "iokiocb_ctx_offset": 80, "kernel_clone_args_exit_signal_offset": 32, "linux_binprm_argc_offset": 88, diff --git a/pkg/security/probe/constantfetch/constant_names.go b/pkg/security/probe/constantfetch/constant_names.go index 2feab9c5e0402..80d8b57c04a25 100644 --- a/pkg/security/probe/constantfetch/constant_names.go +++ b/pkg/security/probe/constantfetch/constant_names.go @@ -31,6 +31,13 @@ const ( OffsetNameDentryDSb = "dentry_d_sb_offset" OffsetNameMountMntID = "mount_id_offset" + // inode + OffsetInodeIno = "inode_ino_offset" + OffsetInodeNlink = "inode_nlink_offset" + OffsetInodeGid = "inode_gid_offset" + OffsetInodeMtime = "inode_mtime_offset" + OffsetInodeCtime = "inode_ctime_offset" + // inode times OffsetNameInodeCtimeSec = "inode_ctime_sec_offset" OffsetNameInodeCtimeNsec = "inode_ctime_nsec_offset" diff --git a/pkg/security/probe/constantfetch/core.go b/pkg/security/probe/constantfetch/core.go index eba5beaea450f..117fb9cf6187d 100644 --- a/pkg/security/probe/constantfetch/core.go +++ b/pkg/security/probe/constantfetch/core.go @@ -89,7 +89,7 @@ func (f *BTFConstantFetcher) runRequest(r constantRequest) { } // AppendSizeofRequest appends a sizeof request -func (f *BTFConstantFetcher) AppendSizeofRequest(id, typeName, _ string) { +func (f *BTFConstantFetcher) AppendSizeofRequest(id, typeName string) { f.runRequest(constantRequest{ id: id, sizeof: true, @@ -98,7 +98,7 @@ func (f *BTFConstantFetcher) AppendSizeofRequest(id, typeName, _ string) { } // AppendOffsetofRequest appends an offset request -func (f *BTFConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName, _ string) { +func (f *BTFConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName string) { f.runRequest(constantRequest{ id: id, sizeof: false, diff --git a/pkg/security/probe/constantfetch/fallback.go b/pkg/security/probe/constantfetch/fallback.go index 531edb59c29d3..c6ecea9a22c3e 100644 --- a/pkg/security/probe/constantfetch/fallback.go +++ b/pkg/security/probe/constantfetch/fallback.go @@ -147,17 +147,27 @@ func (f *FallbackConstantFetcher) appendRequest(id string) { value = getRenameStructOldDentryOffset(f.kernelVersion) case OffsetNameRenameStructNewDentry: value = getRenameStructNewDentryOffset(f.kernelVersion) + case OffsetInodeIno: + value = getInodeInoOffset(f.kernelVersion) + case OffsetInodeGid: + value = getInodeGIDOffset(f.kernelVersion) + case OffsetInodeNlink: + value = getInodeNlinkOffset(f.kernelVersion) + case OffsetInodeMtime: + value = getInodeMtimeOffset(f.kernelVersion) + case OffsetInodeCtime: + value = getInodeCtimeOffset(f.kernelVersion) } f.res[id] = value } // AppendSizeofRequest appends a sizeof request -func (f *FallbackConstantFetcher) AppendSizeofRequest(id, _, _ string) { +func (f *FallbackConstantFetcher) AppendSizeofRequest(id, _ string) { f.appendRequest(id) } // AppendOffsetofRequest appends an offset request -func (f *FallbackConstantFetcher) AppendOffsetofRequest(id, _, _, _ string) { +func (f *FallbackConstantFetcher) AppendOffsetofRequest(id, _, _ string) { f.appendRequest(id) } @@ -236,6 +246,26 @@ func getSizeOfStructInode(kv *kernel.Version) uint64 { return sizeOf } +func getInodeInoOffset(_ *kernel.Version) uint64 { + return uint64(64) +} + +func getInodeGIDOffset(_ *kernel.Version) uint64 { + return uint64(8) +} + +func getInodeNlinkOffset(_ *kernel.Version) uint64 { + return uint64(72) +} + +func getInodeMtimeOffset(_ *kernel.Version) uint64 { + return uint64(104) +} + +func getInodeCtimeOffset(_ *kernel.Version) uint64 { + return uint64(120) +} + func getSuperBlockFlagsOffset(_ *kernel.Version) uint64 { return uint64(80) } diff --git a/pkg/security/probe/constantfetch/fetcher.go b/pkg/security/probe/constantfetch/fetcher.go index cfde02df17e2f..8d03030facbd0 100644 --- a/pkg/security/probe/constantfetch/fetcher.go +++ b/pkg/security/probe/constantfetch/fetcher.go @@ -26,8 +26,8 @@ const ErrorSentinel uint64 = ^uint64(0) // eBPF relocations type ConstantFetcher interface { fmt.Stringer - AppendSizeofRequest(id, typeName, headerName string) - AppendOffsetofRequest(id, typeName, fieldName, headerName string) + AppendSizeofRequest(id, typeName string) + AppendOffsetofRequest(id, typeName, fieldName string) FinishAndGetResults() (map[string]uint64, error) } @@ -57,30 +57,27 @@ func (f *ComposeConstantFetcher) appendRequest(req *composeRequest) { _, _ = io.WriteString(f.hasher, req.id) _, _ = io.WriteString(f.hasher, req.typeName) _, _ = io.WriteString(f.hasher, req.fieldName) - _, _ = io.WriteString(f.hasher, req.headerName) } // AppendSizeofRequest appends a sizeof request -func (f *ComposeConstantFetcher) AppendSizeofRequest(id, typeName, headerName string) { +func (f *ComposeConstantFetcher) AppendSizeofRequest(id, typeName string) { f.appendRequest(&composeRequest{ - id: id, - sizeof: true, - typeName: typeName, - fieldName: "", - headerName: headerName, - value: ErrorSentinel, + id: id, + sizeof: true, + typeName: typeName, + fieldName: "", + value: ErrorSentinel, }) } // AppendOffsetofRequest appends an offset request -func (f *ComposeConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName, headerName string) { +func (f *ComposeConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName string) { f.appendRequest(&composeRequest{ - id: id, - sizeof: false, - typeName: typeName, - fieldName: fieldName, - headerName: headerName, - value: ErrorSentinel, + id: id, + sizeof: false, + typeName: typeName, + fieldName: fieldName, + value: ErrorSentinel, }) } @@ -98,9 +95,9 @@ func (f *ComposeConstantFetcher) fillConstantCacheIfNeeded() { for _, req := range f.requests { if req.value == ErrorSentinel { if req.sizeof { - fetcher.AppendSizeofRequest(req.id, req.typeName, req.headerName) + fetcher.AppendSizeofRequest(req.id, req.typeName) } else { - fetcher.AppendOffsetofRequest(req.id, req.typeName, req.fieldName, req.headerName) + fetcher.AppendOffsetofRequest(req.id, req.typeName, req.fieldName) } } } @@ -166,7 +163,6 @@ type composeRequest struct { id string sizeof bool typeName, fieldName string - headerName string value uint64 fetcherName string } diff --git a/pkg/security/probe/constantfetch/offset_guesser.go b/pkg/security/probe/constantfetch/offset_guesser.go index 903aa95b81a52..ad2dbb09cfe0e 100644 --- a/pkg/security/probe/constantfetch/offset_guesser.go +++ b/pkg/security/probe/constantfetch/offset_guesser.go @@ -160,11 +160,11 @@ func (og *OffsetGuesser) guess(id string) error { } // AppendSizeofRequest appends a sizeof request -func (og *OffsetGuesser) AppendSizeofRequest(_, _, _ string) { +func (og *OffsetGuesser) AppendSizeofRequest(_, _ string) { } // AppendOffsetofRequest appends an offset request -func (og *OffsetGuesser) AppendOffsetofRequest(id, _, _, _ string) { +func (og *OffsetGuesser) AppendOffsetofRequest(id, _, _ string) { og.res[id] = ErrorSentinel } diff --git a/pkg/security/probe/constantfetch/runtime_compiled.go b/pkg/security/probe/constantfetch/runtime_compiled.go deleted file mode 100644 index c847ab821e9a4..0000000000000 --- a/pkg/security/probe/constantfetch/runtime_compiled.go +++ /dev/null @@ -1,178 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build linux && linux_bpf - -// Package constantfetch holds constantfetch related files -package constantfetch - -import ( - "bytes" - "errors" - "fmt" - "sort" - "text/template" - - "github.com/DataDog/datadog-go/v5/statsd" - - "github.com/DataDog/datadog-agent/pkg/ebpf" - "github.com/DataDog/datadog-agent/pkg/ebpf/bytecode/runtime" - "github.com/DataDog/datadog-agent/pkg/security/seclog" - "github.com/DataDog/datadog-agent/pkg/util/safeelf" -) - -type rcSymbolPair struct { - ID string - Operation string -} - -// RuntimeCompilationConstantFetcher is a constant fetcher utilizing runtime compilation -type RuntimeCompilationConstantFetcher struct { - config *ebpf.Config - statsdClient statsd.ClientInterface - headers []string - symbolPairs []rcSymbolPair - result map[string]uint64 -} - -// NewRuntimeCompilationConstantFetcher creates a RuntimeCompilationConstantFetcher -func NewRuntimeCompilationConstantFetcher(config *ebpf.Config, statsdClient statsd.ClientInterface) *RuntimeCompilationConstantFetcher { - return &RuntimeCompilationConstantFetcher{ - config: config, - statsdClient: statsdClient, - result: make(map[string]uint64), - } -} - -func (cf *RuntimeCompilationConstantFetcher) String() string { - return "runtime-compilation" -} - -// AppendSizeofRequest appends a sizeof request -func (cf *RuntimeCompilationConstantFetcher) AppendSizeofRequest(id, typeName, headerName string) { - cf.result[id] = ErrorSentinel - if headerName == "" { - return - } - - cf.headers = append(cf.headers, headerName) - cf.symbolPairs = append(cf.symbolPairs, rcSymbolPair{ - ID: id, - Operation: fmt.Sprintf("sizeof(%s)", typeName), - }) -} - -// AppendOffsetofRequest appends an offset request -func (cf *RuntimeCompilationConstantFetcher) AppendOffsetofRequest(id, typeName, fieldName, headerName string) { - cf.result[id] = ErrorSentinel - if headerName == "" { - return - } - - cf.headers = append(cf.headers, headerName) - cf.symbolPairs = append(cf.symbolPairs, rcSymbolPair{ - ID: id, - Operation: fmt.Sprintf("offsetof(%s, %s)", typeName, fieldName), - }) -} - -const runtimeCompilationTemplate = ` -#include -#ifdef CONFIG_HAVE_ARCH_COMPILER_H -#include -#endif -{{ range .headers }} -#include <{{ . }}> -{{ end }} - -{{ range .symbols }} -size_t {{.ID}} = {{.Operation}}; -{{ end }} -` - -func (cf *RuntimeCompilationConstantFetcher) getCCode() (string, error) { - headers := sortAndDedup(cf.headers) - tmpl, err := template.New("runtimeCompilationTemplate").Parse(runtimeCompilationTemplate) - if err != nil { - return "", err - } - - var buffer bytes.Buffer - if err := tmpl.Execute(&buffer, map[string]interface{}{ - "headers": headers, - "symbols": cf.symbolPairs, - }); err != nil { - return "", err - } - - return buffer.String(), nil -} - -// FinishAndGetResults returns the results -func (cf *RuntimeCompilationConstantFetcher) FinishAndGetResults() (map[string]uint64, error) { - cCode, err := cf.getCCode() - if err != nil { - return nil, err - } - - elfFile, err := runtime.ConstantFetcher.Compile(cf.config, cCode, nil, cf.statsdClient) - if err != nil { - return nil, err - } - - f, err := safeelf.NewFile(elfFile) - if err != nil { - return nil, err - } - - symbols, err := f.Symbols() - if err != nil { - return nil, err - } - for _, sym := range symbols { - if _, present := cf.result[sym.Name]; !present { - continue - } - - section := f.Sections[sym.Section] - if section.ReaderAt == nil { - return nil, errors.New("section not available in random-access form") - } - buf := make([]byte, sym.Size) - if _, err := section.ReadAt(buf, int64(sym.Value)); err != nil { - return nil, fmt.Errorf("unable to read section at %d: %s", int64(sym.Value), err) - } - - var value uint64 - switch sym.Size { - case 4: - value = uint64(f.ByteOrder.Uint32(buf)) - case 8: - value = f.ByteOrder.Uint64(buf) - default: - return nil, fmt.Errorf("unexpected symbol size: `%v`", sym.Size) - } - - cf.result[sym.Name] = value - } - - seclog.Infof("runtime compiled constants: %v", cf.result) - return cf.result, nil -} - -func sortAndDedup(in []string) []string { - // sort and dedup headers - set := make(map[string]bool) - for _, value := range in { - set[value] = true - } - - out := make([]string, 0, len(in)) - for value := range set { - out = append(out, value) - } - sort.Strings(out) - return out -} diff --git a/pkg/security/probe/field_handlers_ebpf.go b/pkg/security/probe/field_handlers_ebpf.go index b76b70a76b833..63215156e4a19 100644 --- a/pkg/security/probe/field_handlers_ebpf.go +++ b/pkg/security/probe/field_handlers_ebpf.go @@ -20,6 +20,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/resolvers" sprocess "github.com/DataDog/datadog-agent/pkg/security/resolvers/process" "github.com/DataDog/datadog-agent/pkg/security/secl/containerutils" + "github.com/DataDog/datadog-agent/pkg/security/utils" "github.com/DataDog/datadog-agent/pkg/security/secl/args" "github.com/DataDog/datadog-agent/pkg/security/secl/model" @@ -230,7 +231,7 @@ func (fh *EBPFFieldHandlers) ResolveRights(_ *model.Event, e *model.FileFields) return int(e.Mode) & (syscall.S_ISUID | syscall.S_ISGID | syscall.S_ISVTX | syscall.S_IRWXU | syscall.S_IRWXG | syscall.S_IRWXO) } -// ResolveChownUID resolves the ResolveProcessCacheEntry id of a chown event to a username +// ResolveChownUID resolves the user id of a chown event to a username func (fh *EBPFFieldHandlers) ResolveChownUID(ev *model.Event, e *model.ChownEvent) string { if len(e.User) == 0 { e.User, _ = fh.resolvers.UserGroupResolver.ResolveUser(int(e.UID), ev.ContainerContext.ContainerID) @@ -731,3 +732,17 @@ func (fh *EBPFFieldHandlers) ResolveOnDemandArg4Str(_ *model.Event, e *model.OnD func (fh *EBPFFieldHandlers) ResolveOnDemandArg4Uint(_ *model.Event, e *model.OnDemandEvent) int { return int(binary.NativeEndian.Uint64(e.Data[192 : 192+8])) } + +// ResolveProcessNSID resolves the process namespace ID +func (fh *EBPFFieldHandlers) ResolveProcessNSID(e *model.Event) (uint64, error) { + if e.ProcessCacheEntry.Process.NSID != 0 { + return e.ProcessCacheEntry.Process.NSID, nil + } + + nsid, err := utils.GetProcessPidNamespace(e.ProcessCacheEntry.Process.Pid) + if err != nil { + return 0, err + } + e.ProcessCacheEntry.Process.NSID = nsid + return nsid, nil +} diff --git a/pkg/security/probe/probe_ebpf.go b/pkg/security/probe/probe_ebpf.go index f06cbe8fa45ae..828a8e0b85ce0 100644 --- a/pkg/security/probe/probe_ebpf.go +++ b/pkg/security/probe/probe_ebpf.go @@ -818,15 +818,11 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { return } - if cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)); err != nil { + cgroupContext, err := p.Resolvers.ResolveCGroupContext(event.CgroupTracing.CGroupContext.CGroupFile, containerutils.CGroupFlags(event.CgroupTracing.CGroupContext.CGroupFlags)) + if err != nil { seclog.Debugf("Failed to resolve cgroup: %s", err) } else { event.CgroupTracing.CGroupContext = *cgroupContext - if cgroupContext.CGroupFlags.IsContainer() { - containerID, _ := containerutils.FindContainerID(cgroupContext.CGroupID) - event.CgroupTracing.ContainerContext.ContainerID = containerID - } - p.profileManagers.activityDumpManager.HandleCGroupTracingEvent(&event.CgroupTracing) } @@ -886,14 +882,10 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { return } - if process.IsKThread(event.ProcessCacheEntry.PPid, event.ProcessCacheEntry.Pid) { + if err := p.Resolvers.ProcessResolver.AddForkEntry(event, newEntryCb); err != nil { + seclog.Errorf("failed to insert fork event: %s (pid %d, offset %d, len %d)", err, event.PIDContext.Pid, offset, len(data)) return } - - p.Resolvers.ProcessResolver.ApplyBootTime(event.ProcessCacheEntry) - event.ProcessCacheEntry.SetSpan(event.SpanContext.SpanID, event.SpanContext.TraceID) - - p.Resolvers.ProcessResolver.AddForkEntry(event.ProcessCacheEntry, event.PIDContext.ExecInode, newEntryCb) case model.ExecEventType: // unmarshal and fill event.processCacheEntry if _, err = p.unmarshalProcessCacheEntry(event, data[offset:]); err != nil { @@ -901,18 +893,12 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { return } - if err = p.Resolvers.ProcessResolver.ResolveNewProcessCacheEntry(event.ProcessCacheEntry, event.ContainerContext); err != nil { - seclog.Debugf("failed to resolve new process cache entry context for pid %d: %s", event.PIDContext.Pid, err) - - var errResolution *path.ErrPathResolution - if errors.As(err, &errResolution) { - event.SetPathResolutionError(&event.ProcessCacheEntry.FileEvent, err) - } - } else { - p.Resolvers.ProcessResolver.AddExecEntry(event.ProcessCacheEntry, event.PIDContext.ExecInode) + err = p.Resolvers.ProcessResolver.AddExecEntry(event) + if err != nil { + seclog.Errorf("failed to insert exec event: %s (pid %d, offset %d, len %d)", err, event.PIDContext.Pid, offset, len(data)) + return } - event.Exec.Process = &event.ProcessCacheEntry.Process } if !p.setProcessContext(eventType, event, newEntryCb) { @@ -1023,26 +1009,13 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { seclog.Errorf("failed to decode exit event: %s (offset %d, len %d)", err, offset, len(data)) return } - - var exists bool - event.ProcessCacheEntry, exists = p.fieldHandlers.GetProcessCacheEntry(event, newEntryCb) - if !exists { - // no need to dispatch an exit event that don't have the corresponding cache entry - return + exists := p.Resolvers.ProcessResolver.ApplyExitEntry(event, newEntryCb) + if exists { + p.Resolvers.MountResolver.DelPid(event.Exit.Pid) + // update action reports + p.processKiller.HandleProcessExited(event) + p.fileHasher.HandleProcessExited(event) } - - // Use the event timestamp as exit time - // The local process cache hasn't been updated yet with the exit time when the exit event is first seen - // The pid_cache kernel map has the exit_time but it's only accessed if there's a local miss - event.ProcessCacheEntry.Process.ExitTime = p.fieldHandlers.ResolveEventTime(event, &event.BaseEvent) - event.Exit.Process = &event.ProcessCacheEntry.Process - - // update mount pid mapping - p.Resolvers.MountResolver.DelPid(event.Exit.Pid) - - // update action reports - p.processKiller.HandleProcessExited(event) - p.fileHasher.HandleProcessExited(event) case model.SetuidEventType: // the process context may be incorrect, do not modify it if event.Error != nil { @@ -1117,7 +1090,14 @@ func (p *EBPFProbe) handleEvent(CPU int, data []byte) { if containerID == "" && event.PTrace.Request != unix.PTRACE_ATTACH { pidToResolve = event.PTrace.NSPID } else { - pid, err := utils.TryToResolveTraceePid(event.ProcessContext.Process.Pid, event.PTrace.NSPID) + nsid, err := p.fieldHandlers.ResolveProcessNSID(event) + if err != nil { + seclog.Debugf("PTrace NSID resolution error for process %s in container %s: %v", + event.ProcessContext.Process.FileEvent.PathnameStr, containerID, err) + return + } + + pid, err := utils.TryToResolveTraceePid(event.ProcessContext.Process.Pid, nsid, event.PTrace.NSPID) if err != nil { seclog.Debugf("PTrace tracee resolution error for process %s in container %s: %v", event.ProcessContext.Process.FileEvent.PathnameStr, containerID, err) @@ -2014,8 +1994,6 @@ func NewEBPFProbe(probe *Probe, config *config.Config, opts Opts) (*EBPFProbe, e p.supportsBPFSendSignal = p.kernelVersion.SupportBPFSendSignal() - p.ensureConfigDefaults() - p.monitors = NewEBPFMonitors(p) p.numCPU, err = utils.NumCPU() @@ -2272,13 +2250,6 @@ func (p *EBPFProbe) GetProfileManagers() *SecurityProfileManagers { return p.profileManagers } -func (p *EBPFProbe) ensureConfigDefaults() { - // enable runtime compiled constants on COS by default - if !p.config.Probe.RuntimeCompiledConstantsIsSet && p.kernelVersion.IsCOSKernel() { - p.config.Probe.RuntimeCompiledConstantsEnabled = true - } -} - const ( netStructHasProcINum uint64 = 0 netStructHasNS uint64 = 1 @@ -2407,123 +2378,135 @@ func getCGroupWriteConstants() manager.ConstantEditor { // GetOffsetConstants returns the offsets and struct sizes constants func (p *EBPFProbe) GetOffsetConstants() (map[string]uint64, error) { - constantFetcher := constantfetch.ComposeConstantFetchers(constantfetch.GetAvailableConstantFetchers(p.config.Probe, p.kernelVersion, p.statsdClient)) + constantFetcher := constantfetch.ComposeConstantFetchers(constantfetch.GetAvailableConstantFetchers(p.config.Probe, p.kernelVersion)) AppendProbeRequestsToFetcher(constantFetcher, p.kernelVersion) return constantFetcher.FinishAndGetResults() } // GetConstantFetcherStatus returns the status of the constant fetcher associated with this probe func (p *EBPFProbe) GetConstantFetcherStatus() (*constantfetch.ConstantFetcherStatus, error) { - constantFetcher := constantfetch.ComposeConstantFetchers(constantfetch.GetAvailableConstantFetchers(p.config.Probe, p.kernelVersion, p.statsdClient)) + constantFetcher := constantfetch.ComposeConstantFetchers(constantfetch.GetAvailableConstantFetchers(p.config.Probe, p.kernelVersion)) AppendProbeRequestsToFetcher(constantFetcher, p.kernelVersion) return constantFetcher.FinishAndGetStatus() } // AppendProbeRequestsToFetcher returns the offsets and struct sizes constants, from a constant fetcher func AppendProbeRequestsToFetcher(constantFetcher constantfetch.ConstantFetcher, kv *kernel.Version) { - constantFetcher.AppendSizeofRequest(constantfetch.SizeOfInode, "struct inode", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSuperBlockStructSFlags, "struct super_block", "s_flags", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSuperBlockStructSMagic, "struct super_block", "s_magic", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDentryStructDSB, "struct dentry", "d_sb", "linux/dcache.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSignalStructStructTTY, "struct signal_struct", "tty", "linux/sched/signal.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTTYStructStructName, "struct tty_struct", "name", "linux/tty.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameCredStructUID, "struct cred", "uid", "linux/cred.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameCredStructCapInheritable, "struct cred", "cap_inheritable", "linux/cred.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmP, "struct linux_binprm", "p", "linux/binfmts.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmArgc, "struct linux_binprm", "argc", "linux/binfmts.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmEnvc, "struct linux_binprm", "envc", "linux/binfmts.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameVMAreaStructFlags, "struct vm_area_struct", "vm_flags", "linux/mm_types.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFileFinode, "struct file", "f_inode", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFileFpath, "struct file", "f_path", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDentryDSb, "struct dentry", "d_sb", "linux/dcache.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameMountMntID, "struct mount", "mnt_id", "") + constantFetcher.AppendSizeofRequest(constantfetch.SizeOfInode, "struct inode") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSuperBlockStructSFlags, "struct super_block", "s_flags") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSuperBlockStructSMagic, "struct super_block", "s_magic") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDentryStructDSB, "struct dentry", "d_sb") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSignalStructStructTTY, "struct signal_struct", "tty") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTTYStructStructName, "struct tty_struct", "name") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameCredStructUID, "struct cred", "uid") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameCredStructCapInheritable, "struct cred", "cap_inheritable") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmP, "struct linux_binprm", "p") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmArgc, "struct linux_binprm", "argc") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmEnvc, "struct linux_binprm", "envc") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameVMAreaStructFlags, "struct vm_area_struct", "vm_flags") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFileFinode, "struct file", "f_inode") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFileFpath, "struct file", "f_path") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDentryDSb, "struct dentry", "d_sb") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameMountMntID, "struct mount", "mnt_id") if kv.Code >= kernel.Kernel5_3 { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameKernelCloneArgsExitSignal, "struct kernel_clone_args", "exit_signal", "linux/sched/task.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameKernelCloneArgsExitSignal, "struct kernel_clone_args", "exit_signal") } // inode time offsets // no runtime compilation for those, we expect them to default to 0 if not there and use the fallback // in the eBPF code itself in that case if kv.Code >= kernel.Kernel6_11 { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeCtimeSec, "struct inode", "i_ctime_sec", "") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeCtimeNsec, "struct inode", "i_ctime_nsec", "") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeMtimeSec, "struct inode", "i_mtime_sec", "") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeMtimeNsec, "struct inode", "i_mtime_nsec", "") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeCtimeSec, "struct inode", "i_ctime_sec") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeCtimeNsec, "struct inode", "i_ctime_nsec") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeMtimeSec, "struct inode", "i_mtime_sec") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameInodeMtimeNsec, "struct inode", "i_mtime_nsec") } // rename offsets if kv.Code >= kernel.Kernel5_12 || (kv.IsInRangeCloseOpen(kernel.Kernel5_10, kernel.Kernel5_11) && kv.Code.Patch() >= 220) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameRenameStructOldDentry, "struct renamedata", "old_dentry", "linux/fs.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameRenameStructNewDentry, "struct renamedata", "new_dentry", "linux/fs.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameRenameStructOldDentry, "struct renamedata", "old_dentry") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameRenameStructNewDentry, "struct renamedata", "new_dentry") } // bpf offsets - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructID, "struct bpf_map", "id", "linux/bpf.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructID, "struct bpf_map", "id") if kv.Code != 0 && (kv.Code >= kernel.Kernel4_15 || kv.IsRH7Kernel()) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructName, "struct bpf_map", "name", "linux/bpf.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructName, "struct bpf_map", "name") } - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructMapType, "struct bpf_map", "map_type", "linux/bpf.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgAuxStructID, "struct bpf_prog_aux", "id", "linux/bpf.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFMapStructMapType, "struct bpf_map", "map_type") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgAuxStructID, "struct bpf_prog_aux", "id") if kv.Code != 0 && (kv.Code >= kernel.Kernel4_15 || kv.IsRH7Kernel()) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgAuxStructName, "struct bpf_prog_aux", "name", "linux/bpf.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgAuxStructName, "struct bpf_prog_aux", "name") } - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructTag, "struct bpf_prog", "tag", "linux/filter.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructAux, "struct bpf_prog", "aux", "linux/filter.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructType, "struct bpf_prog", "type", "linux/filter.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructTag, "struct bpf_prog", "tag") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructAux, "struct bpf_prog", "aux") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructType, "struct bpf_prog", "type") if kv.Code != 0 && (kv.Code > kernel.Kernel4_16 || kv.IsSuse12Kernel() || kv.IsSuse15Kernel()) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructExpectedAttachType, "struct bpf_prog", "expected_attach_type", "linux/filter.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameBPFProgStructExpectedAttachType, "struct bpf_prog", "expected_attach_type") } // namespace nr offsets - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDStructLevel, "struct pid", "level", "linux/pid.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDStructNumbers, "struct pid", "numbers", "linux/pid.h") - constantFetcher.AppendSizeofRequest(constantfetch.SizeOfUPID, "struct upid", "linux/pid.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDStructLevel, "struct pid", "level") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDStructNumbers, "struct pid", "numbers") + constantFetcher.AppendSizeofRequest(constantfetch.SizeOfUPID, "struct upid") if kv.HavePIDLinkStruct() { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTaskStructPIDLink, "struct task_struct", "pids", "linux/sched.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDLinkStructPID, "struct pid_link", "pid", "linux/pid.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTaskStructPIDLink, "struct task_struct", "pids") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePIDLinkStructPID, "struct pid_link", "pid") } else { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTaskStructPID, "struct task_struct", "thread_pid", "linux/sched.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameTaskStructPID, "struct task_struct", "thread_pid") } // splice event - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructBufs, "struct pipe_inode_info", "bufs", "linux/pipe_fs_i.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructBufs, "struct pipe_inode_info", "bufs") if kv.HaveLegacyPipeInodeInfoStruct() { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructNrbufs, "struct pipe_inode_info", "nrbufs", "linux/pipe_fs_i.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructCurbuf, "struct pipe_inode_info", "curbuf", "linux/pipe_fs_i.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructBuffers, "struct pipe_inode_info", "buffers", "linux/pipe_fs_i.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructNrbufs, "struct pipe_inode_info", "nrbufs") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructCurbuf, "struct pipe_inode_info", "curbuf") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructBuffers, "struct pipe_inode_info", "buffers") } else { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructHead, "struct pipe_inode_info", "head", "linux/pipe_fs_i.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructRingsize, "struct pipe_inode_info", "ring_size", "linux/pipe_fs_i.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructHead, "struct pipe_inode_info", "head") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNamePipeInodeInfoStructRingsize, "struct pipe_inode_info", "ring_size") } // network related constants - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetDeviceStructIfIndex, "struct net_device", "ifindex", "linux/netdevice.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetDeviceStructName, "struct net_device", "name", "linux/netdevice.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDeviceStructNdNet, "struct net_device", "nd_net", "linux/netdevice.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSockCommonStructSKCNet, "struct sock_common", "skc_net", "net/sock.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSockCommonStructSKCFamily, "struct sock_common", "skc_family", "net/sock.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI4StructSADDR, "struct flowi4", "saddr", "net/flow.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI4StructULI, "struct flowi4", "uli", "net/flow.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI6StructSADDR, "struct flowi6", "saddr", "net/flow.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI6StructULI, "struct flowi6", "uli", "net/flow.h") - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSocketStructSK, "struct socket", "sk", "linux/net.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetDeviceStructIfIndex, "struct net_device", "ifindex") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetDeviceStructName, "struct net_device", "name") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameDeviceStructNdNet, "struct net_device", "nd_net") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSockCommonStructSKCNet, "struct sock_common", "skc_net") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSockCommonStructSKCFamily, "struct sock_common", "skc_family") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI4StructSADDR, "struct flowi4", "saddr") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI4StructULI, "struct flowi4", "uli") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI6StructSADDR, "struct flowi6", "saddr") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameFlowI6StructULI, "struct flowi6", "uli") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameSocketStructSK, "struct socket", "sk") // Interpreter constants - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmStructFile, "struct linux_binprm", "file", "linux/binfmts.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameLinuxBinprmStructFile, "struct linux_binprm", "file") if !kv.IsRH7Kernel() { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNFConnStructCTNet, "struct nf_conn", "ct_net", "net/netfilter/nf_conntrack.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNFConnStructCTNet, "struct nf_conn", "ct_net") } if getNetStructType(kv) == netStructHasProcINum { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetStructProcInum, "struct net", "proc_inum", "net/net_namespace.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetStructProcInum, "struct net", "proc_inum") } else { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetStructNS, "struct net", "ns", "net/net_namespace.h") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameNetStructNS, "struct net", "ns") } // iouring if kv.Code != 0 && (kv.Code >= kernel.Kernel5_1) { - constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameIoKiocbStructCtx, "struct io_kiocb", "ctx", "") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetNameIoKiocbStructCtx, "struct io_kiocb", "ctx") + } + + // inode + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeIno, "struct inode", "i_ino") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeGid, "struct inode", "i_gid") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeNlink, "struct inode", "i_nlink") + if kv.IsInRangeCloseOpen(kernel.Kernel6_7, kernel.Kernel6_11) { + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeMtime, "struct inode", "__i_mtime") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeCtime, "struct inode", "__i_ctime") + } else if kv.Code < kernel.Kernel6_7 { + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeMtime, "struct inode", "i_mtime") + constantFetcher.AppendOffsetofRequest(constantfetch.OffsetInodeCtime, "struct inode", "i_ctime") } } diff --git a/pkg/security/resolvers/cgroup/resolver.go b/pkg/security/resolvers/cgroup/resolver.go index 2137cccf14506..dcb1de1e6d378 100644 --- a/pkg/security/resolvers/cgroup/resolver.go +++ b/pkg/security/resolvers/cgroup/resolver.go @@ -48,6 +48,7 @@ type ResolverInterface interface { type Resolver struct { *utils.Notifier[Event, *cgroupModel.CacheEntry] sync.Mutex + cgroups *simplelru.LRU[model.PathKey, *model.CGroupContext] hostWorkloads *simplelru.LRU[containerutils.CGroupID, *cgroupModel.CacheEntry] containerWorkloads *simplelru.LRU[containerutils.ContainerID, *cgroupModel.CacheEntry] } @@ -80,6 +81,11 @@ func NewResolver() (*Resolver, error) { return nil, err } + cr.cgroups, err = simplelru.NewLRU(2048, func(_ model.PathKey, _ *model.CGroupContext) {}) + if err != nil { + return nil, err + } + return cr, nil } @@ -121,10 +127,19 @@ func (cr *Resolver) AddPID(process *model.ProcessCacheEntry) { } else { cr.hostWorkloads.Add(process.CGroup.CGroupID, newCGroup) } + cr.cgroups.Add(process.CGroup.CGroupFile, &process.CGroup) cr.NotifyListeners(CGroupCreated, newCGroup) } +// GetCGroupContext returns the cgroup context with the specified path key +func (cr *Resolver) GetCGroupContext(cgroupPath model.PathKey) (*model.CGroupContext, bool) { + cr.Lock() + defer cr.Unlock() + + return cr.cgroups.Get(cgroupPath) +} + // GetWorkload returns the workload referenced by the provided ID func (cr *Resolver) GetWorkload(id containerutils.ContainerID) (*cgroupModel.CacheEntry, bool) { if id == "" { @@ -171,6 +186,7 @@ func (cr *Resolver) deleteWorkloadPID(pid uint32, workload *cgroupModel.CacheEnt // check if the workload should be deleted if len(workload.PIDs) <= 0 { + cr.cgroups.Remove(workload.CGroupFile) cr.hostWorkloads.Remove(workload.CGroupID) if workload.ContainerID != "" { cr.containerWorkloads.Remove(workload.ContainerID) @@ -183,5 +199,5 @@ func (cr *Resolver) Len() int { cr.Lock() defer cr.Unlock() - return cr.hostWorkloads.Len() + cr.containerWorkloads.Len() + return cr.cgroups.Len() } diff --git a/pkg/security/resolvers/process/resolver_ebpf.go b/pkg/security/resolvers/process/resolver_ebpf.go index 5bcccdb52c540..0e80c8aa2ae93 100644 --- a/pkg/security/resolvers/process/resolver_ebpf.go +++ b/pkg/security/resolvers/process/resolver_ebpf.go @@ -290,27 +290,61 @@ func (p *EBPFResolver) UpdateArgsEnvs(event *model.ArgsEnvsEvent) { } // AddForkEntry adds an entry to the local cache and returns the newly created entry -func (p *EBPFResolver) AddForkEntry(entry *model.ProcessCacheEntry, inode uint64, newEntryCb func(*model.ProcessCacheEntry, error)) { - if entry.Pid == 0 { - return +func (p *EBPFResolver) AddForkEntry(event *model.Event, newEntryCb func(*model.ProcessCacheEntry, error)) error { + p.ApplyBootTime(event.ProcessCacheEntry) + event.ProcessCacheEntry.SetSpan(event.SpanContext.SpanID, event.SpanContext.TraceID) + + if event.ProcessCacheEntry.Pid == 0 { + return errors.New("no pid") + } + if IsKThread(event.ProcessCacheEntry.PPid, event.ProcessCacheEntry.Pid) { + return errors.New("process is kthread") } p.Lock() defer p.Unlock() - - p.insertForkEntry(entry, inode, model.ProcessCacheEntryFromEvent, newEntryCb) + p.insertForkEntry(event.ProcessCacheEntry, event.PIDContext.ExecInode, model.ProcessCacheEntryFromEvent, newEntryCb) + return nil } // AddExecEntry adds an entry to the local cache and returns the newly created entry -func (p *EBPFResolver) AddExecEntry(entry *model.ProcessCacheEntry, inode uint64) { - if entry.Pid == 0 { - return - } - +func (p *EBPFResolver) AddExecEntry(event *model.Event) error { p.Lock() defer p.Unlock() - p.insertExecEntry(entry, inode, model.ProcessCacheEntryFromEvent) + var err error + if err := p.ResolveNewProcessCacheEntry(event.ProcessCacheEntry, event.ContainerContext); err != nil { + var errResolution *spath.ErrPathResolution + if errors.As(err, &errResolution) { + event.SetPathResolutionError(&event.ProcessCacheEntry.FileEvent, err) + } + } else { + if event.ProcessCacheEntry.Pid == 0 { + return errors.New("no pid context") + } + p.insertExecEntry(event.ProcessCacheEntry, event.PIDContext.ExecInode, model.ProcessCacheEntryFromEvent) + } + + event.Exec.Process = &event.ProcessCacheEntry.Process + + return err +} + +// ApplyExitEntry delete entry from the local cache if present +func (p *EBPFResolver) ApplyExitEntry(event *model.Event, newEntryCb func(*model.ProcessCacheEntry, error)) bool { + event.ProcessCacheEntry = p.resolve(event.PIDContext.Pid, event.PIDContext.Tid, event.PIDContext.ExecInode, false, newEntryCb) + if event.ProcessCacheEntry == nil { + // no need to dispatch an exit event that don't have the corresponding cache entry + return false + } + + // Use the event timestamp as exit time + // The local process cache hasn't been updated yet with the exit time when the exit event is first seen + // The pid_cache kernel map has the exit_time but it's only accessed if there's a local miss + event.ProcessCacheEntry.ExitTime = event.FieldHandlers.ResolveEventTime(event, &event.BaseEvent) + event.Exit.Process = &event.ProcessCacheEntry.Process + return true + } // enrichEventFromProc uses /proc to enrich a ProcessCacheEntry with additional metadata @@ -334,13 +368,16 @@ func (p *EBPFResolver) enrichEventFromProc(entry *model.ProcessCacheEntry, proc // Get the file fields of the process binary info, err := p.retrieveExecFileFields(procExecPath) if err != nil { + if !os.IsNotExist(err) { + seclog.Errorf("snapshot failed for %d: couldn't retrieve inode info: %s", proc.Pid, err) + } return fmt.Errorf("snapshot failed for %d: couldn't retrieve inode info: %w", proc.Pid, err) } // Retrieve the container ID of the process from /proc containerID, cgroup, err := p.containerResolver.GetContainerContext(pid) if err != nil { - return fmt.Errorf("snapshot failed for %d: couldn't parse container ID: %w", proc.Pid, err) + return fmt.Errorf("snapshot failed for %d: couldn't parse container and cgroup context: %w", proc.Pid, err) } entry.ContainerID = containerID @@ -465,11 +502,11 @@ func (p *EBPFResolver) enrichEventFromProc(entry *model.ProcessCacheEntry, proc func (p *EBPFResolver) retrieveExecFileFields(procExecPath string) (*model.FileFields, error) { fi, err := os.Stat(procExecPath) if err != nil { - return nil, fmt.Errorf("snapshot failed for `%s`: couldn't stat binary: %w", procExecPath, err) + return nil, err } stat, ok := fi.Sys().(*syscall.Stat_t) if !ok { - return nil, fmt.Errorf("snapshot failed for `%s`: couldn't stat binary", procExecPath) + return nil, errors.New("wrong type") } inode := stat.Ino @@ -483,11 +520,11 @@ func (p *EBPFResolver) retrieveExecFileFields(procExecPath string) (*model.FileF var fileFields model.FileFields if _, err := fileFields.UnmarshalBinary(data); err != nil { - return nil, fmt.Errorf("unable to unmarshal entry for inode `%d`", inode) + return nil, fmt.Errorf("unable to unmarshal entry for inode `%d`: %v", inode, err) } if fileFields.Inode == 0 { - return nil, errors.New("not found") + return nil, fmt.Errorf("inode `%d` not found: %v", inode, err) } return &fileFields, nil @@ -520,6 +557,7 @@ func (p *EBPFResolver) insertEntry(entry, prev *model.ProcessCacheEntry, source } func (p *EBPFResolver) insertForkEntry(entry *model.ProcessCacheEntry, inode uint64, source uint64, newEntryCb func(*model.ProcessCacheEntry, error)) { + if entry.Pid == 0 { return } @@ -529,7 +567,6 @@ func (p *EBPFResolver) insertForkEntry(entry *model.ProcessCacheEntry, inode uin // this shouldn't happen but it is better to exit the prev and let the new one replace it prev.Exit(entry.ForkTime) } - if entry.Pid != 1 { parent := p.entryCache[entry.PPid] if entry.PPid >= 1 && inode != 0 && (parent == nil || parent.FileEvent.Inode != inode) { @@ -568,7 +605,6 @@ func (p *EBPFResolver) insertExecEntry(entry *model.ProcessCacheEntry, inode uin prev.ApplyExecTimeOf(entry) return } - prev.Exec(entry) } else { entry.IsParentMissing = true @@ -654,7 +690,6 @@ func (p *EBPFResolver) resolveFileFieldsPath(e *model.FileFields, pce *model.Pro err error maxDepthRetry = 3 ) - for maxDepthRetry > 0 { pathnameStr, mountPath, source, origin, err = p.pathResolver.ResolveFileFieldsPath(e, &pce.PIDContext, ctrCtx) if err == nil { @@ -686,7 +721,6 @@ func (p *EBPFResolver) SetProcessPath(fileEvent *model.FileEvent, pce *model.Pro if fileEvent.Inode == 0 { return onError("", &model.ErrInvalidKeyPath{Inode: fileEvent.Inode, MountID: fileEvent.MountID}) } - pathnameStr, mountPath, source, origin, err := p.resolveFileFieldsPath(&fileEvent.FileFields, pce, ctrCtx) if err != nil { return onError(pathnameStr, err) diff --git a/pkg/security/resolvers/process/resolver_test.go b/pkg/security/resolvers/process/resolver_test.go index c2febee639beb..16432e67338e1 100644 --- a/pkg/security/resolvers/process/resolver_test.go +++ b/pkg/security/resolvers/process/resolver_test.go @@ -16,10 +16,44 @@ import ( "github.com/avast/retry-go/v4" "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/pkg/security/resolvers/cgroup" + "github.com/DataDog/datadog-agent/pkg/security/resolvers/path" + "github.com/DataDog/datadog-agent/pkg/security/resolvers/usergroup" "github.com/DataDog/datadog-agent/pkg/security/secl/model" + "github.com/DataDog/datadog-agent/pkg/util/ktime" "github.com/DataDog/datadog-go/v5/statsd" ) +func newFakeForkEvent(ppid, pid int, inode uint64, resolver *EBPFResolver) *model.Event { + e := model.NewFakeEvent() + e.Type = uint32(model.ForkEventType) + e.ProcessCacheEntry = resolver.NewProcessCacheEntry(model.PIDContext{Pid: uint32(pid), Tid: uint32(pid)}) + e.PIDContext = e.ProcessCacheEntry.PIDContext + e.ProcessContext = &e.ProcessCacheEntry.ProcessContext + e.ProcessCacheEntry.ForkTime = time.Now() + e.ProcessCacheEntry.PPid = uint32(ppid) + e.ProcessCacheEntry.Pid = uint32(pid) + e.ProcessCacheEntry.FileEvent.Inode = inode + return e +} + +func newFakeExecEvent(ppid, pid int, inode uint64, resolver *EBPFResolver) *model.Event { + e := model.NewFakeEvent() + e.Type = uint32(model.ExecEventType) + e.ProcessCacheEntry = resolver.NewProcessCacheEntry(model.PIDContext{Pid: uint32(pid), Tid: uint32(pid)}) + e.PIDContext = e.ProcessCacheEntry.PIDContext + e.ProcessContext = &e.ProcessCacheEntry.ProcessContext + e.ProcessCacheEntry.ExecTime = time.Now() + e.ProcessCacheEntry.PPid = uint32(ppid) + e.ProcessCacheEntry.Pid = uint32(pid) + e.ProcessCacheEntry.FileEvent.Inode = inode + return e +} + +func exit(event *model.Event) { + event.Type = uint32(model.ExitEventType) +} + func testCacheSize(t *testing.T, resolver *EBPFResolver) { err := retry.Do( func() error { @@ -33,701 +67,717 @@ func testCacheSize(t *testing.T, resolver *EBPFResolver) { assert.NoError(t, err) } -func TestFork1st(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) +func newResolver() (*EBPFResolver, error) { + timeResolver, err := ktime.NewResolver() + if err != nil { + return nil, err + } + + cgroupsResolver, err := cgroup.NewResolver() + if err != nil { + return nil, err + } + + userGroupResolver, err := usergroup.NewResolver(cgroupsResolver) + if err != nil { + return nil, err + } + + resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, cgroupsResolver, userGroupResolver, timeResolver, &path.NoOpResolver{}, nil, NewResolverOpts()) if err != nil { - t.Fatal(err) + return nil, err } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() + return resolver, nil +} + +func TestFork1st(t *testing.T) { + + resolver, err := newResolver() + if err != nil { + t.Fatal() + } - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // parent - resolver.DeleteEntry(child.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[child.Pid]) - assert.Equal(t, 1, len(resolver.entryCache)) + // X(pid:3) + exit(child) + resolver.ApplyExitEntry(child, nil) + resolver.DeleteEntry(child.ProcessCacheEntry.Pid, child.ResolveEventTime()) - // nothing - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Zero(t, len(resolver.entryCache)) + assert.Nil(t, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) + // nothing in the entryCache + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessCacheEntry.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 0, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestFork2nd(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent] - // \ child - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - // nothing - resolver.DeleteEntry(child.Pid, time.Now()) - assert.Zero(t, len(resolver.entryCache)) + // nothing in the entryCache + exit(child) + resolver.ApplyExitEntry(child, nil) + resolver.DeleteEntry(child.ProcessContext.Pid, child.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 0, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestForkExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + exec := newFakeExecEvent(3, 4, 456, resolver) - exec := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec.PPid = child.PPid - exec.FileEvent.Inode = 123 - exec.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // parent - // \ [child] -> exec - resolver.AddExecEntry(exec, 0) - assert.Equal(t, exec, resolver.entryCache[exec.Pid]) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec) + assert.Equal(t, exec.ProcessCacheEntry, resolver.entryCache[exec.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, child, exec.Ancestor) - assert.Equal(t, parent, exec.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // [parent] - // \ [child] -> exec - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child, exec.Ancestor) - assert.Equal(t, parent, exec.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor.Ancestor) - // nothing - resolver.DeleteEntry(exec.Pid, time.Now()) + // nothing in the entryCache + exit(child) + resolver.ApplyExitEntry(child, nil) + resolver.DeleteEntry(child.ProcessContext.Pid, child.ResolveEventTime()) assert.Zero(t, len(resolver.entryCache)) - testCacheSize(t, resolver) } func TestOrphanExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + exec := newFakeExecEvent(3, 4, 456, resolver) - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() - - exec := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec.Pid = child.Pid - exec.PPid = child.PPid - exec.FileEvent.Inode = 123 - exec.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent] - // \ child - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - // [parent] - // \ [child] -> exec - resolver.AddExecEntry(exec, 0) - assert.Equal(t, exec, resolver.entryCache[exec.Pid]) + // [X(pid:3)] + // | + // X(pid:4) --> Y(pid:4) + resolver.AddExecEntry(exec) + assert.Equal(t, exec.ProcessCacheEntry, resolver.entryCache[exec.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child, exec.Ancestor) - assert.Equal(t, parent, exec.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // nothing - resolver.DeleteEntry(exec.Pid, time.Now()) + // nothing in the entryCache + exit(exec) + resolver.ApplyExitEntry(exec, nil) + resolver.DeleteEntry(exec.ProcessCacheEntry.Pid, time.Now()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestForkExecExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() - - exec1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec1.PPid = child.PPid - exec1.FileEvent.Inode = 123 - exec1.ExecTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + exec1 := newFakeExecEvent(3, 4, 456, resolver) + exec2 := newFakeExecEvent(3, 4, 789, resolver) - exec2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec2.Pid = child.Pid - exec2.PPid = child.PPid - exec2.FileEvent.Inode = 456 - exec2.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent] - // \ child - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - // [parent] - // \ [child] -> exec1 - resolver.AddExecEntry(exec1, 0) - assert.Equal(t, exec1, resolver.entryCache[exec1.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec1) + assert.Equal(t, exec1.ProcessCacheEntry, resolver.entryCache[exec1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child, exec1.Ancestor) - assert.Equal(t, parent, exec1.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // [parent] - // \ [child] -> [exec1] -> exec2 - resolver.AddExecEntry(exec2, 0) - assert.Equal(t, exec2, resolver.entryCache[exec2.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) -- Z(pid:4) + resolver.AddExecEntry(exec2) + assert.Equal(t, exec2.ProcessCacheEntry, resolver.entryCache[exec2.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, exec1, exec2.Ancestor) - assert.Equal(t, child, exec2.Ancestor.Ancestor) - assert.Equal(t, parent, exec2.Ancestor.Ancestor.Ancestor) + assert.Equal(t, exec1.ProcessCacheEntry, exec2.ProcessCacheEntry.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec2.ProcessCacheEntry.Ancestor.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec2.ProcessCacheEntry.Ancestor.Ancestor.Ancestor) assert.EqualValues(t, 4, resolver.cacheSize.Load()) - // nothing - resolver.DeleteEntry(exec2.Pid, time.Now()) + // nothing in the entryCache in the entryCache + exit(exec2) + resolver.ApplyExitEntry(exec2, nil) + resolver.DeleteEntry(exec1.ProcessCacheEntry.Pid, time.Now()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestForkReuse(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent1.ForkTime = time.Now() - - child1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child1.PPid = parent1.Pid - child1.ForkTime = time.Now() - - exec1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child1.Pid, Tid: child1.Pid}) - exec1.PPid = child1.PPid - exec1.FileEvent.Inode = 123 - exec1.ExecTime = time.Now() + parent1 := newFakeForkEvent(0, 3, 123, resolver) + child1 := newFakeForkEvent(3, 4, 123, resolver) + exec1 := newFakeExecEvent(3, 4, 456, resolver) + parent2 := newFakeForkEvent(0, 3, 123, resolver) + child2 := newFakeForkEvent(3, 5, 123, resolver) - parent2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent2.ForkTime = time.Now() - - child2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 3, Tid: 3}) - child2.PPid = parent2.Pid - child2.ForkTime = time.Now() - - // parent1 - resolver.AddForkEntry(parent1, 0, nil) - assert.Equal(t, parent1, resolver.entryCache[parent1.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent1, nil) + assert.Equal(t, parent1.ProcessCacheEntry, resolver.entryCache[parent1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent1 - // \ child1 - resolver.AddForkEntry(child1, 0, nil) - assert.Equal(t, child1, resolver.entryCache[child1.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child1, nil) + assert.Equal(t, child1.ProcessCacheEntry, resolver.entryCache[child1.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent1, child1.Ancestor) + assert.Equal(t, parent1.ProcessCacheEntry, child1.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent1] - // \ child1 - resolver.DeleteEntry(parent1.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent1.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent1) + resolver.ApplyExitEntry(parent1, nil) + resolver.DeleteEntry(parent1.ProcessContext.Pid, parent1.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent1, child1.Ancestor) + assert.Equal(t, parent1.ProcessCacheEntry, child1.ProcessCacheEntry.Ancestor) - // [parent1] - // \ [child1] -> exec1 - resolver.AddExecEntry(exec1, 0) - assert.Equal(t, exec1, resolver.entryCache[exec1.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec1) + assert.Equal(t, exec1.ProcessCacheEntry, resolver.entryCache[exec1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child1, exec1.Ancestor) - assert.Equal(t, parent1, exec1.Ancestor.Ancestor) + assert.Equal(t, child1.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent1.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // [parent1:pid1] - // \ [child1] -> exec1 + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) // - // parent2:pid1 - resolver.AddForkEntry(parent2, 0, nil) - assert.Equal(t, parent2, resolver.entryCache[parent2.Pid]) + // Z(pid:3) + resolver.AddForkEntry(parent2, nil) + assert.Equal(t, parent2.ProcessCacheEntry, resolver.entryCache[parent2.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) assert.EqualValues(t, 4, resolver.cacheSize.Load()) - // [parent1:pid1] - // \ [child1] -> exec1 + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) // - // parent2:pid1 - // \ child2 - resolver.AddForkEntry(child2, 0, nil) - assert.Equal(t, child2, resolver.entryCache[child2.Pid]) + // Z(pid:3) + // | + // T(pid:5) + resolver.AddForkEntry(child2, nil) + assert.Equal(t, child2.ProcessCacheEntry, resolver.entryCache[child2.ProcessCacheEntry.Pid]) assert.Equal(t, 3, len(resolver.entryCache)) - assert.Equal(t, parent2, child2.Ancestor) + assert.Equal(t, parent2.ProcessCacheEntry, child2.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 5, resolver.cacheSize.Load()) - // parent2:pid1 - // \ child2 - resolver.DeleteEntry(exec1.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[exec1.Pid]) + // Z(pid:3) + // | + // T(pid:5) + exit(exec1) + resolver.ApplyExitEntry(exec1, nil) + resolver.DeleteEntry(exec1.ProcessContext.Pid, exec1.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[exec1.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - // [parent2:pid1] - // \ child2 - resolver.DeleteEntry(parent2.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent2.Pid]) + // [Z(pid:3)] + // | + // T(pid:5) + exit(parent2) + resolver.ApplyExitEntry(parent2, nil) + resolver.DeleteEntry(parent2.ProcessContext.Pid, parent2.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent2.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent2, child2.Ancestor) + assert.Equal(t, parent2.ProcessCacheEntry, child2.ProcessCacheEntry.Ancestor) - // nothing - resolver.DeleteEntry(child2.Pid, time.Now()) + // nothing in the entryCache + exit(child2) + resolver.ApplyExitEntry(child2, nil) + resolver.DeleteEntry(child2.ProcessCacheEntry.Pid, child2.ResolveEventTime()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestForkForkExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + grandChild := newFakeForkEvent(4, 5, 123, resolver) + childExec := newFakeExecEvent(3, 4, 456, resolver) - grandChild := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 3, Tid: 3}) - grandChild.PPid = child.Pid - grandChild.ForkTime = time.Now() - - childExec := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - childExec.Pid = child.Pid - childExec.PPid = child.PPid - childExec.FileEvent.Inode = 123 - childExec.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) - - // parent - // \ child - // \ grandChild - resolver.AddForkEntry(grandChild, 0, nil) - assert.Equal(t, grandChild, resolver.entryCache[grandChild.Pid]) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) + + // X(pid:3) + // | + // X(pid:4) + // | + // X(pid:5) + resolver.AddForkEntry(grandChild, nil) + assert.Equal(t, grandChild.ProcessCacheEntry, resolver.entryCache[grandChild.ProcessCacheEntry.Pid]) assert.Equal(t, 3, len(resolver.entryCache)) - assert.Equal(t, child, grandChild.Ancestor) - assert.Equal(t, parent, grandChild.Ancestor.Ancestor) - - // parent - // \ [child] -> childExec - // \ grandChild - resolver.AddExecEntry(childExec, 0) - assert.Equal(t, childExec, resolver.entryCache[childExec.Pid]) + assert.Equal(t, child.ProcessCacheEntry, grandChild.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, grandChild.ProcessCacheEntry.Ancestor.Ancestor) + + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) + // | + // X(pid:5) + resolver.AddExecEntry(childExec) + assert.Equal(t, childExec.ProcessCacheEntry, resolver.entryCache[childExec.ProcessCacheEntry.Pid]) assert.Equal(t, 3, len(resolver.entryCache)) - assert.Equal(t, child, childExec.Ancestor) - assert.Equal(t, parent, childExec.Ancestor.Ancestor) - assert.Equal(t, child, grandChild.Ancestor) - assert.Equal(t, parent, grandChild.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, childExec.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, childExec.ProcessCacheEntry.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, grandChild.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, grandChild.ProcessCacheEntry.Ancestor.Ancestor) // [parent] // \ [child] -> childExec // \ grandChild - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + // | + // X(pid:5) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - // [parent] - // \ [child] - // \ grandChild - resolver.DeleteEntry(childExec.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[childExec.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + // | + // X(pid:5) + exit(childExec) + resolver.ApplyExitEntry(childExec, nil) + resolver.DeleteEntry(childExec.ProcessContext.Pid, childExec.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[childExec.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - // nothing - resolver.DeleteEntry(grandChild.Pid, time.Now()) + // nothing in the entryCache + exit(grandChild) + resolver.ApplyExitEntry(grandChild, nil) + resolver.DeleteEntry(grandChild.ProcessContext.Pid, grandChild.ResolveEventTime()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestExecBomb(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.ForkTime = time.Now() + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + exec1 := newFakeExecEvent(3, 4, 456, resolver) + exec2 := newFakeExecEvent(3, 4, 456, resolver) - exec1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec1.PPid = child.PPid - exec1.FileEvent.Inode = 123 - exec1.ExecTime = time.Now() - - exec2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: child.Pid, Tid: child.Pid}) - exec2.Pid = child.Pid - exec2.PPid = child.PPid - exec2.FileEvent.Inode = 123 - exec2.ExecTime = time.Now() - - // parent - resolver.AddForkEntry(parent, 0, nil) - assert.Equal(t, parent, resolver.entryCache[parent.Pid]) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.EqualValues(t, 1, resolver.cacheSize.Load()) - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) - assert.Equal(t, child, resolver.entryCache[child.Pid]) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) assert.Equal(t, 2, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) assert.EqualValues(t, 2, resolver.cacheSize.Load()) - // [parent] - // \ child - resolver.DeleteEntry(parent.Pid, time.Now()) - assert.Nil(t, resolver.entryCache[parent.Pid]) + // [X(pid:3)] + // | + // X(pid:4) + exit(parent) + resolver.ApplyExitEntry(parent, nil) + resolver.DeleteEntry(parent.ProcessContext.Pid, parent.ResolveEventTime()) + assert.Nil(t, resolver.entryCache[parent.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, parent, child.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - // [parent] - // \ [child] -> exec1 - resolver.AddExecEntry(exec1, 0) - assert.Equal(t, exec1, resolver.entryCache[exec1.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec1) + assert.Equal(t, exec1.ProcessCacheEntry, resolver.entryCache[exec1.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, child, exec1.Ancestor) - assert.Equal(t, parent, exec1.Ancestor.Ancestor) + assert.Equal(t, child.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor) + assert.Equal(t, parent.ProcessCacheEntry, exec1.ProcessCacheEntry.Ancestor.Ancestor) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // [parent] - // \ [child] -> [exec1] -> exec2 - resolver.AddExecEntry(exec2, 0) - assert.Equal(t, exec1, resolver.entryCache[exec2.Pid]) + // [X(pid:3)] + // | + // X(pid:4) -- Y(pid:4) -- Y(pid:4) + resolver.AddExecEntry(exec2) + assert.Equal(t, exec1.ProcessCacheEntry, resolver.entryCache[exec2.ProcessCacheEntry.Pid]) assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, exec1.ExecTime, exec2.ExecTime) + assert.Equal(t, exec1.ProcessCacheEntry.ExecTime, exec2.ProcessCacheEntry.ExecTime) assert.EqualValues(t, 3, resolver.cacheSize.Load()) - // nothing - resolver.DeleteEntry(exec1.Pid, time.Now()) + // nothing in the entryCache + exit(exec1) + resolver.ApplyExitEntry(exec1, nil) + resolver.DeleteEntry(exec1.ProcessContext.Pid, exec1.ResolveEventTime()) assert.Zero(t, len(resolver.entryCache)) testCacheSize(t, resolver) } func TestExecLostFork(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 11, Tid: 11}) - parent.FileEvent.BasenameStr = "agent" - parent.ForkTime = time.Now() - parent.FileEvent.Inode = 1 - parent.ExecInode = 1 - - // parent - resolver.AddForkEntry(parent, 0, nil) - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 22, Tid: 22}) - child.PPid = parent.Pid - child.FileEvent.Inode = 1 - - // parent - // \ child - resolver.AddForkEntry(child, parent.ExecInode, nil) - - assert.Equal(t, "agent", child.FileEvent.BasenameStr) - assert.False(t, child.IsParentMissing) - - // exec loss with inode 2 - - child1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 33, Tid: 33}) - child1.FileEvent.BasenameStr = "sh" - child1.PPid = child.Pid - child1.ExecInode = 2 - - // parent - // \ child - // \ child1 - resolver.AddForkEntry(child1, child1.ExecInode, nil) + parent := newFakeForkEvent(0, 3, 123, resolver) + parent.ProcessCacheEntry.FileEvent.BasenameStr = "agent" + child := newFakeForkEvent(3, 4, 123, resolver) + child.PIDContext.ExecInode = 123 // ExecInode == Inode Parent + child1 := newFakeForkEvent(4, 5, 123, resolver) + child1.ProcessCacheEntry.FileEvent.BasenameStr = "sh" + child1.PIDContext.ExecInode = 456 // ExecInode != Inode parent + + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, "agent", child1.FileEvent.BasenameStr) - assert.True(t, child1.IsParentMissing) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 2, len(resolver.entryCache)) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) + assert.Equal(t, "agent", child.ProcessCacheEntry.FileEvent.BasenameStr) + assert.False(t, child.ProcessCacheEntry.IsParentMissing) + + // X(pid:3) + // | + // X(pid:4) + // {|} + // X(pid:5) + resolver.AddForkEntry(child1, nil) + assert.Equal(t, "agent", child1.ProcessCacheEntry.FileEvent.BasenameStr) + assert.True(t, child1.ProcessCacheEntry.IsParentMissing) } func TestExecLostExec(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 11, Tid: 11}) - parent.FileEvent.BasenameStr = "agent" - parent.ForkTime = time.Now() - parent.FileEvent.Inode = 1 - parent.ExecInode = 1 - - // parent - resolver.AddForkEntry(parent, 0, nil) - - child1 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 22, Tid: 22}) - child1.PPid = parent.Pid - child1.FileEvent.Inode = 1 - child1.ExecInode = 1 - - // parent - // \ child1 - resolver.AddForkEntry(child1, parent.ExecInode, nil) - - assert.Equal(t, "agent", child1.FileEvent.BasenameStr) - assert.False(t, child1.IsParentMissing) - - // exec loss with inode 2 and pid 22 - - child2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 33, Tid: 33}) - child2.FileEvent.BasenameStr = "sh" - child2.PPid = child1.Pid - child2.ExecInode = 2 - - // parent - // \ child1 - // \ child2 - resolver.AddForkEntry(child2, child2.ExecInode, nil) + parent := newFakeForkEvent(0, 3, 123, resolver) + parent.ProcessCacheEntry.FileEvent.BasenameStr = "agent" + child1 := newFakeForkEvent(3, 4, 123, resolver) + child1.PIDContext.ExecInode = 123 // ExecInode == Inode Parent + child2 := newFakeExecEvent(3, 4, 456, resolver) + child2.ProcessCacheEntry.FileEvent.BasenameStr = "sh" + child2.PIDContext.ExecInode = 456 // ExecInode != Inode Ancestor + + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) - assert.Equal(t, "agent", child2.FileEvent.BasenameStr) - assert.True(t, child2.IsParentMissing) + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child1, nil) + assert.Equal(t, child1.ProcessCacheEntry, resolver.entryCache[child1.ProcessCacheEntry.Pid]) + assert.Equal(t, 2, len(resolver.entryCache)) + assert.Equal(t, parent.ProcessCacheEntry, child1.ProcessCacheEntry.Ancestor) + assert.Equal(t, "agent", child1.ProcessCacheEntry.FileEvent.BasenameStr) + assert.False(t, child1.ProcessCacheEntry.IsParentMissing) + + // X(pid:3) + // | + // X(pid:4) -**- Y(pid:4) + resolver.AddExecEntry(child2) + assert.NotEqual(t, "agent", child2.ProcessCacheEntry.FileEvent.BasenameStr) + assert.True(t, child2.ProcessCacheEntry.IsParentMissing) } func TestIsExecExecRuntime(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - parent.FileEvent.Inode = 1 - - // parent - resolver.AddForkEntry(parent, 0, nil) + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + child2 := newFakeExecEvent(3, 4, 456, resolver) + child3 := newFakeExecEvent(3, 4, 789, resolver) + child4 := newFakeExecEvent(3, 4, 101112, resolver) - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.FileEvent.Inode = 1 - - // parent - // \ child - resolver.AddForkEntry(child, 0, nil) + // X(pid:3) + resolver.AddForkEntry(parent, nil) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) - // parent - // \ child - // \ child2 + // X(pid:3) + // | + // X(pid:4) + resolver.AddForkEntry(child, nil) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 2, len(resolver.entryCache)) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - child2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child2.FileEvent.Inode = 2 - child2.PPid = child.Pid - resolver.AddExecEntry(child2, 0) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(child2) - // parent - // \ child - // \ child2 - // \ child3 + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) -- Z(pid:4) + resolver.AddExecEntry(child3) - child3 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child3.FileEvent.Inode = 3 - child3.PPid = child2.Pid - resolver.AddExecEntry(child3, 0) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) -- Z(pid:4) -- T(pid:4) + resolver.AddExecEntry(child4) - assert.False(t, parent.IsExecExec) - assert.False(t, parent.IsExec) + assert.False(t, parent.ProcessCacheEntry.IsExecExec) + assert.False(t, parent.ProcessCacheEntry.IsExec) - assert.False(t, child.IsExecExec) - assert.False(t, child.IsExec) + assert.False(t, child.ProcessCacheEntry.IsExecExec) + assert.False(t, child.ProcessCacheEntry.IsExec) - assert.False(t, child2.IsExecExec) - assert.True(t, child2.IsExec) + assert.False(t, child2.ProcessCacheEntry.IsExecExec) + assert.True(t, child2.ProcessCacheEntry.IsExec) - assert.True(t, child3.IsExecExec) - assert.True(t, child3.IsExec) + assert.True(t, child3.ProcessCacheEntry.IsExecExec) + assert.True(t, child3.ProcessCacheEntry.IsExec) - child4 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child4.FileEvent.Inode = 3 - child4.PPid = child3.Pid - resolver.AddExecEntry(child4, 0) + assert.True(t, child4.ProcessCacheEntry.IsExecExec) + assert.True(t, child4.ProcessCacheEntry.IsExec) - assert.True(t, child3.IsExecExec) - assert.True(t, child3.IsExec) } func TestIsExecExecSnapshot(t *testing.T) { - resolver, err := NewEBPFResolver(nil, nil, &statsd.NoOpClient{}, nil, nil, nil, nil, nil, nil, nil, nil, NewResolverOpts()) + + resolver, err := newResolver() if err != nil { - t.Fatal(err) + t.Fatal() } - parent := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 1, Tid: 1}) - parent.ForkTime = time.Now() - parent.FileEvent.Inode = 1 - - // parent - resolver.insertEntry(parent, nil, model.ProcessCacheEntryFromSnapshot) - - child := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child.PPid = parent.Pid - child.FileEvent.Inode = 2 + parent := newFakeForkEvent(0, 3, 123, resolver) + child := newFakeForkEvent(3, 4, 123, resolver) + child2 := newFakeExecEvent(3, 4, 456, resolver) + child3 := newFakeExecEvent(3, 4, 769, resolver) - // parent - // \ child - - resolver.setAncestor(child) - resolver.insertEntry(child, nil, model.ProcessCacheEntryFromSnapshot) + // X(pid:3) + resolver.insertEntry(parent.ProcessCacheEntry, nil, model.ProcessCacheEntryFromSnapshot) + assert.Equal(t, parent.ProcessCacheEntry, resolver.entryCache[parent.ProcessCacheEntry.Pid]) + assert.Equal(t, 1, len(resolver.entryCache)) - assert.False(t, parent.IsExecExec) - assert.False(t, parent.IsExec) + // X(pid:3) + // | + // X(pid:4) + resolver.setAncestor(child.ProcessCacheEntry) + resolver.insertEntry(child.ProcessCacheEntry, nil, model.ProcessCacheEntryFromSnapshot) + assert.Equal(t, child.ProcessCacheEntry, resolver.entryCache[child.ProcessCacheEntry.Pid]) + assert.Equal(t, 2, len(resolver.entryCache)) + assert.Equal(t, parent.ProcessCacheEntry, child.ProcessCacheEntry.Ancestor) - assert.False(t, child.IsExecExec) - assert.False(t, child.IsExec) + assert.False(t, parent.ProcessCacheEntry.IsExecExec) + assert.False(t, parent.ProcessCacheEntry.IsExec) - // parent - // \ child - // \ child2 + assert.False(t, child.ProcessCacheEntry.IsExecExec) + assert.False(t, child.ProcessCacheEntry.IsExec) - child2 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child2.FileEvent.Inode = 3 - child2.PPid = child.Pid - resolver.AddExecEntry(child2, 0) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) + resolver.AddExecEntry(child2) - assert.False(t, child2.IsExecExec) - assert.True(t, child2.IsExec) + assert.False(t, child2.ProcessCacheEntry.IsExecExec) + assert.True(t, child2.ProcessCacheEntry.IsExec) - child3 := resolver.NewProcessCacheEntry(model.PIDContext{Pid: 2, Tid: 2}) - child3.FileEvent.Inode = 4 - child3.PPid = child2.Pid - resolver.AddExecEntry(child3, 0) + // X(pid:3) + // | + // X(pid:4) -- Y(pid:4) -- Z(pid:4) + resolver.AddExecEntry(child3) - assert.True(t, child3.IsExecExec) - assert.True(t, child3.IsExec) + assert.True(t, child3.ProcessCacheEntry.IsExecExec) + assert.True(t, child3.ProcessCacheEntry.IsExec) } diff --git a/pkg/security/resolvers/resolvers_ebpf.go b/pkg/security/resolvers/resolvers_ebpf.go index b8899bbd99adf..c9adeb82e557b 100644 --- a/pkg/security/resolvers/resolvers_ebpf.go +++ b/pkg/security/resolvers/resolvers_ebpf.go @@ -219,6 +219,10 @@ func (r *EBPFResolvers) Start(ctx context.Context) error { // ResolveCGroupContext resolves the cgroup context from a cgroup path key func (r *EBPFResolvers) ResolveCGroupContext(pathKey model.PathKey, cgroupFlags containerutils.CGroupFlags) (*model.CGroupContext, error) { + if cgroupContext, found := r.CGroupResolver.GetCGroupContext(pathKey); found { + return cgroupContext, nil + } + path, err := r.DentryResolver.Resolve(pathKey, true) if err != nil { return nil, fmt.Errorf("failed to resolve cgroup file %v: %w", pathKey, err) diff --git a/pkg/security/secl/containerutils/cgroup.go b/pkg/security/secl/containerutils/cgroup.go index 74e7a64540c84..9cc6cca12d884 100644 --- a/pkg/security/secl/containerutils/cgroup.go +++ b/pkg/security/secl/containerutils/cgroup.go @@ -24,6 +24,12 @@ const ( CGroupManagerSystemd // systemd ) +// CGroup flags +const ( + SystemdService CGroupFlags = (0 << 8) + SystemdScope CGroupFlags = (1 << 8) +) + const ( // ContainerRuntimeDocker is used to specify that a container is managed by Docker ContainerRuntimeDocker = "docker" diff --git a/pkg/security/secl/containerutils/helpers.go b/pkg/security/secl/containerutils/helpers.go index 0e46e87af7f2a..311322f94ffc6 100644 --- a/pkg/security/secl/containerutils/helpers.go +++ b/pkg/security/secl/containerutils/helpers.go @@ -29,39 +29,42 @@ func init() { containerIDPattern = regexp.MustCompile(ContainerIDPatternStr) } -func isSystemdCgroup(cgroup CGroupID) bool { - return strings.HasSuffix(string(cgroup), ".service") || strings.HasSuffix(string(cgroup), ".scope") +func isSystemdScope(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".scope") +} + +func isSystemdService(cgroup CGroupID) bool { + return strings.HasSuffix(string(cgroup), ".service") +} + +func getSystemdCGroupFlags(cgroup CGroupID) uint64 { + if isSystemdScope(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdScope) + } else if isSystemdService(cgroup) { + return uint64(CGroupManagerSystemd) | uint64(SystemdService) + } + return 0 } // FindContainerID extracts the first sub string that matches the pattern of a container ID along with the container flags induced from the container runtime prefix func FindContainerID(s CGroupID) (ContainerID, uint64) { match := containerIDPattern.FindIndex([]byte(s)) if match == nil { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - - return "", 0 + return "", getSystemdCGroupFlags(s) } // first, check what's before if match[0] != 0 { previousChar := string(s[match[0]-1]) if strings.ContainsAny(previousChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } // then, check what's after if match[1] < len(s) { nextChar := string(s[match[1]]) if strings.ContainsAny(nextChar, containerIDCoreChars) { - if isSystemdCgroup(s) { - return "", uint64(CGroupManagerSystemd) - } - return "", 0 + return "", getSystemdCGroupFlags(s) } } diff --git a/pkg/security/secl/containerutils/helpers_test.go b/pkg/security/secl/containerutils/helpers_test.go index b5474b0df5016..5a2066e0295f2 100644 --- a/pkg/security/secl/containerutils/helpers_test.go +++ b/pkg/security/secl/containerutils/helpers_test.go @@ -63,7 +63,7 @@ func TestFindContainerID(t *testing.T) { { // Some random path which could match garden format input: "/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-f9176c6a-2a34-4ce2-86af-60d16888ed8e.scope", output: "", - flags: CGroupManagerSystemd, + flags: CGroupManagerSystemd | CGroupManager(SystemdScope), }, { // GARDEN with prefix / suffix input: "prefix01234567-0123-4567-890a-bcdesuffix", diff --git a/pkg/security/secl/go.mod b/pkg/security/secl/go.mod index 7a709c2832e48..da9770680ce9c 100644 --- a/pkg/security/secl/go.mod +++ b/pkg/security/secl/go.mod @@ -41,6 +41,7 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/pkg/security/secl/go.sum b/pkg/security/secl/go.sum index 267d639c9503a..b3daad8d9516f 100644 --- a/pkg/security/secl/go.sum +++ b/pkg/security/secl/go.sum @@ -83,8 +83,8 @@ golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= diff --git a/pkg/security/security_profile/dump/load_controller.go b/pkg/security/security_profile/dump/load_controller.go index ae8767bcfd153..f9bc80cdd3b4f 100644 --- a/pkg/security/security_profile/dump/load_controller.go +++ b/pkg/security/security_profile/dump/load_controller.go @@ -16,6 +16,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/metrics" + "github.com/DataDog/datadog-agent/pkg/security/secl/containerutils" "github.com/DataDog/datadog-agent/pkg/security/secl/model" "github.com/DataDog/datadog-agent/pkg/security/seclog" ) @@ -34,6 +35,7 @@ type ActivityDumpLoadController struct { // eBPF maps activityDumpConfigDefaults *ebpf.Map + activityDumpLoadConfig map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig } // NewActivityDumpLoadController returns a new activity dump load controller @@ -58,7 +60,11 @@ func NewActivityDumpLoadController(adm *ActivityDumpManager) (*ActivityDumpLoadC }, nil } -func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDumpLoadConfig { +func (lc *ActivityDumpLoadController) getDefaultLoadConfigs() (map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig, error) { + if lc.activityDumpLoadConfig != nil { + return lc.activityDumpLoadConfig, nil + } + defaults := NewActivityDumpLoadConfig( lc.adm.config.RuntimeSecurity.ActivityDumpTracedEventTypes, lc.adm.config.RuntimeSecurity.ActivityDumpCgroupDumpTimeout, @@ -68,14 +74,38 @@ func (lc *ActivityDumpLoadController) getDefaultLoadConfig() *model.ActivityDump lc.adm.resolvers.TimeResolver, ) defaults.WaitListTimestampRaw = uint64(lc.adm.config.RuntimeSecurity.ActivityDumpCgroupWaitListTimeout) - return defaults + + allDefaultConfigs := map[string]containerutils.CGroupManager{ + containerutils.CGroupManagerDocker.String(): containerutils.CGroupManagerDocker, + containerutils.CGroupManagerPodman.String(): containerutils.CGroupManagerPodman, + containerutils.CGroupManagerCRI.String(): containerutils.CGroupManagerCRI, + containerutils.CGroupManagerCRIO.String(): containerutils.CGroupManagerCRIO, + containerutils.CGroupManagerSystemd.String(): containerutils.CGroupManagerSystemd, + } + defaultConfigs := make(map[containerutils.CGroupManager]*model.ActivityDumpLoadConfig) + for _, cgroupManager := range lc.adm.config.RuntimeSecurity.ActivityDumpCgroupsManagers { + cgroupManager, found := allDefaultConfigs[cgroupManager] + if !found { + return nil, fmt.Errorf("unsupported cgroup manager '%s'", cgroupManager) + } + defaultConfigs[cgroupManager] = defaults + } + lc.activityDumpLoadConfig = defaultConfigs + return defaultConfigs, nil } -// PushCurrentConfig pushes the current load controller config to kernel space -func (lc *ActivityDumpLoadController) PushCurrentConfig() error { +// PushDefaultCurrentConfigs pushes the current load controller configs to kernel space +func (lc *ActivityDumpLoadController) PushDefaultCurrentConfigs() error { + defaultConfigs, err := lc.getDefaultLoadConfigs() + if err != nil { + return err + } + // push default load config values - if err := lc.activityDumpConfigDefaults.Put(uint32(0), lc.getDefaultLoadConfig()); err != nil { - return fmt.Errorf("couldn't update default activity dump load config: %w", err) + for cgroupManager, defaultConfig := range defaultConfigs { + if err := lc.activityDumpConfigDefaults.Put(uint32(cgroupManager), defaultConfig); err != nil { + return fmt.Errorf("couldn't update default activity dump load config: %w", err) + } } return nil } diff --git a/pkg/security/security_profile/dump/manager.go b/pkg/security/security_profile/dump/manager.go index 079f45c13bc21..be81a28ca5c4d 100644 --- a/pkg/security/security_profile/dump/manager.go +++ b/pkg/security/security_profile/dump/manager.go @@ -321,7 +321,8 @@ func NewActivityDumpManager(config *config.Config, statsdClient statsd.ClientInt if err != nil { return nil, fmt.Errorf("couldn't instantiate the activity dump load controller: %w", err) } - if err = loadController.PushCurrentConfig(); err != nil { + + if err = loadController.PushDefaultCurrentConfigs(); err != nil { return nil, fmt.Errorf("failed to push load controller config settings to kernel space: %w", err) } adm.loadController = loadController @@ -449,8 +450,8 @@ func (adm *ActivityDumpManager) HandleCGroupTracingEvent(event *model.CgroupTrac adm.Lock() defer adm.Unlock() - if len(event.ContainerContext.ContainerID) == 0 { - seclog.Warnf("received a cgroup tracing event with an empty container ID") + if len(event.CGroupContext.CGroupID) == 0 { + seclog.Warnf("received a cgroup tracing event with an empty cgroup ID") return } @@ -514,7 +515,19 @@ workloadLoop: } // if we're still here, we can start tracing this workload - if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *adm.loadController.getDefaultLoadConfig()); err != nil { + defaultConfigs, err := adm.loadController.getDefaultLoadConfigs() + if err != nil { + seclog.Errorf("%v", err) + continue + } + + defaultConfig, found := defaultConfigs[containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags)] + if !found { + seclog.Errorf("Failed to find default activity dump config for %s", containerutils.CGroupManager(workloads[0].CGroupContext.CGroupFlags).String()) + continue + } + + if err := adm.startDumpWithConfig(workloads[0].ContainerID, workloads[0].CGroupContext, utils.NewCookie(), *defaultConfig); err != nil { if !errors.Is(err, unix.E2BIG) { seclog.Debugf("%v", err) break diff --git a/pkg/security/security_profile/dump/storage_manager.go b/pkg/security/security_profile/dump/storage_manager.go index 454309068fad4..ab0382257b6de 100644 --- a/pkg/security/security_profile/dump/storage_manager.go +++ b/pkg/security/security_profile/dump/storage_manager.go @@ -33,12 +33,15 @@ type ActivityDumpStorage interface { type ActivityDumpStorageManager struct { statsdClient statsd.ClientInterface storages map[config.StorageType]ActivityDumpStorage + isFowarder bool } // NewAgentStorageManager returns a new instance of ActivityDumpStorageManager -func NewAgentStorageManager() (*ActivityDumpStorageManager, error) { +func NewAgentStorageManager(statsdClient statsd.ClientInterface) (*ActivityDumpStorageManager, error) { manager := &ActivityDumpStorageManager{ - storages: make(map[config.StorageType]ActivityDumpStorage), + statsdClient: statsdClient, + storages: make(map[config.StorageType]ActivityDumpStorage), + isFowarder: false, } // create remote storage @@ -54,7 +57,8 @@ func NewAgentStorageManager() (*ActivityDumpStorageManager, error) { // NewAgentCommandStorageManager returns a new instance of ActivityDumpStorageManager func NewAgentCommandStorageManager(cfg *config.Config) (*ActivityDumpStorageManager, error) { manager := &ActivityDumpStorageManager{ - storages: make(map[config.StorageType]ActivityDumpStorage), + storages: make(map[config.StorageType]ActivityDumpStorage), + isFowarder: false, } storage, err := NewActivityDumpLocalStorage(cfg, nil) @@ -78,6 +82,7 @@ func NewActivityDumpStorageManager(cfg *config.Config, statsdClient statsd.Clien manager := &ActivityDumpStorageManager{ storages: make(map[config.StorageType]ActivityDumpStorage), statsdClient: statsdClient, + isFowarder: true, } storage, err := NewActivityDumpLocalStorage(cfg, m) @@ -133,7 +138,7 @@ func (manager *ActivityDumpStorageManager) PersistRaw(requests []config.StorageR } // send dump metric - if manager.statsdClient != nil { + if manager.statsdClient != nil && manager.isFowarder { if size := len(raw.Bytes()); size > 0 { tags := []string{"format:" + request.Format.String(), "storage_type:" + request.Type.String(), fmt.Sprintf("compression:%v", request.Compression)} if err := manager.statsdClient.Count(metrics.MetricActivityDumpSizeInBytes, int64(size), tags, 1.0); err != nil { diff --git a/pkg/security/security_profile/dump/storage_manager_unsupported.go b/pkg/security/security_profile/dump/storage_manager_unsupported.go index 885c1f2030808..3fee25e03615c 100644 --- a/pkg/security/security_profile/dump/storage_manager_unsupported.go +++ b/pkg/security/security_profile/dump/storage_manager_unsupported.go @@ -12,6 +12,8 @@ import ( "bytes" "fmt" + "github.com/DataDog/datadog-go/v5/statsd" + "github.com/DataDog/datadog-agent/pkg/security/config" "github.com/DataDog/datadog-agent/pkg/security/proto/api" ) @@ -28,7 +30,7 @@ func (manager *ActivityDumpStorageManager) PersistRaw(_ []config.StorageRequest, func (manager *ActivityDumpStorageManager) SendTelemetry() {} // NewAgentStorageManager returns a new instance of ActivityDumpStorageManager -func NewAgentStorageManager() (*ActivityDumpStorageManager, error) { +func NewAgentStorageManager(_ statsd.ClientInterface) (*ActivityDumpStorageManager, error) { return nil, fmt.Errorf("the activity dump manager is unsupported on this platform") } diff --git a/pkg/security/serializers/serializers_base.go b/pkg/security/serializers/serializers_base.go index 32f84ace271f8..805e52f8c67cc 100644 --- a/pkg/security/serializers/serializers_base.go +++ b/pkg/security/serializers/serializers_base.go @@ -18,15 +18,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/scrubber" ) -// CGroupContextSerializer serializes a cgroup context to JSON -// easyjson:json -type CGroupContextSerializer struct { - // CGroup ID - ID string `json:"id,omitempty"` - // CGroup manager - Manager string `json:"manager,omitempty"` -} - // ContainerContextSerializer serializes a container context to JSON // easyjson:json type ContainerContextSerializer struct { @@ -213,7 +204,6 @@ type BaseEventSerializer struct { *ExitEventSerializer `json:"exit,omitempty"` *ProcessContextSerializer `json:"process,omitempty"` *ContainerContextSerializer `json:"container,omitempty"` - *CGroupContextSerializer `json:"cgroup,omitempty"` } // TLSContextSerializer defines a tls context serializer diff --git a/pkg/security/serializers/serializers_base_linux_easyjson.go b/pkg/security/serializers/serializers_base_linux_easyjson.go index 743795c140527..8502c05b45842 100644 --- a/pkg/security/serializers/serializers_base_linux_easyjson.go +++ b/pkg/security/serializers/serializers_base_linux_easyjson.go @@ -1777,72 +1777,7 @@ func (v ContainerContextSerializer) MarshalEasyJSON(w *jwriter.Writer) { func (v *ContainerContextSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers14(l, v) } -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(in *jlexer.Lexer, out *CGroupContextSerializer) { - isTopLevel := in.IsStart() - if in.IsNull() { - if isTopLevel { - in.Consumed() - } - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeFieldName(false) - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "id": - out.ID = string(in.String()) - case "manager": - out.Manager = string(in.String()) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') - if isTopLevel { - in.Consumed() - } -} -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(out *jwriter.Writer, in CGroupContextSerializer) { - out.RawByte('{') - first := true - _ = first - if in.ID != "" { - const prefix string = ",\"id\":" - first = false - out.RawString(prefix[1:]) - out.String(string(in.ID)) - } - if in.Manager != "" { - const prefix string = ",\"manager\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - out.String(string(in.Manager)) - } - out.RawByte('}') -} - -// MarshalEasyJSON supports easyjson.Marshaler interface -func (v CGroupContextSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(w, v) -} - -// UnmarshalEasyJSON supports easyjson.Unmarshaler interface -func (v *CGroupContextSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(l, v) -} -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(in *jlexer.Lexer, out *BaseEventSerializer) { +func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(in *jlexer.Lexer, out *BaseEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -1855,7 +1790,6 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( out.ExitEventSerializer = new(ExitEventSerializer) out.ProcessContextSerializer = new(ProcessContextSerializer) out.ContainerContextSerializer = new(ContainerContextSerializer) - out.CGroupContextSerializer = new(CGroupContextSerializer) in.Delim('{') for !in.IsDelim('}') { key := in.UnsafeFieldName(false) @@ -1912,16 +1846,6 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( } (*out.ContainerContextSerializer).UnmarshalEasyJSON(in) } - case "cgroup": - if in.IsNull() { - in.Skip() - out.CGroupContextSerializer = nil - } else { - if out.CGroupContextSerializer == nil { - out.CGroupContextSerializer = new(CGroupContextSerializer) - } - (*out.CGroupContextSerializer).UnmarshalEasyJSON(in) - } default: in.SkipRecursive() } @@ -1932,7 +1856,7 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( in.Consumed() } } -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(out *jwriter.Writer, in BaseEventSerializer) { +func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(out *jwriter.Writer, in BaseEventSerializer) { out.RawByte('{') first := true _ = first @@ -1992,29 +1916,19 @@ func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16( } (*in.ContainerContextSerializer).MarshalEasyJSON(out) } - if in.CGroupContextSerializer != nil { - const prefix string = ",\"cgroup\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - (*in.CGroupContextSerializer).MarshalEasyJSON(out) - } out.RawByte('}') } // MarshalEasyJSON supports easyjson.Marshaler interface func (v BaseEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(w, v) + easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BaseEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(l, v) + easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers15(l, v) } -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(in *jlexer.Lexer, out *AWSSecurityCredentialsSerializer) { +func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(in *jlexer.Lexer, out *AWSSecurityCredentialsSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -2053,7 +1967,7 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17( in.Consumed() } } -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(out *jwriter.Writer, in AWSSecurityCredentialsSerializer) { +func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(out *jwriter.Writer, in AWSSecurityCredentialsSerializer) { out.RawByte('{') first := true _ = first @@ -2087,14 +2001,14 @@ func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17( // MarshalEasyJSON supports easyjson.Marshaler interface func (v AWSSecurityCredentialsSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(w, v) + easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *AWSSecurityCredentialsSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(l, v) + easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers16(l, v) } -func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(in *jlexer.Lexer, out *AWSIMDSEventSerializer) { +func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(in *jlexer.Lexer, out *AWSIMDSEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -2135,7 +2049,7 @@ func easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( in.Consumed() } } -func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(out *jwriter.Writer, in AWSIMDSEventSerializer) { +func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(out *jwriter.Writer, in AWSIMDSEventSerializer) { out.RawByte('{') first := true _ = first @@ -2154,10 +2068,10 @@ func easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18( // MarshalEasyJSON supports easyjson.Marshaler interface func (v AWSIMDSEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(w, v) + easyjsonA1e47abeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *AWSIMDSEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers18(l, v) + easyjsonA1e47abeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers17(l, v) } diff --git a/pkg/security/serializers/serializers_linux.go b/pkg/security/serializers/serializers_linux.go index 69b1cf0bd225a..4f38767025b47 100644 --- a/pkg/security/serializers/serializers_linux.go +++ b/pkg/security/serializers/serializers_linux.go @@ -81,6 +81,15 @@ type FileSerializer struct { MountOrigin string `json:"mount_origin,omitempty"` } +// CGroupContextSerializer serializes a cgroup context to JSON +// easyjson:json +type CGroupContextSerializer struct { + // CGroup ID + ID string `json:"id,omitempty"` + // CGroup manager + Manager string `json:"manager,omitempty"` +} + // UserContextSerializer serializes a user context to JSON // easyjson:json type UserContextSerializer struct { @@ -622,6 +631,7 @@ type EventSerializer struct { *NetworkContextSerializer `json:"network,omitempty"` *DDContextSerializer `json:"dd,omitempty"` *SecurityProfileContextSerializer `json:"security_profile,omitempty"` + *CGroupContextSerializer `json:"cgroup,omitempty"` *SELinuxEventSerializer `json:"selinux,omitempty"` *BPFEventSerializer `json:"bpf,omitempty"` diff --git a/pkg/security/serializers/serializers_linux_easyjson.go b/pkg/security/serializers/serializers_linux_easyjson.go index ec264f2cb6de4..e61dfb697a062 100644 --- a/pkg/security/serializers/serializers_linux_easyjson.go +++ b/pkg/security/serializers/serializers_linux_easyjson.go @@ -3595,6 +3595,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( out.NetworkContextSerializer = new(NetworkContextSerializer) out.DDContextSerializer = new(DDContextSerializer) out.SecurityProfileContextSerializer = new(SecurityProfileContextSerializer) + out.CGroupContextSerializer = new(CGroupContextSerializer) out.SELinuxEventSerializer = new(SELinuxEventSerializer) out.BPFEventSerializer = new(BPFEventSerializer) out.MMapEventSerializer = new(MMapEventSerializer) @@ -3652,6 +3653,16 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*out.SecurityProfileContextSerializer).UnmarshalEasyJSON(in) } + case "cgroup": + if in.IsNull() { + in.Skip() + out.CGroupContextSerializer = nil + } else { + if out.CGroupContextSerializer == nil { + out.CGroupContextSerializer = new(CGroupContextSerializer) + } + (*out.CGroupContextSerializer).UnmarshalEasyJSON(in) + } case "selinux": if in.IsNull() { in.Skip() @@ -3889,16 +3900,6 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*out.ContainerContextSerializer).UnmarshalEasyJSON(in) } - case "cgroup": - if in.IsNull() { - in.Skip() - out.CGroupContextSerializer = nil - } else { - if out.CGroupContextSerializer == nil { - out.CGroupContextSerializer = new(CGroupContextSerializer) - } - (*out.CGroupContextSerializer).UnmarshalEasyJSON(in) - } default: in.SkipRecursive() } @@ -3939,6 +3940,16 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*in.SecurityProfileContextSerializer).MarshalEasyJSON(out) } + if in.CGroupContextSerializer != nil { + const prefix string = ",\"cgroup\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + (*in.CGroupContextSerializer).MarshalEasyJSON(out) + } if in.SELinuxEventSerializer != nil { const prefix string = ",\"selinux\":" if first { @@ -4180,16 +4191,6 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers24( } (*in.ContainerContextSerializer).MarshalEasyJSON(out) } - if in.CGroupContextSerializer != nil { - const prefix string = ",\"cgroup\":" - if first { - first = false - out.RawString(prefix[1:]) - } else { - out.RawString(prefix) - } - (*in.CGroupContextSerializer).MarshalEasyJSON(out) - } out.RawByte('}') } @@ -4663,7 +4664,72 @@ func (v CapsetSerializer) MarshalEasyJSON(w *jwriter.Writer) { func (v *CapsetSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers28(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(in *jlexer.Lexer, out *BindEventSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(in *jlexer.Lexer, out *CGroupContextSerializer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "id": + out.ID = string(in.String()) + case "manager": + out.Manager = string(in.String()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(out *jwriter.Writer, in CGroupContextSerializer) { + out.RawByte('{') + first := true + _ = first + if in.ID != "" { + const prefix string = ",\"id\":" + first = false + out.RawString(prefix[1:]) + out.String(string(in.ID)) + } + if in.Manager != "" { + const prefix string = ",\"manager\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.Manager)) + } + out.RawByte('}') +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v CGroupContextSerializer) MarshalEasyJSON(w *jwriter.Writer) { + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(w, v) +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *CGroupContextSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(l, v) +} +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(in *jlexer.Lexer, out *BindEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4696,7 +4762,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(out *jwriter.Writer, in BindEventSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(out *jwriter.Writer, in BindEventSerializer) { out.RawByte('{') first := true _ = first @@ -4715,14 +4781,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BindEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BindEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers29(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(in *jlexer.Lexer, out *BPFProgramSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(in *jlexer.Lexer, out *BPFProgramSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4782,7 +4848,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(out *jwriter.Writer, in BPFProgramSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(out *jwriter.Writer, in BPFProgramSerializer) { out.RawByte('{') first := true _ = first @@ -4846,14 +4912,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BPFProgramSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BPFProgramSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers30(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(in *jlexer.Lexer, out *BPFMapSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(in *jlexer.Lexer, out *BPFMapSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4886,7 +4952,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(out *jwriter.Writer, in BPFMapSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(out *jwriter.Writer, in BPFMapSerializer) { out.RawByte('{') first := true _ = first @@ -4911,14 +4977,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BPFMapSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BPFMapSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers31(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(in *jlexer.Lexer, out *BPFEventSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(in *jlexer.Lexer, out *BPFEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -4969,7 +5035,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(out *jwriter.Writer, in BPFEventSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(out *jwriter.Writer, in BPFEventSerializer) { out.RawByte('{') first := true _ = first @@ -4993,14 +5059,14 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32( // MarshalEasyJSON supports easyjson.Marshaler interface func (v BPFEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *BPFEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers32(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(l, v) } -func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(in *jlexer.Lexer, out *AnomalyDetectionSyscallEventSerializer) { +func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(in *jlexer.Lexer, out *AnomalyDetectionSyscallEventSerializer) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -5031,7 +5097,7 @@ func easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33( in.Consumed() } } -func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(out *jwriter.Writer, in AnomalyDetectionSyscallEventSerializer) { +func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(out *jwriter.Writer, in AnomalyDetectionSyscallEventSerializer) { out.RawByte('{') first := true _ = first @@ -5045,10 +5111,10 @@ func easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33( // MarshalEasyJSON supports easyjson.Marshaler interface func (v AnomalyDetectionSyscallEventSerializer) MarshalEasyJSON(w *jwriter.Writer) { - easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(w, v) + easyjsonDdc0fdbeEncodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *AnomalyDetectionSyscallEventSerializer) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers33(l, v) + easyjsonDdc0fdbeDecodeGithubComDataDogDatadogAgentPkgSecuritySerializers34(l, v) } diff --git a/pkg/security/tests/connect_test.go b/pkg/security/tests/connect_test.go index 1e64fe57e1989..7f56dcfb8956c 100644 --- a/pkg/security/tests/connect_test.go +++ b/pkg/security/tests/connect_test.go @@ -12,7 +12,6 @@ import ( "context" "fmt" "net" - "sync" "testing" "golang.org/x/net/nettest" @@ -50,22 +49,20 @@ func TestConnectEvent(t *testing.T) { } t.Run("connect-af-inet-any-tcp-success", func(t *testing.T) { - var wg sync.WaitGroup - wg.Add(1) - defer wg.Wait() - - done := make(chan struct{}) - defer close(done) - go func() { - defer wg.Done() - err := bindAndAcceptConnection("tcp", ":4242", done) - if err != nil { - t.Error(err) - } - }() + done := make(chan error) + started := make(chan struct{}) + go bindAndAcceptConnection("tcp", ":4242", done, started) + + // Wait until the server is listening before attempting to connect + <-started test.WaitSignal(t, func() error { - return runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET", "any", "tcp") + if err := runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET", "any", "tcp"); err != nil { + return err + } + err := <-done + close(done) + return err }, func(event *model.Event, _ *rules.Rule) { assert.Equal(t, "connect", event.GetType(), "wrong event type") assert.Equal(t, uint16(unix.AF_INET), event.Connect.AddrFamily, "wrong address family") @@ -78,22 +75,19 @@ func TestConnectEvent(t *testing.T) { }) t.Run("connect-af-inet-any-udp-success", func(t *testing.T) { - var wg sync.WaitGroup - wg.Add(1) - defer wg.Wait() - - done := make(chan struct{}) - defer close(done) - go func() { - defer wg.Done() - err := bindAndAcceptConnection("udp", ":4242", done) - if err != nil { - t.Error(err) - } - }() + done := make(chan error) + started := make(chan struct{}) + go bindAndAcceptConnection("udp", ":4242", done, started) + + <-started test.WaitSignal(t, func() error { - return runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET", "any", "udp") + if err := runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET", "any", "udp"); err != nil { + return err + } + err := <-done + close(done) + return err }, func(event *model.Event, _ *rules.Rule) { assert.Equal(t, "connect", event.GetType(), "wrong event type") assert.Equal(t, uint16(unix.AF_INET), event.Connect.AddrFamily, "wrong address family") @@ -110,22 +104,19 @@ func TestConnectEvent(t *testing.T) { t.Skip("IPv6 is not supported") } - var wg sync.WaitGroup - wg.Add(1) - defer wg.Wait() - - done := make(chan struct{}) - defer close(done) - go func() { - defer wg.Done() - err := bindAndAcceptConnection("tcp", ":4242", done) - if err != nil { - t.Error(err) - } - }() + done := make(chan error) + started := make(chan struct{}) + go bindAndAcceptConnection("tcp", ":4242", done, started) + + <-started test.WaitSignal(t, func() error { - return runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET6", "any", "tcp") + if err := runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET6", "any", "tcp"); err != nil { + return err + } + err := <-done + close(done) + return err }, func(event *model.Event, _ *rules.Rule) { assert.Equal(t, "connect", event.GetType(), "wrong event type") assert.Equal(t, uint16(unix.AF_INET6), event.Connect.AddrFamily, "wrong address family") @@ -142,22 +133,19 @@ func TestConnectEvent(t *testing.T) { t.Skip("IPv6 is not supported") } - var wg sync.WaitGroup - wg.Add(1) - defer wg.Wait() - - done := make(chan struct{}) - defer close(done) - go func() { - defer wg.Done() - err := bindAndAcceptConnection("udp", ":4242", done) - if err != nil { - t.Error(err) - } - }() + done := make(chan error) + started := make(chan struct{}) + go bindAndAcceptConnection("udp", ":4242", done, started) + + <-started test.WaitSignal(t, func() error { - return runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET6", "any", "udp") + if err := runSyscallTesterFunc(context.Background(), t, syscallTester, "connect", "AF_INET6", "any", "udp"); err != nil { + return err + } + err := <-done + close(done) + return err }, func(event *model.Event, _ *rules.Rule) { assert.Equal(t, "connect", event.GetType(), "wrong event type") assert.Equal(t, uint16(unix.AF_INET6), event.Connect.AddrFamily, "wrong address family") @@ -168,43 +156,47 @@ func TestConnectEvent(t *testing.T) { test.validateConnectSchema(t, event) }) }) - } -func bindAndAcceptConnection(proto, address string, done chan struct{}) error { +func bindAndAcceptConnection(proto, address string, done chan error, started chan struct{}) { switch proto { case "tcp": listener, err := net.Listen(proto, address) if err != nil { - return fmt.Errorf("failed to bind to %s:%s: %w", proto, address, err) + done <- fmt.Errorf("failed to bind to %s:%s: %w", proto, address, err) + return } defer listener.Close() - // Start a goroutine to accept connections continuously - go func() { - for { - c, err := listener.Accept() - if err != nil { - fmt.Printf("accept error: %v\n", err) - return - } - fmt.Println("Connection accepted") - defer c.Close() - } - }() + // Signal that the server is ready to accept connections + close(started) + + c, err := listener.Accept() + if err != nil { + fmt.Printf("accept error: %v\n", err) + done <- err + return + } + defer c.Close() + + fmt.Println("Connection accepted") case "udp", "unixgram": conn, err := net.ListenPacket(proto, address) if err != nil { - return fmt.Errorf("failed to bind to %s:%s: %w", proto, address, err) + done <- err + return } defer conn.Close() + // For packet-based connections, we can signal readiness immediately + close(started) + default: - return fmt.Errorf("unsupported protocol: %s", proto) + done <- fmt.Errorf("unsupported protocol: %s", proto) + return } // Wait for the test to complete before returning - <-done - return nil + done <- nil } diff --git a/pkg/security/tests/constants_test.go b/pkg/security/tests/constants_test.go index f7a156aac2954..8982f6a1ff686 100644 --- a/pkg/security/tests/constants_test.go +++ b/pkg/security/tests/constants_test.go @@ -19,20 +19,6 @@ import ( "github.com/DataDog/datadog-agent/pkg/security/probe/constantfetch" ) -var BTFHubVsRcPossiblyMissingConstants = []string{ - constantfetch.OffsetNameNFConnStructCTNet, - constantfetch.OffsetNameIoKiocbStructCtx, - constantfetch.OffsetNameMountMntID, -} - -var RCVsFallbackPossiblyMissingConstants = []string{ - constantfetch.OffsetNameIoKiocbStructCtx, - constantfetch.OffsetNameTaskStructPID, - constantfetch.OffsetNameTaskStructPIDLink, - constantfetch.OffsetNameDeviceStructNdNet, - constantfetch.OffsetNameMountMntID, -} - var BTFHubVsFallbackPossiblyMissingConstants = []string{ constantfetch.OffsetNameNFConnStructCTNet, constantfetch.OffsetNameTaskStructPID, @@ -69,40 +55,6 @@ func TestOctogonConstants(t *testing.T) { t.Fatal(err) } - _, secconfig, err := genTestConfigs(dir, testOpts{}) - if err != nil { - t.Fatal(err) - } - - t.Run("rc-vs-fallback", func(t *testing.T) { - checkKernelCompatibility(t, "SLES kernels", func(kv *kernel.Version) bool { - return kv.IsSLESKernel() || (kv.IsAmazonLinux2023Kernel() && (testEnvironment == DockerEnvironment)) - }) - - fallbackFetcher := constantfetch.NewFallbackConstantFetcher(kv) - rcFetcher := constantfetch.NewRuntimeCompilationConstantFetcher(&secconfig.Probe.Config, nil) - - assertConstantsEqual(t, rcFetcher, fallbackFetcher, kv, RCVsFallbackPossiblyMissingConstants) - }) - - t.Run("btfhub-vs-rc", func(t *testing.T) { - checkKernelCompatibility(t, "SLES kernels", func(kv *kernel.Version) bool { - return kv.IsSLESKernel() || (kv.IsAmazonLinux2023Kernel() && (testEnvironment == DockerEnvironment)) - }) - - btfhubFetcher, err := constantfetch.NewBTFHubConstantFetcher(kv) - if err != nil { - t.Skipf("btfhub constant fetcher is not available: %v", err) - } - if !btfhubFetcher.HasConstantsInStore() { - t.Skip("btfhub has no constant for this OS") - } - - rcFetcher := constantfetch.NewRuntimeCompilationConstantFetcher(&secconfig.Probe.Config, nil) - - assertConstantsEqual(t, rcFetcher, btfhubFetcher, kv, BTFHubVsRcPossiblyMissingConstants) - }) - t.Run("btfhub-vs-fallback", func(t *testing.T) { btfhubFetcher, err := constantfetch.NewBTFHubConstantFetcher(kv) if err != nil { @@ -116,17 +68,6 @@ func TestOctogonConstants(t *testing.T) { assertConstantsEqual(t, btfhubFetcher, fallbackFetcher, kv, BTFHubVsFallbackPossiblyMissingConstants) }) - - t.Run("guesser-vs-rc", func(t *testing.T) { - checkKernelCompatibility(t, "SLES kernels", func(kv *kernel.Version) bool { - return kv.IsSLESKernel() || (kv.IsAmazonLinux2023Kernel() && (testEnvironment == DockerEnvironment)) - }) - - rcFetcher := constantfetch.NewRuntimeCompilationConstantFetcher(&secconfig.Probe.Config, nil) - ogFetcher := constantfetch.NewOffsetGuesserFetcher(secconfig.Probe, kv) - - assertConstantContains(t, rcFetcher, ogFetcher, kv, nil) - }) } func getFighterConstants(champion, challenger constantfetch.ConstantFetcher, kv *kernel.Version) (map[string]uint64, map[string]uint64, error) { @@ -177,42 +118,6 @@ func assertConstantsEqual(t *testing.T, champion, challenger constantfetch.Const } } -func assertConstantContains(t *testing.T, champion, challenger constantfetch.ConstantFetcher, kv *kernel.Version, ignoreMissing []string) { - t.Helper() - championConstants, challengerConstants, err := getFighterConstants(champion, challenger, kv) - if err != nil { - t.Error(err) - } - - if len(challengerConstants) == 0 { - t.Errorf("challenger %s has no constant\n", challenger) - } - - // In the case where challenger is missing a constant, we set it to the error sentinel - for _, k := range ignoreMissing { - if _, championPresent := championConstants[k]; !championPresent { - championConstants[k] = constantfetch.ErrorSentinel - } - } - - for k, v := range challengerConstants { - if v == constantfetch.ErrorSentinel { - continue - } - - expected, ok := championConstants[k] - if !ok { - t.Errorf("champion (`%s`) does not contain the expected constant `%s`", champion.String(), k) - } else if v != expected && expected != constantfetch.ErrorSentinel { - t.Errorf("difference between fighters for `%s`: `%s`:%d and `%s`:%d", k, champion.String(), expected, challenger.String(), v) - } - } - - if t.Failed() { - t.Logf("kernel version: %v", kv) - } -} - func getOffsetConstantsFromFetcher(cf constantfetch.ConstantFetcher, kv *kernel.Version) (map[string]uint64, error) { probe.AppendProbeRequestsToFetcher(cf, kv) return cf.FinishAndGetResults() diff --git a/pkg/security/tests/container_test.go b/pkg/security/tests/container_test.go index 7a659481da74b..15d896f1718ef 100644 --- a/pkg/security/tests/container_test.go +++ b/pkg/security/tests/container_test.go @@ -182,3 +182,73 @@ func TestContainerFlagsPodman(t *testing.T) { }) }) } + +func TestContainerVariables(t *testing.T) { + SkipIfNotAvailable(t) + + ruleDefs := []*rules.RuleDefinition{ + { + ID: "test_container_set_variable", + Expression: `container.id != "" && open.file.path == "{{.Root}}/test-open"`, + Actions: []*rules.ActionDefinition{ + { + Set: &rules.SetDefinition{ + Scope: "container", + Value: 1, + Name: "foo", + }, + }, + }, + }, + { + ID: "test_container_check_variable", + Expression: `container.id != "" && open.file.path == "{{.Root}}/test-open2" && ${container.foo} == 1`, + }, + } + test, err := newTestModule(t, nil, ruleDefs) + if err != nil { + t.Fatal(err) + } + defer test.Close() + + testFile, _, err := test.Path("test-open") + if err != nil { + t.Fatal(err) + } + + testFile2, _, err := test.Path("test-open2") + if err != nil { + t.Fatal(err) + } + + dockerWrapper, err := newDockerCmdWrapper(test.Root(), test.Root(), "ubuntu", "") + if err != nil { + t.Skip("Skipping created time in containers tests: Docker not available") + return + } + defer dockerWrapper.stop() + + dockerWrapper.Run(t, "container-variables", func(t *testing.T, _ wrapperType, cmdFunc func(cmd string, args []string, envs []string) *exec.Cmd) { + test.WaitSignal(t, func() error { + cmd := cmdFunc("touch", []string{testFile}, nil) + return cmd.Run() + }, func(event *model.Event, rule *rules.Rule) { + assertTriggeredRule(t, rule, "test_container_set_variable") + assertFieldEqual(t, event, "open.file.path", testFile) + assertFieldNotEmpty(t, event, "container.id", "container id shouldn't be empty") + + test.validateOpenSchema(t, event) + }) + + test.WaitSignal(t, func() error { + cmd := cmdFunc("touch", []string{testFile2}, nil) + return cmd.Run() + }, func(event *model.Event, rule *rules.Rule) { + assertTriggeredRule(t, rule, "test_container_check_variable") + assertFieldEqual(t, event, "open.file.path", testFile2) + assertFieldNotEmpty(t, event, "container.id", "container id shouldn't be empty") + + test.validateOpenSchema(t, event) + }) + }) +} diff --git a/pkg/security/utils/cgroup.go b/pkg/security/utils/cgroup.go index 5dbbb97c58af1..df59e317006a5 100644 --- a/pkg/security/utils/cgroup.go +++ b/pkg/security/utils/cgroup.go @@ -12,6 +12,7 @@ import ( "bufio" "bytes" "crypto/sha256" + "fmt" "os" "strconv" "strings" @@ -49,6 +50,49 @@ func (cg ControlGroup) GetContainerID() containerutils.ContainerID { return containerutils.ContainerID(id) } +// GetLastProcControlGroups returns the first cgroup membership of the specified task. +func GetLastProcControlGroups(tgid, pid uint32) (ControlGroup, error) { + data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) + if err != nil { + return ControlGroup{}, err + } + + data = bytes.TrimSpace(data) + + index := bytes.LastIndexByte(data, '\n') + if index < 0 { + index = 0 + } else { + index++ // to skip the \n + } + if index >= len(data) { + return ControlGroup{}, fmt.Errorf("invalid cgroup data: %s", data) + } + + lastLine := string(data[index:]) + + idstr, rest, ok := strings.Cut(lastLine, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + id, err := strconv.Atoi(idstr) + if err != nil { + return ControlGroup{}, err + } + + controllers, path, ok := strings.Cut(rest, ":") + if !ok { + return ControlGroup{}, fmt.Errorf("invalid cgroup line: %s", lastLine) + } + + return ControlGroup{ + ID: id, + Controllers: strings.Split(controllers, ","), + Path: path, + }, nil +} + // GetProcControlGroups returns the cgroup membership of the specified task. func GetProcControlGroups(tgid, pid uint32) ([]ControlGroup, error) { data, err := os.ReadFile(CgroupTaskPath(tgid, pid)) @@ -85,15 +129,14 @@ func GetProcContainerID(tgid, pid uint32) (containerutils.ContainerID, error) { // GetProcContainerContext returns the container ID which the process belongs to along with its manager. Returns "" if the process does not belong // to a container. func GetProcContainerContext(tgid, pid uint32) (containerutils.ContainerID, model.CGroupContext, error) { - cgroups, err := GetProcControlGroups(tgid, pid) - if err != nil || len(cgroups) == 0 { + cgroup, err := GetLastProcControlGroups(tgid, pid) + if err != nil { return "", model.CGroupContext{}, err } - lastCgroup := len(cgroups) - 1 - containerID, runtime := cgroups[lastCgroup].GetContainerContext() + containerID, runtime := cgroup.GetContainerContext() cgroupContext := model.CGroupContext{ - CGroupID: containerutils.CGroupID(cgroups[lastCgroup].Path), + CGroupID: containerutils.CGroupID(cgroup.Path), CGroupFlags: runtime, } diff --git a/pkg/security/utils/proc_linux.go b/pkg/security/utils/proc_linux.go index 1ee6765ae737e..229a4b16350ed 100644 --- a/pkg/security/utils/proc_linux.go +++ b/pkg/security/utils/proc_linux.go @@ -110,6 +110,11 @@ func StatusPath(pid uint32) string { return procPidPath(pid, "status") } +// TaskStatusPath returns the path to the status file of a task pid in /proc +func TaskStatusPath(pid uint32, task string) string { + return procPidPath3(pid, "task", task, "status") +} + // LoginUIDPath returns the path to the loginuid file of a pid in /proc func LoginUIDPath(pid uint32) string { return procPidPath(pid, "loginuid") @@ -144,6 +149,10 @@ func procPidPath2(pid uint32, path1 string, path2 string) string { return filepath.Join(kernel.ProcFSRoot(), strconv.FormatUint(uint64(pid), 10), path1, path2) } +func procPidPath3(pid uint32, path1, path2, path3 string) string { + return filepath.Join(kernel.ProcFSRoot(), strconv.FormatUint(uint64(pid), 10), path1, path2, path3) +} + // ModulesPath returns the path to the modules file in /proc func ModulesPath() string { return filepath.Join(kernel.ProcFSRoot(), "modules") @@ -410,8 +419,8 @@ func GetProcessPidNamespace(pid uint32) (uint64, error) { } // GetNsPids returns the namespaced pids of the the givent root pid -func GetNsPids(pid uint32) ([]uint32, error) { - statusFile := StatusPath(pid) +func GetNsPids(pid uint32, task string) ([]uint32, error) { + statusFile := TaskStatusPath(pid, task) content, err := os.ReadFile(statusFile) if err != nil { return nil, fmt.Errorf("failed to read status file: %w", err) @@ -442,6 +451,26 @@ func GetNsPids(pid uint32) ([]uint32, error) { return nil, fmt.Errorf("NSpid field not found") } +// GetPidTasks returns the task IDs of a process +func GetPidTasks(pid uint32) ([]string, error) { + taskPath := procPidPath(pid, "task") + + // Read the contents of the task directory + tasks, err := os.ReadDir(taskPath) + if err != nil { + return nil, fmt.Errorf("failed to read task directory: %v", err) + } + + // Collect all task IDs + var taskIDs []string + for _, task := range tasks { + if task.IsDir() { + taskIDs = append(taskIDs, task.Name()) + } + } + return taskIDs, nil +} + // FindPidNamespace search and return the host PID for the given namespaced PID + its namespace func FindPidNamespace(nspid uint32, ns uint64) (uint32, error) { procPids, err := process.Pids() @@ -455,8 +484,17 @@ func FindPidNamespace(nspid uint32, ns uint64) (uint32, error) { continue } - if procNs == ns { - nspids, err := GetNsPids(uint32(procPid)) + if procNs != ns { + continue + } + + tasks, err := GetPidTasks(uint32(procPid)) + if err != nil { + continue + } + + for _, task := range tasks { + nspids, err := GetNsPids(uint32(procPid), task) if err != nil { return 0, err } @@ -529,22 +567,14 @@ var isNsPidAvailable = sync.OnceValue(func() bool { }) // TryToResolveTraceePid tries to resolve and returnt the HOST tracee PID, given the HOST tracer PID and the namespaced tracee PID. -func TryToResolveTraceePid(hostTracerPID, NsTraceePid uint32) (uint32, error) { +func TryToResolveTraceePid(hostTracerPID uint32, tracerNSID uint64, NsTraceePid uint32) (uint32, error) { // Look if the NSpid status field is available or not (it should be, except for Centos7). if isNsPidAvailable() { /* If it's available, we will search for an host pid having the same PID namespace as the tracer, and having the corresponding NS PID in its status field */ - - // 1. get the pid namespace of the tracer - ns, err := GetProcessPidNamespace(hostTracerPID) - if err != nil { - return 0, fmt.Errorf("Failed to resolve PID namespace: %v", err) - } - - // 2. find the host pid matching the arg pid with he tracer namespace - pid, err := FindPidNamespace(NsTraceePid, ns) + pid, err := FindPidNamespace(NsTraceePid, tracerNSID) if err != nil { return 0, fmt.Errorf("Failed to resolve tracee PID namespace: %v", err) } diff --git a/pkg/serializer/go.mod b/pkg/serializer/go.mod index ef22edcd67b2d..0492d99ac7ba5 100644 --- a/pkg/serializer/go.mod +++ b/pkg/serializer/go.mod @@ -178,7 +178,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/pkg/serializer/go.sum b/pkg/serializer/go.sum index 0e4568db35dfe..21381032cc3dc 100644 --- a/pkg/serializer/go.sum +++ b/pkg/serializer/go.sum @@ -323,8 +323,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/serializer/internal/metrics/origin_mapping.go b/pkg/serializer/internal/metrics/origin_mapping.go index 45e433d7135b5..02c54e1913363 100644 --- a/pkg/serializer/internal/metrics/origin_mapping.go +++ b/pkg/serializer/internal/metrics/origin_mapping.go @@ -308,6 +308,7 @@ func metricSourceToOriginCategory(ms metrics.MetricSource) int32 { metrics.MetricSourceZk, metrics.MetricSourceAwsNeuron, metrics.MetricSourceNvidiaNim, + metrics.MetricSourceQuarkus, metrics.MetricSourceMilvus: return 11 // integrationMetrics default: @@ -902,6 +903,8 @@ func metricSourceToOriginService(ms metrics.MetricSource) int32 { return 425 case metrics.MetricSourceNvidiaNim: return 426 + case metrics.MetricSourceQuarkus: + return 427 default: return 0 } diff --git a/pkg/serializer/internal/metrics/service_checks_test.go b/pkg/serializer/internal/metrics/service_checks_test.go index 2a96f3d10072a..2559d7c9ed2cc 100644 --- a/pkg/serializer/internal/metrics/service_checks_test.go +++ b/pkg/serializer/internal/metrics/service_checks_test.go @@ -126,7 +126,7 @@ func TestPayloadsEmptyServiceCheck(t *testing.T) { func TestPayloadsServiceChecks(t *testing.T) { config := mock.New(t) - config.Set("serializer_max_payload_size", 200, pkgconfigmodel.SourceAgentRuntime) + config.Set("serializer_max_payload_size", 250, pkgconfigmodel.SourceAgentRuntime) serviceCheckCollection := []ServiceChecks{ {createServiceCheck("1"), createServiceCheck("2"), createServiceCheck("3")}, diff --git a/pkg/serverless/daemon/routes.go b/pkg/serverless/daemon/routes.go index 733b4a49b050c..fe78651ac4d2f 100644 --- a/pkg/serverless/daemon/routes.go +++ b/pkg/serverless/daemon/routes.go @@ -104,7 +104,17 @@ func (e *EndInvocation) ServeHTTP(w http.ResponseWriter, r *http.Request) { } errorMsg := r.Header.Get(invocationlifecycle.InvocationErrorMsgHeader) + if decodedMsg, err := base64.StdEncoding.DecodeString(errorMsg); err != nil { + log.Debug("Error message header may not be encoded, setting as is") + } else { + errorMsg = string(decodedMsg) + } errorType := r.Header.Get(invocationlifecycle.InvocationErrorTypeHeader) + if decodedType, err := base64.StdEncoding.DecodeString(errorType); err != nil { + log.Debug("Error type header may not be encoded, setting as is") + } else { + errorType = string(decodedType) + } errorStack := r.Header.Get(invocationlifecycle.InvocationErrorStackHeader) if decodedStack, err := base64.StdEncoding.DecodeString(errorStack); err != nil { log.Debug("Could not decode error stack header") diff --git a/pkg/serverless/daemon/routes_test.go b/pkg/serverless/daemon/routes_test.go index 25231204af6c8..e630d2829bd61 100644 --- a/pkg/serverless/daemon/routes_test.go +++ b/pkg/serverless/daemon/routes_test.go @@ -7,6 +7,7 @@ package daemon import ( "bytes" + "encoding/base64" "fmt" "io" "net/http" @@ -104,7 +105,7 @@ func TestEndInvocation(t *testing.T) { assert.Equal(m.lastEndDetails.Runtime, d.ExecutionContext.GetCurrentState().Runtime) } -func TestEndInvocationWithError(t *testing.T) { +func TestEndInvocationWithErrorEncodedHeaders(t *testing.T) { assert := assert.New(t) port := testutil.FreeTCPPort(t) d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) @@ -114,10 +115,52 @@ func TestEndInvocationWithError(t *testing.T) { m := &mockLifecycleProcessor{} d.InvocationProcessor = m + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message \n at TestFunction.Handle(ILambdaContext context)" + + client := &http.Client{} + body := bytes.NewBuffer([]byte(`{}`)) + request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) + request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, base64.StdEncoding.EncodeToString([]byte(errorMessage))) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, base64.StdEncoding.EncodeToString([]byte(errorType))) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, base64.StdEncoding.EncodeToString([]byte(errorStack))) + assert.Nil(err) + res, err := client.Do(request) + assert.Nil(err) + if res != nil { + res.Body.Close() + assert.Equal(res.StatusCode, 200) + } + assert.True(m.OnInvokeEndCalled) + assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) +} + +func TestEndInvocationWithErrorNonEncodedHeaders(t *testing.T) { + assert := assert.New(t) + port := testutil.FreeTCPPort(t) + d := StartDaemon(fmt.Sprintf("127.0.0.1:%d", port)) + time.Sleep(100 * time.Millisecond) + defer d.Stop() + + m := &mockLifecycleProcessor{} + d.InvocationProcessor = m + + errorMessage := "Error message" + errorType := "System.Exception" + errorStack := "System.Exception: Error message at TestFunction.Handle(ILambdaContext context)" + client := &http.Client{} body := bytes.NewBuffer([]byte(`{}`)) request, err := http.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/lambda/end-invocation", port), body) request.Header.Set("x-datadog-invocation-error", "true") + request.Header.Set(invocationlifecycle.InvocationErrorMsgHeader, errorMessage) + request.Header.Set(invocationlifecycle.InvocationErrorTypeHeader, errorType) + request.Header.Set(invocationlifecycle.InvocationErrorStackHeader, errorStack) assert.Nil(err) res, err := client.Do(request) assert.Nil(err) @@ -127,6 +170,9 @@ func TestEndInvocationWithError(t *testing.T) { } assert.True(m.OnInvokeEndCalled) assert.True(m.isError) + assert.Equal(m.lastEndDetails.ErrorMsg, errorMessage) + assert.Equal(m.lastEndDetails.ErrorType, errorType) + assert.Equal(m.lastEndDetails.ErrorStack, errorStack) } func TestTraceContext(t *testing.T) { diff --git a/pkg/serverless/logs/logs_test.go b/pkg/serverless/logs/logs_test.go index 6a8f0d30d3cb0..132efb75bbcaa 100644 --- a/pkg/serverless/logs/logs_test.go +++ b/pkg/serverless/logs/logs_test.go @@ -224,7 +224,7 @@ func TestProcessMessageValid(t *testing.T) { lc.processMessage(&message) - received, timed := demux.WaitForNumberOfSamples(7, 0, 100*time.Millisecond) + received, timed := demux.WaitForNumberOfSamples(7, 0, 125*time.Millisecond) assert.Len(t, received, 7) assert.Len(t, timed, 0) demux.Reset() @@ -232,7 +232,7 @@ func TestProcessMessageValid(t *testing.T) { lc.enhancedMetricsEnabled = false lc.processMessage(&message) - received, timed = demux.WaitForSamples(100 * time.Millisecond) + received, timed = demux.WaitForSamples(125 * time.Millisecond) assert.Len(t, received, 0, "we should NOT have received metrics") assert.Len(t, timed, 0) } @@ -366,7 +366,7 @@ func TestProcessMessageShouldNotProcessArnNotSet(t *testing.T) { go lc.processMessage(message) - received, timed := demux.WaitForSamples(100 * time.Millisecond) + received, timed := demux.WaitForSamples(125 * time.Millisecond) assert.Len(t, received, 0, "We should NOT have received metrics") assert.Len(t, timed, 0) } @@ -394,7 +394,7 @@ func TestProcessMessageShouldNotProcessLogsDropped(t *testing.T) { go lc.processMessage(message) - received, timed := demux.WaitForSamples(100 * time.Millisecond) + received, timed := demux.WaitForSamples(125 * time.Millisecond) assert.Len(t, received, 0, "We should NOT have received metrics") assert.Len(t, timed, 0) } @@ -424,11 +424,13 @@ func TestProcessMessageShouldProcessLogTypeFunctionOutOfMemory(t *testing.T) { go lc.processMessage(message) - received, timed := demux.WaitForNumberOfSamples(2, 0, 100*time.Millisecond) + received, timed := demux.WaitForNumberOfSamples(2, 0, 125*time.Millisecond) assert.Len(t, received, 2) assert.Len(t, timed, 0) - assert.Equal(t, serverlessMetrics.OutOfMemoryMetric, received[0].Name) - assert.Equal(t, serverlessMetrics.ErrorsMetric, received[1].Name) + if len(received) == 2 { + assert.Equal(t, serverlessMetrics.OutOfMemoryMetric, received[0].Name) + assert.Equal(t, serverlessMetrics.ErrorsMetric, received[1].Name) + } } func TestProcessMessageShouldProcessLogTypePlatformReportOutOfMemory(t *testing.T) { @@ -465,7 +467,7 @@ func TestProcessMessageShouldProcessLogTypePlatformReportOutOfMemory(t *testing. go lc.processMessage(message) - received, timed := demux.WaitForNumberOfSamples(2, 0, 100*time.Millisecond) + received, timed := demux.WaitForNumberOfSamples(2, 0, 125*time.Millisecond) assert.Len(t, received, 8) assert.Len(t, timed, 0) assert.Equal(t, serverlessMetrics.OutOfMemoryMetric, received[6].Name) @@ -497,7 +499,7 @@ func TestProcessMessageShouldSendFailoverMetric(t *testing.T) { lc.processMessage(&message) - received, timed := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + received, timed := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, received, 1) assert.Len(t, timed, 0) demux.Reset() @@ -507,7 +509,7 @@ func TestProcessMessageShouldSendFailoverMetric(t *testing.T) { message.stringRecord = "{\"DD_EXTENSION_FALLBACK_REASON\":\"test-reason\"}" // add again bc processing empties it lc.processMessage(&message) - received, timed = demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + received, timed = demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, received, 1) assert.Len(t, timed, 0) } @@ -549,7 +551,7 @@ func TestProcessLogMessageLogsEnabled(t *testing.T) { assert.NotNil(t, received) assert.Equal(t, "my-arn", received.Lambda.ARN) assert.Equal(t, "myRequestID", received.Lambda.RequestID) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -597,7 +599,7 @@ func TestProcessLogMessageLogsEnabledForMixedUnorderedMessages(t *testing.T) { assert.NotNil(t, received) assert.Equal(t, "my-arn", received.Lambda.ARN) assert.Equal(t, expectedRequestIDs[i], received.Lambda.RequestID) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -628,7 +630,7 @@ func TestProcessLogMessageNoStringRecordPlatformLog(t *testing.T) { select { case <-logChannel: assert.Fail(t, "We should not have received logs") - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): // nothing to do here } } @@ -663,7 +665,7 @@ func TestProcessLogMessageNoStringRecordFunctionLog(t *testing.T) { assert.NotNil(t, received) assert.Equal(t, "my-arn", received.Lambda.ARN) assert.Equal(t, "myRequestID", received.Lambda.RequestID) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -700,7 +702,7 @@ func TestProcessLogMessageLogsNotEnabled(t *testing.T) { select { case <-logChannel: assert.Fail(t, "We should not have received logs") - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): // nothing to do here } } @@ -763,7 +765,7 @@ func TestProcessLogMessagesTimeoutLogFromReportLog(t *testing.T) { assert.Equal(t, "myRequestID", received.Lambda.RequestID) assert.Equal(t, expectedStringRecord[i], string(received.Content)) assert.Equal(t, expectedErrors[i], received.IsError) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -858,7 +860,7 @@ func TestProcessMultipleLogMessagesTimeoutLogFromReportLog(t *testing.T) { } else { assert.False(t, received.IsError) } - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -904,7 +906,7 @@ func TestProcessLogMessagesOutOfMemoryError(t *testing.T) { assert.Equal(t, "my-arn", received.Lambda.ARN) assert.Equal(t, "myRequestID", received.Lambda.RequestID) assert.Equal(t, true, received.IsError) - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -943,7 +945,7 @@ func TestProcessLogMessageLogsNoRequestID(t *testing.T) { select { case <-logChannel: assert.Fail(t, "We should not have received logs") - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): // nothing to do here } @@ -952,7 +954,7 @@ func TestProcessLogMessageLogsNoRequestID(t *testing.T) { for _, msg := range received { assert.Equal(t, "", msg.objectRecord.requestID) } - case <-time.After(100 * time.Millisecond): + case <-time.After(125 * time.Millisecond): assert.Fail(t, "We should have received logs") } } @@ -1228,7 +1230,7 @@ func TestRuntimeMetricsMatchLogs(t *testing.T) { lc.processMessage(doneMessage) lc.processMessage(reportMessage) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 125*time.Millisecond) postRuntimeMetricTimestamp := float64(reportLogTime.UnixNano()) / float64(time.Second) runtimeMetricTimestamp := float64(endTime.UnixNano()) / float64(time.Second) assert.Equal(t, generatedMetrics[0], metrics.MetricSample{ @@ -1314,7 +1316,7 @@ func TestRuntimeMetricsMatchLogsProactiveInit(t *testing.T) { lc.processMessage(doneMessage) lc.processMessage(reportMessage) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 125*time.Millisecond) postRuntimeMetricTimestamp := float64(reportLogTime.UnixNano()) / float64(time.Second) runtimeMetricTimestamp := float64(endTime.UnixNano()) / float64(time.Second) assert.Equal(t, generatedMetrics[0], metrics.MetricSample{ @@ -1416,7 +1418,7 @@ func TestRuntimeMetricsOnTimeout(t *testing.T) { assert.Nil(t, restoreErr) lc.processMessage(reportMessage) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(10, 0, 125*time.Millisecond) postRuntimeMetricTimestamp := float64(reportLogTime.UnixNano()) / float64(time.Second) runtimeMetricTimestamp := float64(endTime.UnixNano()) / float64(time.Second) assert.Equal(t, generatedMetrics[0], metrics.MetricSample{ diff --git a/pkg/serverless/metrics/enhanced_metrics_test.go b/pkg/serverless/metrics/enhanced_metrics_test.go index ea6619861f726..9c6f9472ee756 100644 --- a/pkg/serverless/metrics/enhanced_metrics_test.go +++ b/pkg/serverless/metrics/enhanced_metrics_test.go @@ -34,7 +34,7 @@ func TestGenerateEnhancedMetricsFromFunctionLogOutOfMemory(t *testing.T) { GenerateOutOfMemoryEnhancedMetrics(reportLogTime, tags, demux) } - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(2, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(2, 0, 125*time.Millisecond) assert.True(t, isOOM) assert.Len(t, generatedMetrics, 2, "two enhanced metrics should have been generated") assert.Len(t, timedMetrics, 0) @@ -63,7 +63,7 @@ func TestGenerateEnhancedMetricsFromFunctionLogNoMetric(t *testing.T) { GenerateOutOfMemoryEnhancedMetrics(time.Now(), tags, demux) } - generatedMetrics, timedMetrics := demux.WaitForSamples(100 * time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForSamples(125 * time.Millisecond) assert.False(t, isOOM) assert.Len(t, generatedMetrics, 0, "no metrics should have been generated") assert.Len(t, timedMetrics, 0) @@ -89,7 +89,7 @@ func TestGenerateEnhancedMetricsFromReportLogColdStart(t *testing.T) { } go GenerateEnhancedMetricsFromReportLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(7, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(7, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:7], []metrics.MetricSample{{ Name: maxMemoryUsedMetric, @@ -164,7 +164,7 @@ func TestGenerateEnhancedMetricsFromReportLogNoColdStart(t *testing.T) { } go GenerateEnhancedMetricsFromReportLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(6, 0, 0100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(6, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:6], []metrics.MetricSample{{ Name: maxMemoryUsedMetric, @@ -218,7 +218,7 @@ func TestSendTimeoutEnhancedMetric(t *testing.T) { go SendTimeoutEnhancedMetric(tags, demux) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:1], []metrics.MetricSample{{ Name: timeoutsMetric, @@ -238,7 +238,7 @@ func TestSendInvocationEnhancedMetric(t *testing.T) { go SendInvocationEnhancedMetric(tags, demux) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:1], []metrics.MetricSample{{ Name: invocationsMetric, @@ -264,7 +264,7 @@ func TestDisableEnhancedMetrics(t *testing.T) { SendInvocationEnhancedMetric(tags, demux) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -279,7 +279,7 @@ func TestSendOutOfMemoryEnhancedMetric(t *testing.T) { mockTime := time.Now() go SendOutOfMemoryEnhancedMetric(tags, mockTime, demux) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:1], []metrics.MetricSample{{ Name: OutOfMemoryMetric, @@ -298,7 +298,7 @@ func TestSendErrorsEnhancedMetric(t *testing.T) { mockTime := time.Now() go SendErrorsEnhancedMetric(tags, mockTime, demux) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics[:1], []metrics.MetricSample{{ Name: ErrorsMetric, @@ -366,7 +366,7 @@ func TestGenerateEnhancedMetricsFromRuntimeDoneLogNoStartDate(t *testing.T) { Demux: demux, } go GenerateEnhancedMetricsFromRuntimeDoneLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics, []metrics.MetricSample{{ Name: responseLatencyMetric, Value: 19, @@ -407,7 +407,7 @@ func TestGenerateEnhancedMetricsFromRuntimeDoneLogNoEndDate(t *testing.T) { Demux: demux, } go GenerateEnhancedMetricsFromRuntimeDoneLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics, []metrics.MetricSample{{ Name: responseLatencyMetric, Value: 19, @@ -448,7 +448,7 @@ func TestGenerateEnhancedMetricsFromRuntimeDoneLogOK(t *testing.T) { Demux: demux, } go GenerateEnhancedMetricsFromRuntimeDoneLog(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(4, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(4, 0, 125*time.Millisecond) assert.Equal(t, generatedMetrics, []metrics.MetricSample{{ Name: runtimeDurationMetric, Value: 153, @@ -487,7 +487,7 @@ func TestGenerateCPUEnhancedMetrics(t *testing.T) { now := float64(time.Now().UnixNano()) / float64(time.Second) args := generateCPUEnhancedMetricsArgs{100, 53, 200, tags, demux, now} go generateCPUEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(4, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(4, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: cpuSystemTimeMetric, @@ -530,7 +530,7 @@ func TestDisableCPUEnhancedMetrics(t *testing.T) { SendCPUEnhancedMetrics(&proc.CPUData{}, 0, tags, demux) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -560,7 +560,7 @@ func TestGenerateCPUUtilizationEnhancedMetrics(t *testing.T) { Time: now, } go GenerateCPUUtilizationEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(5, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(5, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: cpuTotalUtilizationPctMetric, @@ -621,7 +621,7 @@ func TestGenerateNetworkEnhancedMetrics(t *testing.T) { Time: now, } go generateNetworkEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: rxBytesMetric, @@ -664,7 +664,7 @@ func TestNetworkEnhancedMetricsDisabled(t *testing.T) { SendNetworkEnhancedMetrics(&proc.NetworkData{}, tags, demux) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -685,7 +685,7 @@ func TestSendTmpEnhancedMetrics(t *testing.T) { Time: now, } go generateTmpEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: tmpUsedMetric, @@ -722,7 +722,7 @@ func TestSendTmpEnhancedMetricsDisabled(t *testing.T) { SendTmpEnhancedMetrics(make(chan bool), tags, &metricAgent) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -743,7 +743,7 @@ func TestSendFdEnhancedMetrics(t *testing.T) { Time: now, } go generateFdEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: fdMaxMetric, @@ -779,7 +779,7 @@ func TestSendThreadEnhancedMetrics(t *testing.T) { Time: now, } go generateThreadEnhancedMetrics(args) - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(3, 0, 125*time.Millisecond) assert.Equal(t, []metrics.MetricSample{ { Name: threadsMaxMetric, @@ -816,7 +816,7 @@ func TestSendProcessEnhancedMetricsDisabled(t *testing.T) { SendProcessEnhancedMetrics(make(chan bool), tags, &metricAgent) }() - generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, timedMetrics := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 0) assert.Len(t, timedMetrics, 0) @@ -829,7 +829,7 @@ func TestSendFailoverReasonMetric(t *testing.T) { demux := createDemultiplexer(t) tags := []string{"reason:test-reason"} go SendFailoverReasonMetric(tags, demux) - generatedMetrics, _ := demux.WaitForNumberOfSamples(1, 0, 100*time.Millisecond) + generatedMetrics, _ := demux.WaitForNumberOfSamples(1, 0, 125*time.Millisecond) assert.Len(t, generatedMetrics, 1) } diff --git a/pkg/serverless/proxy/proxy.go b/pkg/serverless/proxy/proxy.go index 7d43dd36db257..1f8a2e3087658 100644 --- a/pkg/serverless/proxy/proxy.go +++ b/pkg/serverless/proxy/proxy.go @@ -8,6 +8,7 @@ package proxy import ( "context" + "errors" "net/http" "net/http/httputil" "net/url" @@ -68,9 +69,44 @@ func newProxy(target string, processor invocationlifecycle.InvocationProcessor) Scheme: "http", Host: target, } + proxy := httputil.NewSingleHostReverseProxy(url) + + // The default error handler logs "http: proxy error: %v" then returns an HTTP 502 (bad gateway) + // response. This is unfortunate because it lacks much any context on the original request that + // failed, and the commonly observed error today is "context deadline exceeded", which is not + // actionnable if you don't know what request it was for. It also logs to STDERR and does not + // honor the agent's log level. + proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { + log.Debugf( + "[serverless/proxy][%T] %s %s -- proxy error: %v", + // The dynamic type of processor informs about what kind of proxy this was (main/appsec) + processor, + // The request method and URL are useful to understand what exactly failed. We won't log + // the body (too large) or headers (risks containing sensitive data, such as API keys) + r.Method, r.URL, + // What happened that caused us to be called? + err, + ) + + // If the error is a [context.DeadlineExceeded], we return an HTTP 504 (gateway timeout) + // instead of the generic HTTP 502 (bad gateway) to give the client a better idea of what is + // going on (this may influence retry behavior, for example). + if errors.Is(err, context.DeadlineExceeded) { + w.WriteHeader(http.StatusGatewayTimeout) + } else { + // Return an HTTP 502 (bad gateway) error response; defer the retrying to the client. + w.WriteHeader(http.StatusBadGateway) + } + + // Writing the error message as best-effort, we simply debug-log any error that occur here. + if _, err := w.Write([]byte(err.Error())); err != nil { + log.Debugf("[serverless/proxy][%T] failed to write error message to response body: %v", processor, err) + } + } + return &runtimeProxy{ target: url, - proxy: httputil.NewSingleHostReverseProxy(url), + proxy: proxy, processor: processor, } } diff --git a/pkg/status/systemprobe/status.go b/pkg/status/systemprobe/status.go index 533b94e1f3981..c6cb485094974 100644 --- a/pkg/status/systemprobe/status.go +++ b/pkg/status/systemprobe/status.go @@ -3,8 +3,6 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build process - // Package systemprobe fetch information about the system probe package systemprobe diff --git a/pkg/status/systemprobe/status_unsupported.go b/pkg/status/systemprobe/status_unsupported.go deleted file mode 100644 index 31c2523258659..0000000000000 --- a/pkg/status/systemprobe/status_unsupported.go +++ /dev/null @@ -1,26 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -//go:build !process - -// Package systemprobe fetch information about the system probe -package systemprobe - -import ( - "github.com/DataDog/datadog-agent/comp/core/status" - "github.com/DataDog/datadog-agent/comp/core/sysprobeconfig" -) - -// GetStatus returns a notice that it is not supported on systems that do not at least build the process agent -func GetStatus(stats map[string]interface{}, _ string) { - stats["systemProbeStats"] = map[string]interface{}{ - "Errors": "System Probe is not supported on this system", - } -} - -// GetProvider returns nil -func GetProvider(_ sysprobeconfig.Component) status.Provider { - return nil -} diff --git a/pkg/trace/api/api.go b/pkg/trace/api/api.go index c74ecbb137fb9..5cb57d2478a27 100644 --- a/pkg/trace/api/api.go +++ b/pkg/trace/api/api.go @@ -141,7 +141,7 @@ func NewHTTPReceiver( } } log.Infof("Receiver configured with %d decoders and a timeout of %dms", semcount, conf.DecoderTimeout) - containerIDProvider := NewIDProvider(conf.ContainerProcRoot) + containerIDProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) telemetryForwarder := NewTelemetryForwarder(conf, containerIDProvider, statsd) return &HTTPReceiver{ Stats: info.NewReceiverStats(), diff --git a/pkg/trace/api/api_test.go b/pkg/trace/api/api_test.go index 9a468dc164487..3d43d0dbe3c6e 100644 --- a/pkg/trace/api/api_test.go +++ b/pkg/trace/api/api_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" pb "github.com/DataDog/datadog-agent/pkg/proto/pbgo/trace" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/trace/config" @@ -576,7 +577,9 @@ func TestDecodeV05(t *testing.T) { req, err := http.NewRequest("POST", "/v0.5/traces", bytes.NewReader(b)) assert.NoError(err) req.Header.Set(header.ContainerID, "abcdef123789456") - tp, err := decodeTracerPayload(v05, req, NewIDProvider(""), "python", "3.8.1", "1.2.3") + tp, err := decodeTracerPayload(v05, req, NewIDProvider("", func(_ origindetection.OriginInfo) (string, error) { + return "abcdef123789456", nil + }), "python", "3.8.1", "1.2.3") assert.NoError(err) assert.EqualValues(tp, &pb.TracerPayload{ ContainerID: "abcdef123789456", @@ -1039,14 +1042,26 @@ func TestExpvar(t *testing.T) { } c := newTestReceiverConfig() - c.DebugServerPort = 5012 + c.DebugServerPort = 6789 info.InitInfo(c) + + // Starting a TLS httptest server to retrieve tlsCert + ts := httptest.NewTLSServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + tlsConfig := ts.TLS.Clone() + // Setting a client with the proper TLS configuration + client := ts.Client() + ts.Close() + + // Starting Debug Server s := NewDebugServer(c) + s.SetTLSConfig(tlsConfig) + + // Starting the Debug server s.Start() defer s.Stop() - resp, err := http.Get("http://127.0.0.1:5012/debug/vars") - assert.NoError(t, err) + resp, err := client.Get(fmt.Sprintf("https://127.0.0.1:%d/debug/vars", c.DebugServerPort)) + require.NoError(t, err) defer resp.Body.Close() t.Run("read-expvars", func(t *testing.T) { diff --git a/pkg/trace/api/container.go b/pkg/trace/api/container.go index 508277c5abaf9..0c789f9d06ca7 100644 --- a/pkg/trace/api/container.go +++ b/pkg/trace/api/container.go @@ -12,6 +12,7 @@ import ( "net" "net/http" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" ) @@ -28,7 +29,7 @@ type IDProvider interface { type idProvider struct{} // NewIDProvider initializes an IDProvider instance, in non-linux environments the procRoot arg is unused. -func NewIDProvider(_ string) IDProvider { +func NewIDProvider(_ string, _ func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { return &idProvider{} } diff --git a/pkg/trace/api/container_linux.go b/pkg/trace/api/container_linux.go index 2129a6377c97f..6dd12d227894d 100644 --- a/pkg/trace/api/container_linux.go +++ b/pkg/trace/api/container_linux.go @@ -18,6 +18,7 @@ import ( "syscall" "time" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/trace/api/internal/header" "github.com/DataDog/datadog-agent/pkg/util/cgroups" "github.com/DataDog/datadog-agent/pkg/util/log" @@ -96,7 +97,7 @@ func (i *noCgroupsProvider) GetContainerID(_ context.Context, h http.Header) str } // NewIDProvider initializes an IDProvider instance using the provided procRoot to perform cgroups lookups in linux environments. -func NewIDProvider(procRoot string) IDProvider { +func NewIDProvider(procRoot string, containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error)) IDProvider { // taken from pkg/util/containers/metrics/system.collector_linux.go var hostPrefix string if strings.HasPrefix(procRoot, "/host") { @@ -120,10 +121,11 @@ func NewIDProvider(procRoot string) IDProvider { } c := NewCache(1 * time.Minute) return &cgroupIDProvider{ - procRoot: procRoot, - controller: cgroupController, - cache: c, - reader: reader, + procRoot: procRoot, + controller: cgroupController, + cache: c, + reader: reader, + containerIDFromOriginInfo: containerIDFromOriginInfo, } } @@ -131,8 +133,9 @@ type cgroupIDProvider struct { procRoot string controller string // reader is used to retrieve the container ID from its cgroup v2 inode. - reader *cgroups.Reader - cache *Cache + reader *cgroups.Reader + cache *Cache + containerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) } // GetContainerID returns the container ID. @@ -157,6 +160,11 @@ func (c *cgroupIDProvider) GetContainerID(ctx context.Context, h http.Header) st return containerID } + // Retrieve container ID from External Data header + if externalData := h.Get(header.ExternalData); externalData != "" { + return c.resolveContainerIDFromExternalData(externalData) + } + return "" } @@ -296,6 +304,27 @@ func (c *cgroupIDProvider) getCachedContainerID(key string, retrievalFunc func() return val, nil } +// resolveContainerIDFromExternalData returns the container ID for the given External Data. +func (c *cgroupIDProvider) resolveContainerIDFromExternalData(rawExternalData string) string { + var generatedContainerID string + + externalData, err := origindetection.ParseExternalData(rawExternalData) + if err != nil { + log.Errorf("Could not parse external data (%s): %v", rawExternalData, err) + return "" + } + generatedContainerID, err = c.containerIDFromOriginInfo(origindetection.OriginInfo{ + ExternalData: externalData, + ProductOrigin: origindetection.ProductOriginAPM, + }) + if err != nil { + log.Errorf("Could not generate container ID from external data (%s): %v", rawExternalData, err) + return "" + } + + return generatedContainerID +} + // The below cache is copied from /pkg/util/containers/v2/metrics/provider/cache.go. It is not // imported to avoid making the datadog-agent module a dependency of the pkg/trace module. The // datadog-agent module contains replace directives which are not inherited by packages that diff --git a/pkg/trace/api/debug_server.go b/pkg/trace/api/debug_server.go index 828d5357330eb..6fd2f39cc011a 100644 --- a/pkg/trace/api/debug_server.go +++ b/pkg/trace/api/debug_server.go @@ -9,6 +9,7 @@ package api import ( "context" + "crypto/tls" "expvar" "fmt" "net" @@ -29,9 +30,10 @@ const ( // DebugServer serves /debug/* endpoints type DebugServer struct { - conf *config.AgentConfig - server *http.Server - mux *http.ServeMux + conf *config.AgentConfig + server *http.Server + mux *http.ServeMux + tlsConfig *tls.Config } // NewDebugServer returns a debug server @@ -53,13 +55,14 @@ func (ds *DebugServer) Start() { WriteTimeout: defaultTimeout, Handler: ds.setupMux(), } - listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", ds.conf.DebugServerPort)) + listener, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(ds.conf.DebugServerPort))) if err != nil { log.Errorf("Error creating debug server listener: %s", err) return } + tlsListener := tls.NewListener(listener, ds.tlsConfig) go func() { - if err := ds.server.Serve(listener); err != nil && err != http.ErrServerClosed { + if err := ds.server.Serve(tlsListener); err != nil && err != http.ErrServerClosed { log.Errorf("Could not start debug server: %s. Debug server disabled.", err) } }() @@ -82,6 +85,11 @@ func (ds *DebugServer) AddRoute(route string, handler http.Handler) { ds.mux.Handle(route, handler) } +// SetTLSConfig adds the provided tls.Config to the internal http.Server +func (ds *DebugServer) SetTLSConfig(config *tls.Config) { + ds.tlsConfig = config +} + func (ds *DebugServer) setupMux() *http.ServeMux { ds.mux.HandleFunc("/debug/pprof/", pprof.Index) ds.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) diff --git a/pkg/trace/api/debugger.go b/pkg/trace/api/debugger.go index 61164f1d3ed02..51816276a3f3b 100644 --- a/pkg/trace/api/debugger.go +++ b/pkg/trace/api/debugger.go @@ -81,7 +81,7 @@ func debuggerErrorHandler(err error) http.Handler { // newDebuggerProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newDebuggerProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/api/evp_proxy.go b/pkg/trace/api/evp_proxy.go index 452d73017aa8d..2795e6304b605 100644 --- a/pkg/trace/api/evp_proxy.go +++ b/pkg/trace/api/evp_proxy.go @@ -86,7 +86,7 @@ func evpProxyForwarder(conf *config.AgentConfig, statsd statsd.ClientInterface) req.Header["X-Forwarded-For"] = nil }, ErrorLog: logger, - Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot), statsd}, + Transport: &evpProxyTransport{conf.NewHTTPTransport(), endpoints, conf, NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo), statsd}, } } diff --git a/pkg/trace/api/internal/header/headers.go b/pkg/trace/api/internal/header/headers.go index 8ae458be834f5..88cb3549b2836 100644 --- a/pkg/trace/api/internal/header/headers.go +++ b/pkg/trace/api/internal/header/headers.go @@ -25,6 +25,16 @@ const ( // * "ci-,in-" LocalData = "Datadog-Entity-ID" + // ExternalData is a list that contain prefixed-items, split by a ','. Current items are: + // * "it-" if the container is an init container. + // * "cn-" for the container name. + // * "pu-" for the pod UID. + // Order does not matter. + // Possible values: + // * "it-false,cn-nginx,pu-3413883c-ac60-44ab-96e0-9e52e4e173e2" + // * "cn-init,pu-cb4aba1d-0129-44f1-9f1b-b4dc5d29a3b3,it-true" + ExternalData = "Datadog-External-Env" + // Lang specifies the name of the header which contains the language from // which the traces originate. Lang = "Datadog-Meta-Lang" diff --git a/pkg/trace/api/otlp.go b/pkg/trace/api/otlp.go index 88491ea9052ec..eb136d267e090 100644 --- a/pkg/trace/api/otlp.go +++ b/pkg/trace/api/otlp.go @@ -9,7 +9,6 @@ import ( "context" "encoding/hex" "fmt" - "github.com/DataDog/datadog-agent/pkg/trace/transform" "math" "net" "net/http" @@ -26,6 +25,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/trace/sampler" "github.com/DataDog/datadog-agent/pkg/trace/timing" "github.com/DataDog/datadog-agent/pkg/trace/traceutil" + "github.com/DataDog/datadog-agent/pkg/trace/transform" "github.com/DataDog/datadog-go/v5/statsd" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" @@ -102,7 +102,7 @@ func NewOTLPReceiver(out chan<- *Payload, cfg *config.AgentConfig, statsd statsd enableReceiveResourceSpansV2Val = 1.0 } _ = statsd.Gauge("datadog.trace_agent.otlp.enable_receive_resource_spans_v2", enableReceiveResourceSpansV2Val, nil, 1) - return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} + return &OTLPReceiver{out: out, conf: cfg, cidProvider: NewIDProvider(cfg.ContainerProcRoot, cfg.ContainerIDFromOriginInfo), statsd: statsd, timing: timing, ignoreResNames: ignoreResNames} } // Start starts the OTLPReceiver, if any of the servers were configured as active. diff --git a/pkg/trace/api/pipeline_stats.go b/pkg/trace/api/pipeline_stats.go index b6fe3f2cd161c..b8daf12590bf7 100644 --- a/pkg/trace/api/pipeline_stats.go +++ b/pkg/trace/api/pipeline_stats.go @@ -71,7 +71,7 @@ func pipelineStatsErrorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. func newPipelineStatsProxy(conf *config.AgentConfig, urls []*url.URL, apiKeys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { log.Debug("[pipeline_stats] Creating reverse proxy") - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/profiles.go b/pkg/trace/api/profiles.go index 9b43efc94de74..19add3e307daa 100644 --- a/pkg/trace/api/profiles.go +++ b/pkg/trace/api/profiles.go @@ -109,7 +109,7 @@ func errorHandler(err error) http.Handler { // The tags will be added as a header to all proxied requests. // For more details please see multiTransport. func newProfileProxy(conf *config.AgentConfig, targets []*url.URL, keys []string, tags string, statsd statsd.ClientInterface) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) director := func(req *http.Request) { req.Header.Set("Via", fmt.Sprintf("trace-agent %s", conf.AgentVersion)) if _, ok := req.Header["User-Agent"]; !ok { diff --git a/pkg/trace/api/symdb.go b/pkg/trace/api/symdb.go index 31b96c494ce5f..b238b478eb65c 100644 --- a/pkg/trace/api/symdb.go +++ b/pkg/trace/api/symdb.go @@ -62,7 +62,7 @@ func symDBErrorHandler(err error) http.Handler { // newSymDBProxy returns a new httputil.ReverseProxy proxying and augmenting requests with headers containing the tags. func newSymDBProxy(conf *config.AgentConfig, transport http.RoundTripper, hostTags string) *httputil.ReverseProxy { - cidProvider := NewIDProvider(conf.ContainerProcRoot) + cidProvider := NewIDProvider(conf.ContainerProcRoot, conf.ContainerIDFromOriginInfo) logger := log.NewThrottled(5, 10*time.Second) // limit to 5 messages every 10 seconds return &httputil.ReverseProxy{ Director: getSymDBDirector(hostTags, cidProvider, conf.ContainerTags), diff --git a/pkg/trace/config/config.go b/pkg/trace/config/config.go index 5d0e59f059e39..cbf0b0ea5c3a3 100644 --- a/pkg/trace/config/config.go +++ b/pkg/trace/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" + "github.com/DataDog/datadog-agent/comp/core/tagger/origindetection" "github.com/DataDog/datadog-agent/pkg/obfuscate" "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" "github.com/DataDog/datadog-agent/pkg/trace/log" @@ -445,6 +446,9 @@ type AgentConfig struct { // ContainerTags ... ContainerTags func(cid string) ([]string, error) `json:"-"` + // ContainerIDFromOriginInfo ... + ContainerIDFromOriginInfo func(originInfo origindetection.OriginInfo) (string, error) `json:"-"` + // ContainerProcRoot is the root dir for `proc` info ContainerProcRoot string diff --git a/pkg/trace/go.mod b/pkg/trace/go.mod index fa1ca487a3dc8..e0e3ee4906d99 100644 --- a/pkg/trace/go.mod +++ b/pkg/trace/go.mod @@ -52,6 +52,7 @@ require ( ) require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 github.com/shirou/gopsutil/v4 v4.24.11 go.opentelemetry.io/collector/component/componenttest v0.115.0 ) @@ -106,16 +107,16 @@ require ( go.opentelemetry.io/otel/trace v1.32.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( github.com/DataDog/datadog-agent => ../../ + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../comp/trace/compression/def github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip => ../../comp/trace/compression/impl-gzip github.com/DataDog/datadog-agent/comp/trace/compression/impl-zstd => ../../comp/trace/compression/impl-zstd diff --git a/pkg/trace/go.sum b/pkg/trace/go.sum index 4613790f7cc34..44ab376fcf70f 100644 --- a/pkg/trace/go.sum +++ b/pkg/trace/go.sum @@ -335,8 +335,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -402,8 +402,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/trace/info/info.go b/pkg/trace/info/info.go index df86c40587fdc..4198100ee6a5b 100644 --- a/pkg/trace/info/info.go +++ b/pkg/trace/info/info.go @@ -8,6 +8,7 @@ package info import ( "bytes" + "crypto/tls" "encoding/json" "expvar" // automatically publish `/debug/vars` on HTTP port "fmt" @@ -236,8 +237,9 @@ func getProgramBanner(version string) (string, string) { // If error is nil, means the program is running. // If not, it displays a pretty-printed message anyway (for support) func Info(w io.Writer, conf *config.AgentConfig) error { - url := fmt.Sprintf("http://127.0.0.1:%d/debug/vars", conf.DebugServerPort) - client := http.Client{Timeout: 3 * time.Second} + url := fmt.Sprintf("https://127.0.0.1:%d/debug/vars", conf.DebugServerPort) + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + client := http.Client{Timeout: 3 * time.Second, Transport: tr} resp, err := client.Get(url) if err != nil { // OK, here, we can't even make an http call on the agent port, diff --git a/pkg/trace/info/info_test.go b/pkg/trace/info/info_test.go index 01c369cac403e..25b7cb42b13ff 100644 --- a/pkg/trace/info/info_test.go +++ b/pkg/trace/info/info_test.go @@ -63,7 +63,7 @@ func (h *testServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func testServer(t *testing.T, testFile string) *httptest.Server { t.Helper() - server := httptest.NewServer(&testServerHandler{t: t, testFile: testFile}) + server := httptest.NewTLSServer(&testServerHandler{t: t, testFile: testFile}) t.Logf("test server (serving fake yet valid data) listening on %s", server.URL) return server } @@ -94,7 +94,7 @@ func (h *testServerWarningHandler) ServeHTTP(w http.ResponseWriter, r *http.Requ } func testServerWarning(t *testing.T) *httptest.Server { - server := httptest.NewServer(&testServerWarningHandler{t: t}) + server := httptest.NewTLSServer(&testServerWarningHandler{t: t}) t.Logf("test server (serving data containing worrying values) listening on %s", server.URL) return server } @@ -119,7 +119,7 @@ func (h *testServerErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques } func testServerError(t *testing.T) *httptest.Server { - server := httptest.NewServer(&testServerErrorHandler{t: t}) + server := httptest.NewTLSServer(&testServerErrorHandler{t: t}) t.Logf("test server (serving bad data to trigger errors) listening on %s", server.URL) return server } @@ -331,7 +331,7 @@ func TestError(t *testing.T) { assert.Equal(len(lines[1]), len(lines[2])) assert.Equal("", lines[3]) assert.Regexp(regexp.MustCompile(`^ Error: .*$`), lines[4]) - assert.Equal(fmt.Sprintf(" URL: http://127.0.0.1:%d/debug/vars", port), lines[5]) + assert.Equal(fmt.Sprintf(" URL: https://127.0.0.1:%d/debug/vars", port), lines[5]) assert.Equal("", lines[6]) assert.Equal("", lines[7]) } diff --git a/pkg/trace/stats/oteltest/go.mod b/pkg/trace/stats/oteltest/go.mod index 306e0217b22ad..3f97dcf41808c 100644 --- a/pkg/trace/stats/oteltest/go.mod +++ b/pkg/trace/stats/oteltest/go.mod @@ -20,6 +20,7 @@ require ( require go.opentelemetry.io/collector/component v0.115.0 // indirect require ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/def v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/trace/compression/impl-gzip v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.56.0-rc.3 // indirect @@ -76,18 +77,18 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.8.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect google.golang.org/grpc v1.67.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) replace ( + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/metricsclient => ../../../../comp/otelcol/otlp/components/metricsclient github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/statsprocessor => ../../../../comp/otelcol/otlp/components/statsprocessor github.com/DataDog/datadog-agent/comp/trace/compression/def => ../../../../comp/trace/compression/def diff --git a/pkg/trace/stats/oteltest/go.sum b/pkg/trace/stats/oteltest/go.sum index 25a9e6fccac3c..abd898f9bed2a 100644 --- a/pkg/trace/stats/oteltest/go.sum +++ b/pkg/trace/stats/oteltest/go.sum @@ -196,8 +196,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -245,8 +245,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/cgroups/go.mod b/pkg/util/cgroups/go.mod index 47a3c2c368f5f..0be6406d65503 100644 --- a/pkg/util/cgroups/go.mod +++ b/pkg/util/cgroups/go.mod @@ -31,7 +31,6 @@ require ( go.uber.org/atomic v1.11.0 // indirect golang.org/x/sys v0.28.0 // indirect google.golang.org/protobuf v1.35.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/cgroups/go.sum b/pkg/util/cgroups/go.sum index 69f8c3e58e44b..d2a0373b5e095 100644 --- a/pkg/util/cgroups/go.sum +++ b/pkg/util/cgroups/go.sum @@ -38,7 +38,5 @@ google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/defaultpaths/go.mod b/pkg/util/defaultpaths/go.mod index ad84aa45db6e9..0f0c498c8154e 100644 --- a/pkg/util/defaultpaths/go.mod +++ b/pkg/util/defaultpaths/go.mod @@ -22,7 +22,7 @@ require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/util/defaultpaths/go.sum b/pkg/util/defaultpaths/go.sum index 767c365bf98df..82beecfd023fb 100644 --- a/pkg/util/defaultpaths/go.sum +++ b/pkg/util/defaultpaths/go.sum @@ -21,7 +21,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/filesystem/go.mod b/pkg/util/filesystem/go.mod index 292bb06f164b9..31b6f1fbd4ab0 100644 --- a/pkg/util/filesystem/go.mod +++ b/pkg/util/filesystem/go.mod @@ -29,7 +29,6 @@ require ( github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/filesystem/go.sum b/pkg/util/filesystem/go.sum index 14cc56a3421e8..ebdb96f4beb1d 100644 --- a/pkg/util/filesystem/go.sum +++ b/pkg/util/filesystem/go.sum @@ -36,7 +36,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/grpc/go.mod b/pkg/util/grpc/go.mod index 9598ddb031dea..8c98d5f5187ee 100644 --- a/pkg/util/grpc/go.mod +++ b/pkg/util/grpc/go.mod @@ -40,7 +40,7 @@ require ( github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 google.golang.org/grpc v1.67.1 ) diff --git a/pkg/util/grpc/go.sum b/pkg/util/grpc/go.sum index f5e59fbfc1ba8..c91536ab269b2 100644 --- a/pkg/util/grpc/go.sum +++ b/pkg/util/grpc/go.sum @@ -292,8 +292,8 @@ golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= diff --git a/pkg/util/hostname/validate/go.mod b/pkg/util/hostname/validate/go.mod index 6ec1ae79a177d..9c1a227476e2c 100644 --- a/pkg/util/hostname/validate/go.mod +++ b/pkg/util/hostname/validate/go.mod @@ -19,7 +19,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/hostname/validate/go.sum b/pkg/util/hostname/validate/go.sum index c57375d7e46df..d45d69118e85e 100644 --- a/pkg/util/hostname/validate/go.sum +++ b/pkg/util/hostname/validate/go.sum @@ -17,7 +17,5 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/http/go.mod b/pkg/util/http/go.mod index 74a5f34311844..56e028a8317d6 100644 --- a/pkg/util/http/go.mod +++ b/pkg/util/http/go.mod @@ -35,7 +35,7 @@ require ( github.com/DataDog/datadog-agent/pkg/config/model v0.59.0 github.com/DataDog/datadog-agent/pkg/util/log v0.59.1 github.com/stretchr/testify v1.10.0 - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 ) require ( diff --git a/pkg/util/http/go.sum b/pkg/util/http/go.sum index 0ba8e47583917..1051ef2578f7d 100644 --- a/pkg/util/http/go.sum +++ b/pkg/util/http/go.sum @@ -254,8 +254,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go index 45ea62c6e2d26..79e8a89895beb 100644 --- a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go +++ b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection.go @@ -228,9 +228,15 @@ func (le *LeaderEngine) EnsureLeaderElectionRuns() error { func (le *LeaderEngine) runLeaderElection() { for { - log.Infof("Starting leader election process for %q...", le.HolderIdentity) - le.leaderElector.Run(le.ctx) - log.Info("Leader election lost") + select { + case <-le.ctx.Done(): + log.Infof("Quitting leader election process: context was cancelled") + return + default: + log.Infof("Starting leader election process for %q...", le.HolderIdentity) + le.leaderElector.Run(le.ctx) + log.Info("Leader election lost") + } } } diff --git a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go index af604ec84e37c..eefce053e2e22 100644 --- a/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go +++ b/pkg/util/kubernetes/apiserver/leaderelection/leaderelection_test.go @@ -211,7 +211,7 @@ func TestSubscribe(t *testing.T) { coreClient: client.CoreV1(), coordClient: client.CoordinationV1(), leaderMetric: &dummyGauge{}, - lockType: cmLock.ConfigMapsResourceLock, + lockType: tc.lockType, } notif1 := le.Subscribe() @@ -227,6 +227,9 @@ func TestSubscribe(t *testing.T) { le.EnsureLeaderElectionRuns() require.True(t, le.IsLeader()) + err = tc.getTokenFunc(client) + require.NoError(t, err) + counter1, counter2 := 0, 0 for { select { diff --git a/pkg/util/log/go.mod b/pkg/util/log/go.mod index 26f759935f92e..a439a62283f76 100644 --- a/pkg/util/log/go.mod +++ b/pkg/util/log/go.mod @@ -17,7 +17,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect go.uber.org/multierr v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/log/go.sum b/pkg/util/log/go.sum index f46b720dc7374..ffca41d546584 100644 --- a/pkg/util/log/go.sum +++ b/pkg/util/log/go.sum @@ -23,7 +23,5 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/scrubber/default_test.go b/pkg/util/scrubber/default_test.go index 86134fbc90328..0bb2d721c3983 100644 --- a/pkg/util/scrubber/default_test.go +++ b/pkg/util/scrubber/default_test.go @@ -6,16 +6,13 @@ package scrubber import ( - "encoding/json" "os" "path/filepath" - "reflect" "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" ) func assertClean(t *testing.T, contents, cleanContents string) { @@ -26,96 +23,6 @@ func assertClean(t *testing.T, contents, cleanContents string) { assert.Equal(t, strings.TrimSpace(cleanContents), strings.TrimSpace(cleanedString)) } -func TestConfigScrubbedValidYaml(t *testing.T) { - wd, _ := os.Getwd() - - inputConf := filepath.Join(wd, "test", "conf.yaml") - inputConfData, err := os.ReadFile(inputConf) - require.NoError(t, err) - - outputConf := filepath.Join(wd, "test", "conf_scrubbed.yaml") - outputConfData, err := os.ReadFile(outputConf) - require.NoError(t, err) - - cleaned, err := ScrubBytes([]byte(inputConfData)) - require.NoError(t, err) - - // First test that the a scrubbed yaml is still a valid yaml - var out interface{} - err = yaml.Unmarshal(cleaned, &out) - assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") - - // We replace windows line break by linux so the tests pass on every OS - trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) - trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) - - assert.Equal(t, trimmedOutput, trimmedCleaned) -} - -func TestConfigScrubbedYaml(t *testing.T) { - wd, _ := os.Getwd() - - inputConf := filepath.Join(wd, "test", "conf_multiline.yaml") - inputConfData, err := os.ReadFile(inputConf) - require.NoError(t, err) - - outputConf := filepath.Join(wd, "test", "conf_multiline_scrubbed.yaml") - outputConfData, err := os.ReadFile(outputConf) - require.NoError(t, err) - - cleaned, err := ScrubYaml([]byte(inputConfData)) - require.NoError(t, err) - - // First test that the a scrubbed yaml is still a valid yaml - var out interface{} - err = yaml.Unmarshal(cleaned, &out) - assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") - - // We replace windows line break by linux so the tests pass on every OS - trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) - trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) - - assert.Equal(t, trimmedOutput, trimmedCleaned) -} - -func TestConfigScrubbedJson(t *testing.T) { - wd, _ := os.Getwd() - - inputConf := filepath.Join(wd, "test", "config.json") - inputConfData, err := os.ReadFile(inputConf) - require.NoError(t, err) - cleaned, err := ScrubJSON([]byte(inputConfData)) - require.NoError(t, err) - // First test that the a scrubbed json is still valid - var actualOutJSON map[string]interface{} - err = json.Unmarshal(cleaned, &actualOutJSON) - assert.NoError(t, err, "Could not load JSON configuration after being scrubbed") - - outputConf := filepath.Join(wd, "test", "config_scrubbed.json") - outputConfData, err := os.ReadFile(outputConf) - require.NoError(t, err) - var expectedOutJSON map[string]interface{} - err = json.Unmarshal(outputConfData, &expectedOutJSON) - require.NoError(t, err) - assert.Equal(t, reflect.DeepEqual(expectedOutJSON, actualOutJSON), true) -} - -func TestEmptyYaml(t *testing.T) { - cleaned, err := ScrubYaml(nil) - require.NoError(t, err) - assert.Equal(t, "", string(cleaned)) - - cleaned, err = ScrubYaml([]byte("")) - require.NoError(t, err) - assert.Equal(t, "", string(cleaned)) -} - -func TestEmptyYamlString(t *testing.T) { - cleaned, err := ScrubYamlString("") - require.NoError(t, err) - assert.Equal(t, "", string(cleaned)) -} - func TestConfigStripApiKey(t *testing.T) { assertClean(t, `api_key: aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb`, @@ -567,39 +474,8 @@ func TestAddStrippedKeys(t *testing.T) { AddStrippedKeys([]string{"foobar"}) assertClean(t, contents, `foobar: "********"`) -} -func TestAddStrippedKeysExceptions(t *testing.T) { - t.Run("single key", func(t *testing.T) { - contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'` - - AddStrippedKeys([]string{"api_key"}) - - scrubbed, err := ScrubYamlString(contents) - require.Nil(t, err) - require.YAMLEq(t, `api_key: '***************************aaaaa'`, scrubbed) - }) - - t.Run("multiple keys", func(t *testing.T) { - contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' -some_other_key: 'bbbb' -app_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaacccc' -yet_another_key: 'dddd'` - - keys := []string{"api_key", "some_other_key", "app_key"} - AddStrippedKeys(keys) - - // check that AddStrippedKeys didn't modify the parameter slice - assert.Equal(t, []string{"api_key", "some_other_key", "app_key"}, keys) - - scrubbed, err := ScrubYamlString(contents) - require.Nil(t, err) - expected := `api_key: '***************************aaaaa' -some_other_key: '********' -app_key: '***********************************acccc' -yet_another_key: 'dddd'` - require.YAMLEq(t, expected, scrubbed) - }) + dynamicReplacers = []Replacer{} } func TestAddStrippedKeysNewReplacer(t *testing.T) { @@ -612,6 +488,8 @@ func TestAddStrippedKeysNewReplacer(t *testing.T) { cleaned, err := newScrubber.ScrubBytes([]byte(contents)) require.NoError(t, err) assert.Equal(t, strings.TrimSpace(`foobar: "********"`), strings.TrimSpace(string(cleaned))) + + dynamicReplacers = []Replacer{} } func TestCertConfig(t *testing.T) { @@ -686,33 +564,6 @@ network_devices: log_level: info`) } -func TestConfigFile(t *testing.T) { - cleanedConfigFile := `dd_url: https://app.datadoghq.com - -api_key: "***************************aaaaa" - -proxy: http://user:********@host:port - - - - - - -dogstatsd_port : 8125 - - -log_level: info -` - - wd, _ := os.Getwd() - filePath := filepath.Join(wd, "test", "datadog.yaml") - cleaned, err := ScrubFile(filePath) - assert.NoError(t, err) - cleanedString := string(cleaned) - - assert.Equal(t, cleanedConfigFile, cleanedString) -} - func TestBearerToken(t *testing.T) { assertClean(t, `Bearer 2fe663014abcd1850076f6d68c0355666db98758262870811cace007cd4a62ba`, @@ -785,3 +636,30 @@ func TestScrubCommandsEnv(t *testing.T) { }) } } + +func TestConfigFile(t *testing.T) { + cleanedConfigFile := `dd_url: https://app.datadoghq.com + +api_key: "***************************aaaaa" + +proxy: http://user:********@host:port + + + + + + +dogstatsd_port : 8125 + + +log_level: info +` + + wd, _ := os.Getwd() + filePath := filepath.Join(wd, "test", "datadog.yaml") + cleaned, err := ScrubFile(filePath) + assert.NoError(t, err) + cleanedString := string(cleaned) + + assert.Equal(t, cleanedConfigFile, cleanedString) +} diff --git a/pkg/util/scrubber/go.mod b/pkg/util/scrubber/go.mod index 6f10e986954d3..ff0eb09afc0ef 100644 --- a/pkg/util/scrubber/go.mod +++ b/pkg/util/scrubber/go.mod @@ -5,14 +5,13 @@ go 1.22.0 require ( github.com/DataDog/datadog-agent/pkg/version v0.59.1 github.com/stretchr/testify v1.10.0 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/util/scrubber/go.sum b/pkg/util/scrubber/go.sum index def44d1759f5c..878d5c29e2e5d 100644 --- a/pkg/util/scrubber/go.sum +++ b/pkg/util/scrubber/go.sum @@ -14,7 +14,5 @@ github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/scrubber/json_scrubber.go b/pkg/util/scrubber/json_scrubber.go index 34c65dd357cfa..dbf624b119533 100644 --- a/pkg/util/scrubber/json_scrubber.go +++ b/pkg/util/scrubber/json_scrubber.go @@ -22,7 +22,7 @@ func (c *Scrubber) ScrubJSON(input []byte) ([]byte, error) { if len(input) != 0 && err == nil { c.ScrubDataObj(data) - newInput, err := json.Marshal(data) + newInput, err := json.MarshalIndent(data, "", " ") if err == nil { return newInput, nil } diff --git a/pkg/util/scrubber/json_scrubber_test.go b/pkg/util/scrubber/json_scrubber_test.go index 007da8d3f34cb..b45d0a8d08f78 100644 --- a/pkg/util/scrubber/json_scrubber_test.go +++ b/pkg/util/scrubber/json_scrubber_test.go @@ -6,9 +6,14 @@ package scrubber import ( + "encoding/json" + "os" + "path/filepath" + "reflect" "regexp" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -68,3 +73,25 @@ func TestScrubJSON(t *testing.T) { require.Equal(t, expected, string(actual)) }) } + +func TestConfigScrubbedJson(t *testing.T) { + wd, _ := os.Getwd() + + inputConf := filepath.Join(wd, "test", "config.json") + inputConfData, err := os.ReadFile(inputConf) + require.NoError(t, err) + cleaned, err := ScrubJSON([]byte(inputConfData)) + require.NoError(t, err) + // First test that the a scrubbed json is still valid + var actualOutJSON map[string]interface{} + err = json.Unmarshal(cleaned, &actualOutJSON) + assert.NoError(t, err, "Could not load JSON configuration after being scrubbed") + + outputConf := filepath.Join(wd, "test", "config_scrubbed.json") + outputConfData, err := os.ReadFile(outputConf) + require.NoError(t, err) + var expectedOutJSON map[string]interface{} + err = json.Unmarshal(outputConfData, &expectedOutJSON) + require.NoError(t, err) + assert.Equal(t, reflect.DeepEqual(expectedOutJSON, actualOutJSON), true) +} diff --git a/pkg/util/scrubber/scrubber_test.go b/pkg/util/scrubber/scrubber_test.go index dc23ec52d1971..5a6245d81cddd 100644 --- a/pkg/util/scrubber/scrubber_test.go +++ b/pkg/util/scrubber/scrubber_test.go @@ -28,6 +28,16 @@ func TestNewWithDefaults(t *testing.T) { assert.Equal(t, len(scrubberEmpty.multiLineReplacers), len(scrubber.multiLineReplacers)) } +func TestLastUpdated(t *testing.T) { + scrubber := NewWithDefaults() + for _, replacer := range scrubber.singleLineReplacers { + assert.NotNil(t, replacer.LastUpdated, "single line replacer has no LastUpdated: %v", replacer) + } + for _, replacer := range scrubber.multiLineReplacers { + assert.NotNil(t, replacer.LastUpdated, "multi line replacer has no LastUpdated: %v", replacer) + } +} + func TestRepl(t *testing.T) { scrubber := New() scrubber.AddReplacer(SingleLine, Replacer{ diff --git a/pkg/util/scrubber/test/conf.yaml b/pkg/util/scrubber/test/conf.yaml index cf7fc3d34d3f5..d2bb11a2a9fe3 100644 --- a/pkg/util/scrubber/test/conf.yaml +++ b/pkg/util/scrubber/test/conf.yaml @@ -14,7 +14,7 @@ list_api_key: [aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, aaaaaaaaaaaaaaaaaaaaaaaaaaaabbb list_app_key: [aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb] dict_api_key: {one: aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, two: aaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb} -dict_app_key: {two: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, two: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb} +dict_app_key: {one: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb, two: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbb} additional_endpoints: "https://app.datadoghq.com": @@ -29,15 +29,15 @@ random_url_key1: http://user:p@ssw0r)@host:port random_url_key2: protocol://user:p@ssw0r)@host:port random_url_key3: "http://user:password@host:port" random_domain_key: 'user:password@host:port' -random_url_key1: | +random_url_key4: | http://user:password@host:port -random_url_key2: > +random_url_key5: > http://user:password@host:port -random_url_key3: 'http://user:password@host:port' -random_url_key4: 'mongodb+s.r-v://user:password@host:port' -random_url_key5: 'mongodb+srv://user:pass-with-hyphen@abc.example.com/database' -random_url_key6: 'http://user-with-hyphen:pass-with-hyphen@abc.example.com/database' -random_url_key7: 'http://user-with-hyphen:pass@abc.example.com/database' +random_url_key6: 'http://user:password@host:port' +random_url_key7: 'mongodb+s.r-v://user:password@host:port' +random_url_key8: 'mongodb+srv://user:pass-with-hyphen@abc.example.com/database' +random_url_key9: 'http://user-with-hyphen:pass-with-hyphen@abc.example.com/database' +random_url_key10: 'http://user-with-hyphen:pass@abc.example.com/database' mysql_password1: password mysql_password3: "password" diff --git a/pkg/util/scrubber/test/conf_scrubbed.yaml b/pkg/util/scrubber/test/conf_scrubbed.yaml index c4c987bba5df4..5bb34c750ac3c 100644 --- a/pkg/util/scrubber/test/conf_scrubbed.yaml +++ b/pkg/util/scrubber/test/conf_scrubbed.yaml @@ -14,7 +14,7 @@ list_api_key: ["***************************abbbb", "***************************a list_app_key: ["***********************************abbbb", "***********************************abbbb"] dict_api_key: {one: "***************************abbbb", two: "***************************abbbb"} -dict_app_key: {two: "***********************************abbbb", two: "***********************************abbbb"} +dict_app_key: {one: "***********************************abbbb", two: "***********************************abbbb"} additional_endpoints: "https://app.datadoghq.com": @@ -29,15 +29,15 @@ random_url_key1: http://user:********@host:port random_url_key2: protocol://user:********@host:port random_url_key3: "http://user:********@host:port" random_domain_key: 'user:********@host:port' -random_url_key1: | +random_url_key4: | http://user:********@host:port -random_url_key2: > +random_url_key5: > http://user:********@host:port -random_url_key3: 'http://user:********@host:port' -random_url_key4: 'mongodb+s.r-v://user:********@host:port' -random_url_key5: 'mongodb+srv://user:********@abc.example.com/database' -random_url_key6: 'http://user-with-hyphen:********@abc.example.com/database' -random_url_key7: 'http://user-with-hyphen:********@abc.example.com/database' +random_url_key6: 'http://user:********@host:port' +random_url_key7: 'mongodb+s.r-v://user:********@host:port' +random_url_key8: 'mongodb+srv://user:********@abc.example.com/database' +random_url_key9: 'http://user-with-hyphen:********@abc.example.com/database' +random_url_key10: 'http://user-with-hyphen:********@abc.example.com/database' mysql_password1: "********" mysql_password3: "********" diff --git a/pkg/util/scrubber/yaml_scrubber.go b/pkg/util/scrubber/yaml_scrubber.go index f01d93e064ec5..90f144c6617eb 100644 --- a/pkg/util/scrubber/yaml_scrubber.go +++ b/pkg/util/scrubber/yaml_scrubber.go @@ -6,10 +6,11 @@ package scrubber import ( + "bytes" "fmt" "os" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) type scrubCallback = func(string, interface{}) (bool, interface{}) @@ -110,13 +111,16 @@ func (c *Scrubber) ScrubYaml(input []byte) ([]byte, error) { // if we can't load the yaml run the default scrubber on the input if len(input) != 0 && err == nil { c.ScrubDataObj(data) - newInput, err := yaml.Marshal(data) - if err == nil { - input = newInput - } else { - // Since the scrubber is a dependency of the logger we can use it here. + + var buffer bytes.Buffer + encoder := yaml.NewEncoder(&buffer) + encoder.SetIndent(2) + if err := encoder.Encode(&data); err != nil { fmt.Fprintf(os.Stderr, "error scrubbing YAML, falling back on text scrubber: %s\n", err) + } else { + input = buffer.Bytes() } + encoder.Close() } return c.ScrubBytes(input) } diff --git a/pkg/util/scrubber/yaml_scrubber_test.go b/pkg/util/scrubber/yaml_scrubber_test.go index 68f45dbcb9aaa..d576472186bd2 100644 --- a/pkg/util/scrubber/yaml_scrubber_test.go +++ b/pkg/util/scrubber/yaml_scrubber_test.go @@ -6,9 +6,15 @@ package scrubber import ( + "os" + "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "gopkg.in/yaml.v3" ) func TestScrubDataObj(t *testing.T) { @@ -62,3 +68,104 @@ func TestScrubDataObj(t *testing.T) { }) } } + +func TestConfigScrubbedValidYaml(t *testing.T) { + wd, _ := os.Getwd() + + inputConf := filepath.Join(wd, "test", "conf.yaml") + inputConfData, err := os.ReadFile(inputConf) + require.NoError(t, err) + + outputConf := filepath.Join(wd, "test", "conf_scrubbed.yaml") + outputConfData, err := os.ReadFile(outputConf) + require.NoError(t, err) + + cleaned, err := ScrubBytes([]byte(inputConfData)) + require.NoError(t, err) + + // First test that the a scrubbed yaml is still a valid yaml + var out interface{} + err = yaml.Unmarshal(cleaned, &out) + assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") + + // We replace windows line break by linux so the tests pass on every OS + trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) + trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) + + assert.Equal(t, trimmedOutput, trimmedCleaned) +} + +func TestConfigScrubbedYaml(t *testing.T) { + wd, _ := os.Getwd() + + inputConf := filepath.Join(wd, "test", "conf_multiline.yaml") + inputConfData, err := os.ReadFile(inputConf) + require.NoError(t, err) + + outputConf := filepath.Join(wd, "test", "conf_multiline_scrubbed.yaml") + outputConfData, err := os.ReadFile(outputConf) + require.NoError(t, err) + + cleaned, err := ScrubYaml([]byte(inputConfData)) + require.NoError(t, err) + + // First test that the a scrubbed yaml is still a valid yaml + var out interface{} + err = yaml.Unmarshal(cleaned, &out) + assert.NoError(t, err, "Could not load YAML configuration after being scrubbed") + + // We replace windows line break by linux so the tests pass on every OS + trimmedOutput := strings.TrimSpace(strings.Replace(string(outputConfData), "\r\n", "\n", -1)) + trimmedCleaned := strings.TrimSpace(strings.Replace(string(cleaned), "\r\n", "\n", -1)) + + assert.Equal(t, trimmedOutput, trimmedCleaned) +} + +func TestEmptyYaml(t *testing.T) { + cleaned, err := ScrubYaml(nil) + require.NoError(t, err) + assert.Equal(t, "", string(cleaned)) + + cleaned, err = ScrubYaml([]byte("")) + require.NoError(t, err) + assert.Equal(t, "", string(cleaned)) +} + +func TestEmptyYamlString(t *testing.T) { + cleaned, err := ScrubYamlString("") + require.NoError(t, err) + assert.Equal(t, "", string(cleaned)) +} + +func TestAddStrippedKeysExceptions(t *testing.T) { + t.Run("single key", func(t *testing.T) { + contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'` + + AddStrippedKeys([]string{"api_key"}) + + scrubbed, err := ScrubYamlString(contents) + require.Nil(t, err) + require.YAMLEq(t, `api_key: '***************************aaaaa'`, scrubbed) + }) + + t.Run("multiple keys", func(t *testing.T) { + contents := `api_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' +some_other_key: 'bbbb' +app_key: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaacccc' +yet_another_key: 'dddd'` + + keys := []string{"api_key", "some_other_key", "app_key"} + AddStrippedKeys(keys) + + // check that AddStrippedKeys didn't modify the parameter slice + assert.Equal(t, []string{"api_key", "some_other_key", "app_key"}, keys) + + scrubbed, err := ScrubYamlString(contents) + require.Nil(t, err) + expected := `api_key: '***************************aaaaa' +some_other_key: '********' +app_key: '***********************************acccc' +yet_another_key: 'dddd'` + require.YAMLEq(t, expected, scrubbed) + }) +} diff --git a/pkg/util/system/go.mod b/pkg/util/system/go.mod index 78e3380d124ed..69183e1155599 100644 --- a/pkg/util/system/go.mod +++ b/pkg/util/system/go.mod @@ -37,7 +37,6 @@ require ( github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/system/go.sum b/pkg/util/system/go.sum index 47e0acb25ee1f..5eac83e6a0bd2 100644 --- a/pkg/util/system/go.sum +++ b/pkg/util/system/go.sum @@ -45,7 +45,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/uuid/go.mod b/pkg/util/uuid/go.mod index ad2e7ef2c7f0a..0f8d47fc3bb8c 100644 --- a/pkg/util/uuid/go.mod +++ b/pkg/util/uuid/go.mod @@ -28,7 +28,7 @@ require ( github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/atomic v1.11.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/DataDog/datadog-agent/pkg/version => ../../version diff --git a/pkg/util/uuid/go.sum b/pkg/util/uuid/go.sum index 2e5f8b7759901..7399205747094 100644 --- a/pkg/util/uuid/go.sum +++ b/pkg/util/uuid/go.sum @@ -44,7 +44,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/util/winutil/go.mod b/pkg/util/winutil/go.mod index 1facedf93843c..172db988efae7 100644 --- a/pkg/util/winutil/go.mod +++ b/pkg/util/winutil/go.mod @@ -21,7 +21,6 @@ require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/util/winutil/go.sum b/pkg/util/winutil/go.sum index 99e864f633ef6..788135f619bbb 100644 --- a/pkg/util/winutil/go.sum +++ b/pkg/util/winutil/go.sum @@ -21,7 +21,5 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml b/releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml new file mode 100644 index 0000000000000..0e5a6cd7f5c92 --- /dev/null +++ b/releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG-DCA.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + The `kubernetes_state_core` check now collects only metadata for configmaps, + reducing memory, CPU, and network usage in the Cluster Agent while preserving + full metric functionality. diff --git a/releasenotes/notes/Add-metrics-origins-for-Quarkus-integration.-e2ff9ac00b7ea6df.yaml b/releasenotes/notes/Add-metrics-origins-for-Quarkus-integration.-e2ff9ac00b7ea6df.yaml new file mode 100644 index 0000000000000..6a23ab8155f8c --- /dev/null +++ b/releasenotes/notes/Add-metrics-origins-for-Quarkus-integration.-e2ff9ac00b7ea6df.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +other: + - | + Add metrics origins for Quarkus integration. diff --git a/releasenotes/notes/add-integration-field-to-topology-link-metadata-8a35c0a70ac80b1e.yaml b/releasenotes/notes/add-integration-field-to-topology-link-metadata-8a35c0a70ac80b1e.yaml new file mode 100644 index 0000000000000..003ca577e3799 --- /dev/null +++ b/releasenotes/notes/add-integration-field-to-topology-link-metadata-8a35c0a70ac80b1e.yaml @@ -0,0 +1,12 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Emit new field: integration for TopologyLinkMetadata for better metrics collection. + diff --git a/releasenotes/notes/add_system_cpu_total_metrics-769eed5d476ba712.yaml b/releasenotes/notes/add_system_cpu_total_metrics-769eed5d476ba712.yaml new file mode 100644 index 0000000000000..9d2899b7bfcc1 --- /dev/null +++ b/releasenotes/notes/add_system_cpu_total_metrics-769eed5d476ba712.yaml @@ -0,0 +1,15 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Add new system.cpu.user.total, system.cpu.nice.total, + system.cpu.system.total, system.cpu.idle.total, system.cpu.iowait.total, + system.cpu.irq.total, system.cpu.softirq.total, system.cpu.steal.total, + system.cpu.guest.total, system.cpu.guestnice.total metrics + with core tag for each one of them. diff --git a/releasenotes/notes/agent-sidecar-security-cbfd5ea9f72124d0.yaml b/releasenotes/notes/agent-sidecar-security-cbfd5ea9f72124d0.yaml new file mode 100644 index 0000000000000..a1f2095a0d121 --- /dev/null +++ b/releasenotes/notes/agent-sidecar-security-cbfd5ea9f72124d0.yaml @@ -0,0 +1,14 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +upgrade: +enhancements: + - | + The Datadog Cluster Agent admission controller agent sidecar injection now sets up + Agent sidecars to run with securityContext of `readOnlyRootFilesystem:false` by default. + Advanced users can customize the securityContext via clusterAgent.admissionController.agentSidecarInjection.profiles. diff --git a/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml new file mode 100644 index 0000000000000..4aa3e9ef23e70 --- /dev/null +++ b/releasenotes/notes/external_data_apm-6538531e3f34f305.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Implement External Data resolution for APM. This is needed to support the + latest Origin Detection spec and resolution with nested virtualization. + diff --git a/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml b/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml new file mode 100644 index 0000000000000..2baa2dea73281 --- /dev/null +++ b/releasenotes/notes/flare-semodule-list-883aecc886cd62ac.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Added the output of ``semodule -l`` to the Agent flare; this information appears in ``system-probe/selinux_semodule_list.log``. diff --git a/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml b/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml index 53476dcfa0bb9..e9c1031ef5d3b 100644 --- a/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml +++ b/releasenotes/notes/force-http1-for-log-agent-2cbfba763697ab42.yaml @@ -9,7 +9,7 @@ features: - | Introduced a new configuration variable `logs_config.http_protocol`, allowing users to enforce HTTP/1.1 for outgoing HTTP connections in the Datadog Agent. This provides better control over transport protocols and improves compatibility with systems that do not support HTTP/2. - By default, the log agent will now attempt to use HTTP/2 and fall back to the best available protocol if HTTP/2 is not supported. + By default, the log agent will now attempt to use HTTP/2 (unless a proxy is configured) and fall back to the best available protocol if HTTP/2 is not supported. enhancements: - | - Improved logging to add visiblity for latency and transport protocol \ No newline at end of file + Improved logging to add visibility for latency and transport protocol \ No newline at end of file diff --git a/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml new file mode 100644 index 0000000000000..7d9a8653d8f0d --- /dev/null +++ b/releasenotes/notes/metrics-compression-default-zstd-c786c2d28eb51b1f.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Metric payloads are compressed using `zstd` compression by default. + This can be reverted to the previous compression kind by adding + ``serializer_compressor_kind: zlib`` to the configuration. diff --git a/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml new file mode 100644 index 0000000000000..2957ae7f9441d --- /dev/null +++ b/releasenotes/notes/none-cardinality-tag-f2ceec7ec571387e.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Adds support for the `none` cardinality type in the tagger. diff --git a/releasenotes/notes/process-agent-config-auth-09a3123ac6496052.yaml b/releasenotes/notes/process-agent-config-auth-09a3123ac6496052.yaml new file mode 100644 index 0000000000000..f95ff5b472dac --- /dev/null +++ b/releasenotes/notes/process-agent-config-auth-09a3123ac6496052.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + All process agent endpoints now require authentication. diff --git a/tasks/agent.py b/tasks/agent.py index 19cd6170b96c6..84d1134310965 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -352,9 +352,8 @@ def refresh_assets(_, build_tags, development=True, flavor=AgentFlavor.base.name check_dir = os.path.join(dist_folder, f"conf.d/{check}.d/") shutil.copytree(f"./cmd/agent/dist/conf.d/{check}.d/", check_dir, dirs_exist_ok=True) - if "apm" in build_tags: + if sys.platform == 'darwin': shutil.copy("./cmd/agent/dist/conf.d/apm.yaml.default", os.path.join(dist_folder, "conf.d/apm.yaml.default")) - if "process" in build_tags: shutil.copy( "./cmd/agent/dist/conf.d/process_agent.yaml.default", os.path.join(dist_folder, "conf.d/process_agent.yaml.default"), diff --git a/tasks/build_tags.py b/tasks/build_tags.py index c400d2560aa94..9f58d3e45c60a 100644 --- a/tasks/build_tags.py +++ b/tasks/build_tags.py @@ -17,7 +17,6 @@ # ALL_TAGS lists all available build tags. # Used to remove unknown tags from provided tag lists. ALL_TAGS = { - "apm", "clusterchecks", "consul", "containerd", @@ -40,7 +39,6 @@ "otlp", "pcap", # used by system-probe to compile packet filters using google/gopacket/pcap, which requires cgo to link libpcap "podman", - "process", "python", "sds", "serverless", @@ -58,7 +56,6 @@ # AGENT_TAGS lists the tags needed when building the agent. AGENT_TAGS = { - "apm", "consul", "containerd", "cri", @@ -76,7 +73,6 @@ "orchestrator", "otlp", "podman", - "process", "python", "systemd", "trivy", @@ -166,7 +162,6 @@ "netcgo", "npm", "pcap", - "process", "trivy", "zlib", "zstd", diff --git a/tasks/buildimages.py b/tasks/buildimages.py index e2bf5316e18c7..06ea84211ac1a 100644 --- a/tasks/buildimages.py +++ b/tasks/buildimages.py @@ -46,13 +46,23 @@ def update_test_infra_definitions(ctx: Context, commit_sha: str, go_mod_only: bo """ Update the test-infra-definition image version in the Gitlab CI as well as in the e2e go.mod """ - if not go_mod_only: - update_test_infra_def(".gitlab/common/test_infra_version.yml", commit_sha[:12], is_dev_image) - + print(f"Updating test-infra-definitions to {commit_sha}") with ctx.cd("test/new-e2e"): ctx.run(f"go get github.com/DataDog/test-infra-definitions@{commit_sha}") ctx.run("go mod tidy") + if not go_mod_only: + prefix_comment = """# File generated by inv buildimages.update-test-infra-definitions +# Please do not edit this file manually +# To update the test-infra-definitions version, run `inv buildimages.update-test-infra-definitions --commit-sha ` [--is-dev-image] +""" + update_test_infra_def( + file_path=".gitlab/common/test_infra_version.yml", + image_tag=commit_sha[:12], + is_dev_image=is_dev_image, + prefix_comment=prefix_comment, + ) + @task( help={ diff --git a/tasks/github_tasks.py b/tasks/github_tasks.py index 7cc67757a61c3..bf2c546c8ecb4 100644 --- a/tasks/github_tasks.py +++ b/tasks/github_tasks.py @@ -22,7 +22,6 @@ from tasks.libs.common.constants import DEFAULT_INTEGRATIONS_CORE_BRANCH from tasks.libs.common.datadog_api import create_gauge, send_event, send_metrics from tasks.libs.common.git import get_default_branch -from tasks.libs.common.junit_upload_core import repack_macos_junit_tar from tasks.libs.common.utils import get_git_pretty_ref from tasks.libs.owners.linter import codeowner_has_orphans, directory_has_packages_without_owner from tasks.libs.owners.parsing import read_owners @@ -76,8 +75,6 @@ def trigger_macos( version_cache=None, retry_download=3, retry_interval=10, - fast_tests=None, - test_washer=False, integrations_core_ref=DEFAULT_INTEGRATIONS_CORE_BRANCH, ): """ @@ -107,26 +104,8 @@ def trigger_macos( version_cache_file_content=version_cache, integrations_core_ref=integrations_core_ref, ) - elif workflow_type == "test": - conclusion = _trigger_macos_workflow( - release_version, - destination, - retry_download, - retry_interval, - workflow_name="test.yaml", - datadog_agent_ref=datadog_agent_ref, - version_cache_file_content=version_cache, - fast_tests=fast_tests, - test_washer=test_washer, - ) - repack_macos_junit_tar(conclusion, "junit-tests_macos.tgz", "junit-tests_macos-repacked.tgz") - elif workflow_type == "lint": - conclusion = _trigger_macos_workflow( - release_version, - workflow_name="lint.yaml", - datadog_agent_ref=datadog_agent_ref, - version_cache_file_content=version_cache, - ) + else: + raise Exit(f"Unsupported workflow type: {workflow_type}", code=1) if conclusion != "success": raise Exit(message=f"Macos {workflow_type} workflow {conclusion}", code=1) diff --git a/tasks/go.py b/tasks/go.py index 13f451cce050f..5bb3fcad7d656 100644 --- a/tasks/go.py +++ b/tasks/go.py @@ -78,11 +78,16 @@ def lint_module(target): concurrency_arg = "" if concurrency is None else f"--concurrency {concurrency}" tags_arg = " ".join(sorted(set(tags))) timeout_arg_value = "25m0s" if not timeout else f"{timeout}m0s" - return ctx.run( + res = ctx.run( f'golangci-lint run {verbosity} --timeout {timeout_arg_value} {concurrency_arg} --build-tags "{tags_arg}" --path-prefix "{module_path}" {golangci_lint_kwargs} {target}/...', env=env, warn=True, ) + # early stop on SIGINT: exit code is 128 + signal number, SIGINT is 2, so 130 + # for some reason this becomes -2 here + if res is not None and (res.exited == -2 or res.exited == 130): + raise KeyboardInterrupt() + return res target_path = Path(module_path) / target result, time_result = TimedOperationResult.run( diff --git a/tasks/gotest.py b/tasks/gotest.py index 4aa9a4b3e151f..af889e8b6dcae 100644 --- a/tasks/gotest.py +++ b/tasks/gotest.py @@ -145,6 +145,9 @@ def command(test_results, module, module_result): out_stream=test_profiler, warn=True, ) + # early stop on SIGINT: exit code is 128 + signal number, SIGINT is 2, so 130 + if res is not None and res.exited == 130: + raise KeyboardInterrupt() module_result.result_json_path = os.path.join(module_path, GO_TEST_RESULT_TMP_JSON) diff --git a/tasks/kmt.py b/tasks/kmt.py index 18f5f9370893b..b284e6d56c91b 100644 --- a/tasks/kmt.py +++ b/tasks/kmt.py @@ -738,17 +738,18 @@ def prepare( ) if ci: - domains = None stack = "ci" return _prepare(ctx, stack, component, arch_obj, packages, verbose, ci, compile_only) - if alien_vms is not None: - err_msg = f"no alient VMs discovered from provided profile {alien_vms}." - else: - err_msg = f"no vms found from list {vms}. Run `inv -e kmt.status` to see all VMs in current stack" - stack = get_kmt_or_alien_stack(ctx, stack, vms, alien_vms) - domains = get_target_domains(ctx, stack, ssh_key, arch_obj, vms, alien_vms) - assert len(domains) > 0, err_msg + domains = None + if not compile_only: + if alien_vms is not None: + err_msg = f"no alient VMs discovered from provided profile {alien_vms}." + else: + err_msg = f"no vms found from list {vms}. Run `inv -e kmt.status` to see all VMs in current stack" + stack = get_kmt_or_alien_stack(ctx, stack, vms, alien_vms) + domains = get_target_domains(ctx, stack, ssh_key, arch_obj, vms, alien_vms) + assert len(domains) > 0, err_msg _prepare(ctx, stack, component, arch_obj, packages, verbose, ci, compile_only, domains=domains) diff --git a/tasks/libs/ciproviders/github_api.py b/tasks/libs/ciproviders/github_api.py index 8a4d5e69027d1..54a3ba49a43f1 100644 --- a/tasks/libs/ciproviders/github_api.py +++ b/tasks/libs/ciproviders/github_api.py @@ -332,7 +332,9 @@ def workflow_run_for_ref_after_date(self, workflow_name, ref, oldest_date): return sorted(recent_runs, key=lambda run: run.created_at, reverse=True) - def latest_release(self) -> str: + def latest_release(self, major_version=7) -> str: + if major_version == 6: + return max((r for r in self.get_releases() if r.title.startswith('6.53')), key=lambda r: r.created_at).title release = self._repository.get_latest_release() return release.title diff --git a/tasks/libs/ciproviders/gitlab_api.py b/tasks/libs/ciproviders/gitlab_api.py index adfd8aacfb569..f1c0b4784d2f6 100644 --- a/tasks/libs/ciproviders/gitlab_api.py +++ b/tasks/libs/ciproviders/gitlab_api.py @@ -1258,12 +1258,10 @@ def full_config_get_all_stages(full_config: dict) -> set[str]: return all_stages -def update_test_infra_def(file_path, image_tag, is_dev_image=False): +def update_test_infra_def(file_path, image_tag, is_dev_image=False, prefix_comment=""): """ Updates TEST_INFRA_DEFINITIONS_BUILDIMAGES in `.gitlab/common/test_infra_version.yml` file """ - import yaml - test_infra_def = {} with open(file_path) as test_infra_version_file: try: @@ -1276,6 +1274,7 @@ def update_test_infra_def(file_path, image_tag, is_dev_image=False): except yaml.YAMLError as e: raise Exit(f"Error while loading {file_path}: {e}") from e with open(file_path, "w") as test_infra_version_file: + test_infra_version_file.write(prefix_comment + ('\n\n' if prefix_comment else '')) # Add explicit_start=True to keep the document start marker --- # See "Document Start" in https://www.yaml.info/learn/document.html for more details yaml.dump(test_infra_def, test_infra_version_file, explicit_start=True) diff --git a/tasks/libs/common/datadog_api.py b/tasks/libs/common/datadog_api.py index 55b888678b768..7b8b9df7f6a8a 100644 --- a/tasks/libs/common/datadog_api.py +++ b/tasks/libs/common/datadog_api.py @@ -16,8 +16,12 @@ def create_metric(metric_type, metric_name, timestamp, value, tags, unit=None, m unit = unit or unset metadata = unset - if metric_origin: - metadata = MetricMetadata(origin=MetricOrigin(**metric_origin)) + origin_metadata = metric_origin or { + "origin_product": 10, # Agent + "origin_sub_product": 54, # Agent CI + "origin_product_detail": 64, # Gitlab + } + metadata = MetricMetadata(origin=MetricOrigin(**origin_metadata)) return MetricSeries( metric=metric_name, diff --git a/tasks/libs/common/junit_upload_core.py b/tasks/libs/common/junit_upload_core.py index b7282d9bf4ebc..cebbe0a1c2b75 100644 --- a/tasks/libs/common/junit_upload_core.py +++ b/tasks/libs/common/junit_upload_core.py @@ -377,25 +377,3 @@ def produce_junit_tar(files, result_path): job_env_info.size = job_env_file.getbuffer().nbytes job_env_file.seek(0) tgz.addfile(job_env_info, job_env_file) - - -def repack_macos_junit_tar(workflow_conclusion, infile, outfile): - """ - Repacks JUnit tgz file from macOS Github Action run, so it would - contain correct job name and job URL. - """ - if workflow_conclusion == "cancelled" and not os.path.exists(infile): - print(f"Skipping repacking of JUnit tarball due to {workflow_conclusion} workflow") - return - - with tarfile.open(infile) as infp, tarfile.open(outfile, "w:gz") as outfp, tempfile.TemporaryDirectory() as tempd: - infp.extractall(tempd) - - # write the proper job url and job name - with open(os.path.join(tempd, JOB_ENV_FILE_NAME), "w") as fp: - fp.write(f'CI_JOB_URL={os.environ.get("CI_JOB_URL", "")}\n') - fp.write(f'CI_JOB_NAME={os.environ.get("CI_JOB_NAME", "")}') - - # pack all files to a new tarball - for f in os.listdir(tempd): - outfp.add(os.path.join(tempd, f), arcname=f) diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index 38208c6a4d1fe..1887cffb97133 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -116,6 +116,7 @@ def env_filter(item): "GET_SOURCES_ATTEMPTS", "GO_TEST_SKIP_FLAKE", "GONOSUMDB", + "GOPROXY", "HELM_HOOKS_CI_IMAGE", "HELM_HOOKS_PERIODICAL_REBUILD_CONDUCTOR_ENV", "HOME", diff --git a/tasks/libs/package/size.py b/tasks/libs/package/size.py index 4743dfcd906de..2323d40fdb56b 100644 --- a/tasks/libs/package/size.py +++ b/tasks/libs/package/size.py @@ -37,22 +37,22 @@ # The below template contains the relative increase threshold for each package type PACKAGE_SIZE_TEMPLATE = { 'amd64': { - 'datadog-agent': {'deb': 140 * pow(10, 6)}, - 'datadog-iot-agent': {'deb': 10 * pow(10, 6)}, - 'datadog-dogstatsd': {'deb': 10 * pow(10, 6)}, - 'datadog-heroku-agent': {'deb': 70 * pow(10, 6)}, + 'datadog-agent': {'deb': int(140e6)}, + 'datadog-iot-agent': {'deb': int(10e6)}, + 'datadog-dogstatsd': {'deb': int(10e6)}, + 'datadog-heroku-agent': {'deb': int(70e6)}, }, 'x86_64': { - 'datadog-agent': {'rpm': 140 * pow(10, 6), 'suse': 140 * pow(10, 6)}, - 'datadog-iot-agent': {'rpm': 10 * pow(10, 6), 'suse': 10 * pow(10, 6)}, - 'datadog-dogstatsd': {'rpm': 10 * pow(10, 6), 'suse': 10 * pow(10, 6)}, + 'datadog-agent': {'rpm': int(140e6), 'suse': int(140e6)}, + 'datadog-iot-agent': {'rpm': int(10e6), 'suse': int(10e6)}, + 'datadog-dogstatsd': {'rpm': int(10e6), 'suse': int(10e6)}, }, 'arm64': { - 'datadog-agent': {'deb': 140 * pow(10, 6)}, - 'datadog-iot-agent': {'deb': 10 * pow(10, 6)}, - 'datadog-dogstatsd': {'deb': 10 * pow(10, 6)}, + 'datadog-agent': {'deb': int(140e6)}, + 'datadog-iot-agent': {'deb': int(10e6)}, + 'datadog-dogstatsd': {'deb': int(10e6)}, }, - 'aarch64': {'datadog-agent': {'rpm': 140 * pow(10, 6)}, 'datadog-iot-agent': {'rpm': 10 * pow(10, 6)}}, + 'aarch64': {'datadog-agent': {'rpm': int(140e6)}, 'datadog-iot-agent': {'rpm': int(10e6)}}, } @@ -177,25 +177,11 @@ def compare(ctx, package_sizes, ancestor, pkg_size): return pkg_size -def mb(value): - return f"{value / 1000000:.2f}MB" - - def _get_uncompressed_size(ctx, package, os_name): if os_name == 'deb': - return _get_deb_uncompressed_size(ctx, package) + return ( + int(ctx.run(f'dpkg-deb --info {package} | grep Installed-Size | cut -d : -f 2 | xargs', hide=True).stdout) + * 1024 + ) else: - return _get_rpm_uncompressed_size(ctx, package) - - -def _get_deb_uncompressed_size(ctx, package): - # the size returned by dpkg is a number of bytes divided by 1024 - # so we multiply it back to get the same unit as RPM or stat - return ( - int(ctx.run(f'dpkg-deb --info {package} | grep Installed-Size | cut -d : -f 2 | xargs', hide=True).stdout) - * 1024 - ) - - -def _get_rpm_uncompressed_size(ctx, package): - return int(ctx.run(f'rpm -qip {package} | grep Size | cut -d : -f 2 | xargs', hide=True).stdout) + return int(ctx.run(f'rpm -qip {package} | grep Size | cut -d : -f 2 | xargs', hide=True).stdout) diff --git a/tasks/libs/package/utils.py b/tasks/libs/package/utils.py index ec8363fcfa134..8f43eaa9b4e33 100644 --- a/tasks/libs/package/utils.py +++ b/tasks/libs/package/utils.py @@ -20,6 +20,7 @@ def __init__(self, arch, flavor, os_name, threshold): self.size = 0 self.ancestor_size = 0 self.diff = 0 + self.mb_diff = 0 self.threshold = threshold self.emoji = "✅" @@ -48,14 +49,15 @@ def compare(self, size, ancestor_size): self.size = size self.ancestor_size = ancestor_size self.diff = self.size - self.ancestor_size + self.mb_diff = float(f"{self.diff / pow(10, 6):.2f}") if self.ko(): self.emoji = "❌" - elif self.diff > 0: + elif self.mb_diff > 0: self.emoji = "⚠️" @staticmethod def mb(value): - return f"{value / 1000000:.2f}MB" + return f"{value / 1e6:.2f}MB" def log(self): return f"{self.emoji} - {self.name} size {self.mb(self.size)}: {self.mb(self.diff)} diff[{self.diff}] with previous {self.mb(self.ancestor_size)} (max: {self.mb(self.threshold)})" diff --git a/tasks/libs/releasing/documentation.py b/tasks/libs/releasing/documentation.py index 13c7f0d148252..9834983792cbd 100644 --- a/tasks/libs/releasing/documentation.py +++ b/tasks/libs/releasing/documentation.py @@ -31,7 +31,7 @@ def release_entry_for(agent_major_version): return f"release-a{agent_major_version}" -def create_release_page(version, freeze_date): +def create_release_page(version, cutoff_date): username = os.environ['ATLASSIAN_USERNAME'] password = os.environ['ATLASSIAN_PASSWORD'] parent_page_id = "2244936127" @@ -44,7 +44,7 @@ def create_release_page(version, freeze_date): page = confluence.create_page( space=SPACE_KEY, title=page_title, - body=create_release_table(version, freeze_date, teams), + body=create_release_table(version, cutoff_date, teams), parent_id=parent_page_id, editor="v2", ) @@ -52,7 +52,7 @@ def create_release_page(version, freeze_date): confluence.create_page( space=SPACE_KEY, title=f"{page_title} Notes", - body=create_release_notes(freeze_date, teams), + body=create_release_notes(cutoff_date, teams), parent_id=release_page["id"], ) return release_page @@ -118,12 +118,13 @@ def get_releasing_teams(): 'agent-platform', 'agent-release-management', 'container-ecosystems', + 'apm-trace-storage', } owners = set(list_owners()) return sorted(owners - non_releasing_teams) -def create_release_table(version, freeze_date, teams): +def create_release_table(version, cutoff_date, teams): from yattag import Doc doc, tag, text, line = Doc().ttl() @@ -153,7 +154,7 @@ def create_release_table(version, freeze_date, teams): with ( tag('td', colspan="2"), tag('p', style="text-align: center;"), - tag('time', datetime=f"{freeze_date + timedelta(days=26)}"), + tag('time', datetime=f"{cutoff_date + timedelta(days=26)}"), ): pass with tag('tr'): @@ -164,11 +165,11 @@ def create_release_table(version, freeze_date, teams): text(f'https://github.com/DataDog/datadog-agent/releases/tag/{version}') with tag('tr'): with tag('td'), tag('p'): - text('Code freeze date') + text('Cut-off date') with ( tag('td', colspan="2"), tag('p', style="text-align: center;"), - tag('time', datetime=f"{freeze_date}"), + tag('time', datetime=f"{cutoff_date}"), ): pass with tag('tr'): @@ -201,17 +202,17 @@ def create_release_table(version, freeze_date, teams): return doc.getvalue() -def create_release_notes(freeze_date, teams): +def create_release_notes(cutoff_date, teams): from yattag import Doc doc, tag, text, line = Doc().ttl() milestones = { - '"Code freeze"': freeze_date, - '"RC.1 built"': freeze_date + timedelta(days=1), - '"Staging deployment"': freeze_date + timedelta(days=3), - '"Prod deployment start"': freeze_date + timedelta(days=11), - '"Full prod deployment"': freeze_date + timedelta(days=20), - '"Release"': freeze_date + timedelta(days=26), + '"Cut-off"': cutoff_date, + '"RC.1 built"': cutoff_date + timedelta(days=1), + '"Staging deployment"': cutoff_date + timedelta(days=3), + '"Prod deployment start"': cutoff_date + timedelta(days=11), + '"Full prod deployment"': cutoff_date + timedelta(days=20), + '"Release"': cutoff_date + timedelta(days=26), } line('h2', 'Schedule') diff --git a/tasks/libs/releasing/version.py b/tasks/libs/releasing/version.py index 23472d57c0cca..85e66ea283080 100644 --- a/tasks/libs/releasing/version.py +++ b/tasks/libs/releasing/version.py @@ -6,6 +6,7 @@ from invoke import Exit from tasks.libs.ciproviders.github_api import GithubAPI +from tasks.libs.common.color import Color, color_message from tasks.libs.common.constants import ( AGENT_VERSION_CACHE_NAME, ALLOWED_REPO_NIGHTLY_BRANCHES, @@ -441,3 +442,80 @@ def get_matching_pattern(ctx, major_version, release=False): ) pattern = max(tags, key=cmp_to_key(semver.compare)) return pattern + + +def deduce_version(ctx, branch, as_str: bool = True, trust: bool = False, next_version: bool = True) -> str | Version: + """Deduces the version from the release branch name. + + Args: + next_version: If True, will return the next tag version, otherwise will return the current tag version. Example: If there are 7.60.0 and 7.60.1 tags, it will return 7.60.2 if next_tag is True, 7.60.1 otherwise. + """ + release_version = get_next_version_from_branch(ctx, branch, as_str=as_str, next_version=next_version) + + print( + f'{color_message("Info", Color.BLUE)}: Version {release_version} deduced from branch {branch}', file=sys.stderr + ) + + if ( + trust + or not os.isatty(sys.stdin.fileno()) + or yes_no_question( + 'Is this the version you want to use?', + color="orange", + default=False, + ) + ): + return release_version + + raise Exit(color_message("Aborting.", "red"), code=1) + + +def get_version_major(branch: str) -> int: + """Get the major version from a branch name.""" + + return 7 if branch == 'main' else int(branch.split('.')[0]) + + +def get_all_version_tags(ctx) -> list[str]: + """Returns the tags for all the versions of the Agent in git.""" + + cmd = "bash -c 'git tag | grep -E \"^[0-9]\\.[0-9]+\\.[0-9]+$\"'" + + return ctx.run(cmd, hide=True).stdout.strip().split('\n') + + +def get_next_version_from_branch(ctx, branch: str, as_str: bool = True, next_version: bool = True) -> str | Version: + """Returns the latest version + 1 belonging to a branch. + + Args: + next_version: If True, will return the next tag version, otherwise will return the current tag version. Example: If there are 7.60.0 and 7.60.1 tags, it will return 7.60.2 if next_tag is True, 7.60.1 otherwise. + + Example: + get_latest_version_from_branch("7.55.x") -> Version(7, 55, 4) if there are 7.55.0, 7.55.1, 7.55.2, 7.55.3 tags. + get_latest_version_from_branch("6.99.x") -> Version(6, 99, 0) if there are no 6.99.* tags. + """ + + re_branch = re.compile(r"^([0-9]\.[0-9]+\.)x$") + + try: + matched = re_branch.match(branch).group(1) + except Exception as e: + raise Exit( + f'{color_message("Error:", "red")}: Branch {branch} is not a release branch (should be X.Y.x)', code=1 + ) from e + + tags = [tuple(map(int, tag.split('.'))) for tag in get_all_version_tags(ctx) if tag.startswith(matched)] + versions = sorted(Version(*tag) for tag in tags) + + minor, major = tuple(map(int, branch.split('.')[:2])) + + if next_version: + # Get version after the latest one + version = versions[-1].next_version(bump_patch=True) if versions else Version(minor, major, 0) + else: + # Get current latest version + assert versions, f"No tags found for branch {branch} (expected at least one tag)" + + version = versions[-1] + + return str(version) if as_str else version diff --git a/tasks/package.py b/tasks/package.py index f2d49130152dc..3a831a24225fc 100644 --- a/tasks/package.py +++ b/tasks/package.py @@ -8,15 +8,12 @@ from tasks.libs.common.git import get_default_branch from tasks.libs.package.size import ( PACKAGE_SIZE_TEMPLATE, - _get_deb_uncompressed_size, - _get_rpm_uncompressed_size, compare, compute_package_size_metrics, ) from tasks.libs.package.utils import ( PackageSize, display_message, - find_package, get_ancestor, list_packages, retrieve_package_sizes, @@ -61,47 +58,6 @@ def check_size(ctx, filename: str = 'package_sizes.json', dry_run: bool = False) raise Exit(code=1) -@task -def compare_size(ctx, new_package, stable_package, package_type, last_stable, threshold): - mb = 1000000 - - if package_type.endswith('deb'): - new_package_size = _get_deb_uncompressed_size(ctx, find_package(new_package)) - stable_package_size = _get_deb_uncompressed_size(ctx, find_package(stable_package)) - else: - new_package_size = _get_rpm_uncompressed_size(ctx, find_package(new_package)) - stable_package_size = _get_rpm_uncompressed_size(ctx, find_package(stable_package)) - - threshold = int(threshold) - - diff = new_package_size - stable_package_size - - # For printing purposes - new_package_size_mb = new_package_size / mb - stable_package_size_mb = stable_package_size / mb - threshold_mb = threshold / mb - diff_mb = diff / mb - - if diff > threshold: - print( - color_message( - f"""{package_type} size increase is too large: - New package size is {new_package_size_mb:.2f}MB - Stable package ({last_stable}) size is {stable_package_size_mb:.2f}MB - Diff is {diff_mb:.2f}MB > {threshold_mb:.2f}MB (max allowed diff)""", - "red", - ) - ) - raise Exit(code=1) - - print( - f"""{package_type} size increase is OK: - New package size is {new_package_size_mb:.2f}MB - Stable package ({last_stable}) size is {stable_package_size_mb:.2f}MB - Diff is {diff_mb:.2f}MB (max allowed diff: {threshold_mb:.2f}MB)""" - ) - - @task def send_size( ctx, diff --git a/tasks/release.py b/tasks/release.py index 077834b279923..39274fc027bc1 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -70,10 +70,11 @@ VERSION_RE, _create_version_from_match, current_version, + deduce_version, + get_version_major, next_final_version, next_rc_version, ) -from tasks.libs.types.version import Version from tasks.pipeline import edit_schedule, run from tasks.release_metrics.metrics import get_prs_metrics, get_release_lead_time @@ -85,68 +86,6 @@ BACKPORT_LABEL_COLOR = "5319e7" -def deduce_version(ctx, branch, as_str=True, trust=False) -> str | Version: - release_version = get_next_version_from_branch(ctx, branch, as_str=as_str) - - print( - f'{color_message("Info", Color.BLUE)}: Version {release_version} deduced from branch {branch}', file=sys.stderr - ) - - if ( - trust - or not os.isatty(sys.stdin.fileno()) - or yes_no_question( - 'Is this the version you want to use?', - color="orange", - default=False, - ) - ): - return release_version - - raise Exit(color_message("Aborting.", "red"), code=1) - - -def get_version_major(branch: str) -> int: - """Get the major version from a branch name.""" - - return 7 if branch == 'main' else int(branch.split('.')[0]) - - -def get_all_version_tags(ctx) -> list[str]: - """Returns the tags for all the versions of the Agent in git.""" - - cmd = "bash -c 'git tag | grep -E \"^[0-9]\\.[0-9]+\\.[0-9]+$\"'" - - return ctx.run(cmd, hide=True).stdout.strip().split('\n') - - -def get_next_version_from_branch(ctx, branch: str, as_str=True) -> str | Version: - """Returns the latest version + 1 belonging to a branch. - - Example: - get_latest_version_from_branch("7.55.x") -> Version(7, 55, 4) if there are 7.55.0, 7.55.1, 7.55.2, 7.55.3 tags. - get_latest_version_from_branch("6.99.x") -> Version(6, 99, 0) if there are no 6.99.* tags. - """ - - re_branch = re.compile(r"^([0-9]\.[0-9]+\.)x$") - - try: - matched = re_branch.match(branch).group(1) - except Exception as e: - raise Exit( - f'{color_message("Error:", "red")}: Branch {branch} is not a release branch (should be X.Y.x)', code=1 - ) from e - - tags = [tuple(map(int, tag.split('.'))) for tag in get_all_version_tags(ctx) if tag.startswith(matched)] - versions = sorted(Version(*tag) for tag in tags) - - minor, major = tuple(map(int, branch.split('.')[:2])) - - latest = versions[-1].next_version(bump_patch=True) if versions else Version(minor, major, 0) - - return str(latest) if as_str else latest - - @task def list_major_change(_, milestone): """List all PR labeled "major_changed" for this release.""" @@ -837,10 +776,6 @@ def _update_last_stable(_, version, major_version: int = 7): return release_json["current_milestone"] -def _get_agent6_latest_release(gh): - return max((r for r in gh.get_releases() if r.title.startswith('6.53')), key=lambda r: r.created_at).title - - @task def cleanup(ctx, release_branch): """Perform the post release cleanup steps @@ -850,13 +785,13 @@ def cleanup(ctx, release_branch): - Updates the release.json last_stable fields """ - with agent_context(ctx, release_branch): + # This task will create a PR to update the last_stable field in release.json + # It must create the PR against the default branch (6 or 7), so setting the context on it + main_branch = get_default_branch() + with agent_context(ctx, main_branch): gh = GithubAPI() major_version = get_version_major(release_branch) - if major_version == 6: - latest_release = _get_agent6_latest_release(gh) - else: - latest_release = gh.latest_release() + latest_release = gh.latest_release(major_version) match = VERSION_RE.search(latest_release) if not match: raise Exit(f'Unexpected version fetched from github {latest_release}', code=1) @@ -865,7 +800,6 @@ def cleanup(ctx, release_branch): current_milestone = _update_last_stable(ctx, version, major_version=major_version) # create pull request to update last stable version - main_branch = get_default_branch() cleanup_branch = f"release/{version}-cleanup" ctx.run(f"git checkout -b {cleanup_branch}") ctx.run("git add release.json") @@ -1034,7 +968,7 @@ def get_active_release_branch(ctx, release_branch): with agent_context(ctx, branch=release_branch): gh = GithubAPI() - next_version = get_next_version(gh, latest_release=_get_agent6_latest_release(gh) if is_agent6(ctx) else None) + next_version = get_next_version(gh, latest_release=gh.latest_release(6) if is_agent6(ctx) else None) release_branch = gh.get_branch(next_version.branch()) if release_branch: print(f"{release_branch.name}") @@ -1095,12 +1029,13 @@ def generate_release_metrics(ctx, milestone, freeze_date, release_date): print(code_stats) +# TODO rename to freeze_date to cutoff_date @task def create_schedule(_, version, freeze_date): """Create confluence pages for the release schedule. Args: - freeze_date: Date when the code freeze was started. Expected format YYYY-MM-DD, like '2022-02-01' + freeze_date: Date when the code cut-off happened. Expected format YYYY-MM-DD, like '2022-02-01' """ required_environment_variables = ["ATLASSIAN_USERNAME", "ATLASSIAN_PASSWORD"] @@ -1226,7 +1161,7 @@ def create_github_release(ctx, release_branch, draft=True): ) notes = [] - version = deduce_version(ctx, release_branch) + version = deduce_version(ctx, release_branch, next_version=False) with agent_context(ctx, release_branch): for section, filename in sections: diff --git a/tasks/system_probe.py b/tasks/system_probe.py index 6f8e4e1f7457e..ddd731b535bea 100644 --- a/tasks/system_probe.py +++ b/tasks/system_probe.py @@ -2024,3 +2024,40 @@ def copy_ebpf_and_related_files(ctx: Context, target: Path | str, arch: Arch | N ctx.run(f"chmod 0444 {target}/*.o {target}/*.c {target}/co-re/*.o") ctx.run(f"cp /opt/datadog-agent/embedded/bin/clang-bpf {target}") ctx.run(f"cp /opt/datadog-agent/embedded/bin/llc-bpf {target}") + + +@task +def build_usm_debugger( + ctx, + arch: str = CURRENT_ARCH, + strip_binary=False, +): + build_object_files(ctx) + + build_dir = os.path.join("pkg", "ebpf", "bytecode", "build", arch) + + # copy compilation artifacts to the debugger root directory for the purposes of embedding + usm_programs = [ + os.path.join(build_dir, "co-re", "usm-debug.o"), + os.path.join(build_dir, "co-re", "shared-libraries-debug.o"), + ] + + embedded_dir = os.path.join(".", "pkg", "network", "usm", "debugger", "cmd") + + for p in usm_programs: + print(p) + shutil.copy(p, embedded_dir) + + arch_obj = Arch.from_str(arch) + ldflags, gcflags, env = get_build_flags(ctx, arch=arch_obj) + + cmd = 'go build -tags="linux_bpf,usm_debugger" -o bin/usm-debugger -ldflags="{ldflags}" ./pkg/network/usm/debugger/cmd/' + + if strip_binary: + ldflags += ' -s -w' + + args = { + "ldflags": ldflags, + } + + ctx.run(cmd.format(**args), env=env) diff --git a/tasks/unit_tests/package_lib_tests.py b/tasks/unit_tests/package_lib_tests.py index 984e4cc4f871c..72f3c6c190252 100644 --- a/tasks/unit_tests/package_lib_tests.py +++ b/tasks/unit_tests/package_lib_tests.py @@ -238,6 +238,26 @@ def test_on_branch_warning(self, mock_print): f"⚠️ - {flavor}-{arch}-{os_name} size 69.00MB: 1.00MB diff[1000000] with previous 68.00MB (max: 70.00MB)" ) + @patch.dict( + 'os.environ', + {'OMNIBUS_PACKAGE_DIR_SUSE': 'tasks/unit_tests/testdata/packages', 'CI_COMMIT_REF_NAME': 'pikachu'}, + ) + @patch('builtins.print') + def test_on_branch_ok_small_diff(self, mock_print): + flavor, arch, os_name = 'datadog-agent', 'aarch64', 'suse' + s = PackageSize(arch, flavor, os_name, 70000000) + c = MockContext( + run={ + 'git merge-base HEAD origin/main': Result('25'), + f"rpm -qip {self.pkg_root}/{flavor}-7.{arch}.rpm | grep Size | cut -d : -f 2 | xargs": Result(68004999), + } + ) + res = compare(c, self.package_sizes, '25', s) + self.assertEqual(res.markdown(), "|datadog-agent-aarch64-suse|0.00MB|✅|68.00MB|68.00MB|70.00MB|") + mock_print.assert_called_with( + f"✅ - {flavor}-{arch}-{os_name} size 68.00MB: 0.00MB diff[4999] with previous 68.00MB (max: 70.00MB)" + ) + @patch.dict( 'os.environ', {'OMNIBUS_PACKAGE_DIR': 'tasks/unit_tests/testdata/packages', 'CI_COMMIT_REF_NAME': 'pikachu'} ) diff --git a/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 b/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 index 991317602152f..41bf71748cc2b 100644 --- a/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 +++ b/tasks/winbuildscripts/Generate-Chocolatey-Package.ps1 @@ -2,11 +2,8 @@ .SYNOPSIS Generates a Chocolatey package for the Datadog Agent. -.PARAMETER installMethod -Specifies the installation method. Valid values are "offline" and "online". This parameter is mandatory. - .PARAMETER msiDirectory -Specifies the directory containing the MSI file that will be used to calculate the checksum. This parameter is mandatory when the installMethod is "online". +Specifies the directory containing the MSI file that will be used to calculate the checksum. .PARAMETER Flavor Specifies the flavor of the Datadog Agent. The default value is "datadog-agent". @@ -18,23 +15,18 @@ Overrides the Agent version when building packages locally for testing. Indicates whether to install dependencies. The default value is $true. .EXAMPLE -.\Generate-Chocolatey-Package.ps1 -installMethod online -Flavor datadog-agent -VersionOverride "7.62.0" -msiDirectory C:\mnt\omnibus\pkg\ +.\Generate-Chocolatey-Package.ps1 -Flavor datadog-agent -VersionOverride "7.62.0" -msiDirectory C:\mnt\omnibus\pkg Generates a chocolatey package for 7.62.0, requires the MSI file to be present in MSIDirectory. .EXAMPLE -$env:CI_PIPELINE_ID="50910739"; .\Generate-Chocolatey-Package.ps1 -installMethod online -Flavor datadog-agent -VersionOverride "7.62.0-devel.git.276.e59b1b3.pipeline.50910739" -msiDirectory C:\mnt\omnibus\pkg +$env:CI_PIPELINE_ID="50910739"; .\Generate-Chocolatey-Package.ps1 -Flavor datadog-agent -VersionOverride "7.62.0-devel.git.276.e59b1b3.pipeline.50910739" -msiDirectory C:\mnt\omnibus\pkg Generates a chocolatey package for PR/devel build 7.62.0-devel.git.276.e59b1b3.pipeline.50910739, requires the MSI file to be present in MSIDirectory. The generated chocolatey package requires the MSI be uploaded to the dd-agent-mstesting bucket. #> Param( [Parameter(Mandatory=$true)] - [ValidateSet("offline", "online")] - [String] - $installMethod, - - [Parameter(Mandatory=$false)] [String] $msiDirectory, @@ -69,9 +61,9 @@ if (![string]::IsNullOrEmpty($VersionOverride)) { } $copyright = "Datadog {0}" -f (Get-Date).Year -$releasePattern = "(\d+\.\d+\.\d+)" -$releaseCandidatePattern = "(\d+\.\d+\.\d+)-rc\.(\d+)" -$develPattern = "(\d+\.\d+\.\d+)-devel\.git\.\d+\.(.+)" +$releasePattern = "^(\d+\.\d+\.\d+)$" +$releaseCandidatePattern = "^(\d+\.\d+\.\d+)-rc\.(\d+)$" +$develPattern = "^(\d+\.\d+\.\d+)-devel\.git\.\d+\.(.+)" # Build the package in a temporary directory # Some of the build steps modify the package source, so we don't want to do this in the source directory @@ -87,21 +79,16 @@ try { # For historical reasons, use a different artifact name for the datadog-agent flavor # See agent-release-management for more details $artifactName = "ddagent-cli" - $packageSource = "$repoRoot\chocolatey\datadog-agent\$installMethod" - $nuspecFile = "datadog-agent-$installMethod.nuspec" } elseif ($Flavor -eq "datadog-fips-agent") { - if ($installMethod -eq "offline") { - Write-Error "Offline install method not supported for flavor $Flavor" - exit 1 - } $artifactName = "datadog-fips-agent" - $packageSource = "$repoRoot\chocolatey\datadog-fips-agent\online" - $nuspecFile = "datadog-fips-agent-online.nuspec" } else { Write-Error "Unknown flavor $Flavor" exit 1 } + $packageSource = "$repoRoot\chocolatey\$Flavor" + $nuspecFile = "$Flavor.nuspec" + # These files/directories are referenced in the nuspec file $licensePath = "tools\LICENSE.txt" $installScript = "tools\chocolateyinstall.ps1" @@ -118,24 +105,22 @@ try { $releaseNotes = "https://github.com/DataDog/datadog-agent/releases/tag/{0}-rc.{1}" -f $agentVersionMatches.Matches.Groups[1], $agentVersionMatches.Matches.Groups[2] $url = "https://s3.amazonaws.com/dd-agent-mstesting/builds/beta/$artifactName-$($agentVersionMatches.Matches.Groups[1])-rc.$($agentVersionMatches.Matches.Groups[2]).msi" } elseif ($rawAgentVersion -match $develPattern) { - if ($installMethod -eq "online") { - # For devel builds/branches, use the dd-agent-mstesting bucket URL - # This allows us to build and test the package in PRs, and locally - # by using the `-VersionOverride` param. - if ([string]::IsNullOrEmpty($env:CI_PIPELINE_ID)) { - Write-Error "CI_PIPELINE_ID is not set, aborting" - exit 1 - } else { - if ($rawAgentVersion -notmatch $env:CI_PIPELINE_ID) { - Write-Error "CI_PIPELINE_ID is not found in the agent version, aborting" -ErrorAction Continue - if ([string]::IsNullOrEmpty($env:BUCKET_BRANCH)) { - # inv agent.version requires BUCKET_BRANCH to be set when including pipeline in version - Write-Error "BUCKET_BRANCH is not set, if you are running this locally, set `$env:BUCKET_BRANCH='dev' or pass the -VersionOverride parameter" -ErrorAction Continue - } - exit 1 + # For devel builds/branches, use the dd-agent-mstesting bucket URL + # This allows us to build and test the package in PRs, and locally + # by using the `-VersionOverride` param. + if ([string]::IsNullOrEmpty($env:CI_PIPELINE_ID)) { + Write-Error "CI_PIPELINE_ID is not set, aborting" + exit 1 + } else { + if ($rawAgentVersion -notmatch $env:CI_PIPELINE_ID) { + Write-Error "CI_PIPELINE_ID is not found in the agent version, aborting" -ErrorAction Continue + if ([string]::IsNullOrEmpty($env:BUCKET_BRANCH)) { + # inv agent.version requires BUCKET_BRANCH to be set when including pipeline in version + Write-Error "BUCKET_BRANCH is not set, if you are running this locally, set `$env:BUCKET_BRANCH='dev' or pass the -VersionOverride parameter" -ErrorAction Continue } - $url = "https://s3.amazonaws.com/dd-agent-mstesting/pipelines/A7/$env:CI_PIPELINE_ID/$flavor-$rawAgentVersion-1-x86_64.msi" + exit 1 } + $url = "https://s3.amazonaws.com/dd-agent-mstesting/pipelines/A7/$env:CI_PIPELINE_ID/$flavor-$rawAgentVersion-1-x86_64.msi" } $agentVersionMatches = $rawAgentVersion | Select-String -Pattern $develPattern $agentVersion = "{0}-devel-{1}" -f $agentVersionMatches.Matches.Groups[1], $agentVersionMatches.Matches.Groups[2].Value @@ -151,26 +136,24 @@ try { exit 1 } - Write-Host "Generating Chocolatey $installMethod package $flavor version $agentVersion in $(Get-Location)" + Write-Host "Generating Chocolatey package $flavor version $agentVersion in $(Get-Location)" # Template the install script with the URL and checksum - if ($installMethod -eq "online") { - try { - $tempMsi = Join-Path -Path "$msiDirectory" "$flavor-$rawAgentVersion-1-x86_64.msi" - if (!(Test-Path $tempMsi)) { - Write-Host "Error: Could not find MSI file in $tempMsi" - Get-ChildItem "$msiDirectory" - exit 1 - } - $checksum = (Get-FileHash $tempMsi -Algorithm SHA256).Hash - } - catch { - Write-Host "Error: Could not generate checksum for package $($tempMsi): $($_)" + try { + $msiPath = Join-Path -Path "$msiDirectory" "$flavor-$rawAgentVersion-1-x86_64.msi" + if (!(Test-Path $msiPath)) { + Write-Host "Error: Could not find MSI file in $msiPath" + Get-ChildItem "$msiDirectory" exit 1 } - # Set the $url in the install script - (Get-Content $installScript).replace('$__url_from_ci__', '"' + $url + '"').replace('$__checksum_from_ci__', '"' + $checksum + '"') | Set-Content $installScript + $checksum = (Get-FileHash $msiPath -Algorithm SHA256).Hash + } + catch { + Write-Host "Error: Could not generate checksum for package $($msiPath): $($_)" + exit 1 } + # Set the $url in the install script + (Get-Content $installScript).replace('$__url_from_ci__', '"' + $url + '"').replace('$__checksum_from_ci__', '"' + $checksum + '"') | Set-Content $installScript Write-Host "Generated nuspec file:" Write-Host (Get-Content $installScript | Out-String) diff --git a/tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 b/tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 deleted file mode 100644 index e8879285da1ee..0000000000000 --- a/tasks/winbuildscripts/Publish-Chocolatey-Package.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -$ErrorActionPreference = 'Stop'; - -# Install chocolatey binary -$env:chocolateyUseWindowsCompression = 'true'; Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) - -Set-Location c:\mnt -$nupkgs = Get-ChildItem .\nupkg\datadog-agent*.nupkg -foreach($nupkg in $nupkgs) { - Write-Host "Publishing Chocolatey package $($nupkg.Name) for agent version $agentVersion" - choco push $nupkg.FullName --verbose --key $env:CHOCOLATEY_API_KEY --source https://push.chocolatey.org/ - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } -} diff --git a/tasks/winbuildscripts/chocopush.bat b/tasks/winbuildscripts/chocopush.bat deleted file mode 100644 index d359e3c5c2dec..0000000000000 --- a/tasks/winbuildscripts/chocopush.bat +++ /dev/null @@ -1,11 +0,0 @@ -if not exist c:\mnt\ goto nomntdir - -@echo c:\mnt found, continuing - -Powershell -C "C:\mnt\tasks\winbuildscripts\Publish-Chocolatey-Package.ps1" || exit /b 1 -goto :EOF - -:nomntdir -@echo directory not mounted, parameters incorrect -exit /b 2 -goto :EOF \ No newline at end of file diff --git a/test/integration/serverless_perf/compute.sh b/test/integration/serverless_perf/compute.sh index c8421c24a84c9..06be249e90867 100755 --- a/test/integration/serverless_perf/compute.sh +++ b/test/integration/serverless_perf/compute.sh @@ -2,7 +2,7 @@ set -o pipefail -STARTUP_TIME_THRESHOLD=25 +STARTUP_TIME_THRESHOLD=28 calculate_median() { local sorted=($(printf "%s\n" "${@}" | sort -n)) diff --git a/test/new-e2e/examples/agentenv_checkruns_test.go b/test/new-e2e/examples/agentenv_checkruns_test.go index 7a7b73b77aab5..d62d0cfc05bc5 100644 --- a/test/new-e2e/examples/agentenv_checkruns_test.go +++ b/test/new-e2e/examples/agentenv_checkruns_test.go @@ -9,11 +9,12 @@ import ( "testing" "time" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type agentSuiteEx5 struct { diff --git a/test/new-e2e/examples/agentenv_copyfolder_test.go b/test/new-e2e/examples/agentenv_copyfolder_test.go index 0aff0e3273db3..2c93589669d22 100644 --- a/test/new-e2e/examples/agentenv_copyfolder_test.go +++ b/test/new-e2e/examples/agentenv_copyfolder_test.go @@ -10,11 +10,12 @@ import ( "path" "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type agentSuiteEx6 struct { diff --git a/test/new-e2e/examples/agentenv_file_permissions_test.go b/test/new-e2e/examples/agentenv_file_permissions_test.go index a00fe133b4ba1..516498473859d 100644 --- a/test/new-e2e/examples/agentenv_file_permissions_test.go +++ b/test/new-e2e/examples/agentenv_file_permissions_test.go @@ -8,13 +8,14 @@ package examples import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" perms "github.com/DataDog/test-infra-definitions/components/datadog/agentparams/filepermissions" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) type filePermissionsTestSuite struct { diff --git a/test/new-e2e/examples/agentenv_file_permissions_win_test.go b/test/new-e2e/examples/agentenv_file_permissions_win_test.go index b35e4cceb003a..faf7fae0f2e96 100644 --- a/test/new-e2e/examples/agentenv_file_permissions_win_test.go +++ b/test/new-e2e/examples/agentenv_file_permissions_win_test.go @@ -17,7 +17,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/examples/agentenv_logs_test.go b/test/new-e2e/examples/agentenv_logs_test.go index 1f9ae6f347b34..0306d113c06de 100644 --- a/test/new-e2e/examples/agentenv_logs_test.go +++ b/test/new-e2e/examples/agentenv_logs_test.go @@ -13,7 +13,7 @@ import ( fi "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" diff --git a/test/new-e2e/examples/agentenv_metrics_test.go b/test/new-e2e/examples/agentenv_metrics_test.go index 86150b5a5b26c..d3fb580e17f1e 100644 --- a/test/new-e2e/examples/agentenv_metrics_test.go +++ b/test/new-e2e/examples/agentenv_metrics_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/stretchr/testify/assert" ) diff --git a/test/new-e2e/examples/agentenv_updateenv_test.go b/test/new-e2e/examples/agentenv_updateenv_test.go index b1f19447229b2..b13705226530e 100644 --- a/test/new-e2e/examples/agentenv_updateenv_test.go +++ b/test/new-e2e/examples/agentenv_updateenv_test.go @@ -8,10 +8,11 @@ package examples import ( "testing" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/stretchr/testify/assert" ) diff --git a/test/new-e2e/examples/aks_test.go b/test/new-e2e/examples/aks_test.go index 4df8cfc7728c8..991999c300b81 100644 --- a/test/new-e2e/examples/aks_test.go +++ b/test/new-e2e/examples/aks_test.go @@ -17,7 +17,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - azurekubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/azure/kubernetes" + azurekubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/azure/kubernetes" ) type aksSuite struct { diff --git a/test/new-e2e/examples/azure_vm_test.go b/test/new-e2e/examples/azure_vm_test.go index 227b4b2ea28e7..64c59f9c156bf 100644 --- a/test/new-e2e/examples/azure_vm_test.go +++ b/test/new-e2e/examples/azure_vm_test.go @@ -6,9 +6,10 @@ package examples import ( - azurehost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/azure/host/windows" "testing" + azurehost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/azure/host/windows" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" ) diff --git a/test/new-e2e/examples/customenv_with_docker_app_test.go b/test/new-e2e/examples/customenv_with_docker_app_test.go index 8eb86a38054fe..71815e2a9631a 100644 --- a/test/new-e2e/examples/customenv_with_docker_app_test.go +++ b/test/new-e2e/examples/customenv_with_docker_app_test.go @@ -11,11 +11,13 @@ import ( "testing" "time" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/test-infra-definitions/components/docker" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/resources/aws" @@ -47,7 +49,7 @@ var lighttpdConfigContent string //go:embed testfixtures/lighttpd_integration.conf.yaml var lighttpdIntegrationConfigContent string -func vmPlusDockerEnvProvisioner() e2e.PulumiEnvRunFunc[vmPlusDockerEnv] { +func vmPlusDockerEnvProvisioner() provisioners.PulumiEnvRunFunc[vmPlusDockerEnv] { return func(ctx *pulumi.Context, env *vmPlusDockerEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/examples/customenv_with_filemanager_test.go b/test/new-e2e/examples/customenv_with_filemanager_test.go index 219a36fdb0331..40938ba05c957 100644 --- a/test/new-e2e/examples/customenv_with_filemanager_test.go +++ b/test/new-e2e/examples/customenv_with_filemanager_test.go @@ -10,9 +10,11 @@ import ( "path" "testing" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" osComp "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/resources/aws" @@ -25,7 +27,7 @@ type fileManagerSuiteEx7 struct { e2e.BaseSuite[environments.Host] } -func customProvisionerFileManager(localFolderPath string, remoteFolderPath string) e2e.PulumiEnvRunFunc[environments.Host] { +func customProvisionerFileManager(localFolderPath string, remoteFolderPath string) provisioners.PulumiEnvRunFunc[environments.Host] { return func(ctx *pulumi.Context, env *environments.Host) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/examples/customenv_with_two_vm_test.go b/test/new-e2e/examples/customenv_with_two_vm_test.go index 4d45a35a62c04..c400927784c19 100644 --- a/test/new-e2e/examples/customenv_with_two_vm_test.go +++ b/test/new-e2e/examples/customenv_with_two_vm_test.go @@ -10,6 +10,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/resources/aws" @@ -23,7 +24,7 @@ type multiVMEnv struct { AppVM *components.RemoteHost } -func multiVMEnvProvisioner() e2e.PulumiEnvRunFunc[multiVMEnv] { +func multiVMEnvProvisioner() provisioners.PulumiEnvRunFunc[multiVMEnv] { return func(ctx *pulumi.Context, env *multiVMEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/examples/dockerenv_test.go b/test/new-e2e/examples/dockerenv_test.go index 22913d26f4251..729ea7b1d50a0 100644 --- a/test/new-e2e/examples/dockerenv_test.go +++ b/test/new-e2e/examples/dockerenv_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/examples/dockerenv_with_extra_compose_test.go b/test/new-e2e/examples/dockerenv_with_extra_compose_test.go index cf416c968b966..a99d8153bf3a3 100644 --- a/test/new-e2e/examples/dockerenv_with_extra_compose_test.go +++ b/test/new-e2e/examples/dockerenv_with_extra_compose_test.go @@ -10,13 +10,14 @@ import ( "strings" "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" "github.com/DataDog/test-infra-definitions/components/datadog/dockeragentparams" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" ) type dockerSuiteWithExtraCompose struct { diff --git a/test/new-e2e/examples/ecs_test.go b/test/new-e2e/examples/ecs_test.go index 7a30eb6dfd7b4..e7e21ebb09c97 100644 --- a/test/new-e2e/examples/ecs_test.go +++ b/test/new-e2e/examples/ecs_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/ecs" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/ecs" tifEcs "github.com/DataDog/test-infra-definitions/scenarios/aws/ecs" awsecs "github.com/aws/aws-sdk-go-v2/service/ecs" diff --git a/test/new-e2e/examples/gcp_vm_test.go b/test/new-e2e/examples/gcp_vm_test.go index 1f897a87bbfe3..ec1d5fdb3a716 100644 --- a/test/new-e2e/examples/gcp_vm_test.go +++ b/test/new-e2e/examples/gcp_vm_test.go @@ -6,9 +6,10 @@ package examples import ( - gcphost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/gcp/host/linux" "testing" + gcphost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/gcp/host/linux" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" ) diff --git a/test/new-e2e/examples/gke_autopilot_test.go b/test/new-e2e/examples/gke_autopilot_test.go index 9317ace5b67be..82ced93302b4a 100644 --- a/test/new-e2e/examples/gke_autopilot_test.go +++ b/test/new-e2e/examples/gke_autopilot_test.go @@ -16,7 +16,7 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - gcpkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/gcp/kubernetes" + gcpkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/gcp/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" diff --git a/test/new-e2e/examples/gke_test.go b/test/new-e2e/examples/gke_test.go index 5f01481914652..d2ffd1dcf444c 100644 --- a/test/new-e2e/examples/gke_test.go +++ b/test/new-e2e/examples/gke_test.go @@ -6,13 +6,15 @@ package examples import ( "context" - gcpkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/gcp/kubernetes" + "strings" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "strings" - "testing" + + gcpkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/gcp/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" diff --git a/test/new-e2e/examples/kind_local_test.go b/test/new-e2e/examples/kind_local_test.go index dd5aa8f878c3a..a559f78be50a1 100644 --- a/test/new-e2e/examples/kind_local_test.go +++ b/test/new-e2e/examples/kind_local_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - localkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/local/kubernetes" + localkubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/local/kubernetes" ) type myLocalKindSuite struct { diff --git a/test/new-e2e/examples/kind_test.go b/test/new-e2e/examples/kind_test.go index dcfa50dbc3f64..00677a5f673f9 100644 --- a/test/new-e2e/examples/kind_test.go +++ b/test/new-e2e/examples/kind_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/components/datadog/apps/dogstatsd" diff --git a/test/new-e2e/examples/vm_localpodman_test.go b/test/new-e2e/examples/vm_localpodman_test.go index 92941a6d7a8a4..c7157468e6219 100644 --- a/test/new-e2e/examples/vm_localpodman_test.go +++ b/test/new-e2e/examples/vm_localpodman_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - localhost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/local/host" + localhost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/local/host" ) type vmLocalPodmanSuite struct { diff --git a/test/new-e2e/examples/vm_test.go b/test/new-e2e/examples/vm_test.go index a2fea62f01969..d3d8375a4937d 100644 --- a/test/new-e2e/examples/vm_test.go +++ b/test/new-e2e/examples/vm_test.go @@ -10,7 +10,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type vmSuite struct { diff --git a/test/new-e2e/examples/vm_with_file_operations_test.go b/test/new-e2e/examples/vm_with_file_operations_test.go index 02f8592388bd6..d97764dac8daa 100644 --- a/test/new-e2e/examples/vm_with_file_operations_test.go +++ b/test/new-e2e/examples/vm_with_file_operations_test.go @@ -11,12 +11,13 @@ import ( "golang.org/x/crypto/ssh" + "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" - "github.com/DataDog/test-infra-definitions/components/os" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/stretchr/testify/assert" ) diff --git a/test/new-e2e/examples/vmenv_withami_test.go b/test/new-e2e/examples/vmenv_withami_test.go index 1bf202cd16d86..eaae1d898408a 100644 --- a/test/new-e2e/examples/vmenv_withami_test.go +++ b/test/new-e2e/examples/vmenv_withami_test.go @@ -10,7 +10,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/test-infra-definitions/components/os" diff --git a/test/new-e2e/examples/vmenv_withos_test.go b/test/new-e2e/examples/vmenv_withos_test.go index 6ec8775df5905..aa64c8483f8f0 100644 --- a/test/new-e2e/examples/vmenv_withos_test.go +++ b/test/new-e2e/examples/vmenv_withos_test.go @@ -8,12 +8,13 @@ package examples import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + "github.com/stretchr/testify/assert" ) diff --git a/test/new-e2e/go.mod b/test/new-e2e/go.mod index 2bd2138c3ea69..26734c2ec8418 100644 --- a/test/new-e2e/go.mod +++ b/test/new-e2e/go.mod @@ -58,7 +58,7 @@ require ( // `TEST_INFRA_DEFINITIONS_BUILDIMAGES` matches the commit sha in the module version // Example: github.com/DataDog/test-infra-definitions v0.0.0-YYYYMMDDHHmmSS-0123456789AB // => TEST_INFRA_DEFINITIONS_BUILDIMAGES: 0123456789AB - github.com/DataDog/test-infra-definitions v0.0.0-20241217110507-4b4112f5f64d + github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 github.com/aws/aws-sdk-go-v2 v1.32.6 github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/service/ec2 v1.190.0 @@ -72,10 +72,10 @@ require ( github.com/kr/pretty v0.3.1 github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c github.com/pkg/sftp v1.13.7 - github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 - github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 - github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 - github.com/pulumi/pulumi/sdk/v3 v3.140.0 + github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 + github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 + github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 + github.com/pulumi/pulumi/sdk/v3 v3.142.0 github.com/samber/lo v1.47.0 github.com/stretchr/testify v1.10.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -171,7 +171,7 @@ require ( github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl/v2 v2.20.1 // indirect + github.com/hashicorp/hcl/v2 v2.22.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -253,7 +253,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.32.0 + golang.org/x/net v0.33.0 golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/text v0.21.0 diff --git a/test/new-e2e/go.sum b/test/new-e2e/go.sum index b08902fb2e378..c79b919019bf8 100644 --- a/test/new-e2e/go.sum +++ b/test/new-e2e/go.sum @@ -17,8 +17,8 @@ github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEU github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49 h1:EbzDX8HPk5uE2FsJYxD74QmMw0/3CqSKhEr6teh0ncQ= github.com/DataDog/mmh3 v0.0.0-20210722141835-012dc69a9e49/go.mod h1:SvsjzyJlSg0rKsqYgdcFxeEVflx3ZNAyFfkUHP0TxXg= -github.com/DataDog/test-infra-definitions v0.0.0-20241217110507-4b4112f5f64d h1:F1yqGiWtXVsHkMiNUhs8bgaoZ1WlV3rYGRZ1CtCxpm8= -github.com/DataDog/test-infra-definitions v0.0.0-20241217110507-4b4112f5f64d/go.mod h1:1PAUwGjC25ACjfft4HrLEmHliuajlvjzcLFWpuqAIyk= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266 h1:w+uoG7RdtPiW6YvJvWrzWOC78eNzCemcH8ZM6tJoUBw= +github.com/DataDog/test-infra-definitions v0.0.0-20241218140851-221bbc806266/go.mod h1:+13pRFKChJo9VZ0WcHFm5GTPCNYyim3hxFjy6cpcLG8= github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd_0 v0.0.0-20210310093942-586c1286621f h1:5Vuo4niPKFkfwW55jV4vY0ih3VQ9RaQqeqY67fvRn8A= @@ -268,8 +268,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/hcl/v2 v2.22.0 h1:hkZ3nCtqeJsDhPRFz5EA9iwcG1hNWGePOTw6oyul12M= +github.com/hashicorp/hcl/v2 v2.22.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -403,10 +403,10 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435 github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE= github.com/pulumi/esc v0.10.0 h1:jzBKzkLVW0mePeanDRfqSQoCJ5yrkux0jIwAkUxpRKE= github.com/pulumi/esc v0.10.0/go.mod h1:2Bfa+FWj/xl8CKqRTWbWgDX0SOD4opdQgvYSURTGK2c= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 h1:wA38Ep4sEphX+3YGwFfaxRHs7NQv8dNObFepX6jaRa4= -github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1/go.mod h1:m/ejZ2INurqq/ncDjJfgC1Ff/lnbt0J/uO33BnPVots= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1 h1:6082hB+ILpPB/0V5F+LTmHbX1BO54tCVOQCVOL/FYI4= -github.com/pulumi/pulumi-awsx/sdk/v2 v2.16.1/go.mod h1:z2bnBPHNYfk72IW1P01H9qikBtBSBhCwi3QpH6Y/38Q= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0 h1:OvCLqUueOja9YE2WEGPYAw+lKHFRbLQ7QjwX55+uNsA= +github.com/pulumi/pulumi-aws/sdk/v6 v6.65.0/go.mod h1:FFzye44v9E0BgaFXVB/9X7KH0S0MapoXEy2YonrQfz4= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0 h1:jil2EBzZnKsRDrLfvx2gnAaq17HQLrTbpPsIb3h+98U= +github.com/pulumi/pulumi-awsx/sdk/v2 v2.19.0/go.mod h1:r+K4M7jnLqvvQDeR/0mBRq2EPZaqsDg24Ciy3ml/thA= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1 h1:miIJy4njnFYw7VxMLvEztoMPr9zYC2kqBTwRlaFAf48= github.com/pulumi/pulumi-azure-native-sdk/authorization/v2 v2.73.1/go.mod h1:LR1QBq0C1NIhmD9E0uKozCAu32j5qsamhrIsTSNVMS8= github.com/pulumi/pulumi-azure-native-sdk/compute/v2 v2.73.1 h1:79HTKSE1uJQolCRUHRFnIbSPNSIhxekIhznHnjpLi6s= @@ -427,16 +427,16 @@ github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 h1:s2Cpu6E2lmADNUbutbJGm6O+O9j0mBLlrh github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 h1:21oSj+TKlKTzQcxN9Hik7iSNNHPUQXN4s3itOnahy/w= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0/go.mod h1:YaEZms1NgXFqGhObKVofcAeWXu2V+3t/BAXdHQZq7fU= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 h1:quqoGsLbF7lpGpGU4mi5WfVLIAo4gfvoQeYYmemx1Dg= -github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3/go.mod h1:9dBA6+rtpKmyZB3k1XryUOHDOuNdoTODFKEEZZCtrz8= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0 h1:7AjJpUyW6YHHpZr0bI6Fy1A3/b7ERxq1LAo5mlyNN1Y= +github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.19.0/go.mod h1:ATS+UN8pguMxypQAK+SaPewesU+UN5dpf93PNqVuHzs= github.com/pulumi/pulumi-libvirt/sdk v0.5.3 h1:CiUGTweLLIxbAbADxxnwPv4BK8pxXfU8urokJvK1ihM= github.com/pulumi/pulumi-libvirt/sdk v0.5.3/go.mod h1:gAhyIZKtzs4rknrl8fu8BQnyqijAmViFbaUkyuHt4xY= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7 h1:39rhOe/PTUGMYia8pR5T2wbxxMt2pwrlonf0ncYKSzE= github.com/pulumi/pulumi-random/sdk/v4 v4.16.7/go.mod h1:cxxDhJzUPt/YElfvlWa15Q4NGF6XXS8kUs4OQsCxSBk= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1 h1:tXemWrzeVTqG8zq6hBdv1TdPFXjgZ+dob63a/6GlF1o= github.com/pulumi/pulumi-tls/sdk/v4 v4.11.1/go.mod h1:hODo3iEmmXDFOXqPK+V+vwI0a3Ww7BLjs5Tgamp86Ng= -github.com/pulumi/pulumi/sdk/v3 v3.140.0 h1:+Z/RBvdYg7tBNkBwk4p/FzlV7niBT3TbLAICq/Y0LDU= -github.com/pulumi/pulumi/sdk/v3 v3.140.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= +github.com/pulumi/pulumi/sdk/v3 v3.142.0 h1:SmcVddGuvwAh3g3XUVQQ5gVRQUKH1yZ6iETpDNHIHlw= +github.com/pulumi/pulumi/sdk/v3 v3.142.0/go.mod h1:PvKsX88co8XuwuPdzolMvew5lZV+4JmZfkeSjj7A6dI= github.com/pulumiverse/pulumi-time/sdk v0.1.0 h1:xfi9HKDgV+GgDxQ23oSv9KxC3DQqViGTcMrJICRgJv0= github.com/pulumiverse/pulumi-time/sdk v0.1.0/go.mod h1:NUa1zA74DF002WrM6iF111A6UjX9knPpXufVRvBwNyg= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -595,8 +595,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= diff --git a/test/new-e2e/pkg/e2e/suite.go b/test/new-e2e/pkg/e2e/suite.go index bb15e92e7ce2e..586f87bb8cd76 100644 --- a/test/new-e2e/pkg/e2e/suite.go +++ b/test/new-e2e/pkg/e2e/suite.go @@ -154,6 +154,7 @@ import ( "github.com/DataDog/test-infra-definitions/components" "gopkg.in/zorkian/go-datadog-api.v2" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/infra" @@ -175,7 +176,7 @@ type Suite[Env any] interface { init(params []SuiteOption, self Suite[Env]) - UpdateEnv(...Provisioner) + UpdateEnv(...provisioners.Provisioner) Env() *Env } @@ -189,8 +190,8 @@ type BaseSuite[Env any] struct { datadogClient *datadog.Client params suiteParams - originalProvisioners ProvisionerMap - currentProvisioners ProvisionerMap + originalProvisioners provisioners.ProvisionerMap + currentProvisioners provisioners.ProvisionerMap firstFailTest string startTime time.Time @@ -211,9 +212,9 @@ func (bs *BaseSuite[Env]) Env() *Env { } // UpdateEnv updates the environment with new provisioners. -func (bs *BaseSuite[Env]) UpdateEnv(newProvisioners ...Provisioner) { +func (bs *BaseSuite[Env]) UpdateEnv(newProvisioners ...provisioners.Provisioner) { uniqueIDs := make(map[string]struct{}) - targetProvisioners := make(ProvisionerMap, len(newProvisioners)) + targetProvisioners := make(provisioners.ProvisionerMap, len(newProvisioners)) for _, provisioner := range newProvisioners { if _, found := uniqueIDs[provisioner.ID()]; found { panic(fmt.Errorf("Multiple providers with same id found, provisioner with id %s already exists", provisioner.ID())) @@ -275,7 +276,7 @@ func (bs *BaseSuite[Env]) init(options []SuiteOption, self Suite[Env]) { bs.originalProvisioners = bs.params.provisioners } -func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error { +func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners provisioners.ProvisionerMap) error { if reflect.DeepEqual(bs.currentProvisioners, targetProvisioners) { bs.T().Logf("No change in provisioners, skipping environment update") return nil @@ -303,23 +304,23 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error } // Then we provision new resources - resources := make(RawResources) + resources := make(provisioners.RawResources) for id, provisioner := range targetProvisioners { - var provisionerResources RawResources + var provisionerResources provisioners.RawResources var err error bs.T().Logf("Provisioning environment stack %s with provisioner %s", bs.params.stackName, id) switch pType := provisioner.(type) { - case TypedProvisioner[Env]: + case provisioners.TypedProvisioner[Env]: provisionerResources, err = pType.ProvisionEnv(ctx, bs.params.stackName, logger, newEnv) - case UntypedProvisioner: + case provisioners.UntypedProvisioner: provisionerResources, err = pType.Provision(ctx, bs.params.stackName, logger) default: return fmt.Errorf("provisioner of type %T does not implement UntypedProvisioner nor TypedProvisioner", provisioner) } if err != nil { - if diagnosableProvisioner, ok := provisioner.(Diagnosable); ok { + if diagnosableProvisioner, ok := provisioner.(provisioners.Diagnosable); ok { stackName, err := infra.GetStackManager().GetPulumiStackName(bs.params.stackName) if err != nil { bs.T().Logf("unable to get stack name for diagnose, err: %v", err) @@ -359,7 +360,7 @@ func (bs *BaseSuite[Env]) reconcileEnv(targetProvisioners ProvisionerMap) error // On success we update the current environment // We need top copy provisioners to protect against external modifications - bs.currentProvisioners = copyProvisioners(targetProvisioners) + bs.currentProvisioners = provisioners.CopyProvisioners(targetProvisioners) bs.env = newEnv return nil } @@ -407,7 +408,7 @@ func (bs *BaseSuite[Env]) createEnv() (*Env, []reflect.StructField, []reflect.Va return &env, retainedFields, retainedValues, nil } -func (bs *BaseSuite[Env]) buildEnvFromResources(resources RawResources, fields []reflect.StructField, values []reflect.Value) error { +func (bs *BaseSuite[Env]) buildEnvFromResources(resources provisioners.RawResources, fields []reflect.StructField, values []reflect.Value) error { if len(fields) != len(values) { panic("fields and values must have the same length") } @@ -582,7 +583,7 @@ func (bs *BaseSuite[Env]) TearDownSuite() { for id, provisioner := range bs.originalProvisioners { // Run provisioner Diagnose before tearing down the stack - if diagnosableProvisioner, ok := provisioner.(Diagnosable); ok { + if diagnosableProvisioner, ok := provisioner.(provisioners.Diagnosable); ok { stackName, err := infra.GetStackManager().GetPulumiStackName(bs.params.stackName) if err != nil { bs.T().Logf("unable to get stack name for diagnose, err: %v", err) diff --git a/test/new-e2e/pkg/e2e/suite_params.go b/test/new-e2e/pkg/e2e/suite_params.go index bc38e26005a45..111da630670d2 100644 --- a/test/new-e2e/pkg/e2e/suite_params.go +++ b/test/new-e2e/pkg/e2e/suite_params.go @@ -8,8 +8,10 @@ package e2e import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" ) // Params implements [BaseSuite] options @@ -22,7 +24,7 @@ type suiteParams struct { skipDeleteOnFailure bool - provisioners ProvisionerMap + provisioners provisioners.ProvisionerMap } // SuiteOption is an optional function parameter type for e2e options @@ -53,14 +55,14 @@ func WithSkipDeleteOnFailure() SuiteOption { } // WithProvisioner adds a provisioner to the suite -func WithProvisioner(provisioner Provisioner) SuiteOption { +func WithProvisioner(provisioner provisioners.Provisioner) SuiteOption { return func(options *suiteParams) { if _, found := options.provisioners[provisioner.ID()]; found { panic(fmt.Sprintf("Duplicate provider in test Suite: %s", provisioner.ID())) } if options.provisioners == nil { - options.provisioners = make(ProvisionerMap) + options.provisioners = make(provisioners.ProvisionerMap) } options.provisioners[provisioner.ID()] = provisioner @@ -69,10 +71,10 @@ func WithProvisioner(provisioner Provisioner) SuiteOption { // WithUntypedPulumiProvisioner adds an untyped Pulumi provisioner to the suite func WithUntypedPulumiProvisioner(runFunc pulumi.RunFunc, configMap runner.ConfigMap) SuiteOption { - return WithProvisioner(NewUntypedPulumiProvisioner("", runFunc, configMap)) + return WithProvisioner(provisioners.NewUntypedPulumiProvisioner("", runFunc, configMap)) } // WithPulumiProvisioner adds a typed Pulumi provisioner to the suite -func WithPulumiProvisioner[Env any](runFunc PulumiEnvRunFunc[Env], configMap runner.ConfigMap) SuiteOption { - return WithProvisioner(NewTypedPulumiProvisioner("", runFunc, configMap)) +func WithPulumiProvisioner[Env any](runFunc provisioners.PulumiEnvRunFunc[Env], configMap runner.ConfigMap) SuiteOption { + return WithProvisioner(provisioners.NewTypedPulumiProvisioner("", runFunc, configMap)) } diff --git a/test/new-e2e/pkg/e2e/suite_test.go b/test/new-e2e/pkg/e2e/suite_test.go index 37219945b2389..e86152a30a395 100644 --- a/test/new-e2e/pkg/e2e/suite_test.go +++ b/test/new-e2e/pkg/e2e/suite_test.go @@ -14,6 +14,8 @@ import ( "github.com/DataDog/test-infra-definitions/components" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" ) type testTypeOutput struct { @@ -47,8 +49,8 @@ type testSuite struct { BaseSuite[testEnv] } -func testRawResources(key, value string) RawResources { - return RawResources{key: []byte(fmt.Sprintf(`{"myField":"%s"}`, value))} +func testRawResources(key, value string) provisioners.RawResources { + return provisioners.RawResources{key: []byte(fmt.Sprintf(`{"myField":"%s"}`, value))} } func TestCreateEnv(t *testing.T) { @@ -69,16 +71,16 @@ type testProvisioner struct { mock.Mock } -var _ UntypedProvisioner = &testProvisioner{} +var _ provisioners.UntypedProvisioner = &testProvisioner{} func (m *testProvisioner) ID() string { args := m.Called() return args.Get(0).(string) } -func (m *testProvisioner) Provision(arg0 context.Context, arg1 string, arg2 io.Writer) (RawResources, error) { +func (m *testProvisioner) Provision(arg0 context.Context, arg1 string, arg2 io.Writer) (provisioners.RawResources, error) { args := m.Called(arg0, arg1, arg2) - return args.Get(0).(RawResources), args.Error(1) + return args.Get(0).(provisioners.RawResources), args.Error(1) } func (m *testProvisioner) Destroy(arg0 context.Context, arg1 string, arg2 io.Writer) error { diff --git a/test/new-e2e/pkg/environments/doc.go b/test/new-e2e/pkg/environments/environments.go similarity index 100% rename from test/new-e2e/pkg/environments/doc.go rename to test/new-e2e/pkg/environments/environments.go diff --git a/test/new-e2e/pkg/environments/aws/docker/host.go b/test/new-e2e/pkg/provisioners/aws/docker/host.go similarity index 95% rename from test/new-e2e/pkg/environments/aws/docker/host.go rename to test/new-e2e/pkg/provisioners/aws/docker/host.go index 0e39ed8b74b20..0a730bdcf5e72 100644 --- a/test/new-e2e/pkg/environments/aws/docker/host.go +++ b/test/new-e2e/pkg/provisioners/aws/docker/host.go @@ -9,8 +9,8 @@ package awsdocker import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -224,10 +224,10 @@ func Run(ctx *pulumi.Context, env *environments.DockerHost, runParams RunParams) // Provisioner creates a VM environment with an EC2 VM with Docker, an ECS Fargate FakeIntake and a Docker Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.DockerHost] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.DockerHost] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.DockerHost) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.DockerHost) error { return Run(ctx, env, RunParams{ProvisionerParams: params}) }, params.extraConfigParams) diff --git a/test/new-e2e/pkg/environments/aws/ecs/ecs.go b/test/new-e2e/pkg/provisioners/aws/ecs/ecs.go similarity index 97% rename from test/new-e2e/pkg/environments/aws/ecs/ecs.go rename to test/new-e2e/pkg/provisioners/aws/ecs/ecs.go index 280920f2196a8..8e11e8a3ad0b9 100644 --- a/test/new-e2e/pkg/environments/aws/ecs/ecs.go +++ b/test/new-e2e/pkg/provisioners/aws/ecs/ecs.go @@ -28,8 +28,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ecs" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) @@ -300,10 +300,10 @@ func Run(ctx *pulumi.Context, env *environments.ECS, params *ProvisionerParams) // Provisioner creates a VM environment with an EC2 VM with Docker, an ECS Fargate FakeIntake and a Docker Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.ECS] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.ECS] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.ECS) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.ECS) error { return Run(ctx, env, params) }, params.extraConfigParams) diff --git a/test/new-e2e/pkg/environments/aws/host/host.go b/test/new-e2e/pkg/provisioners/aws/host/host.go similarity index 94% rename from test/new-e2e/pkg/environments/aws/host/host.go rename to test/new-e2e/pkg/provisioners/aws/host/host.go index 4668905ac83a8..ea652309c06e2 100644 --- a/test/new-e2e/pkg/environments/aws/host/host.go +++ b/test/new-e2e/pkg/provisioners/aws/host/host.go @@ -9,8 +9,8 @@ package awshost import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -151,7 +151,7 @@ func WithDocker() ProvisionerOption { } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -160,7 +160,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) @@ -283,11 +283,11 @@ func Run(ctx *pulumi.Context, env *environments.Host, runParams RunParams) error // Provisioner creates a VM environment with an EC2 VM, an ECS Fargate FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := GetProvisionerParams(opts...) diff --git a/test/new-e2e/pkg/environments/aws/host/windows/host.go b/test/new-e2e/pkg/provisioners/aws/host/windows/host.go similarity index 93% rename from test/new-e2e/pkg/environments/aws/host/windows/host.go rename to test/new-e2e/pkg/provisioners/aws/host/windows/host.go index 1fd5885a88c4d..642a1a7dcc2e2 100644 --- a/test/new-e2e/pkg/environments/aws/host/windows/host.go +++ b/test/new-e2e/pkg/provisioners/aws/host/windows/host.go @@ -19,8 +19,8 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" installer "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components/datadog-installer" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -253,10 +253,10 @@ func getProvisionerParams(opts ...ProvisionerOption) *ProvisionerParams { // Provisioner creates a VM environment with a Windows EC2 VM, an ECS Fargate FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { // We need to build params here to be able to use params.name in the provisioner name params := getProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.WindowsHost) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.WindowsHost) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := getProvisionerParams(opts...) @@ -267,7 +267,7 @@ func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Wi } // ProvisionerNoAgent wraps Provisioner with hardcoded WithoutAgent options. -func ProvisionerNoAgent(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoAgent(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent()) @@ -276,7 +276,7 @@ func ProvisionerNoAgent(opts ...ProvisionerOption) e2e.TypedProvisioner[environm } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -285,7 +285,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/eks.go b/test/new-e2e/pkg/provisioners/aws/kubernetes/eks.go similarity index 95% rename from test/new-e2e/pkg/environments/aws/kubernetes/eks.go rename to test/new-e2e/pkg/provisioners/aws/kubernetes/eks.go index 22971ae20f9ed..1e44a7019a711 100644 --- a/test/new-e2e/pkg/environments/aws/kubernetes/eks.go +++ b/test/new-e2e/pkg/provisioners/aws/kubernetes/eks.go @@ -26,8 +26,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/eks" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -42,13 +42,13 @@ func eksDiagnoseFunc(ctx context.Context, stackName string) (string, error) { } // EKSProvisioner creates a new provisioner -func EKSProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func EKSProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/kind.go b/test/new-e2e/pkg/provisioners/aws/kubernetes/kind.go similarity index 95% rename from test/new-e2e/pkg/environments/aws/kubernetes/kind.go rename to test/new-e2e/pkg/provisioners/aws/kubernetes/kind.go index 60da0620167f9..44f5716bc92fa 100644 --- a/test/new-e2e/pkg/environments/aws/kubernetes/kind.go +++ b/test/new-e2e/pkg/provisioners/aws/kubernetes/kind.go @@ -12,8 +12,8 @@ import ( "github.com/DataDog/test-infra-definitions/common/utils" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" "github.com/DataDog/test-infra-definitions/components/datadog/agent/helm" @@ -51,13 +51,13 @@ func kindDiagnoseFunc(ctx context.Context, stackName string) (string, error) { } // KindProvisioner creates a new provisioner -func KindProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func KindProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/kubernetes_dump.go b/test/new-e2e/pkg/provisioners/aws/kubernetes/kubernetes_dump.go similarity index 100% rename from test/new-e2e/pkg/environments/aws/kubernetes/kubernetes_dump.go rename to test/new-e2e/pkg/provisioners/aws/kubernetes/kubernetes_dump.go diff --git a/test/new-e2e/pkg/environments/aws/kubernetes/params.go b/test/new-e2e/pkg/provisioners/aws/kubernetes/params.go similarity index 100% rename from test/new-e2e/pkg/environments/aws/kubernetes/params.go rename to test/new-e2e/pkg/provisioners/aws/kubernetes/params.go diff --git a/test/new-e2e/pkg/environments/azure/host/linux/host.go b/test/new-e2e/pkg/provisioners/azure/host/linux/host.go similarity index 92% rename from test/new-e2e/pkg/environments/azure/host/linux/host.go rename to test/new-e2e/pkg/provisioners/azure/host/linux/host.go index 5006c4c4750e3..30aa4cecedafa 100644 --- a/test/new-e2e/pkg/environments/azure/host/linux/host.go +++ b/test/new-e2e/pkg/provisioners/azure/host/linux/host.go @@ -7,13 +7,13 @@ package azurehost import ( - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/resources/azure" "github.com/DataDog/test-infra-definitions/scenarios/azure/compute" "github.com/DataDog/test-infra-definitions/scenarios/azure/fakeintake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/components/datadog/agent" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" @@ -28,11 +28,11 @@ const ( // Provisioner creates a VM environment with an VM, a FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard-to-debug issues. params := GetProvisionerParams(opts...) diff --git a/test/new-e2e/pkg/environments/azure/host/linux/params.go b/test/new-e2e/pkg/provisioners/azure/host/linux/params.go similarity index 94% rename from test/new-e2e/pkg/environments/azure/host/linux/params.go rename to test/new-e2e/pkg/provisioners/azure/host/linux/params.go index ff32a326f06eb..923d4ac1778d8 100644 --- a/test/new-e2e/pkg/environments/azure/host/linux/params.go +++ b/test/new-e2e/pkg/provisioners/azure/host/linux/params.go @@ -7,15 +7,17 @@ package azurehost import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/resources/azure" "github.com/DataDog/test-infra-definitions/scenarios/azure/compute" "github.com/DataDog/test-infra-definitions/scenarios/azure/fakeintake" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) // ProvisionerParams is a set of parameters for the Provisioner. @@ -128,7 +130,7 @@ func WithUpdater() ProvisionerOption { } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -137,7 +139,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) diff --git a/test/new-e2e/pkg/environments/azure/host/windows/host.go b/test/new-e2e/pkg/provisioners/azure/host/windows/host.go similarity index 89% rename from test/new-e2e/pkg/environments/azure/host/windows/host.go rename to test/new-e2e/pkg/provisioners/azure/host/windows/host.go index 414c6e8e469b0..dc32cff8ed9a4 100644 --- a/test/new-e2e/pkg/environments/azure/host/windows/host.go +++ b/test/new-e2e/pkg/provisioners/azure/host/windows/host.go @@ -7,7 +7,6 @@ package winazurehost import ( - installer "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components/datadog-installer" "github.com/DataDog/test-infra-definitions/components/activedirectory" "github.com/DataDog/test-infra-definitions/components/datadog/agent" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" @@ -16,7 +15,9 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/azure/fakeintake" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + installer "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components/datadog-installer" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/components/defender" ) @@ -28,10 +29,10 @@ const ( // Provisioner creates a VM environment with a Windows VM, a FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { // We need to build params here to be able to use params.name in the provisioner name params := getProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.WindowsHost) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.WindowsHost) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard-to-debug issues. params := getProvisionerParams(opts...) @@ -42,7 +43,7 @@ func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Wi } // ProvisionerNoAgent wraps Provisioner with hardcoded WithoutAgent options. -func ProvisionerNoAgent(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoAgent(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent()) @@ -51,7 +52,7 @@ func ProvisionerNoAgent(opts ...ProvisionerOption) e2e.TypedProvisioner[environm } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -60,7 +61,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.WindowsHost] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.WindowsHost] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) diff --git a/test/new-e2e/pkg/environments/azure/host/windows/params.go b/test/new-e2e/pkg/provisioners/azure/host/windows/params.go similarity index 100% rename from test/new-e2e/pkg/environments/azure/host/windows/params.go rename to test/new-e2e/pkg/provisioners/azure/host/windows/params.go diff --git a/test/new-e2e/pkg/environments/azure/kubernetes/aks.go b/test/new-e2e/pkg/provisioners/azure/kubernetes/aks.go similarity index 91% rename from test/new-e2e/pkg/environments/azure/kubernetes/aks.go rename to test/new-e2e/pkg/provisioners/azure/kubernetes/aks.go index 829a76675469c..53075f5f230fd 100644 --- a/test/new-e2e/pkg/environments/azure/kubernetes/aks.go +++ b/test/new-e2e/pkg/provisioners/azure/kubernetes/aks.go @@ -15,8 +15,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/azure/aks" "github.com/DataDog/test-infra-definitions/scenarios/azure/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) @@ -25,13 +25,13 @@ const ( ) // AKSProvisioner creates a new provisioner for AKS on Azure -func AKSProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func AKSProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/azure/kubernetes/params.go b/test/new-e2e/pkg/provisioners/azure/kubernetes/params.go similarity index 100% rename from test/new-e2e/pkg/environments/azure/kubernetes/params.go rename to test/new-e2e/pkg/provisioners/azure/kubernetes/params.go diff --git a/test/new-e2e/pkg/e2e/file_provisioner.go b/test/new-e2e/pkg/provisioners/file_provisioner.go similarity index 98% rename from test/new-e2e/pkg/e2e/file_provisioner.go rename to test/new-e2e/pkg/provisioners/file_provisioner.go index de8d0a4c22bd3..c0284816be6ef 100644 --- a/test/new-e2e/pkg/e2e/file_provisioner.go +++ b/test/new-e2e/pkg/provisioners/file_provisioner.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -package e2e +package provisioners import ( "context" diff --git a/test/new-e2e/pkg/environments/gcp/host/linux/host.go b/test/new-e2e/pkg/provisioners/gcp/host/linux/host.go similarity index 92% rename from test/new-e2e/pkg/environments/gcp/host/linux/host.go rename to test/new-e2e/pkg/provisioners/gcp/host/linux/host.go index 0e479d8a51bdf..09e7be211d665 100644 --- a/test/new-e2e/pkg/environments/gcp/host/linux/host.go +++ b/test/new-e2e/pkg/provisioners/gcp/host/linux/host.go @@ -7,12 +7,12 @@ package gcphost import ( - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/test-infra-definitions/resources/gcp" "github.com/DataDog/test-infra-definitions/scenarios/gcp/compute" "github.com/DataDog/test-infra-definitions/scenarios/gcp/fakeintake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/components/datadog/agent" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" @@ -27,11 +27,11 @@ const ( // Provisioner creates a VM environment with an VM, a FakeIntake and a Host Agent configured to talk to each other. // FakeIntake and Agent creation can be deactivated by using [WithoutFakeIntake] and [WithoutAgent] options. -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { // We need to build params here to be able to use params.name in the provisioner name params := GetProvisionerParams(opts...) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard-to-debug issues. params := GetProvisionerParams(opts...) diff --git a/test/new-e2e/pkg/environments/gcp/host/linux/params.go b/test/new-e2e/pkg/provisioners/gcp/host/linux/params.go similarity index 94% rename from test/new-e2e/pkg/environments/gcp/host/linux/params.go rename to test/new-e2e/pkg/provisioners/gcp/host/linux/params.go index 442fd28b889b0..0542b3e045a66 100644 --- a/test/new-e2e/pkg/environments/gcp/host/linux/params.go +++ b/test/new-e2e/pkg/provisioners/gcp/host/linux/params.go @@ -7,15 +7,17 @@ package gcphost import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/resources/gcp" "github.com/DataDog/test-infra-definitions/scenarios/gcp/compute" "github.com/DataDog/test-infra-definitions/scenarios/gcp/fakeintake" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) // ProvisionerParams is a set of parameters for the Provisioner. @@ -128,7 +130,7 @@ func WithUpdater() ProvisionerOption { } // ProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -137,7 +139,7 @@ func ProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisio } // ProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func ProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func ProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) diff --git a/test/new-e2e/pkg/environments/gcp/kubernetes/gke.go b/test/new-e2e/pkg/provisioners/gcp/kubernetes/gke.go similarity index 89% rename from test/new-e2e/pkg/environments/gcp/kubernetes/gke.go rename to test/new-e2e/pkg/provisioners/gcp/kubernetes/gke.go index 10c315fb7b254..80107a6369ce0 100644 --- a/test/new-e2e/pkg/environments/gcp/kubernetes/gke.go +++ b/test/new-e2e/pkg/provisioners/gcp/kubernetes/gke.go @@ -15,8 +15,8 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" "github.com/DataDog/test-infra-definitions/scenarios/gcp/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" ) @@ -25,13 +25,13 @@ const ( ) // GKEProvisioner creates a new provisioner for GKE on GCP -func GKEProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func GKEProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/gcp/kubernetes/params.go b/test/new-e2e/pkg/provisioners/gcp/kubernetes/params.go similarity index 100% rename from test/new-e2e/pkg/environments/gcp/kubernetes/params.go rename to test/new-e2e/pkg/provisioners/gcp/kubernetes/params.go diff --git a/test/new-e2e/pkg/environments/local/host/podman.go b/test/new-e2e/pkg/provisioners/local/host/podman.go similarity index 91% rename from test/new-e2e/pkg/environments/local/host/podman.go rename to test/new-e2e/pkg/provisioners/local/host/podman.go index 911a91f36ab76..85e73718dbcbe 100644 --- a/test/new-e2e/pkg/environments/local/host/podman.go +++ b/test/new-e2e/pkg/provisioners/local/host/podman.go @@ -7,8 +7,8 @@ package localhost import ( - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -89,7 +89,7 @@ func WithExtraConfigParams(configMap runner.ConfigMap) ProvisionerOption { } // PodmanProvisionerNoAgentNoFakeIntake wraps Provisioner with hardcoded WithoutAgent and WithoutFakeIntake options. -func PodmanProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func PodmanProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+2) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutAgent(), WithoutFakeIntake()) @@ -98,7 +98,7 @@ func PodmanProvisionerNoAgentNoFakeIntake(opts ...ProvisionerOption) e2e.TypedPr } // PodmanProvisionerNoFakeIntake wraps Provisioner with hardcoded WithoutFakeIntake option. -func PodmanProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func PodmanProvisionerNoFakeIntake(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { mergedOpts := make([]ProvisionerOption, 0, len(opts)+1) mergedOpts = append(mergedOpts, opts...) mergedOpts = append(mergedOpts, WithoutFakeIntake()) @@ -107,13 +107,13 @@ func PodmanProvisionerNoFakeIntake(opts ...ProvisionerOption) e2e.TypedProvision } // PodmanProvisioner creates a new provisioner -func PodmanProvisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Host] { +func PodmanProvisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Host] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Host) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/environments/local/kubernetes/kind.go b/test/new-e2e/pkg/provisioners/local/kubernetes/kind.go similarity index 95% rename from test/new-e2e/pkg/environments/local/kubernetes/kind.go rename to test/new-e2e/pkg/provisioners/local/kubernetes/kind.go index 99703f64eb402..e754b6f57af91 100644 --- a/test/new-e2e/pkg/environments/local/kubernetes/kind.go +++ b/test/new-e2e/pkg/provisioners/local/kubernetes/kind.go @@ -9,8 +9,8 @@ package localkubernetes import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -105,13 +105,13 @@ func WithWorkloadApp(appFunc WorkloadAppFunc) ProvisionerOption { } // Provisioner creates a new provisioner -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to hard to debug issues. params := newProvisionerParams() diff --git a/test/new-e2e/pkg/e2e/provisioner.go b/test/new-e2e/pkg/provisioners/provisioners.go similarity index 71% rename from test/new-e2e/pkg/e2e/provisioner.go rename to test/new-e2e/pkg/provisioners/provisioners.go index a537cc7a05ce4..7dd64c4468740 100644 --- a/test/new-e2e/pkg/e2e/provisioner.go +++ b/test/new-e2e/pkg/provisioners/provisioners.go @@ -3,7 +3,8 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -package e2e +// Package provisioners contains the definitions of the different provisioners that can be used in a test to setup an environment. +package provisioners import ( "context" @@ -21,6 +22,16 @@ type Provisioner interface { Destroy(context.Context, string, io.Writer) error } +// RawResources is the common types returned by provisioners +type RawResources map[string][]byte + +// Merge merges two RawResources maps +func (rr RawResources) Merge(in RawResources) { + for k, v := range in { + rr[k] = v + } +} + // UntypedProvisioner defines the interface for a provisioner without env binding type UntypedProvisioner interface { Provisioner @@ -36,7 +47,8 @@ type TypedProvisioner[Env any] interface { // ProvisionerMap is a map of provisioners. type ProvisionerMap map[string]Provisioner -func copyProvisioners(in ProvisionerMap) ProvisionerMap { +// CopyProvisioners copies a map of provisioners +func CopyProvisioners(in ProvisionerMap) ProvisionerMap { out := make(ProvisionerMap, len(in)) for k, v := range in { out[k] = v diff --git a/test/new-e2e/pkg/e2e/pulumi_provisioner.go b/test/new-e2e/pkg/provisioners/pulumi_provisioner.go similarity index 99% rename from test/new-e2e/pkg/e2e/pulumi_provisioner.go rename to test/new-e2e/pkg/provisioners/pulumi_provisioner.go index ef8aba2b8d295..ab9e6e69ca03c 100644 --- a/test/new-e2e/pkg/e2e/pulumi_provisioner.go +++ b/test/new-e2e/pkg/provisioners/pulumi_provisioner.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -package e2e +package provisioners import ( "context" diff --git a/test/new-e2e/pkg/utils/infra/retriable_errors.go b/test/new-e2e/pkg/utils/infra/retriable_errors.go index b940cbcd2e064..211c9179511cb 100644 --- a/test/new-e2e/pkg/utils/infra/retriable_errors.go +++ b/test/new-e2e/pkg/utils/infra/retriable_errors.go @@ -55,5 +55,11 @@ func getKnownErrors() []knownError { errorMessage: `ssh: handshake failed: ssh: unable to authenticate`, retryType: ReCreate, }, + { + // https://datadoghq.atlassian.net/browse/ADXT-798 + // https://datadoghq.atlassian.net/browse/ADXT-813 + errorMessage: `error: awsx:ecs:FargateTaskDefinition resource '.+fakeintake.+' has a problem: grpc: the client connection is closing`, + retryType: ReCreate, + }, } } diff --git a/test/new-e2e/system-probe/test-runner/files/cws_fentry.json b/test/new-e2e/system-probe/test-runner/files/cws_fentry.json deleted file mode 100644 index 8330f6d6048af..0000000000000 --- a/test/new-e2e/system-probe/test-runner/files/cws_fentry.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "filters": { - "*": { - "exclude": false - } - }, - "additional_env_vars": [ - "DD_EVENT_MONITORING_CONFIG_EVENT_STREAM_USE_FENTRY=true" - ] -} diff --git a/test/new-e2e/test-infra-definition/agent_test.go b/test/new-e2e/test-infra-definition/agent_test.go index 0452841024795..deae9e07d8916 100644 --- a/test/new-e2e/test-infra-definition/agent_test.go +++ b/test/new-e2e/test-infra-definition/agent_test.go @@ -8,11 +8,12 @@ package testinfradefinition import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type agentSuite struct { diff --git a/test/new-e2e/test-infra-definition/docker_test.go b/test/new-e2e/test-infra-definition/docker_test.go index 1ede9e48f9d66..a3c18fa65262d 100644 --- a/test/new-e2e/test-infra-definition/docker_test.go +++ b/test/new-e2e/test-infra-definition/docker_test.go @@ -12,12 +12,13 @@ import ( "testing" "time" + "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" - "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" - "github.com/stretchr/testify/assert" ) type dockerSuite struct { diff --git a/test/new-e2e/test-infra-definition/kind_test.go b/test/new-e2e/test-infra-definition/kind_test.go index 5312afb891f34..e31fa3c5c87f3 100644 --- a/test/new-e2e/test-infra-definition/kind_test.go +++ b/test/new-e2e/test-infra-definition/kind_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/components/datadog/apps/nginx" diff --git a/test/new-e2e/test-infra-definition/vm_test.go b/test/new-e2e/test-infra-definition/vm_test.go index a396b65a7eb25..0d5c098bc8d9b 100644 --- a/test/new-e2e/test-infra-definition/vm_test.go +++ b/test/new-e2e/test-infra-definition/vm_test.go @@ -10,7 +10,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/test-infra-definitions/components/os" @@ -46,7 +47,7 @@ type vmSuiteWithUserData struct { type vmTestCase struct { testName string - provisioner e2e.Provisioner + provisioner provisioners.Provisioner suite e2e.Suite[environments.Host] } diff --git a/test/new-e2e/tests/agent-metrics-logs/jmxfetch/jmxfetch_nix_test.go b/test/new-e2e/tests/agent-metrics-logs/jmxfetch/jmxfetch_nix_test.go index c8f573f95307d..7e13406586659 100644 --- a/test/new-e2e/tests/agent-metrics-logs/jmxfetch/jmxfetch_nix_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/jmxfetch/jmxfetch_nix_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/test-infra-definitions/components/datadog/apps/jmxfetch" diff --git a/test/new-e2e/tests/agent-metrics-logs/kindfilelogging/kind.go b/test/new-e2e/tests/agent-metrics-logs/kindfilelogging/kind.go index d66fa4f20746d..f863a3142877c 100644 --- a/test/new-e2e/tests/agent-metrics-logs/kindfilelogging/kind.go +++ b/test/new-e2e/tests/agent-metrics-logs/kindfilelogging/kind.go @@ -10,8 +10,8 @@ package kindfilelogging import ( "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/optional" @@ -112,13 +112,13 @@ func WithExtraConfigParams(configMap runner.ConfigMap) ProvisionerOption { } // Provisioner creates a new provisioner -func Provisioner(opts ...ProvisionerOption) e2e.TypedProvisioner[environments.Kubernetes] { +func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[environments.Kubernetes] { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to issues that are hard to debug. params := newProvisionerParams() _ = optional.ApplyOptions(params, opts) - provisioner := e2e.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { + provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+params.name, func(ctx *pulumi.Context, env *environments.Kubernetes) error { // We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times. // and it's easy to forget about it, leading to issues that are hard to debug. params := newProvisionerParams() diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/file_tailing_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/file_tailing_test.go index e2067a24f8cd7..9f442afa76893 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/file_tailing_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/file_tailing_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" ) diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/multi_line_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/multi_line_test.go index 47c155ad6cc44..c8fb16ac43dde 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/multi_line_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/multi_line_test.go @@ -17,7 +17,7 @@ import ( fi "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" ) diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/utf_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/utf_test.go index 1851a6e8d4ba6..d038e2624109b 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/utf_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/file-tailing/utf_test.go @@ -17,7 +17,7 @@ import ( fi "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) //go:embed config/utf-16-le.yaml diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/integrations/logs_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/integrations/logs_test.go index bd70db87ca4c3..e29aaef83d61e 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/integrations/logs_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/integrations/logs_test.go @@ -9,11 +9,12 @@ import ( _ "embed" "testing" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" - "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" ) type IntegrationsLogsSuite struct { diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/journald/journald_tailing_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/journald/journald_tailing_test.go index 3a37f3ccda671..7472f18ad5d0d 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/journald/journald_tailing_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/linux-log/journald/journald_tailing_test.go @@ -15,11 +15,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" - "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" ) const eventuallyWithTickDuration = 5 * time.Second diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/listener/listener_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/listener/listener_test.go index 6a11c2f7d2852..8dc817a250776 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/listener/listener_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/listener/listener_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" appslogger "github.com/DataDog/test-infra-definitions/components/datadog/apps/logger" "github.com/DataDog/test-infra-definitions/components/datadog/dockeragentparams" diff --git a/test/new-e2e/tests/agent-metrics-logs/log-agent/windows-log/file-tailing/file_tailing_test.go b/test/new-e2e/tests/agent-metrics-logs/log-agent/windows-log/file-tailing/file_tailing_test.go index bd61e8cebde0d..f4bd94512b90e 100644 --- a/test/new-e2e/tests/agent-metrics-logs/log-agent/windows-log/file-tailing/file_tailing_test.go +++ b/test/new-e2e/tests/agent-metrics-logs/log-agent/windows-log/file-tailing/file_tailing_test.go @@ -20,7 +20,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-metrics-logs/log-agent/utils" ) diff --git a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go index 150dc1792462c..83ae2407654d4 100644 --- a/test/new-e2e/tests/agent-platform/install-script/install_script_test.go +++ b/test/new-e2e/tests/agent-platform/install-script/install_script_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" filemanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/file-manager" diff --git a/test/new-e2e/tests/agent-platform/package-signing/package_signing_test.go b/test/new-e2e/tests/agent-platform/package-signing/package_signing_test.go index 0613fba20236b..52aa09e27a60e 100644 --- a/test/new-e2e/tests/agent-platform/package-signing/package_signing_test.go +++ b/test/new-e2e/tests/agent-platform/package-signing/package_signing_test.go @@ -10,13 +10,14 @@ import ( "fmt" "regexp" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/platforms" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" - "github.com/stretchr/testify/assert" "testing" diff --git a/test/new-e2e/tests/agent-platform/persisting-integrations/persisting_integrations_test.go b/test/new-e2e/tests/agent-platform/persisting-integrations/persisting_integrations_test.go index 425f9b88d33c8..39fd4e5327ef1 100644 --- a/test/new-e2e/tests/agent-platform/persisting-integrations/persisting_integrations_test.go +++ b/test/new-e2e/tests/agent-platform/persisting-integrations/persisting_integrations_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" diff --git a/test/new-e2e/tests/agent-platform/rpm/rpm_test.go b/test/new-e2e/tests/agent-platform/rpm/rpm_test.go index feeaaa5bfb5bf..19a5d221a58dd 100644 --- a/test/new-e2e/tests/agent-platform/rpm/rpm_test.go +++ b/test/new-e2e/tests/agent-platform/rpm/rpm_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" filemanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/file-manager" diff --git a/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go b/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go index 01ba9d04182a5..6c88086a0ab2b 100644 --- a/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go +++ b/test/new-e2e/tests/agent-platform/step-by-step/step_by_step_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" filemanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/file-manager" diff --git a/test/new-e2e/tests/agent-platform/upgrade/upgrade_test.go b/test/new-e2e/tests/agent-platform/upgrade/upgrade_test.go index 06ecdf8d39273..1ac124114add3 100644 --- a/test/new-e2e/tests/agent-platform/upgrade/upgrade_test.go +++ b/test/new-e2e/tests/agent-platform/upgrade/upgrade_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" filemanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/file-manager" diff --git a/test/new-e2e/tests/agent-shared-components/api/api_test.go b/test/new-e2e/tests/agent-shared-components/api/api_test.go index 97a0ee3553fa2..db7b22ffcdc49 100644 --- a/test/new-e2e/tests/agent-shared-components/api/api_test.go +++ b/test/new-e2e/tests/agent-shared-components/api/api_test.go @@ -26,7 +26,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/agent-shared-components/config-refresh/config_endpoint.go b/test/new-e2e/tests/agent-shared-components/config-refresh/config_endpoint.go index 47c732d2569dc..356f76386e644 100644 --- a/test/new-e2e/tests/agent-shared-components/config-refresh/config_endpoint.go +++ b/test/new-e2e/tests/agent-shared-components/config-refresh/config_endpoint.go @@ -20,7 +20,7 @@ type agentConfigEndpointInfo struct { } func traceConfigEndpoint(port int) agentConfigEndpointInfo { - return agentConfigEndpointInfo{"trace-agent", "http", port, "/config"} + return agentConfigEndpointInfo{"trace-agent", "https", port, "/config"} } func processConfigEndpoint(port int) agentConfigEndpointInfo { diff --git a/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_nix_test.go b/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_nix_test.go index 6ffb0a57472c6..d12913b0ea296 100644 --- a/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_nix_test.go +++ b/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_nix_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" diff --git a/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_win_test.go b/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_win_test.go index 5499a802a5e65..a68e94ec1d6fe 100644 --- a/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_win_test.go +++ b/test/new-e2e/tests/agent-shared-components/config-refresh/non_core_agents_sync_win_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" diff --git a/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go b/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go index 373bd61ce39af..12ef0c145b839 100644 --- a/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go +++ b/test/new-e2e/tests/agent-shared-components/forwarder/nss_failover_test.go @@ -31,6 +31,7 @@ import ( fi "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) @@ -82,7 +83,7 @@ var customLogsConfigTmplFile string //go:embed testfixtures/config.yaml.tmpl var configTmplFile string -func multiFakeIntakeAWS(agentOptions ...agentparams.Option) e2e.Provisioner { +func multiFakeIntakeAWS(agentOptions ...agentparams.Option) provisioners.Provisioner { runFunc := func(ctx *pulumi.Context, env *multiFakeIntakeEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -116,7 +117,7 @@ func multiFakeIntakeAWS(agentOptions ...agentparams.Option) e2e.Provisioner { return nil } - return e2e.NewTypedPulumiProvisioner("aws-nssfailover", runFunc, nil) + return provisioners.NewTypedPulumiProvisioner("aws-nssfailover", runFunc, nil) } type multiFakeIntakeSuite struct { diff --git a/test/new-e2e/tests/agent-shared-components/gui/gui_nix_test.go b/test/new-e2e/tests/agent-shared-components/gui/gui_nix_test.go index c0396429d896d..2f1656483cfe1 100644 --- a/test/new-e2e/tests/agent-shared-components/gui/gui_nix_test.go +++ b/test/new-e2e/tests/agent-shared-components/gui/gui_nix_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" ) diff --git a/test/new-e2e/tests/agent-shared-components/gui/gui_win_test.go b/test/new-e2e/tests/agent-shared-components/gui/gui_win_test.go index b20b3fb5ce01e..c20ff3f1be9a3 100644 --- a/test/new-e2e/tests/agent-shared-components/gui/gui_win_test.go +++ b/test/new-e2e/tests/agent-shared-components/gui/gui_win_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" ) diff --git a/test/new-e2e/tests/agent-shared-components/hostname/imdsv2_transition_common_test.go b/test/new-e2e/tests/agent-shared-components/hostname/imdsv2_transition_common_test.go index 5fa5c05dffc73..1e5ae1e964feb 100644 --- a/test/new-e2e/tests/agent-shared-components/hostname/imdsv2_transition_common_test.go +++ b/test/new-e2e/tests/agent-shared-components/hostname/imdsv2_transition_common_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/tests/agent-shared-components/inventory/inventory_agent_test.go b/test/new-e2e/tests/agent-shared-components/inventory/inventory_agent_test.go index c06ca14096431..415493d5c7ebd 100644 --- a/test/new-e2e/tests/agent-shared-components/inventory/inventory_agent_test.go +++ b/test/new-e2e/tests/agent-shared-components/inventory/inventory_agent_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/tests/agent-shared-components/secret/secret_nix_test.go b/test/new-e2e/tests/agent-shared-components/secret/secret_nix_test.go index b60e9a1bf594f..1cb11d89fffa0 100644 --- a/test/new-e2e/tests/agent-shared-components/secret/secret_nix_test.go +++ b/test/new-e2e/tests/agent-shared-components/secret/secret_nix_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/agent-shared-components/secret/secret_win_test.go b/test/new-e2e/tests/agent-shared-components/secret/secret_win_test.go index 0528334d553c0..f5bc2fbea85b0 100644 --- a/test/new-e2e/tests/agent-shared-components/secret/secret_win_test.go +++ b/test/new-e2e/tests/agent-shared-components/secret/secret_win_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go b/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go index 253ecc8075cc3..69dde70ecfc02 100644 --- a/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/check/check_nix_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/tests/agent-subcommands/check/check_win_test.go b/test/new-e2e/tests/agent-subcommands/check/check_win_test.go index 383663c37f0e6..c9398ad1e8d80 100644 --- a/test/new-e2e/tests/agent-subcommands/check/check_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/check/check_win_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsCheckSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/config/config_common_test.go b/test/new-e2e/tests/agent-subcommands/config/config_common_test.go index b7b289647a097..9c0c0e589f1d1 100644 --- a/test/new-e2e/tests/agent-subcommands/config/config_common_test.go +++ b/test/new-e2e/tests/agent-subcommands/config/config_common_test.go @@ -10,14 +10,15 @@ import ( _ "embed" "fmt" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) type baseConfigSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/config/config_nix_test.go b/test/new-e2e/tests/agent-subcommands/config/config_nix_test.go index 595f088a3d644..139bf58d4c2c2 100644 --- a/test/new-e2e/tests/agent-subcommands/config/config_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/config/config_nix_test.go @@ -9,10 +9,11 @@ package config import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxConfigSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/config/config_win_test.go b/test/new-e2e/tests/agent-subcommands/config/config_win_test.go index d9b8684de8ec1..7d56eba5c8c82 100644 --- a/test/new-e2e/tests/agent-subcommands/config/config_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/config/config_win_test.go @@ -13,7 +13,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsConfigSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_nix_test.go b/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_nix_test.go index 6f6e744e09bcb..23df3f5210443 100644 --- a/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_nix_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxConfigCheckSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_win_test.go b/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_win_test.go index c82fa2c31f5fb..b53ea9629f151 100644 --- a/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/configcheck/configcheck_win_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/os" diff --git a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go index bc32af4f6e4ba..88203b0db4931 100644 --- a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_nix_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" svcmanager "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/svc-manager" ) diff --git a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go index bce881ffc8179..a8537e2887690 100644 --- a/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/diagnose/diagnose_win_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsDiagnoseSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go b/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go index a027956ba8939..c0935a712cd28 100644 --- a/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/flare/flare_nix_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" ) diff --git a/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go b/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go index 45c68f64a1507..144ffa170e54f 100644 --- a/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/flare/flare_win_test.go @@ -13,7 +13,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/stretchr/testify/assert" diff --git a/test/new-e2e/tests/agent-subcommands/health/health_common_test.go b/test/new-e2e/tests/agent-subcommands/health/health_common_test.go index 572282b7e1875..8aec70e72c28c 100644 --- a/test/new-e2e/tests/agent-subcommands/health/health_common_test.go +++ b/test/new-e2e/tests/agent-subcommands/health/health_common_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/api" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/cenkalti/backoff" "github.com/stretchr/testify/assert" diff --git a/test/new-e2e/tests/agent-subcommands/health/health_nix_test.go b/test/new-e2e/tests/agent-subcommands/health/health_nix_test.go index 4a064a2d88f93..50e5ecef69919 100644 --- a/test/new-e2e/tests/agent-subcommands/health/health_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/health/health_nix_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxHealthSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/health/health_win_test.go b/test/new-e2e/tests/agent-subcommands/health/health_win_test.go index 84da19fdcadc5..33b0285dd31ea 100644 --- a/test/new-e2e/tests/agent-subcommands/health/health_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/health/health_win_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsHealthSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_azure_nix_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_azure_nix_test.go index 7ddd73a1b1bdc..c22d8e08c4abe 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_azure_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_azure_nix_test.go @@ -17,7 +17,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - azurehost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/azure/host/linux" + azurehost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/azure/host/linux" ) type linuxAzureHostnameSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_common_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_common_test.go index f691a97c6bfd5..6636a8a00c0bb 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_common_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_common_test.go @@ -6,11 +6,12 @@ package hostname import ( - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type baseHostnameSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go index 2437c5a4306ad..f8e181b6608f2 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_nix_test.go @@ -8,13 +8,14 @@ package hostname import ( "testing" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" ) type linuxHostnameSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_win_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_win_test.go index 3429569e29e7a..d07857f633922 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_ec2_win_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" ) diff --git a/test/new-e2e/tests/agent-subcommands/hostname/hostname_gcp_nix_test.go b/test/new-e2e/tests/agent-subcommands/hostname/hostname_gcp_nix_test.go index 72cdc02690437..540baa3fcfa4f 100644 --- a/test/new-e2e/tests/agent-subcommands/hostname/hostname_gcp_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/hostname/hostname_gcp_nix_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - gcphost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/gcp/host/linux" + gcphost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/gcp/host/linux" ) type linuxGCPHostnameSuite struct { diff --git a/test/new-e2e/tests/agent-subcommands/run/run_nix_test.go b/test/new-e2e/tests/agent-subcommands/run/run_nix_test.go index 10376b4e30096..9bc5744a03765 100644 --- a/test/new-e2e/tests/agent-subcommands/run/run_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/run/run_nix_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common/process" diff --git a/test/new-e2e/tests/agent-subcommands/run/run_win_test.go b/test/new-e2e/tests/agent-subcommands/run/run_win_test.go index 521eb1400fe07..04673cadd6934 100644 --- a/test/new-e2e/tests/agent-subcommands/run/run_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/run/run_win_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" ) diff --git a/test/new-e2e/tests/agent-subcommands/secret/secret_nix_test.go b/test/new-e2e/tests/agent-subcommands/secret/secret_nix_test.go index 69de472cab0f4..7050698056943 100644 --- a/test/new-e2e/tests/agent-subcommands/secret/secret_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/secret/secret_nix_test.go @@ -10,7 +10,7 @@ import ( "time" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" diff --git a/test/new-e2e/tests/agent-subcommands/secret/secret_win_test.go b/test/new-e2e/tests/agent-subcommands/secret/secret_win_test.go index d4e099410dda6..bbeeef59ec1dd 100644 --- a/test/new-e2e/tests/agent-subcommands/secret/secret_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/secret/secret_win_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" secrets "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go b/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go index ab805b2d90db7..35841716f6463 100644 --- a/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go +++ b/test/new-e2e/tests/agent-subcommands/status/status_nix_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" ) diff --git a/test/new-e2e/tests/agent-subcommands/status/status_win_test.go b/test/new-e2e/tests/agent-subcommands/status/status_win_test.go index eef19d962c363..2a66687fe7303 100644 --- a/test/new-e2e/tests/agent-subcommands/status/status_win_test.go +++ b/test/new-e2e/tests/agent-subcommands/status/status_win_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" ) diff --git a/test/new-e2e/tests/apm/docker_test.go b/test/new-e2e/tests/apm/docker_test.go index 04cdf2045e79f..4c7fc8451145f 100644 --- a/test/new-e2e/tests/apm/docker_test.go +++ b/test/new-e2e/tests/apm/docker_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/test-infra-definitions/components/datadog/dockeragentparams" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" diff --git a/test/new-e2e/tests/apm/vm_test.go b/test/new-e2e/tests/apm/vm_test.go index 90f78554db9e2..006cd6353bd79 100644 --- a/test/new-e2e/tests/apm/vm_test.go +++ b/test/new-e2e/tests/apm/vm_test.go @@ -26,7 +26,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclient" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-shared-components/secretsutils" ) diff --git a/test/new-e2e/tests/containers/k8s_test.go b/test/new-e2e/tests/containers/k8s_test.go index ea2e14c8d3742..9561b71fb7a26 100644 --- a/test/new-e2e/tests/containers/k8s_test.go +++ b/test/new-e2e/tests/containers/k8s_test.go @@ -827,7 +827,7 @@ func (suite *k8sSuite) TestDogstatsdInAgent() { // Test with UDP + DD_ENTITY_ID suite.testDogstatsd(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDP) // Test with UDP + External Data - suite.testDogstatsdExternalData(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDPExternalData) + suite.testDogstatsd(kubeNamespaceDogstatsWorkload, kubeDeploymentDogstatsdUDPExternalData) } func (suite *k8sSuite) TestDogstatsdStandalone() { @@ -873,78 +873,6 @@ func (suite *k8sSuite) testDogstatsd(kubeNamespace, kubeDeployment string) { }) } -// testDogstatsdExternalData tests that the External Data origin resolution works for the dogstatsd. -func (suite *k8sSuite) testDogstatsdExternalData(kubeNamespace, kubeDeployment string) { - ctx := context.Background() - - // Record old pod, so we can be sure we are not looking at the incorrect one after deletion - oldPods, err := suite.K8sClient.CoreV1().Pods(kubeNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", kubeDeployment).String(), - }) - suite.Require().NoError(err) - suite.Require().Len(oldPods.Items, 1) - oldPod := oldPods.Items[0] - - // Delete the pod to ensure it is recreated after the admission controller is deployed - err = suite.K8sClient.CoreV1().Pods(kubeNamespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", kubeDeployment).String(), - }) - suite.Require().NoError(err) - - // Wait for the fresh pod to be created - var pod corev1.Pod - suite.Require().EventuallyWithTf(func(c *assert.CollectT) { - pods, err := suite.K8sClient.CoreV1().Pods(kubeNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", kubeDeployment).String(), - }) - if !assert.NoError(c, err) { - return - } - if !assert.Len(c, pods.Items, 1) { - return - } - pod = pods.Items[0] - if !assert.NotEqual(c, oldPod.Name, pod.Name) { - return - } - }, 2*time.Minute, 10*time.Second, "Failed to witness the creation of pod with name %s in namespace %s", kubeDeployment, kubeNamespace) - - suite.Require().Len(pod.Spec.Containers, 1) - - suite.testMetric(&testMetricArgs{ - Filter: testMetricFilterArgs{ - Name: "custom.metric", - Tags: []string{ - "^kube_deployment:" + regexp.QuoteMeta(kubeDeployment) + "$", - "^kube_namespace:" + regexp.QuoteMeta(kubeNamespace) + "$", - }, - }, - Expect: testMetricExpectArgs{ - Tags: &[]string{ - `^container_id:`, - `^container_name:dogstatsd$`, - `^display_container_name:dogstatsd`, - `^git.commit.sha:`, // org.opencontainers.image.revision docker image label - `^git.repository_url:https://github.com/DataDog/test-infra-definitions$`, // org.opencontainers.image.source docker image label - `^image_id:ghcr.io/datadog/apps-dogstatsd@sha256:`, - `^image_name:ghcr.io/datadog/apps-dogstatsd$`, - `^image_tag:main$`, - `^kube_container_name:dogstatsd$`, - `^kube_deployment:` + regexp.QuoteMeta(kubeDeployment) + `$`, - "^kube_namespace:" + regexp.QuoteMeta(kubeNamespace) + "$", - `^kube_ownerref_kind:replicaset$`, - `^kube_ownerref_name:` + regexp.QuoteMeta(kubeDeployment) + `-[[:alnum:]]+$`, - `^kube_qos:Burstable$`, - `^kube_replica_set:` + regexp.QuoteMeta(kubeDeployment) + `-[[:alnum:]]+$`, - `^pod_name:` + regexp.QuoteMeta(kubeDeployment) + `-[[:alnum:]]+-[[:alnum:]]+$`, - `^pod_phase:running$`, - `^series:`, - `^short_image:apps-dogstatsd$`, - }, - }, - }) -} - func (suite *k8sSuite) TestPrometheus() { // Test Prometheus check suite.testMetric(&testMetricArgs{ @@ -1020,37 +948,12 @@ func (suite *k8sSuite) testAdmissionControllerPod(namespace string, name string, }, 5*time.Minute, 10*time.Second, "The deployment with name %s in namespace %s does not exist or does not have the auto detected languages annotation", name, namespace) } - // Record old pod, so we can be sure we are not looking at the incorrect one after deletion - oldPods, err := suite.K8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", name).String(), - }) - suite.Require().NoError(err) - suite.Require().Len(oldPods.Items, 1) - oldPod := oldPods.Items[0] - - // Delete the pod to ensure it is recreated after the admission controller is deployed - err = suite.K8sClient.CoreV1().Pods(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{ + pods, err := suite.K8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ LabelSelector: fields.OneTermEqualSelector("app", name).String(), }) suite.Require().NoError(err) - - // Wait for the fresh pod to be created - var pod corev1.Pod - suite.Require().EventuallyWithTf(func(c *assert.CollectT) { - pods, err := suite.K8sClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector("app", name).String(), - }) - if !assert.NoError(c, err) { - return - } - if !assert.Len(c, pods.Items, 1) { - return - } - pod = pods.Items[0] - if !assert.NotEqual(c, oldPod.Name, pod.Name) { - return - } - }, 2*time.Minute, 10*time.Second, "Failed to witness the creation of pod with name %s in namespace %s", name, namespace) + suite.Require().Len(pods.Items, 1) + pod := pods.Items[0] suite.Require().Len(pod.Spec.Containers, 1) diff --git a/test/new-e2e/tests/cspm/cspm_test.go b/test/new-e2e/tests/cspm/cspm_test.go index 6f5d881214691..929027e7595cd 100644 --- a/test/new-e2e/tests/cspm/cspm_test.go +++ b/test/new-e2e/tests/cspm/cspm_test.go @@ -24,7 +24,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" ) type cspmTestSuite struct { diff --git a/test/new-e2e/tests/cws/ec2_test.go b/test/new-e2e/tests/cws/ec2_test.go index 9266609f74ce0..c9fa1febb4efc 100644 --- a/test/new-e2e/tests/cws/ec2_test.go +++ b/test/new-e2e/tests/cws/ec2_test.go @@ -23,7 +23,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters" "github.com/DataDog/datadog-agent/test/new-e2e/tests/cws/api" diff --git a/test/new-e2e/tests/cws/kind_test.go b/test/new-e2e/tests/cws/kind_test.go index 9dcfd5ef7da3b..31c93018509f7 100644 --- a/test/new-e2e/tests/cws/kind_test.go +++ b/test/new-e2e/tests/cws/kind_test.go @@ -18,7 +18,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/platforms" "github.com/DataDog/datadog-agent/test/new-e2e/tests/cws/api" diff --git a/test/new-e2e/tests/cws/windows_test.go b/test/new-e2e/tests/cws/windows_test.go index 4533befdf1653..ce840e6fe05dc 100644 --- a/test/new-e2e/tests/cws/windows_test.go +++ b/test/new-e2e/tests/cws/windows_test.go @@ -19,10 +19,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + testos "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - testos "github.com/DataDog/test-infra-definitions/components/os" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters" diff --git a/test/new-e2e/tests/discovery/linux_test.go b/test/new-e2e/tests/discovery/linux_test.go index 0df272712dedd..a833bb7457346 100644 --- a/test/new-e2e/tests/discovery/linux_test.go +++ b/test/new-e2e/tests/discovery/linux_test.go @@ -20,7 +20,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) //go:embed testdata/config/agent_config.yaml diff --git a/test/new-e2e/tests/gpu/provisioner.go b/test/new-e2e/tests/gpu/provisioner.go index ee552dcca7d8d..bc6db36edc6da 100644 --- a/test/new-e2e/tests/gpu/provisioner.go +++ b/test/new-e2e/tests/gpu/provisioner.go @@ -22,8 +22,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" ) // gpuEnabledAMI is an AMI that has GPU drivers pre-installed. In this case it's @@ -89,8 +89,8 @@ func getDefaultProvisionerParams() *provisionerParams { } } -func gpuInstanceProvisioner(params *provisionerParams) e2e.Provisioner { - return e2e.NewTypedPulumiProvisioner[environments.Host]("gpu", func(ctx *pulumi.Context, env *environments.Host) error { +func gpuInstanceProvisioner(params *provisionerParams) provisioners.Provisioner { + return provisioners.NewTypedPulumiProvisioner[environments.Host]("gpu", func(ctx *pulumi.Context, env *environments.Host) error { name := "gpuvm" awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/tests/ha-agent/haagent_test.go b/test/new-e2e/tests/ha-agent/haagent_test.go index cc173d2da7bae..12019455df4ab 100644 --- a/test/new-e2e/tests/ha-agent/haagent_test.go +++ b/test/new-e2e/tests/ha-agent/haagent_test.go @@ -20,7 +20,7 @@ import ( fakeintakeclient "github.com/DataDog/datadog-agent/test/fakeintake/client" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type haAgentTestSuite struct { diff --git a/test/new-e2e/tests/installer/script/all_scripts_test.go b/test/new-e2e/tests/installer/script/all_scripts_test.go index b79d99de89f1c..e50d01b0e18d7 100644 --- a/test/new-e2e/tests/installer/script/all_scripts_test.go +++ b/test/new-e2e/tests/installer/script/all_scripts_test.go @@ -12,14 +12,15 @@ import ( "strings" "testing" + e2eos "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/host" - e2eos "github.com/DataDog/test-infra-definitions/components/os" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" - "github.com/stretchr/testify/require" ) type installerScriptTests func(os e2eos.Descriptor, arch e2eos.Architecture) installerScriptSuite diff --git a/test/new-e2e/tests/installer/script/databricks_test.go b/test/new-e2e/tests/installer/script/databricks_test.go index b6f038ab96803..820de6a627152 100644 --- a/test/new-e2e/tests/installer/script/databricks_test.go +++ b/test/new-e2e/tests/installer/script/databricks_test.go @@ -8,8 +8,9 @@ package installscript import ( "fmt" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" e2eos "github.com/DataDog/test-infra-definitions/components/os" + + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) const ( diff --git a/test/new-e2e/tests/installer/unix/all_packages_test.go b/test/new-e2e/tests/installer/unix/all_packages_test.go index f213a0fda8394..8fe357e74e741 100644 --- a/test/new-e2e/tests/installer/unix/all_packages_test.go +++ b/test/new-e2e/tests/installer/unix/all_packages_test.go @@ -21,7 +21,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/host" ) diff --git a/test/new-e2e/tests/installer/unix/package_agent_test.go b/test/new-e2e/tests/installer/unix/package_agent_test.go index 70ddeff75c676..d419ba8f02585 100644 --- a/test/new-e2e/tests/installer/unix/package_agent_test.go +++ b/test/new-e2e/tests/installer/unix/package_agent_test.go @@ -12,11 +12,12 @@ import ( "strings" "time" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/host" e2eos "github.com/DataDog/test-infra-definitions/components/os" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" + + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/host" ) const ( diff --git a/test/new-e2e/tests/installer/unix/package_installer_test.go b/test/new-e2e/tests/installer/unix/package_installer_test.go index 8588ee018d210..304cd12383e7b 100644 --- a/test/new-e2e/tests/installer/unix/package_installer_test.go +++ b/test/new-e2e/tests/installer/unix/package_installer_test.go @@ -6,9 +6,13 @@ package installer import ( - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "strings" + "time" + e2eos "github.com/DataDog/test-infra-definitions/components/os" "github.com/stretchr/testify/assert" + + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type packageInstallerSuite struct { @@ -102,20 +106,33 @@ func (s *packageInstallerSuite) TestReInstall() { func (s *packageInstallerSuite) TestUpdateInstallerOCI() { // Install prod err := s.RunInstallScriptProdOci( + "DD_REMOTE_UPDATES=true", envForceVersion("datadog-installer", "7.58.0-installer-0.5.1-1"), ) defer s.Purge() assert.NoError(s.T(), err) - version := s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version") - assert.Equal(s.T(), "7.58.0-installer-0.5.1\n", version) + versionDisk := s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version") + assert.Equal(s.T(), "7.58.0-installer-0.5.1\n", versionDisk) + assert.Eventually(s.T(), func() bool { + versionRunning, err := s.Env().RemoteHost.Execute("sudo datadog-installer status") + s.T().Logf("checking version: %s, err: %v", versionRunning, err) + return err == nil && strings.Contains(versionRunning, "7.58.0-installer-0.5.1") + }, 30*time.Second, 1*time.Second) // Install from QA registry - err = s.RunInstallScriptWithError() + err = s.RunInstallScriptWithError( + "DD_REMOTE_UPDATES=true", + ) assert.NoError(s.T(), err) - version = s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version") - assert.NotEqual(s.T(), "7.58.0-installer-0.5.1\n", version) + versionDisk = s.Env().RemoteHost.MustExecute("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer version") + assert.NotEqual(s.T(), "7.58.0-installer-0.5.1\n", versionDisk) + assert.Eventually(s.T(), func() bool { + versionRunning, err := s.Env().RemoteHost.Execute("sudo datadog-installer status") + s.T().Logf("checking version: %s, err: %v", versionRunning, err) + return err == nil && !strings.Contains(versionRunning, "7.58.0-installer-0.5.1") + }, 30*time.Second, 1*time.Second) } func (s *packageInstallerSuite) TestInstallWithUmask() { diff --git a/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go b/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go index 6230ef741410c..5fdf6a24c60eb 100644 --- a/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go +++ b/test/new-e2e/tests/installer/unix/upgrade_scenario_test.go @@ -410,8 +410,8 @@ func (s *upgradeScenarioSuite) TestConfigUpgradeSuccessful() { state.AssertDirExists("/etc/datadog-agent/managed/datadog-agent", 0755, "root", "root") state.AssertSymlinkExists("/etc/datadog-agent/managed/datadog-agent/stable", "/etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d", "root", "root") // Verify metadata - state.AssertFileExists("/etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d/policy.metadata", 0440, "dd-agent", "dd-agent") - file := s.Env().RemoteHost.MustExecute("sudo cat /etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d/policy.metadata") + state.AssertFileExists("/etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d/policy.metadata", 0444, "root", "root") + file := s.Env().RemoteHost.MustExecute("cat /etc/datadog-agent/managed/datadog-agent/e94406c45ae766b7d34d2793e4759b9c4d15ed5d5e2b7f73ce1bf0e6836f728d/policy.metadata") policiesState := &pbgo.PoliciesState{} err := json.Unmarshal([]byte(file), policiesState) require.NoError(s.T(), err) diff --git a/test/new-e2e/tests/installer/windows/suites/agent-package/agent-user_test.go b/test/new-e2e/tests/installer/windows/suites/agent-package/agent-user_test.go index 9c7d4d716e468..6a1029294f235 100644 --- a/test/new-e2e/tests/installer/windows/suites/agent-package/agent-user_test.go +++ b/test/new-e2e/tests/installer/windows/suites/agent-package/agent-user_test.go @@ -8,13 +8,14 @@ package agenttests import ( installer "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components/datadog-installer" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" - "github.com/stretchr/testify/require" "testing" + + "github.com/stretchr/testify/require" ) type testAgentInstallWithAgentUserSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/agent-package/install_test.go b/test/new-e2e/tests/installer/windows/suites/agent-package/install_test.go index 38fd858783940..a7c186c31cac2 100644 --- a/test/new-e2e/tests/installer/windows/suites/agent-package/install_test.go +++ b/test/new-e2e/tests/installer/windows/suites/agent-package/install_test.go @@ -10,7 +10,7 @@ import ( "path/filepath" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" diff --git a/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go b/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go index cad8fab208066..af28bc4d66b66 100644 --- a/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go +++ b/test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installer "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/unix" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" ) diff --git a/test/new-e2e/tests/installer/windows/suites/install-script/agent-user_test.go b/test/new-e2e/tests/installer/windows/suites/install-script/agent-user_test.go index ad2ab623c5b57..b8aceac530e26 100644 --- a/test/new-e2e/tests/installer/windows/suites/install-script/agent-user_test.go +++ b/test/new-e2e/tests/installer/windows/suites/install-script/agent-user_test.go @@ -7,13 +7,14 @@ package agenttests import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" - "github.com/stretchr/testify/require" "testing" + + "github.com/stretchr/testify/require" ) type testInstallScriptWithAgentUserSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/installer-package/install_test.go b/test/new-e2e/tests/installer/windows/suites/installer-package/install_test.go index 7bfa29e502797..9d2e4f4799351 100644 --- a/test/new-e2e/tests/installer/windows/suites/installer-package/install_test.go +++ b/test/new-e2e/tests/installer/windows/suites/installer-package/install_test.go @@ -7,11 +7,12 @@ package installertests import ( + "testing" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" - "testing" ) type testInstallerSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/installer-package/rollback_test.go b/test/new-e2e/tests/installer/windows/suites/installer-package/rollback_test.go index b211dce84ead1..8863214eca148 100644 --- a/test/new-e2e/tests/installer/windows/suites/installer-package/rollback_test.go +++ b/test/new-e2e/tests/installer/windows/suites/installer-package/rollback_test.go @@ -6,10 +6,11 @@ package installertests import ( + "testing" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" - "testing" ) type testInstallerRollbackSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/installer-package/system_integrity_test.go b/test/new-e2e/tests/installer/windows/suites/installer-package/system_integrity_test.go index 088bd9f80aee7..568e1159adc27 100644 --- a/test/new-e2e/tests/installer/windows/suites/installer-package/system_integrity_test.go +++ b/test/new-e2e/tests/installer/windows/suites/installer-package/system_integrity_test.go @@ -6,12 +6,13 @@ package installertests import ( + "testing" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" - "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/install-test" - "testing" + installtest "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/install-test" ) type testSystemIntegrityInstallerSuite struct { diff --git a/test/new-e2e/tests/installer/windows/suites/installer-package/upgrade_test.go b/test/new-e2e/tests/installer/windows/suites/installer-package/upgrade_test.go index a2fc1d9970507..e145109d62b75 100644 --- a/test/new-e2e/tests/installer/windows/suites/installer-package/upgrade_test.go +++ b/test/new-e2e/tests/installer/windows/suites/installer-package/upgrade_test.go @@ -6,13 +6,14 @@ package installertests import ( + "testing" + agentVersion "github.com/DataDog/datadog-agent/pkg/version" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + winawshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" installerwindows "github.com/DataDog/datadog-agent/test/new-e2e/tests/installer/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/pipeline" - "testing" ) type testInstallerUpgradesSuite struct { diff --git a/test/new-e2e/tests/language-detection/language_detection_test.go b/test/new-e2e/tests/language-detection/language_detection_test.go index 29ce07db52c7a..52d06d382ec5a 100644 --- a/test/new-e2e/tests/language-detection/language_detection_test.go +++ b/test/new-e2e/tests/language-detection/language_detection_test.go @@ -17,10 +17,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" - "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) //go:embed etc/process_config.yaml diff --git a/test/new-e2e/tests/language-detection/python_test.go b/test/new-e2e/tests/language-detection/python_test.go index 2be850b1da5a9..875ac519ce602 100644 --- a/test/new-e2e/tests/language-detection/python_test.go +++ b/test/new-e2e/tests/language-detection/python_test.go @@ -11,7 +11,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/require" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) func (s *languageDetectionSuite) installPython() { diff --git a/test/new-e2e/tests/ndm/netflow/netflow_test.go b/test/new-e2e/tests/ndm/netflow/netflow_test.go index 3694560cbbe44..cf2564c974db5 100644 --- a/test/new-e2e/tests/ndm/netflow/netflow_test.go +++ b/test/new-e2e/tests/ndm/netflow/netflow_test.go @@ -13,9 +13,11 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/common/utils" "github.com/DataDog/test-infra-definitions/components/datadog/agent" @@ -37,8 +39,8 @@ var datadogYaml string // netflowDockerProvisioner defines a stack with a docker agent on an AmazonLinuxECS VM // with the netflow-generator running and sending netflow payloads to the agent -func netflowDockerProvisioner() e2e.Provisioner { - return e2e.NewTypedPulumiProvisioner[environments.DockerHost]("", func(ctx *pulumi.Context, env *environments.DockerHost) error { +func netflowDockerProvisioner() provisioners.Provisioner { + return provisioners.NewTypedPulumiProvisioner[environments.DockerHost]("", func(ctx *pulumi.Context, env *environments.DockerHost) error { name := "netflowvm" awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/tests/ndm/snmp/snmp_test.go b/test/new-e2e/tests/ndm/snmp/snmp_test.go index e4b2b56e53324..abdc91b3a69b8 100644 --- a/test/new-e2e/tests/ndm/snmp/snmp_test.go +++ b/test/new-e2e/tests/ndm/snmp/snmp_test.go @@ -13,11 +13,13 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/test-infra-definitions/common/utils" "github.com/DataDog/test-infra-definitions/components/datadog/agent" @@ -43,8 +45,8 @@ const ( // snmpDockerProvisioner defines a stack with a docker agent on an AmazonLinuxECS VM // with snmpsim installed and configured with snmp recordings -func snmpDockerProvisioner() e2e.Provisioner { - return e2e.NewTypedPulumiProvisioner("", func(ctx *pulumi.Context, env *environments.DockerHost) error { +func snmpDockerProvisioner() provisioners.Provisioner { + return provisioners.NewTypedPulumiProvisioner("", func(ctx *pulumi.Context, env *environments.DockerHost) error { name := "snmpvm" awsEnv, err := aws.NewEnvironment(ctx) if err != nil { diff --git a/test/new-e2e/tests/netpath/network-path-integration/netpath_int_nix_test.go b/test/new-e2e/tests/netpath/network-path-integration/netpath_int_nix_test.go index de5bc31f363ef..964382b9af13a 100644 --- a/test/new-e2e/tests/netpath/network-path-integration/netpath_int_nix_test.go +++ b/test/new-e2e/tests/netpath/network-path-integration/netpath_int_nix_test.go @@ -11,10 +11,11 @@ import ( "testing" "time" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxNetworkPathIntegrationTestSuite struct { diff --git a/test/new-e2e/tests/netpath/network-path-integration/netpath_int_win_test.go b/test/new-e2e/tests/netpath/network-path-integration/netpath_int_win_test.go index eb6efd844ab68..ae449f2b8ca5b 100644 --- a/test/new-e2e/tests/netpath/network-path-integration/netpath_int_win_test.go +++ b/test/new-e2e/tests/netpath/network-path-integration/netpath_int_win_test.go @@ -11,12 +11,13 @@ import ( "testing" "time" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/stretchr/testify/assert" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsNetworkPathIntegrationTestSuite struct { diff --git a/test/new-e2e/tests/npm/agentenv_npm_test.go b/test/new-e2e/tests/npm/agentenv_npm_test.go index 1dae47d257bfb..7fd062b40ac97 100644 --- a/test/new-e2e/tests/npm/agentenv_npm_test.go +++ b/test/new-e2e/tests/npm/agentenv_npm_test.go @@ -11,7 +11,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" diff --git a/test/new-e2e/tests/npm/ec2_1host_containerized_test.go b/test/new-e2e/tests/npm/ec2_1host_containerized_test.go index 68e1d68a92107..8fd4b25051591 100644 --- a/test/new-e2e/tests/npm/ec2_1host_containerized_test.go +++ b/test/new-e2e/tests/npm/ec2_1host_containerized_test.go @@ -13,7 +13,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" "github.com/DataDog/test-infra-definitions/components/docker" "github.com/DataDog/test-infra-definitions/resources/aws" @@ -30,7 +31,7 @@ type ec2VMContainerizedSuite struct { e2e.BaseSuite[dockerHostNginxEnv] } -func dockerHostHttpbinEnvProvisioner() e2e.PulumiEnvRunFunc[dockerHostNginxEnv] { +func dockerHostHttpbinEnvProvisioner() provisioners.PulumiEnvRunFunc[dockerHostNginxEnv] { return func(ctx *pulumi.Context, env *dockerHostNginxEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -76,7 +77,7 @@ func TestEC2VMContainerizedSuite(t *testing.T) { t.Parallel() s := &ec2VMContainerizedSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("dockerHostHttpbin", dockerHostHttpbinEnvProvisioner(), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("dockerHostHttpbin", dockerHostHttpbinEnvProvisioner(), nil))} // Source of our kitchen CI images test/kitchen/platforms.json // Other VM image can be used, our kitchen CI images test/kitchen/platforms.json diff --git a/test/new-e2e/tests/npm/ec2_1host_selinux_test.go b/test/new-e2e/tests/npm/ec2_1host_selinux_test.go index 67afaa540c9a4..2aaf28eb7138c 100644 --- a/test/new-e2e/tests/npm/ec2_1host_selinux_test.go +++ b/test/new-e2e/tests/npm/ec2_1host_selinux_test.go @@ -9,7 +9,8 @@ import ( "testing" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" compos "github.com/DataDog/test-infra-definitions/components/os" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" @@ -25,7 +26,7 @@ func TestEC2VMSELinuxSuite(t *testing.T) { s := &ec2VMSELinuxSuite{} e2eParams := []e2e.SuiteOption{e2e.WithProvisioner( - e2e.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(awshost.WithEC2InstanceOptions( + provisioners.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(awshost.WithEC2InstanceOptions( // RHEL9 ec2.WithAMI("ami-0fe630eb857a6ec83", compos.AmazonLinux2, compos.AMD64Arch), ec2.WithInstanceType("t3.medium"))), nil)), } diff --git a/test/new-e2e/tests/npm/ec2_1host_test.go b/test/new-e2e/tests/npm/ec2_1host_test.go index ba07540f58ca7..78776323315a7 100644 --- a/test/new-e2e/tests/npm/ec2_1host_test.go +++ b/test/new-e2e/tests/npm/ec2_1host_test.go @@ -13,7 +13,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/test-infra-definitions/components/datadog/agentparams" "github.com/DataDog/test-infra-definitions/components/docker" @@ -30,7 +31,7 @@ type ec2VMSuite struct { e2e.BaseSuite[hostHttpbinEnv] } -func hostDockerHttpbinEnvProvisioner(opt ...awshost.ProvisionerOption) e2e.PulumiEnvRunFunc[hostHttpbinEnv] { +func hostDockerHttpbinEnvProvisioner(opt ...awshost.ProvisionerOption) provisioners.PulumiEnvRunFunc[hostHttpbinEnv] { return func(ctx *pulumi.Context, env *hostHttpbinEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -77,7 +78,7 @@ func TestEC2VMSuite(t *testing.T) { t.Parallel() s := &ec2VMSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(), nil))} // Source of our E2E CI images test/new-e2e/tests/agent-platform/platforms.json // Other VM image can be used, our E2E CI images test/new-e2e/tests/agent-platform/platforms.json diff --git a/test/new-e2e/tests/npm/ec2_1host_wkit_test.go b/test/new-e2e/tests/npm/ec2_1host_wkit_test.go index 1dd04c1541990..e6d02c3c07186 100644 --- a/test/new-e2e/tests/npm/ec2_1host_wkit_test.go +++ b/test/new-e2e/tests/npm/ec2_1host_wkit_test.go @@ -12,7 +12,8 @@ import ( "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type ec2VMWKitSuite struct { @@ -25,7 +26,7 @@ func TestEC2VMWKitSuite(t *testing.T) { s := &ec2VMWKitSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(awshost.WithEC2InstanceOptions(ec2.WithOS(os.WindowsDefault))), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("hostHttpbin", hostDockerHttpbinEnvProvisioner(awshost.WithEC2InstanceOptions(ec2.WithOS(os.WindowsDefault))), nil))} e2e.Run(t, s, e2eParams...) } diff --git a/test/new-e2e/tests/npm/ecs_1host_test.go b/test/new-e2e/tests/npm/ecs_1host_test.go index ae617ac4040ae..c829ccdaa1b46 100644 --- a/test/new-e2e/tests/npm/ecs_1host_test.go +++ b/test/new-e2e/tests/npm/ecs_1host_test.go @@ -15,7 +15,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - envecs "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/ecs" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + envecs "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/ecs" npmtools "github.com/DataDog/test-infra-definitions/components/datadog/apps/npm-tools" "github.com/DataDog/test-infra-definitions/components/datadog/ecsagentparams" @@ -39,7 +40,7 @@ type ecsVMSuite struct { e2e.BaseSuite[ecsHttpbinEnv] } -func ecsHttpbinEnvProvisioner() e2e.PulumiEnvRunFunc[ecsHttpbinEnv] { +func ecsHttpbinEnvProvisioner() provisioners.PulumiEnvRunFunc[ecsHttpbinEnv] { return func(ctx *pulumi.Context, env *ecsHttpbinEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -86,7 +87,7 @@ func ecsHttpbinEnvProvisioner() e2e.PulumiEnvRunFunc[ecsHttpbinEnv] { func TestECSVMSuite(t *testing.T) { t.Parallel() s := &ecsVMSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("ecsHttpbin", ecsHttpbinEnvProvisioner(), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("ecsHttpbin", ecsHttpbinEnvProvisioner(), nil))} e2e.Run(t, s, e2eParams...) } diff --git a/test/new-e2e/tests/npm/eks_1host_test.go b/test/new-e2e/tests/npm/eks_1host_test.go index 12f86c5fd37e2..bfdd551266b86 100644 --- a/test/new-e2e/tests/npm/eks_1host_test.go +++ b/test/new-e2e/tests/npm/eks_1host_test.go @@ -22,7 +22,8 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - envkube "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" + envkube "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" ) @@ -38,7 +39,7 @@ type eksVMSuite struct { e2e.BaseSuite[eksHttpbinEnv] } -func eksHttpbinEnvProvisioner(opts ...envkube.ProvisionerOption) e2e.PulumiEnvRunFunc[eksHttpbinEnv] { +func eksHttpbinEnvProvisioner(opts ...envkube.ProvisionerOption) provisioners.PulumiEnvRunFunc[eksHttpbinEnv] { return func(ctx *pulumi.Context, env *eksHttpbinEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -94,7 +95,7 @@ func TestEKSVMSuite(t *testing.T) { t.Parallel() s := &eksVMSuite{} - e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(e2e.NewTypedPulumiProvisioner("eksHttpbin", eksHttpbinEnvProvisioner(), nil))} + e2eParams := []e2e.SuiteOption{e2e.WithProvisioner(provisioners.NewTypedPulumiProvisioner("eksHttpbin", eksHttpbinEnvProvisioner(), nil))} e2e.Run(t, s, e2eParams...) } diff --git a/test/new-e2e/tests/otel/otel-agent/complete_test.go b/test/new-e2e/tests/otel/otel-agent/complete_test.go index d3359aae3cb87..37789d8b59766 100644 --- a/test/new-e2e/tests/otel/otel-agent/complete_test.go +++ b/test/new-e2e/tests/otel/otel-agent/complete_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/config/no-dd-exporter.yml b/test/new-e2e/tests/otel/otel-agent/config/no-dd-exporter.yml new file mode 100644 index 0000000000000..b952fe8d38811 --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/config/no-dd-exporter.yml @@ -0,0 +1,23 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +exporters: + debug: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + metrics: + receivers: [otlp] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] diff --git a/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go b/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go index 81b57d87917d9..2003c8c6e210e 100644 --- a/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go +++ b/test/new-e2e/tests/otel/otel-agent/infraattributes_eks_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/infraattributes_test.go b/test/new-e2e/tests/otel/otel-agent/infraattributes_test.go index b0706f90e10ba..f97d9dfa62cb8 100644 --- a/test/new-e2e/tests/otel/otel-agent/infraattributes_test.go +++ b/test/new-e2e/tests/otel/otel-agent/infraattributes_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/minimal_test.go b/test/new-e2e/tests/otel/otel-agent/minimal_test.go index 9377c56fd514f..ee33ec9d6cbac 100644 --- a/test/new-e2e/tests/otel/otel-agent/minimal_test.go +++ b/test/new-e2e/tests/otel/otel-agent/minimal_test.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/comp/core/tagger/types" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) @@ -32,8 +32,8 @@ var minimalProvidedConfig string //go:embed testdata/minimal-full-config.yml var minimalFullConfig string -//go:embed testdata/minimal-sources.json -var minimalSources string +//go:embed testdata/sources.json +var sources string func TestOTelAgentMinimal(t *testing.T) { values := ` @@ -82,7 +82,7 @@ func (s *minimalTestSuite) TestOTelAgentInstalled() { } func (s *minimalTestSuite) TestOTelFlareExtensionResponse() { - utils.TestOTelFlareExtensionResponse(s, minimalProvidedConfig, minimalFullConfig, minimalSources) + utils.TestOTelFlareExtensionResponse(s, minimalProvidedConfig, minimalFullConfig, sources) } func (s *minimalTestSuite) TestOTelFlareFiles() { diff --git a/test/new-e2e/tests/otel/otel-agent/no_dd_exporter_test.go b/test/new-e2e/tests/otel/otel-agent/no_dd_exporter_test.go new file mode 100644 index 0000000000000..471e0c41d9dbd --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/no_dd_exporter_test.go @@ -0,0 +1,58 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package otelagent contains e2e otel agent tests +package otelagent + +import ( + _ "embed" + "testing" + + "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" + + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" + "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" +) + +type noDDExporterTestSuite struct { + e2e.BaseSuite[environments.Kubernetes] +} + +//go:embed config/no-dd-exporter.yml +var noDDExporterConfig string + +//go:embed testdata/no-dd-exporter-provided-config.yml +var noDDExporterProvidedConfig string + +//go:embed testdata/no-dd-exporter-full-config.yml +var noDDExporterFullConfig string + +func TestOTelAgentWithNoDDExporter(t *testing.T) { + values := ` +datadog: + logs: + containerCollectAll: false + containerCollectUsingFiles: false +` + t.Parallel() + e2e.Run(t, &noDDExporterTestSuite{}, + e2e.WithProvisioner( + awskubernetes.KindProvisioner( + awskubernetes.WithAgentOptions( + kubernetesagentparams.WithHelmValues(values), + kubernetesagentparams.WithOTelAgent(), + kubernetesagentparams.WithOTelConfig(noDDExporterConfig), + )))) +} + +func (s *noDDExporterTestSuite) TestOTelAgentInstalled() { + utils.TestOTelAgentInstalled(s) +} + +func (s *noDDExporterTestSuite) TestOTelFlareExtensionResponse() { + utils.TestOTelFlareExtensionResponse(s, noDDExporterProvidedConfig, noDDExporterFullConfig, sources) +} diff --git a/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go b/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go index aa9e8163e1664..6dee2b5052230 100644 --- a/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go +++ b/test/new-e2e/tests/otel/otel-agent/receive_resource_spans_v2_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/sampling_test.go b/test/new-e2e/tests/otel/otel-agent/sampling_test.go index 99f92bc32cc63..6fc08ce59c78b 100644 --- a/test/new-e2e/tests/otel/otel-agent/sampling_test.go +++ b/test/new-e2e/tests/otel/otel-agent/sampling_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-full-config.yml b/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-full-config.yml new file mode 100644 index 0000000000000..e0276e1bb2aa3 --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-full-config.yml @@ -0,0 +1,281 @@ +connectors: {} +exporters: + datadog: + api: + fail_on_invalid_key: false + key: '[REDACTED]' + site: datadoghq.com + auth: null + compression: "" + cookies: null + disable_keep_alives: false + endpoint: "" + headers: {} + host_metadata: + enabled: false + hostname_source: config_or_system + reporter_period: 30m0s + tags: [] + hostname: "" + http2_ping_timeout: 0s + http2_read_idle_timeout: 0s + idle_conn_timeout: 1m30s + logs: + batch_wait: 5 + compression_level: 6 + dialer: + timeout: 0s + dump_payloads: false + use_compression: true + max_conns_per_host: 0 + max_idle_conns: 100 + max_idle_conns_per_host: 0 + metrics: + delta_ttl: 3600 + dialer: + timeout: 0s + histograms: + mode: distributions + send_aggregation_metrics: false + send_count_sum_metrics: false + instrumentation_scope_metadata_as_tags: false + resource_attributes_as_tags: false + summaries: + mode: gauges + sums: + cumulative_monotonic_mode: to_delta + initial_cumulative_monotonic_value: auto + only_metadata: false + proxy_url: "" + read_buffer_size: 0 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 1000 + storage: null + timeout: 15s + tls: + ca_file: "" + ca_pem: '[REDACTED]' + cert_file: "" + cert_pem: '[REDACTED]' + cipher_suites: [] + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + key_pem: '[REDACTED]' + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + traces: + compute_stats_by_span_kind: false + compute_top_level_by_span_kind: true + dialer: + timeout: 0s + ignore_resources: [] + peer_service_aggregation: false + peer_tags: [] + peer_tags_aggregation: false + span_name_as_resource_name: false + span_name_remappings: {} + trace_buffer: 0 + write_buffer_size: 0 + debug: + sampling_initial: 2 + sampling_thereafter: 1 + use_internal_logger: true + verbosity: Detailed +extensions: + ddflare/dd-autoconfigured: + auth: null + compression_algorithms: [] + cors: null + endpoint: localhost:7777 + idle_timeout: 0s + include_metadata: false + max_request_body_size: 0 + read_header_timeout: 0s + read_timeout: 0s + response_headers: {} + tls: null + write_timeout: 0s + health_check/dd-autoconfigured: + auth: null + check_collector_pipeline: + enabled: false + exporter_failure_threshold: 5 + interval: 5m + compression_algorithms: [] + cors: null + endpoint: localhost:13133 + idle_timeout: 0s + include_metadata: false + max_request_body_size: 0 + path: / + read_header_timeout: 0s + read_timeout: 0s + response_body: null + response_headers: {} + tls: null + write_timeout: 0s + pprof/dd-autoconfigured: + block_profile_fraction: 0 + dialer: + timeout: 0s + endpoint: localhost:1777 + mutex_profile_fraction: 0 + save_to_file: "" + zpages/dd-autoconfigured: + auth: null + compression_algorithms: [] + cors: null + endpoint: localhost:55679 + idle_timeout: 0s + include_metadata: false + max_request_body_size: 0 + read_header_timeout: 0s + read_timeout: 0s + response_headers: {} + tls: null + write_timeout: 0s +processors: {} +receivers: + otlp: + protocols: + grpc: + auth: null + dialer: + timeout: 0s + endpoint: 0.0.0.0:4317 + include_metadata: false + keepalive: null + max_concurrent_streams: 0 + max_recv_msg_size_mib: 0 + read_buffer_size: 524288 + tls: null + transport: tcp + write_buffer_size: 0 + http: + auth: null + compression_algorithms: [] + cors: null + endpoint: 0.0.0.0:4318 + idle_timeout: 0s + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + read_header_timeout: 0s + read_timeout: 0s + response_headers: {} + tls: null + traces_url_path: /v1/traces + write_timeout: 0s + prometheus/dd-autoconfigured: + config: + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_protocols: + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText0.0.4 + scrape_timeout: 10s + runtime: + gogc: 75 + scrape_configs: + - enable_compression: true + enable_http2: true + follow_redirects: true + honor_timestamps: true + job_name: datadog-agent + metrics_path: /metrics + scheme: http + scrape_interval: 10s + scrape_protocols: + - OpenMetricsText1.0.0 + - OpenMetricsText0.0.1 + - PrometheusText0.0.4 + scrape_timeout: 10s + static_configs: + - targets: + - 0.0.0.0:8888 + track_timestamps_staleness: false + report_extra_scrape_metrics: false + start_time_metric_regex: "" + target_allocator: null + trim_metric_suffixes: false + use_start_time_metric: false +service: + extensions: + - pprof/dd-autoconfigured + - zpages/dd-autoconfigured + - health_check/dd-autoconfigured + - ddflare/dd-autoconfigured + pipelines: + logs: + exporters: + - debug + processors: [] + receivers: + - otlp + metrics: + exporters: + - debug + processors: [] + receivers: + - otlp + metrics/dd-autoconfigured/datadog: + exporters: + - datadog + processors: [] + receivers: + - prometheus/dd-autoconfigured + traces: + exporters: + - debug + processors: [] + receivers: + - otlp + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + error_output_paths: + - stderr + initial_fields: {} + level: info + output_paths: + - stderr + processors: [] + sampling: + enabled: true + initial: 10 + thereafter: 100 + tick: 10s + metrics: + address: "" + level: Normal + readers: + - pull: + exporter: + additionalproperties: null + prometheus: + host: localhost + port: 8888 + resource: {} + traces: + level: Basic + processors: [] + propagators: [] diff --git a/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-provided-config.yml b/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-provided-config.yml new file mode 100644 index 0000000000000..b97863a32c71c --- /dev/null +++ b/test/new-e2e/tests/otel/otel-agent/testdata/no-dd-exporter-provided-config.yml @@ -0,0 +1,184 @@ +connectors: {} +exporters: + datadog: + api: + fail_on_invalid_key: false + key: '[REDACTED]' + site: datadoghq.com + auth: null + compression: "" + cookies: null + disable_keep_alives: false + endpoint: "" + headers: {} + host_metadata: + enabled: false + hostname_source: config_or_system + reporter_period: 30m0s + tags: [] + hostname: "" + http2_ping_timeout: 0s + http2_read_idle_timeout: 0s + idle_conn_timeout: 1m30s + logs: + batch_wait: 5 + compression_level: 6 + dialer: + timeout: 0s + dump_payloads: false + use_compression: true + max_conns_per_host: 0 + max_idle_conns: 100 + max_idle_conns_per_host: 0 + metrics: + delta_ttl: 3600 + dialer: + timeout: 0s + histograms: + mode: distributions + send_aggregation_metrics: false + send_count_sum_metrics: false + instrumentation_scope_metadata_as_tags: false + resource_attributes_as_tags: false + summaries: + mode: gauges + sums: + cumulative_monotonic_mode: to_delta + initial_cumulative_monotonic_value: auto + only_metadata: false + proxy_url: "" + read_buffer_size: 0 + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + enabled: true + num_consumers: 10 + queue_size: 1000 + storage: null + timeout: 15s + tls: + ca_file: "" + ca_pem: '[REDACTED]' + cert_file: "" + cert_pem: '[REDACTED]' + cipher_suites: [] + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + key_pem: '[REDACTED]' + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + traces: + compute_stats_by_span_kind: false + compute_top_level_by_span_kind: true + dialer: + timeout: 0s + ignore_resources: [] + peer_service_aggregation: false + peer_tags: [] + peer_tags_aggregation: false + span_name_as_resource_name: false + span_name_remappings: {} + trace_buffer: 0 + write_buffer_size: 0 + debug: + sampling_initial: 2 + sampling_thereafter: 1 + use_internal_logger: true + verbosity: Detailed +extensions: {} +processors: {} +receivers: + otlp: + protocols: + grpc: + auth: null + dialer: + timeout: 0s + endpoint: 0.0.0.0:4317 + include_metadata: false + keepalive: null + max_concurrent_streams: 0 + max_recv_msg_size_mib: 0 + read_buffer_size: 524288 + tls: null + transport: tcp + write_buffer_size: 0 + http: + auth: null + compression_algorithms: [] + cors: null + endpoint: 0.0.0.0:4318 + idle_timeout: 0s + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + read_header_timeout: 0s + read_timeout: 0s + response_headers: {} + tls: null + traces_url_path: /v1/traces + write_timeout: 0s +service: + extensions: [] + pipelines: + logs: + exporters: + - debug + processors: [] + receivers: + - otlp + metrics: + exporters: + - debug + processors: [] + receivers: + - otlp + traces: + exporters: + - debug + processors: [] + receivers: + - otlp + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + error_output_paths: + - stderr + initial_fields: {} + level: info + output_paths: + - stderr + processors: [] + sampling: + enabled: true + initial: 10 + thereafter: 100 + tick: 10s + metrics: + address: "" + level: Normal + readers: + - pull: + exporter: + additionalproperties: null + prometheus: + host: localhost + port: 8888 + resource: {} + traces: + level: Basic + processors: [] + propagators: [] \ No newline at end of file diff --git a/test/new-e2e/tests/otel/otel-agent/testdata/minimal-sources.json b/test/new-e2e/tests/otel/otel-agent/testdata/sources.json similarity index 100% rename from test/new-e2e/tests/otel/otel-agent/testdata/minimal-sources.json rename to test/new-e2e/tests/otel/otel-agent/testdata/sources.json diff --git a/test/new-e2e/tests/otel/otlp-ingest/operation_and_resource_name_logic_v2_test.go b/test/new-e2e/tests/otel/otlp-ingest/operation_and_resource_name_logic_v2_test.go index 89a30686706be..dff8498f51fa0 100644 --- a/test/new-e2e/tests/otel/otlp-ingest/operation_and_resource_name_logic_v2_test.go +++ b/test/new-e2e/tests/otel/otlp-ingest/operation_and_resource_name_logic_v2_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otlp-ingest/pipelines_sampling_test.go b/test/new-e2e/tests/otel/otlp-ingest/pipelines_sampling_test.go index 62b882e4def47..acbf90ad5a02a 100644 --- a/test/new-e2e/tests/otel/otlp-ingest/pipelines_sampling_test.go +++ b/test/new-e2e/tests/otel/otlp-ingest/pipelines_sampling_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otlp-ingest/pipelines_test.go b/test/new-e2e/tests/otel/otlp-ingest/pipelines_test.go index eb1e879491051..9be18d44c52e3 100644 --- a/test/new-e2e/tests/otel/otlp-ingest/pipelines_test.go +++ b/test/new-e2e/tests/otel/otlp-ingest/pipelines_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go b/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go index 5f83e4dba06cf..74f5bcfa679d0 100644 --- a/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go +++ b/test/new-e2e/tests/otel/otlp-ingest/receive_resource_spans_v2_test.go @@ -12,7 +12,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" "github.com/DataDog/datadog-agent/test/new-e2e/tests/otel/utils" ) diff --git a/test/new-e2e/tests/otel/utils/config_utils.go b/test/new-e2e/tests/otel/utils/config_utils.go index ddee2793648fa..3d10d5267bc2e 100644 --- a/test/new-e2e/tests/otel/utils/config_utils.go +++ b/test/new-e2e/tests/otel/utils/config_utils.go @@ -211,13 +211,15 @@ func validateConfigs(t *testing.T, expectedCfg string, actualCfg string) { // Traces, metrics and logs endpoints are set dynamically to the fake intake address in the config // These endpoints vary from test to test and should be ignored in the comparison exps, _ := actualConfRaw["exporters"].(map[string]any) - ddExp, _ := exps["datadog"].(map[string]any) - tcfg := ddExp["traces"].(map[string]any) - delete(tcfg, "endpoint") - mcfg := ddExp["metrics"].(map[string]any) - delete(mcfg, "endpoint") - lcfg := ddExp["logs"].(map[string]any) - delete(lcfg, "endpoint") + if ddExp, ok := exps["datadog"]; ok { + ddExpCfg := ddExp.(map[string]any) + tcfg := ddExpCfg["traces"].(map[string]any) + delete(tcfg, "endpoint") + mcfg := ddExpCfg["metrics"].(map[string]any) + delete(mcfg, "endpoint") + lcfg := ddExpCfg["logs"].(map[string]any) + delete(lcfg, "endpoint") + } actualCfgBytes, err := yaml.Marshal(actualConfRaw) require.NoError(t, err) diff --git a/test/new-e2e/tests/process/docker_test.go b/test/new-e2e/tests/process/docker_test.go index c33a48f0a7473..536529b180750 100644 --- a/test/new-e2e/tests/process/docker_test.go +++ b/test/new-e2e/tests/process/docker_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/docker" + awsdocker "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/docker" ) type dockerTestSuite struct { diff --git a/test/new-e2e/tests/process/ecs_test.go b/test/new-e2e/tests/process/ecs_test.go index 45881701fde06..a1c1e3d1b9e44 100644 --- a/test/new-e2e/tests/process/ecs_test.go +++ b/test/new-e2e/tests/process/ecs_test.go @@ -21,9 +21,10 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/ecs" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/ecs" ) type ECSEC2Suite struct { @@ -34,7 +35,7 @@ type ecsCPUStressEnv struct { environments.ECS } -func ecsEC2CPUStressProvisioner(runInCoreAgent bool) e2e.PulumiEnvRunFunc[ecsCPUStressEnv] { +func ecsEC2CPUStressProvisioner(runInCoreAgent bool) provisioners.PulumiEnvRunFunc[ecsCPUStressEnv] { return func(ctx *pulumi.Context, env *ecsCPUStressEnv) error { awsEnv, err := aws.NewEnvironment(ctx) if err != nil { @@ -65,7 +66,7 @@ func TestECSEC2TestSuite(t *testing.T) { t.Parallel() s := ECSEC2Suite{} e2eParams := []e2e.SuiteOption{e2e.WithProvisioner( - e2e.NewTypedPulumiProvisioner("ecsEC2CPUStress", ecsEC2CPUStressProvisioner(false), nil))} + provisioners.NewTypedPulumiProvisioner("ecsEC2CPUStress", ecsEC2CPUStressProvisioner(false), nil))} e2e.Run(t, &s, e2eParams...) } @@ -98,7 +99,7 @@ func TestECSEC2CoreAgentSuite(t *testing.T) { t.Parallel() s := ECSEC2CoreAgentSuite{} e2eParams := []e2e.SuiteOption{e2e.WithProvisioner( - e2e.NewTypedPulumiProvisioner("ecsEC2CoreAgentCPUStress", ecsEC2CPUStressProvisioner(true), nil))} + provisioners.NewTypedPulumiProvisioner("ecsEC2CoreAgentCPUStress", ecsEC2CPUStressProvisioner(true), nil))} e2e.Run(t, &s, e2eParams...) } diff --git a/test/new-e2e/tests/process/fargate_test.go b/test/new-e2e/tests/process/fargate_test.go index 695500ca7a8fe..11aa844f1170a 100644 --- a/test/new-e2e/tests/process/fargate_test.go +++ b/test/new-e2e/tests/process/fargate_test.go @@ -15,6 +15,7 @@ import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/stretchr/testify/assert" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" @@ -25,14 +26,14 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/ecs" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/ecs" ) type ECSFargateSuite struct { e2e.BaseSuite[environments.ECS] } -func getFargateProvisioner(configMap runner.ConfigMap) e2e.TypedProvisioner[environments.ECS] { +func getFargateProvisioner(configMap runner.ConfigMap) provisioners.TypedProvisioner[environments.ECS] { return ecs.Provisioner( ecs.WithECSOptions(tifEcs.WithFargateCapacityProvider()), ecs.WithFargateWorkloadApp(func(e aws.Environment, clusterArn pulumi.StringInput, apiKeySSMParamName pulumi.StringInput, fakeIntake *fakeintakeComp.Fakeintake) (*ecsComp.Workload, error) { diff --git a/test/new-e2e/tests/process/k8s_test.go b/test/new-e2e/tests/process/k8s_test.go index 7ec7ccff16888..8d72e17fb535f 100644 --- a/test/new-e2e/tests/process/k8s_test.go +++ b/test/new-e2e/tests/process/k8s_test.go @@ -30,7 +30,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/kubernetes" + awskubernetes "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/kubernetes" ) // helmTemplate define the embedded minimal configuration for NPM diff --git a/test/new-e2e/tests/process/linux_test.go b/test/new-e2e/tests/process/linux_test.go index 7b5e093d5bf83..1934d33d6cbd6 100644 --- a/test/new-e2e/tests/process/linux_test.go +++ b/test/new-e2e/tests/process/linux_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type linuxTestSuite struct { @@ -117,7 +117,7 @@ func (s *linuxTestSuite) TestProcessChecksInCoreAgent() { // Verify that the process agent is not running assert.EventuallyWithT(t, func(c *assert.CollectT) { - status := s.Env().RemoteHost.MustExecute("/opt/datadog-agent/embedded/bin/process-agent status") + status := s.Env().RemoteHost.MustExecute("sudo /opt/datadog-agent/embedded/bin/process-agent status") assert.Contains(c, status, "The Process Agent is not running") }, 1*time.Minute, 5*time.Second) diff --git a/test/new-e2e/tests/process/windows_test.go b/test/new-e2e/tests/process/windows_test.go index 5c7f7ef84711b..ba27cfc014703 100644 --- a/test/new-e2e/tests/process/windows_test.go +++ b/test/new-e2e/tests/process/windows_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/fakeintake/aggregator" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type windowsTestSuite struct { diff --git a/test/new-e2e/tests/remote-config/rc_ssl_config_test.go b/test/new-e2e/tests/remote-config/rc_ssl_config_test.go index 6fadad1dc47ff..b542f5fb18e36 100644 --- a/test/new-e2e/tests/remote-config/rc_ssl_config_test.go +++ b/test/new-e2e/tests/remote-config/rc_ssl_config_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type sslConfigSuite struct { diff --git a/test/new-e2e/tests/remote-config/tracer_test.go b/test/new-e2e/tests/remote-config/tracer_test.go index 6abc19e49c0ad..51150329e3caa 100644 --- a/test/new-e2e/tests/remote-config/tracer_test.go +++ b/test/new-e2e/tests/remote-config/tracer_test.go @@ -16,7 +16,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" ) type tracerSuite struct { diff --git a/test/new-e2e/tests/security-agent-functional/security_agent_test.go b/test/new-e2e/tests/security-agent-functional/security_agent_test.go index 1b693e1b43b21..81f737f39f953 100644 --- a/test/new-e2e/tests/security-agent-functional/security_agent_test.go +++ b/test/new-e2e/tests/security-agent-functional/security_agent_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/util/testutil/flake" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" ) diff --git a/test/new-e2e/tests/sysprobe-functional/apmtags_test.go b/test/new-e2e/tests/sysprobe-functional/apmtags_test.go index 3aff8ab90c5ae..7895154561a10 100644 --- a/test/new-e2e/tests/sysprobe-functional/apmtags_test.go +++ b/test/new-e2e/tests/sysprobe-functional/apmtags_test.go @@ -23,7 +23,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" ) diff --git a/test/new-e2e/tests/sysprobe-functional/sysprobe_test.go b/test/new-e2e/tests/sysprobe-functional/sysprobe_test.go index 888f9427e4413..710f3541de0fc 100644 --- a/test/new-e2e/tests/sysprobe-functional/sysprobe_test.go +++ b/test/new-e2e/tests/sysprobe-functional/sysprobe_test.go @@ -14,14 +14,15 @@ import ( "testing" "time" + componentsos "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/stretchr/testify/require" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" - componentsos "github.com/DataDog/test-infra-definitions/components/os" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" - "github.com/stretchr/testify/require" ) type vmSuite struct { diff --git a/test/new-e2e/tests/windows/domain-test/domain_test.go b/test/new-e2e/tests/windows/domain-test/domain_test.go index 134dc68d30683..47c779edf3acb 100644 --- a/test/new-e2e/tests/windows/domain-test/domain_test.go +++ b/test/new-e2e/tests/windows/domain-test/domain_test.go @@ -14,7 +14,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/activedirectory" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" "github.com/stretchr/testify/assert" diff --git a/test/new-e2e/tests/windows/fips-test/fips_test.go b/test/new-e2e/tests/windows/fips-test/fips_test.go index e46be55c5a941..48313099eed29 100644 --- a/test/new-e2e/tests/windows/fips-test/fips_test.go +++ b/test/new-e2e/tests/windows/fips-test/fips_test.go @@ -13,7 +13,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" diff --git a/test/new-e2e/tests/windows/install-test/base.go b/test/new-e2e/tests/windows/install-test/base.go index 566ee7c6225cc..958c171160f05 100644 --- a/test/new-e2e/tests/windows/install-test/base.go +++ b/test/new-e2e/tests/windows/install-test/base.go @@ -15,7 +15,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/runner/parameters" "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows" diff --git a/test/new-e2e/tests/windows/powershell-module-test/install_test.go b/test/new-e2e/tests/windows/powershell-module-test/install_test.go index 1e289c567cb02..1bf62393b9e48 100644 --- a/test/new-e2e/tests/windows/powershell-module-test/install_test.go +++ b/test/new-e2e/tests/windows/powershell-module-test/install_test.go @@ -14,14 +14,15 @@ import ( "testing" "time" + "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" + "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host" + awshost "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host" "github.com/DataDog/datadog-agent/test/new-e2e/tests/agent-platform/common" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" - "github.com/DataDog/test-infra-definitions/components/os" - "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" ) var ( diff --git a/test/new-e2e/tests/windows/service-test/startstop_test.go b/test/new-e2e/tests/windows/service-test/startstop_test.go index 837766c78f5e7..d4c3090f3d69d 100644 --- a/test/new-e2e/tests/windows/service-test/startstop_test.go +++ b/test/new-e2e/tests/windows/service-test/startstop_test.go @@ -19,7 +19,7 @@ import ( "github.com/DataDog/datadog-agent/test/new-e2e/pkg/components" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments" - awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/environments/aws/host/windows" + awsHostWindows "github.com/DataDog/datadog-agent/test/new-e2e/pkg/provisioners/aws/host/windows" "github.com/DataDog/datadog-agent/test/new-e2e/pkg/utils/e2e/client/agentclientparams" windowsCommon "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common" windowsAgent "github.com/DataDog/datadog-agent/test/new-e2e/tests/windows/common/agent" diff --git a/test/otel/go.mod b/test/otel/go.mod index a4523c0b33b0b..7689db77e10e6 100644 --- a/test/otel/go.mod +++ b/test/otel/go.mod @@ -12,6 +12,7 @@ replace ( github.com/DataDog/datadog-agent/comp/core/log/mock => ./../../comp/core/log/mock github.com/DataDog/datadog-agent/comp/core/secrets => ./../../comp/core/secrets github.com/DataDog/datadog-agent/comp/core/status => ../../comp/core/status + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection => ../../comp/core/tagger/origindetection github.com/DataDog/datadog-agent/comp/core/telemetry => ./../../comp/core/telemetry github.com/DataDog/datadog-agent/comp/def => ./../../comp/def github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder => ../../comp/forwarder/defaultforwarder @@ -129,6 +130,7 @@ require ( github.com/DataDog/datadog-agent/comp/core/flare/builder v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/flare/types v0.56.0-rc.3 // indirect github.com/DataDog/datadog-agent/comp/core/secrets v0.59.0 // indirect + github.com/DataDog/datadog-agent/comp/core/tagger/origindetection v0.0.0-20241217122454-175edb6c74f2 // indirect github.com/DataDog/datadog-agent/comp/core/telemetry v0.57.1 // indirect github.com/DataDog/datadog-agent/comp/def v0.59.0 // indirect github.com/DataDog/datadog-agent/comp/logs/agent/config v0.56.0-rc.3 // indirect @@ -292,7 +294,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/test/otel/go.sum b/test/otel/go.sum index bce88ff15bfa8..a996d7f3007b6 100644 --- a/test/otel/go.sum +++ b/test/otel/go.sum @@ -516,8 +516,8 @@ golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= diff --git a/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml index b445834c940d6..0d3dba0eb57f2 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 30g + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml index c51505add2bb6..0d3dba0eb57f2 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http1/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 1.5GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml b/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml index c51505add2bb6..0d3dba0eb57f2 100644 --- a/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_0ms_latency_http2/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 1.5GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml index ccca15933f243..a1b83cb9c0a98 100644 --- a/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_1000ms_latency/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_1000ms_latency_linear_load/experiment.yaml b/test/regression/cases/file_to_blackhole_1000ms_latency_linear_load/experiment.yaml index 215dcfa86cc48..b252bbe75d094 100644 --- a/test/regression/cases/file_to_blackhole_1000ms_latency_linear_load/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_1000ms_latency_linear_load/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 30g + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml index 68432ca735b1c..1db5b405767ff 100644 --- a/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_100ms_latency/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml b/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml index 68432ca735b1c..1db5b405767ff 100644 --- a/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml +++ b/test/regression/cases/file_to_blackhole_500ms_latency/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/file_tree/experiment.yaml b/test/regression/cases/file_tree/experiment.yaml index 6c67c51ef14a5..1edff05390f25 100644 --- a/test/regression/cases/file_tree/experiment.yaml +++ b/test/regression/cases/file_tree/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/otel_to_otel_logs/experiment.yaml b/test/regression/cases/otel_to_otel_logs/experiment.yaml index dfbe54658435f..e4b260015988f 100644 --- a/test/regression/cases/otel_to_otel_logs/experiment.yaml +++ b/test/regression/cases/otel_to_otel_logs/experiment.yaml @@ -3,7 +3,6 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh cpu_allotment: 8 memory_allotment: 24GiB diff --git a/test/regression/cases/quality_gate_idle/experiment.yaml b/test/regression/cases/quality_gate_idle/experiment.yaml index 1c120a0d23187..d17ce5548c2fe 100644 --- a/test/regression/cases/quality_gate_idle/experiment.yaml +++ b/test/regression/cases/quality_gate_idle/experiment.yaml @@ -8,8 +8,7 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 + cpu_allotment: 4 memory_allotment: 500MiB environment: @@ -36,7 +35,7 @@ checks: description: "Memory usage quality gate. This puts a bound on the total agent memory usage." bounds: series: total_rss_bytes - upper_bound: "365.0 MiB" + upper_bound: "372.0 MiB" report_links: - text: "bounds checks dashboard" diff --git a/test/regression/cases/quality_gate_idle_all_features/experiment.yaml b/test/regression/cases/quality_gate_idle_all_features/experiment.yaml index 2b16f4988c0b0..34e8e63fd4cf5 100644 --- a/test/regression/cases/quality_gate_idle_all_features/experiment.yaml +++ b/test/regression/cases/quality_gate_idle_all_features/experiment.yaml @@ -8,8 +8,7 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 + cpu_allotment: 4 memory_allotment: 1GiB environment: @@ -47,7 +46,7 @@ checks: description: "Memory usage quality gate. This puts a bound on the total agent memory usage." bounds: series: total_rss_bytes - upper_bound: "755.0 MiB" + upper_bound: "779.0 MiB" report_links: - text: "bounds checks dashboard" diff --git a/test/regression/cases/quality_gate_logs/experiment.yaml b/test/regression/cases/quality_gate_logs/experiment.yaml index 761cae4678a8e..182f50e8679c4 100644 --- a/test/regression/cases/quality_gate_logs/experiment.yaml +++ b/test/regression/cases/quality_gate_logs/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 500MiB environment: DD_API_KEY: 00000001 @@ -29,7 +28,7 @@ checks: description: "Memory usage" bounds: series: total_rss_bytes - upper_bound: 420MiB + upper_bound: 429MiB - name: lost_bytes description: "Allowable bytes not polled by log Agent" diff --git a/test/regression/cases/tcp_dd_logs_filter_exclude/experiment.yaml b/test/regression/cases/tcp_dd_logs_filter_exclude/experiment.yaml index 17a5adb011dd1..461bbc607eb8b 100644 --- a/test/regression/cases/tcp_dd_logs_filter_exclude/experiment.yaml +++ b/test/regression/cases/tcp_dd_logs_filter_exclude/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/tcp_syslog_to_blackhole/experiment.yaml b/test/regression/cases/tcp_syslog_to_blackhole/experiment.yaml index f9168734cf9e7..5de7351418822 100644 --- a/test/regression/cases/tcp_syslog_to_blackhole/experiment.yaml +++ b/test/regression/cases/tcp_syslog_to_blackhole/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/uds_dogstatsd_to_api/experiment.yaml b/test/regression/cases/uds_dogstatsd_to_api/experiment.yaml index fe93432493ecf..b8e40dac08462 100644 --- a/test/regression/cases/uds_dogstatsd_to_api/experiment.yaml +++ b/test/regression/cases/uds_dogstatsd_to_api/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/cases/uds_dogstatsd_to_api_cpu/experiment.yaml b/test/regression/cases/uds_dogstatsd_to_api_cpu/experiment.yaml index ccb9f7b4e4866..c51909b2f72ee 100644 --- a/test/regression/cases/uds_dogstatsd_to_api_cpu/experiment.yaml +++ b/test/regression/cases/uds_dogstatsd_to_api_cpu/experiment.yaml @@ -3,9 +3,8 @@ erratic: false target: name: datadog-agent - command: /bin/entrypoint.sh - cpu_allotment: 8 - memory_allotment: 4GiB + cpu_allotment: 4 + memory_allotment: 2GiB environment: DD_API_KEY: 00000001 diff --git a/test/regression/config.yaml b/test/regression/config.yaml index 362301357f864..7a9684510426a 100644 --- a/test/regression/config.yaml +++ b/test/regression/config.yaml @@ -1,9 +1,7 @@ lading: - version: 0.24.0 + version: 0.25.2 target: - cpu_allotment: 8 - memory_allotment: 30g # Link templates for reports. #